meshagent-api 0.56.2__tar.gz → 0.57.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {meshagent_api-0.56.2/meshagent_api.egg-info → meshagent_api-0.57.1}/PKG-INFO +1 -1
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/client.py +48 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/files.py +7 -1
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/files_test.py +31 -2
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/room_server_client.py +21 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/room_server_client_test.py +26 -0
- meshagent_api-0.57.1/meshagent/api/storage_connections.py +196 -0
- meshagent_api-0.57.1/meshagent/api/storage_connections_test.py +127 -0
- meshagent_api-0.57.1/meshagent/api/version.py +1 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1/meshagent_api.egg-info}/PKG-INFO +1 -1
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent_api.egg-info/SOURCES.txt +2 -0
- meshagent_api-0.56.2/meshagent/api/version.py +0 -1
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/LICENSE +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/MANIFEST.in +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/README.md +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/__init__.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/agent_content.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/aiohttp_redaction.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/chan.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/client_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/crdt.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/dataset_schema.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/entrypoint.js +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/error_codes.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/file_permissions.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/helpers.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/http.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/image_runtime.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/keys.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/llm_router_config.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/managed_agents.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/managed_agents_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/messaging.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/messaging_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/mime_types.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/mime_types_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/oauth.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/oauth_branding.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/oauth_scopes.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/oauth_scopes_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/participant.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/participant_token.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/participant_token_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/port_forward.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/port_forward_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/protocol.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/protocol_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/py.typed +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/reasoning_schema.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/registry_auth.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/room_ports.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/runtime.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/runtime_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/schema.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/schema_document.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/schema_document_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/schema_registry.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/schema_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/schema_util.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/service_spec_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/service_template_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/services.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/services_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/specs/service.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/sql.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/sql_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/sqlite_batch_integration_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/sqlite_replication_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/sqlite_transfer.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/sqlite_transfer_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/storage_policy_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/token_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/urls.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/webhooks.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/webhooks_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/websocket_protocol.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/websocket_protocol_test.py +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent_api.egg-info/dependency_links.txt +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent_api.egg-info/requires.txt +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent_api.egg-info/top_level.txt +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/pyproject.toml +0 -0
- {meshagent_api-0.56.2 → meshagent_api-0.57.1}/setup.cfg +0 -0
|
@@ -38,6 +38,12 @@ from meshagent.api.specs.service import (
|
|
|
38
38
|
ServiceTemplateSpec,
|
|
39
39
|
)
|
|
40
40
|
from meshagent.api.managed_agents import ManagedAgentSpec
|
|
41
|
+
from meshagent.api.storage_connections import (
|
|
42
|
+
CreateStorageConnection,
|
|
43
|
+
UpdateStorageConnection,
|
|
44
|
+
StorageConnection,
|
|
45
|
+
StorageConnectionsResponse,
|
|
46
|
+
)
|
|
41
47
|
import os
|
|
42
48
|
|
|
43
49
|
|
|
@@ -4668,6 +4674,48 @@ class Meshagent:
|
|
|
4668
4674
|
async with self._session.delete(url, headers=self._get_headers()) as resp:
|
|
4669
4675
|
await self._raise_for_status(resp)
|
|
4670
4676
|
|
|
4677
|
+
async def list_room_storage_connections(
|
|
4678
|
+
self, *, project_id: str, room: str
|
|
4679
|
+
) -> list[StorageConnection]:
|
|
4680
|
+
url = f"{self.base_url}/accounts/projects/{project_id}/rooms/{room}/storage-connections"
|
|
4681
|
+
async with self._session.get(url, headers=self._get_headers()) as resp:
|
|
4682
|
+
await self._raise_for_status(resp)
|
|
4683
|
+
return (
|
|
4684
|
+
await self._read_model(resp, StorageConnectionsResponse)
|
|
4685
|
+
).connections
|
|
4686
|
+
|
|
4687
|
+
async def create_room_storage_connection(
|
|
4688
|
+
self, *, project_id: str, room: str, connection: CreateStorageConnection
|
|
4689
|
+
) -> StorageConnection:
|
|
4690
|
+
url = f"{self.base_url}/accounts/projects/{project_id}/rooms/{room}/storage-connections"
|
|
4691
|
+
async with self._session.post(
|
|
4692
|
+
url, headers=self._get_headers(), json=connection.model_dump(mode="json")
|
|
4693
|
+
) as resp:
|
|
4694
|
+
await self._raise_for_status(resp)
|
|
4695
|
+
return await self._read_model(resp, StorageConnection)
|
|
4696
|
+
|
|
4697
|
+
async def update_room_storage_connection(
|
|
4698
|
+
self,
|
|
4699
|
+
*,
|
|
4700
|
+
project_id: str,
|
|
4701
|
+
room: str,
|
|
4702
|
+
connection_id: str,
|
|
4703
|
+
connection: UpdateStorageConnection,
|
|
4704
|
+
) -> StorageConnection:
|
|
4705
|
+
url = f"{self.base_url}/accounts/projects/{project_id}/rooms/{room}/storage-connections/{connection_id}"
|
|
4706
|
+
async with self._session.put(
|
|
4707
|
+
url, headers=self._get_headers(), json=connection.model_dump(mode="json")
|
|
4708
|
+
) as resp:
|
|
4709
|
+
await self._raise_for_status(resp)
|
|
4710
|
+
return await self._read_model(resp, StorageConnection)
|
|
4711
|
+
|
|
4712
|
+
async def delete_room_storage_connection(
|
|
4713
|
+
self, *, project_id: str, room: str, connection_id: str
|
|
4714
|
+
) -> None:
|
|
4715
|
+
url = f"{self.base_url}/accounts/projects/{project_id}/rooms/{room}/storage-connections/{connection_id}"
|
|
4716
|
+
async with self._session.delete(url, headers=self._get_headers()) as resp:
|
|
4717
|
+
await self._raise_for_status(resp)
|
|
4718
|
+
|
|
4671
4719
|
async def list_room_volumes(
|
|
4672
4720
|
self,
|
|
4673
4721
|
*,
|
|
@@ -261,12 +261,18 @@ class TargetFilesClient:
|
|
|
261
261
|
*,
|
|
262
262
|
content_type: str = "application/octet-stream",
|
|
263
263
|
overwrite: bool = True,
|
|
264
|
+
if_match: str | None = None,
|
|
264
265
|
) -> None:
|
|
266
|
+
if if_match is not None and not if_match.strip():
|
|
267
|
+
raise ValueError("If-Match ETag must not be empty")
|
|
265
268
|
client = self._files._client
|
|
269
|
+
headers = {**client._get_headers(), "Content-Type": content_type}
|
|
270
|
+
if if_match is not None:
|
|
271
|
+
headers["If-Match"] = if_match
|
|
266
272
|
async with client._session.put(
|
|
267
273
|
await self._files._url(self.target, path),
|
|
268
274
|
params={"overwrite": str(overwrite).lower()},
|
|
269
|
-
headers=
|
|
275
|
+
headers=headers,
|
|
270
276
|
data=data,
|
|
271
277
|
allow_redirects=False,
|
|
272
278
|
) as response:
|
|
@@ -19,6 +19,7 @@ class FilesApiFixture:
|
|
|
19
19
|
calls: list[tuple[str, str, dict[str, str], str]] = field(default_factory=list)
|
|
20
20
|
blobs: dict[str, bytes] = field(default_factory=dict)
|
|
21
21
|
actions: list[dict] = field(default_factory=list)
|
|
22
|
+
etags: dict[str, int] = field(default_factory=dict)
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
@pytest_asyncio.fixture
|
|
@@ -73,10 +74,23 @@ async def files_api() -> AsyncIterator[FilesApiFixture]:
|
|
|
73
74
|
state.actions.append(await request.json())
|
|
74
75
|
return web.json_response({})
|
|
75
76
|
if request.method == "PUT":
|
|
77
|
+
current = state.etags.get(request.path)
|
|
78
|
+
if (
|
|
79
|
+
request.headers.get("If-Match") is not None
|
|
80
|
+
and request.headers["If-Match"] != f'"v{current}"'
|
|
81
|
+
):
|
|
82
|
+
return web.Response(status=412)
|
|
83
|
+
if (
|
|
84
|
+
not request.query.get("overwrite", "true") == "true"
|
|
85
|
+
and current is not None
|
|
86
|
+
):
|
|
87
|
+
return web.Response(status=409)
|
|
76
88
|
state.blobs[request.path] = await request.read()
|
|
89
|
+
state.etags[request.path] = (current or 0) + 1
|
|
77
90
|
return web.Response(status=204)
|
|
78
91
|
if request.method == "DELETE":
|
|
79
92
|
state.blobs.pop(request.path, None)
|
|
93
|
+
state.etags.pop(request.path, None)
|
|
80
94
|
return web.Response(status=204)
|
|
81
95
|
data = state.blobs.get(request.path)
|
|
82
96
|
if data is None:
|
|
@@ -84,7 +98,7 @@ async def files_api() -> AsyncIterator[FilesApiFixture]:
|
|
|
84
98
|
return web.Response(
|
|
85
99
|
body=data,
|
|
86
100
|
content_type="application/octet-stream",
|
|
87
|
-
headers={"ETag": '"
|
|
101
|
+
headers={"ETag": f'"v{state.etags[request.path]}"'},
|
|
88
102
|
)
|
|
89
103
|
|
|
90
104
|
app.router.add_get("/config", config)
|
|
@@ -126,7 +140,7 @@ async def test_room_agent_streams_share_discovery_session_and_current_credential
|
|
|
126
140
|
)
|
|
127
141
|
metadata = await room.stat(path)
|
|
128
142
|
assert metadata.size == len(b"\x00\xfffirstsecond")
|
|
129
|
-
assert metadata.etag == '"
|
|
143
|
+
assert metadata.etag == '"v1"'
|
|
130
144
|
assert await agent.exists(path)
|
|
131
145
|
assert not await agent.exists("missing")
|
|
132
146
|
client.token = "rotated-token"
|
|
@@ -233,3 +247,18 @@ async def test_invalid_paths_fail_before_discovery(files_api):
|
|
|
233
247
|
def test_target_cannot_escape_namespace(name):
|
|
234
248
|
with pytest.raises(ValidationError):
|
|
235
249
|
FilesTarget(project_id="p1", kind="agent", name=name)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
@pytest.mark.asyncio
|
|
253
|
+
async def test_conditional_upload_rejects_stale_etag(files_api):
|
|
254
|
+
async with Meshagent(base_url=files_api.base_url, token="test-token") as client:
|
|
255
|
+
files = client.files.for_room(project_id="p", room_name="r")
|
|
256
|
+
await files.upload("workbook", b"first", overwrite=False)
|
|
257
|
+
etag = (await files.stat("workbook")).etag
|
|
258
|
+
assert etag == '"v1"'
|
|
259
|
+
await files.upload("workbook", b"second", if_match=etag)
|
|
260
|
+
with pytest.raises(RoomException):
|
|
261
|
+
await files.upload("workbook", b"stale", if_match=etag)
|
|
262
|
+
assert await files.download("workbook") == b"second"
|
|
263
|
+
with pytest.raises(ValueError):
|
|
264
|
+
await files.upload("workbook", b"invalid", if_match=" ")
|
|
@@ -7495,6 +7495,27 @@ class DatasetsClient:
|
|
|
7495
7495
|
input_stream=input_stream,
|
|
7496
7496
|
)
|
|
7497
7497
|
|
|
7498
|
+
async def register_table(
|
|
7499
|
+
self,
|
|
7500
|
+
*,
|
|
7501
|
+
name: str,
|
|
7502
|
+
connection_id: str,
|
|
7503
|
+
path: str,
|
|
7504
|
+
namespace: Optional[list[str]] = None,
|
|
7505
|
+
branch: Optional[str] = None,
|
|
7506
|
+
) -> None:
|
|
7507
|
+
"""Register an existing, read-only Lance dataset from room external storage."""
|
|
7508
|
+
await self._invoke(
|
|
7509
|
+
operation="register_table",
|
|
7510
|
+
input={
|
|
7511
|
+
"name": name,
|
|
7512
|
+
"connection_id": connection_id,
|
|
7513
|
+
"path": path,
|
|
7514
|
+
"namespace": namespace,
|
|
7515
|
+
"branch": branch,
|
|
7516
|
+
},
|
|
7517
|
+
)
|
|
7518
|
+
|
|
7498
7519
|
async def drop_table(
|
|
7499
7520
|
self,
|
|
7500
7521
|
*,
|
|
@@ -6621,3 +6621,29 @@ async def test_sqlite_batch_rejects_malformed_results(payload):
|
|
|
6621
6621
|
database="app", commands=[room_server_client.SqliteSqlCommand("SELECT 1")]
|
|
6622
6622
|
)
|
|
6623
6623
|
assert error.value.code == ErrorCode.UNEXPECTED_RESPONSE_TYPE
|
|
6624
|
+
|
|
6625
|
+
|
|
6626
|
+
@pytest.mark.asyncio
|
|
6627
|
+
async def test_datasets_client_registers_external_source_without_credentials():
|
|
6628
|
+
from unittest.mock import AsyncMock
|
|
6629
|
+
|
|
6630
|
+
room = SimpleNamespace(invoke=AsyncMock(return_value=EmptyContent()))
|
|
6631
|
+
client = room_server_client.DatasetsClient(room)
|
|
6632
|
+
await client.register_table(
|
|
6633
|
+
name="events",
|
|
6634
|
+
connection_id="connection-id",
|
|
6635
|
+
path="events.lance",
|
|
6636
|
+
namespace=["analytics"],
|
|
6637
|
+
branch="experiment",
|
|
6638
|
+
)
|
|
6639
|
+
room.invoke.assert_awaited_once_with(
|
|
6640
|
+
toolkit="dataset",
|
|
6641
|
+
tool="register_table",
|
|
6642
|
+
input={
|
|
6643
|
+
"name": "events",
|
|
6644
|
+
"connection_id": "connection-id",
|
|
6645
|
+
"path": "events.lance",
|
|
6646
|
+
"namespace": ["analytics"],
|
|
6647
|
+
"branch": "experiment",
|
|
6648
|
+
},
|
|
6649
|
+
)
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Room-owned external storage configuration and credential wire models."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
import json
|
|
5
|
+
from typing import Annotated, Any, Literal
|
|
6
|
+
from uuid import UUID
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class _StorageModel(BaseModel):
|
|
12
|
+
model_config = ConfigDict(extra="forbid")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class _Location(_StorageModel):
|
|
16
|
+
anonymous: bool = False
|
|
17
|
+
prefix: str = ""
|
|
18
|
+
|
|
19
|
+
@field_validator("prefix")
|
|
20
|
+
@classmethod
|
|
21
|
+
def validate_prefix(cls, value: str) -> str:
|
|
22
|
+
if value and (
|
|
23
|
+
value.startswith("/")
|
|
24
|
+
or any(char in value for char in "\\:?#%")
|
|
25
|
+
or any(part in {"", ".", ".."} for part in value.split("/"))
|
|
26
|
+
or any(ord(char) < 32 for char in value)
|
|
27
|
+
):
|
|
28
|
+
raise ValueError("prefix must be a relative object storage path")
|
|
29
|
+
return value
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
StorageName = Annotated[str, Field(pattern=r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$")]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class GCSStorageConfig(_Location):
|
|
36
|
+
provider: Literal["gcs"] = "gcs"
|
|
37
|
+
bucket: StorageName
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class S3StorageConfig(_Location):
|
|
41
|
+
provider: Literal["s3"] = "s3"
|
|
42
|
+
bucket: StorageName
|
|
43
|
+
region: Annotated[str, Field(pattern=r"^[a-z0-9-]+$")]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class AzureStorageConfig(_Location):
|
|
47
|
+
provider: Literal["azure"] = "azure"
|
|
48
|
+
account: Annotated[str, Field(pattern=r"^[a-z0-9]{3,24}$")]
|
|
49
|
+
container: StorageName
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
StorageConfig = Annotated[
|
|
53
|
+
GCSStorageConfig | S3StorageConfig | AzureStorageConfig,
|
|
54
|
+
Field(discriminator="provider"),
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class GCSStorageCredentials(_StorageModel):
|
|
59
|
+
provider: Literal["gcs"] = "gcs"
|
|
60
|
+
service_account_json: dict[str, Any] = Field(repr=False)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class S3StorageCredentials(_StorageModel):
|
|
64
|
+
provider: Literal["s3"] = "s3"
|
|
65
|
+
access_key_id: str = Field(min_length=1, repr=False)
|
|
66
|
+
secret_access_key: str = Field(min_length=1, repr=False)
|
|
67
|
+
session_token: str | None = Field(default=None, repr=False)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class AzureStorageCredentials(_StorageModel):
|
|
71
|
+
provider: Literal["azure"] = "azure"
|
|
72
|
+
account_key: str | None = Field(default=None, min_length=1, repr=False)
|
|
73
|
+
sas_token: str | None = Field(default=None, min_length=1, repr=False)
|
|
74
|
+
|
|
75
|
+
@model_validator(mode="after")
|
|
76
|
+
def require_one_credential(self):
|
|
77
|
+
if (self.account_key is None) == (self.sas_token is None):
|
|
78
|
+
raise ValueError("provide exactly one of account_key or sas_token")
|
|
79
|
+
return self
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
StorageCredentials = Annotated[
|
|
83
|
+
GCSStorageCredentials | S3StorageCredentials | AzureStorageCredentials,
|
|
84
|
+
Field(discriminator="provider"),
|
|
85
|
+
]
|
|
86
|
+
ConnectionName = Annotated[
|
|
87
|
+
str, Field(pattern=r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class CreateStorageConnection(_StorageModel):
|
|
92
|
+
name: ConnectionName
|
|
93
|
+
config: StorageConfig
|
|
94
|
+
credentials: StorageCredentials | None = Field(default=None, repr=False)
|
|
95
|
+
|
|
96
|
+
@model_validator(mode="after")
|
|
97
|
+
def matching_provider(self):
|
|
98
|
+
if self.config.anonymous:
|
|
99
|
+
if self.credentials is not None:
|
|
100
|
+
raise ValueError("anonymous connections must not contain credentials")
|
|
101
|
+
elif self.credentials is None:
|
|
102
|
+
raise ValueError("credentials are required for authenticated connections")
|
|
103
|
+
elif self.config.provider != self.credentials.provider:
|
|
104
|
+
raise ValueError("configuration and credentials must use the same provider")
|
|
105
|
+
return self
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class UpdateStorageConnection(_StorageModel):
|
|
109
|
+
name: ConnectionName
|
|
110
|
+
config: StorageConfig
|
|
111
|
+
credentials: StorageCredentials | None = Field(default=None, repr=False)
|
|
112
|
+
|
|
113
|
+
@model_validator(mode="after")
|
|
114
|
+
def matching_provider(self):
|
|
115
|
+
if self.config.anonymous and self.credentials is not None:
|
|
116
|
+
raise ValueError("anonymous connections must not contain credentials")
|
|
117
|
+
if self.credentials and self.config.provider != self.credentials.provider:
|
|
118
|
+
raise ValueError("configuration and credentials must use the same provider")
|
|
119
|
+
return self
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class StorageConnection(_StorageModel):
|
|
123
|
+
id: UUID
|
|
124
|
+
room_id: UUID
|
|
125
|
+
name: str
|
|
126
|
+
config: StorageConfig
|
|
127
|
+
revision: int
|
|
128
|
+
created_at: datetime
|
|
129
|
+
updated_at: datetime
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class ResolvedStorageConnection(_StorageModel):
|
|
133
|
+
id: UUID
|
|
134
|
+
config: StorageConfig
|
|
135
|
+
revision: int
|
|
136
|
+
credentials: StorageCredentials | None = Field(default=None, repr=False)
|
|
137
|
+
|
|
138
|
+
def location_and_options(self, path: str) -> tuple[str, dict[str, str]]:
|
|
139
|
+
validate_external_dataset_path(path)
|
|
140
|
+
options: dict[str, str] = {}
|
|
141
|
+
config, credentials = self.config, self.credentials
|
|
142
|
+
if config.anonymous:
|
|
143
|
+
if credentials is not None:
|
|
144
|
+
raise ValueError("anonymous connections must not contain credentials")
|
|
145
|
+
if isinstance(config, GCSStorageConfig):
|
|
146
|
+
root = f"gs://{config.bucket}"
|
|
147
|
+
options["google_skip_signature"] = "true"
|
|
148
|
+
elif isinstance(config, S3StorageConfig):
|
|
149
|
+
root = f"s3://{config.bucket}"
|
|
150
|
+
options.update(aws_region=config.region, aws_skip_signature="true")
|
|
151
|
+
else:
|
|
152
|
+
root = f"az://{config.container}"
|
|
153
|
+
options.update(
|
|
154
|
+
azure_storage_account_name=config.account,
|
|
155
|
+
azure_skip_signature="true",
|
|
156
|
+
)
|
|
157
|
+
elif isinstance(config, GCSStorageConfig) and isinstance(
|
|
158
|
+
credentials, GCSStorageCredentials
|
|
159
|
+
):
|
|
160
|
+
root = f"gs://{config.bucket}"
|
|
161
|
+
options["service_account_key"] = json.dumps(
|
|
162
|
+
credentials.service_account_json
|
|
163
|
+
)
|
|
164
|
+
elif isinstance(config, S3StorageConfig) and isinstance(
|
|
165
|
+
credentials, S3StorageCredentials
|
|
166
|
+
):
|
|
167
|
+
root = f"s3://{config.bucket}"
|
|
168
|
+
options.update(
|
|
169
|
+
aws_region=config.region,
|
|
170
|
+
aws_access_key_id=credentials.access_key_id,
|
|
171
|
+
aws_secret_access_key=credentials.secret_access_key,
|
|
172
|
+
)
|
|
173
|
+
if credentials.session_token is not None:
|
|
174
|
+
options["aws_session_token"] = credentials.session_token
|
|
175
|
+
elif isinstance(config, AzureStorageConfig) and isinstance(
|
|
176
|
+
credentials, AzureStorageCredentials
|
|
177
|
+
):
|
|
178
|
+
root = f"az://{config.container}"
|
|
179
|
+
options["azure_storage_account_name"] = config.account
|
|
180
|
+
if credentials.account_key is not None:
|
|
181
|
+
options["azure_storage_account_key"] = credentials.account_key
|
|
182
|
+
elif credentials.sas_token is not None:
|
|
183
|
+
options["azure_storage_sas_key"] = credentials.sas_token
|
|
184
|
+
else:
|
|
185
|
+
raise ValueError("storage connection credential provider mismatch")
|
|
186
|
+
return "/".join(part for part in (root, config.prefix, path) if part), options
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def validate_external_dataset_path(path: str) -> str:
|
|
190
|
+
if not path:
|
|
191
|
+
raise ValueError("external dataset path must not be empty")
|
|
192
|
+
return _Location.validate_prefix(path)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class StorageConnectionsResponse(_StorageModel):
|
|
196
|
+
connections: list[StorageConnection]
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from pydantic import ValidationError
|
|
3
|
+
|
|
4
|
+
from meshagent.api.storage_connections import CreateStorageConnection
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@pytest.mark.parametrize(
|
|
8
|
+
"prefix",
|
|
9
|
+
[
|
|
10
|
+
"../private",
|
|
11
|
+
"/absolute",
|
|
12
|
+
"a//b",
|
|
13
|
+
"a/../b",
|
|
14
|
+
"a%2fb",
|
|
15
|
+
"a?token=secret",
|
|
16
|
+
"gs://other-bucket",
|
|
17
|
+
],
|
|
18
|
+
)
|
|
19
|
+
def test_connection_rejects_unsafe_prefixes(prefix):
|
|
20
|
+
with pytest.raises(ValidationError):
|
|
21
|
+
CreateStorageConnection.model_validate(
|
|
22
|
+
{
|
|
23
|
+
"name": "analytics",
|
|
24
|
+
"config": {
|
|
25
|
+
"provider": "gcs",
|
|
26
|
+
"bucket": "test-bucket",
|
|
27
|
+
"prefix": prefix,
|
|
28
|
+
},
|
|
29
|
+
"credentials": {"provider": "gcs", "service_account_json": {}},
|
|
30
|
+
}
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_connection_rejects_mismatched_credentials():
|
|
35
|
+
with pytest.raises(ValidationError):
|
|
36
|
+
CreateStorageConnection.model_validate(
|
|
37
|
+
{
|
|
38
|
+
"name": "analytics",
|
|
39
|
+
"config": {"provider": "gcs", "bucket": "test-bucket"},
|
|
40
|
+
"credentials": {"provider": "azure", "account_key": "test-only"},
|
|
41
|
+
}
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@pytest.mark.parametrize(
|
|
46
|
+
"credentials", [{}, {"account_key": "test", "sas_token": "test"}]
|
|
47
|
+
)
|
|
48
|
+
def test_azure_requires_exactly_one_authentication_method(credentials):
|
|
49
|
+
with pytest.raises(ValidationError):
|
|
50
|
+
CreateStorageConnection.model_validate(
|
|
51
|
+
{
|
|
52
|
+
"name": "analytics",
|
|
53
|
+
"config": {
|
|
54
|
+
"provider": "azure",
|
|
55
|
+
"account": "testaccount",
|
|
56
|
+
"container": "test-container",
|
|
57
|
+
},
|
|
58
|
+
"credentials": {"provider": "azure", **credentials},
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_credential_repr_is_redacted():
|
|
64
|
+
connection = CreateStorageConnection.model_validate(
|
|
65
|
+
{
|
|
66
|
+
"name": "analytics",
|
|
67
|
+
"config": {
|
|
68
|
+
"provider": "s3",
|
|
69
|
+
"bucket": "test-bucket",
|
|
70
|
+
"region": "us-east-1",
|
|
71
|
+
},
|
|
72
|
+
"credentials": {
|
|
73
|
+
"provider": "s3",
|
|
74
|
+
"access_key_id": "test-key",
|
|
75
|
+
"secret_access_key": "never-display-this",
|
|
76
|
+
},
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
assert "never-display-this" not in repr(connection)
|
|
80
|
+
assert "never-display-this" not in repr(connection.credentials)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@pytest.mark.parametrize(
|
|
84
|
+
"config,expected_options",
|
|
85
|
+
[
|
|
86
|
+
(
|
|
87
|
+
{"provider": "gcs", "bucket": "test-bucket"},
|
|
88
|
+
{"google_skip_signature": "true"},
|
|
89
|
+
),
|
|
90
|
+
(
|
|
91
|
+
{"provider": "s3", "bucket": "test-bucket", "region": "us-east-1"},
|
|
92
|
+
{"aws_skip_signature": "true", "aws_region": "us-east-1"},
|
|
93
|
+
),
|
|
94
|
+
(
|
|
95
|
+
{
|
|
96
|
+
"provider": "azure",
|
|
97
|
+
"account": "testaccount",
|
|
98
|
+
"container": "test-container",
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"azure_skip_signature": "true",
|
|
102
|
+
"azure_storage_account_name": "testaccount",
|
|
103
|
+
},
|
|
104
|
+
),
|
|
105
|
+
],
|
|
106
|
+
)
|
|
107
|
+
def test_public_connections_explicitly_disable_signing(config, expected_options):
|
|
108
|
+
from uuid import uuid4
|
|
109
|
+
from meshagent.api.storage_connections import (
|
|
110
|
+
ResolvedStorageConnection,
|
|
111
|
+
UpdateStorageConnection,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
public = {**config, "anonymous": True}
|
|
115
|
+
connection = CreateStorageConnection(name="public", config=public)
|
|
116
|
+
assert connection.credentials is None
|
|
117
|
+
resolved = ResolvedStorageConnection(id=uuid4(), revision=1, config=public)
|
|
118
|
+
assert resolved.location_and_options("events.lance")[1] == expected_options
|
|
119
|
+
with pytest.raises(ValidationError):
|
|
120
|
+
CreateStorageConnection(name="private", config=config)
|
|
121
|
+
for model in (CreateStorageConnection, UpdateStorageConnection):
|
|
122
|
+
with pytest.raises(ValidationError):
|
|
123
|
+
model(
|
|
124
|
+
name="public",
|
|
125
|
+
config=public,
|
|
126
|
+
credentials={"provider": "gcs", "service_account_json": {}},
|
|
127
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.57.1"
|
|
@@ -61,6 +61,8 @@ meshagent/api/sqlite_batch_integration_test.py
|
|
|
61
61
|
meshagent/api/sqlite_replication_test.py
|
|
62
62
|
meshagent/api/sqlite_transfer.py
|
|
63
63
|
meshagent/api/sqlite_transfer_test.py
|
|
64
|
+
meshagent/api/storage_connections.py
|
|
65
|
+
meshagent/api/storage_connections_test.py
|
|
64
66
|
meshagent/api/storage_policy_test.py
|
|
65
67
|
meshagent/api/token_test.py
|
|
66
68
|
meshagent/api/urls.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.56.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{meshagent_api-0.56.2 → meshagent_api-0.57.1}/meshagent/api/sqlite_batch_integration_test.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|