meshagent-api 0.56.1__tar.gz → 0.57.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {meshagent_api-0.56.1/meshagent_api.egg-info → meshagent_api-0.57.0}/PKG-INFO +1 -1
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/client.py +50 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/client_test.py +2 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/room_server_client.py +21 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/room_server_client_test.py +26 -0
- meshagent_api-0.57.0/meshagent/api/storage_connections.py +196 -0
- meshagent_api-0.57.0/meshagent/api/storage_connections_test.py +127 -0
- meshagent_api-0.57.0/meshagent/api/version.py +1 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0/meshagent_api.egg-info}/PKG-INFO +1 -1
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent_api.egg-info/SOURCES.txt +2 -0
- meshagent_api-0.56.1/meshagent/api/version.py +0 -1
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/LICENSE +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/MANIFEST.in +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/README.md +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/__init__.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/agent_content.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/aiohttp_redaction.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/chan.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/crdt.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/dataset_schema.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/entrypoint.js +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/error_codes.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/file_permissions.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/files.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/files_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/helpers.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/http.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/image_runtime.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/keys.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/llm_router_config.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/managed_agents.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/managed_agents_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/messaging.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/messaging_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/mime_types.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/mime_types_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/oauth.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/oauth_branding.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/oauth_scopes.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/oauth_scopes_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/participant.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/participant_token.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/participant_token_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/port_forward.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/port_forward_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/protocol.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/protocol_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/py.typed +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/reasoning_schema.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/registry_auth.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/room_ports.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/runtime.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/runtime_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/schema.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/schema_document.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/schema_document_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/schema_registry.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/schema_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/schema_util.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/service_spec_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/service_template_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/services.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/services_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/specs/service.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/sql.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/sql_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/sqlite_batch_integration_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/sqlite_replication_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/sqlite_transfer.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/sqlite_transfer_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/storage_policy_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/token_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/urls.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/webhooks.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/webhooks_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/websocket_protocol.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/websocket_protocol_test.py +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent_api.egg-info/dependency_links.txt +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent_api.egg-info/requires.txt +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent_api.egg-info/top_level.txt +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/pyproject.toml +0 -0
- {meshagent_api-0.56.1 → meshagent_api-0.57.0}/setup.cfg +0 -0
|
@@ -38,6 +38,12 @@ from meshagent.api.specs.service import (
|
|
|
38
38
|
ServiceTemplateSpec,
|
|
39
39
|
)
|
|
40
40
|
from meshagent.api.managed_agents import ManagedAgentSpec
|
|
41
|
+
from meshagent.api.storage_connections import (
|
|
42
|
+
CreateStorageConnection,
|
|
43
|
+
UpdateStorageConnection,
|
|
44
|
+
StorageConnection,
|
|
45
|
+
StorageConnectionsResponse,
|
|
46
|
+
)
|
|
41
47
|
import os
|
|
42
48
|
|
|
43
49
|
|
|
@@ -4668,6 +4674,48 @@ class Meshagent:
|
|
|
4668
4674
|
async with self._session.delete(url, headers=self._get_headers()) as resp:
|
|
4669
4675
|
await self._raise_for_status(resp)
|
|
4670
4676
|
|
|
4677
|
+
async def list_room_storage_connections(
|
|
4678
|
+
self, *, project_id: str, room: str
|
|
4679
|
+
) -> list[StorageConnection]:
|
|
4680
|
+
url = f"{self.base_url}/accounts/projects/{project_id}/rooms/{room}/storage-connections"
|
|
4681
|
+
async with self._session.get(url, headers=self._get_headers()) as resp:
|
|
4682
|
+
await self._raise_for_status(resp)
|
|
4683
|
+
return (
|
|
4684
|
+
await self._read_model(resp, StorageConnectionsResponse)
|
|
4685
|
+
).connections
|
|
4686
|
+
|
|
4687
|
+
async def create_room_storage_connection(
|
|
4688
|
+
self, *, project_id: str, room: str, connection: CreateStorageConnection
|
|
4689
|
+
) -> StorageConnection:
|
|
4690
|
+
url = f"{self.base_url}/accounts/projects/{project_id}/rooms/{room}/storage-connections"
|
|
4691
|
+
async with self._session.post(
|
|
4692
|
+
url, headers=self._get_headers(), json=connection.model_dump(mode="json")
|
|
4693
|
+
) as resp:
|
|
4694
|
+
await self._raise_for_status(resp)
|
|
4695
|
+
return await self._read_model(resp, StorageConnection)
|
|
4696
|
+
|
|
4697
|
+
async def update_room_storage_connection(
|
|
4698
|
+
self,
|
|
4699
|
+
*,
|
|
4700
|
+
project_id: str,
|
|
4701
|
+
room: str,
|
|
4702
|
+
connection_id: str,
|
|
4703
|
+
connection: UpdateStorageConnection,
|
|
4704
|
+
) -> StorageConnection:
|
|
4705
|
+
url = f"{self.base_url}/accounts/projects/{project_id}/rooms/{room}/storage-connections/{connection_id}"
|
|
4706
|
+
async with self._session.put(
|
|
4707
|
+
url, headers=self._get_headers(), json=connection.model_dump(mode="json")
|
|
4708
|
+
) as resp:
|
|
4709
|
+
await self._raise_for_status(resp)
|
|
4710
|
+
return await self._read_model(resp, StorageConnection)
|
|
4711
|
+
|
|
4712
|
+
async def delete_room_storage_connection(
|
|
4713
|
+
self, *, project_id: str, room: str, connection_id: str
|
|
4714
|
+
) -> None:
|
|
4715
|
+
url = f"{self.base_url}/accounts/projects/{project_id}/rooms/{room}/storage-connections/{connection_id}"
|
|
4716
|
+
async with self._session.delete(url, headers=self._get_headers()) as resp:
|
|
4717
|
+
await self._raise_for_status(resp)
|
|
4718
|
+
|
|
4671
4719
|
async def list_room_volumes(
|
|
4672
4720
|
self,
|
|
4673
4721
|
*,
|
|
@@ -5227,12 +5275,14 @@ class Meshagent:
|
|
|
5227
5275
|
project_id: str,
|
|
5228
5276
|
subject: AccessSubject,
|
|
5229
5277
|
resource: AccessResource,
|
|
5278
|
+
roles: list[str],
|
|
5230
5279
|
relations: list[str] | None = None,
|
|
5231
5280
|
) -> ResourceAccessSummary:
|
|
5232
5281
|
"""POST /accounts/projects/{project_id}/access:effective."""
|
|
5233
5282
|
payload: dict[str, Any] = {
|
|
5234
5283
|
"subject": subject.model_dump(mode="json", exclude_none=True),
|
|
5235
5284
|
"resource": resource.model_dump(mode="json", exclude_none=True),
|
|
5285
|
+
"roles": roles,
|
|
5236
5286
|
}
|
|
5237
5287
|
if relations is not None:
|
|
5238
5288
|
payload["relations"] = relations
|
|
@@ -2655,6 +2655,7 @@ async def test_access_evaluator_methods_use_access_routes():
|
|
|
2655
2655
|
project_id="proj_123",
|
|
2656
2656
|
subject=AccessSubject(type="user", id="user-1"),
|
|
2657
2657
|
resource=AccessResource(type="room", id="room-1"),
|
|
2658
|
+
roles=["developer"],
|
|
2658
2659
|
relations=["can_use", "can_manage"],
|
|
2659
2660
|
)
|
|
2660
2661
|
bindings_page = await client.list_access_bindings_page(
|
|
@@ -2704,6 +2705,7 @@ async def test_access_evaluator_methods_use_access_routes():
|
|
|
2704
2705
|
{
|
|
2705
2706
|
"subject": {"type": "user", "id": "user-1"},
|
|
2706
2707
|
"resource": {"type": "room", "id": "room-1"},
|
|
2708
|
+
"roles": ["developer"],
|
|
2707
2709
|
"relations": ["can_use", "can_manage"],
|
|
2708
2710
|
},
|
|
2709
2711
|
),
|
|
@@ -7495,6 +7495,27 @@ class DatasetsClient:
|
|
|
7495
7495
|
input_stream=input_stream,
|
|
7496
7496
|
)
|
|
7497
7497
|
|
|
7498
|
+
async def register_table(
|
|
7499
|
+
self,
|
|
7500
|
+
*,
|
|
7501
|
+
name: str,
|
|
7502
|
+
connection_id: str,
|
|
7503
|
+
path: str,
|
|
7504
|
+
namespace: Optional[list[str]] = None,
|
|
7505
|
+
branch: Optional[str] = None,
|
|
7506
|
+
) -> None:
|
|
7507
|
+
"""Register an existing, read-only Lance dataset from room external storage."""
|
|
7508
|
+
await self._invoke(
|
|
7509
|
+
operation="register_table",
|
|
7510
|
+
input={
|
|
7511
|
+
"name": name,
|
|
7512
|
+
"connection_id": connection_id,
|
|
7513
|
+
"path": path,
|
|
7514
|
+
"namespace": namespace,
|
|
7515
|
+
"branch": branch,
|
|
7516
|
+
},
|
|
7517
|
+
)
|
|
7518
|
+
|
|
7498
7519
|
async def drop_table(
|
|
7499
7520
|
self,
|
|
7500
7521
|
*,
|
|
@@ -6621,3 +6621,29 @@ async def test_sqlite_batch_rejects_malformed_results(payload):
|
|
|
6621
6621
|
database="app", commands=[room_server_client.SqliteSqlCommand("SELECT 1")]
|
|
6622
6622
|
)
|
|
6623
6623
|
assert error.value.code == ErrorCode.UNEXPECTED_RESPONSE_TYPE
|
|
6624
|
+
|
|
6625
|
+
|
|
6626
|
+
@pytest.mark.asyncio
|
|
6627
|
+
async def test_datasets_client_registers_external_source_without_credentials():
|
|
6628
|
+
from unittest.mock import AsyncMock
|
|
6629
|
+
|
|
6630
|
+
room = SimpleNamespace(invoke=AsyncMock(return_value=EmptyContent()))
|
|
6631
|
+
client = room_server_client.DatasetsClient(room)
|
|
6632
|
+
await client.register_table(
|
|
6633
|
+
name="events",
|
|
6634
|
+
connection_id="connection-id",
|
|
6635
|
+
path="events.lance",
|
|
6636
|
+
namespace=["analytics"],
|
|
6637
|
+
branch="experiment",
|
|
6638
|
+
)
|
|
6639
|
+
room.invoke.assert_awaited_once_with(
|
|
6640
|
+
toolkit="dataset",
|
|
6641
|
+
tool="register_table",
|
|
6642
|
+
input={
|
|
6643
|
+
"name": "events",
|
|
6644
|
+
"connection_id": "connection-id",
|
|
6645
|
+
"path": "events.lance",
|
|
6646
|
+
"namespace": ["analytics"],
|
|
6647
|
+
"branch": "experiment",
|
|
6648
|
+
},
|
|
6649
|
+
)
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Room-owned external storage configuration and credential wire models."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
import json
|
|
5
|
+
from typing import Annotated, Any, Literal
|
|
6
|
+
from uuid import UUID
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class _StorageModel(BaseModel):
|
|
12
|
+
model_config = ConfigDict(extra="forbid")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class _Location(_StorageModel):
|
|
16
|
+
anonymous: bool = False
|
|
17
|
+
prefix: str = ""
|
|
18
|
+
|
|
19
|
+
@field_validator("prefix")
|
|
20
|
+
@classmethod
|
|
21
|
+
def validate_prefix(cls, value: str) -> str:
|
|
22
|
+
if value and (
|
|
23
|
+
value.startswith("/")
|
|
24
|
+
or any(char in value for char in "\\:?#%")
|
|
25
|
+
or any(part in {"", ".", ".."} for part in value.split("/"))
|
|
26
|
+
or any(ord(char) < 32 for char in value)
|
|
27
|
+
):
|
|
28
|
+
raise ValueError("prefix must be a relative object storage path")
|
|
29
|
+
return value
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
StorageName = Annotated[str, Field(pattern=r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$")]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class GCSStorageConfig(_Location):
|
|
36
|
+
provider: Literal["gcs"] = "gcs"
|
|
37
|
+
bucket: StorageName
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class S3StorageConfig(_Location):
|
|
41
|
+
provider: Literal["s3"] = "s3"
|
|
42
|
+
bucket: StorageName
|
|
43
|
+
region: Annotated[str, Field(pattern=r"^[a-z0-9-]+$")]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class AzureStorageConfig(_Location):
|
|
47
|
+
provider: Literal["azure"] = "azure"
|
|
48
|
+
account: Annotated[str, Field(pattern=r"^[a-z0-9]{3,24}$")]
|
|
49
|
+
container: StorageName
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
StorageConfig = Annotated[
|
|
53
|
+
GCSStorageConfig | S3StorageConfig | AzureStorageConfig,
|
|
54
|
+
Field(discriminator="provider"),
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class GCSStorageCredentials(_StorageModel):
|
|
59
|
+
provider: Literal["gcs"] = "gcs"
|
|
60
|
+
service_account_json: dict[str, Any] = Field(repr=False)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class S3StorageCredentials(_StorageModel):
|
|
64
|
+
provider: Literal["s3"] = "s3"
|
|
65
|
+
access_key_id: str = Field(min_length=1, repr=False)
|
|
66
|
+
secret_access_key: str = Field(min_length=1, repr=False)
|
|
67
|
+
session_token: str | None = Field(default=None, repr=False)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class AzureStorageCredentials(_StorageModel):
|
|
71
|
+
provider: Literal["azure"] = "azure"
|
|
72
|
+
account_key: str | None = Field(default=None, min_length=1, repr=False)
|
|
73
|
+
sas_token: str | None = Field(default=None, min_length=1, repr=False)
|
|
74
|
+
|
|
75
|
+
@model_validator(mode="after")
|
|
76
|
+
def require_one_credential(self):
|
|
77
|
+
if (self.account_key is None) == (self.sas_token is None):
|
|
78
|
+
raise ValueError("provide exactly one of account_key or sas_token")
|
|
79
|
+
return self
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
StorageCredentials = Annotated[
|
|
83
|
+
GCSStorageCredentials | S3StorageCredentials | AzureStorageCredentials,
|
|
84
|
+
Field(discriminator="provider"),
|
|
85
|
+
]
|
|
86
|
+
ConnectionName = Annotated[
|
|
87
|
+
str, Field(pattern=r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class CreateStorageConnection(_StorageModel):
|
|
92
|
+
name: ConnectionName
|
|
93
|
+
config: StorageConfig
|
|
94
|
+
credentials: StorageCredentials | None = Field(default=None, repr=False)
|
|
95
|
+
|
|
96
|
+
@model_validator(mode="after")
|
|
97
|
+
def matching_provider(self):
|
|
98
|
+
if self.config.anonymous:
|
|
99
|
+
if self.credentials is not None:
|
|
100
|
+
raise ValueError("anonymous connections must not contain credentials")
|
|
101
|
+
elif self.credentials is None:
|
|
102
|
+
raise ValueError("credentials are required for authenticated connections")
|
|
103
|
+
elif self.config.provider != self.credentials.provider:
|
|
104
|
+
raise ValueError("configuration and credentials must use the same provider")
|
|
105
|
+
return self
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class UpdateStorageConnection(_StorageModel):
|
|
109
|
+
name: ConnectionName
|
|
110
|
+
config: StorageConfig
|
|
111
|
+
credentials: StorageCredentials | None = Field(default=None, repr=False)
|
|
112
|
+
|
|
113
|
+
@model_validator(mode="after")
|
|
114
|
+
def matching_provider(self):
|
|
115
|
+
if self.config.anonymous and self.credentials is not None:
|
|
116
|
+
raise ValueError("anonymous connections must not contain credentials")
|
|
117
|
+
if self.credentials and self.config.provider != self.credentials.provider:
|
|
118
|
+
raise ValueError("configuration and credentials must use the same provider")
|
|
119
|
+
return self
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class StorageConnection(_StorageModel):
|
|
123
|
+
id: UUID
|
|
124
|
+
room_id: UUID
|
|
125
|
+
name: str
|
|
126
|
+
config: StorageConfig
|
|
127
|
+
revision: int
|
|
128
|
+
created_at: datetime
|
|
129
|
+
updated_at: datetime
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class ResolvedStorageConnection(_StorageModel):
|
|
133
|
+
id: UUID
|
|
134
|
+
config: StorageConfig
|
|
135
|
+
revision: int
|
|
136
|
+
credentials: StorageCredentials | None = Field(default=None, repr=False)
|
|
137
|
+
|
|
138
|
+
def location_and_options(self, path: str) -> tuple[str, dict[str, str]]:
|
|
139
|
+
validate_external_dataset_path(path)
|
|
140
|
+
options: dict[str, str] = {}
|
|
141
|
+
config, credentials = self.config, self.credentials
|
|
142
|
+
if config.anonymous:
|
|
143
|
+
if credentials is not None:
|
|
144
|
+
raise ValueError("anonymous connections must not contain credentials")
|
|
145
|
+
if isinstance(config, GCSStorageConfig):
|
|
146
|
+
root = f"gs://{config.bucket}"
|
|
147
|
+
options["google_skip_signature"] = "true"
|
|
148
|
+
elif isinstance(config, S3StorageConfig):
|
|
149
|
+
root = f"s3://{config.bucket}"
|
|
150
|
+
options.update(aws_region=config.region, aws_skip_signature="true")
|
|
151
|
+
else:
|
|
152
|
+
root = f"az://{config.container}"
|
|
153
|
+
options.update(
|
|
154
|
+
azure_storage_account_name=config.account,
|
|
155
|
+
azure_skip_signature="true",
|
|
156
|
+
)
|
|
157
|
+
elif isinstance(config, GCSStorageConfig) and isinstance(
|
|
158
|
+
credentials, GCSStorageCredentials
|
|
159
|
+
):
|
|
160
|
+
root = f"gs://{config.bucket}"
|
|
161
|
+
options["service_account_key"] = json.dumps(
|
|
162
|
+
credentials.service_account_json
|
|
163
|
+
)
|
|
164
|
+
elif isinstance(config, S3StorageConfig) and isinstance(
|
|
165
|
+
credentials, S3StorageCredentials
|
|
166
|
+
):
|
|
167
|
+
root = f"s3://{config.bucket}"
|
|
168
|
+
options.update(
|
|
169
|
+
aws_region=config.region,
|
|
170
|
+
aws_access_key_id=credentials.access_key_id,
|
|
171
|
+
aws_secret_access_key=credentials.secret_access_key,
|
|
172
|
+
)
|
|
173
|
+
if credentials.session_token is not None:
|
|
174
|
+
options["aws_session_token"] = credentials.session_token
|
|
175
|
+
elif isinstance(config, AzureStorageConfig) and isinstance(
|
|
176
|
+
credentials, AzureStorageCredentials
|
|
177
|
+
):
|
|
178
|
+
root = f"az://{config.container}"
|
|
179
|
+
options["azure_storage_account_name"] = config.account
|
|
180
|
+
if credentials.account_key is not None:
|
|
181
|
+
options["azure_storage_account_key"] = credentials.account_key
|
|
182
|
+
elif credentials.sas_token is not None:
|
|
183
|
+
options["azure_storage_sas_key"] = credentials.sas_token
|
|
184
|
+
else:
|
|
185
|
+
raise ValueError("storage connection credential provider mismatch")
|
|
186
|
+
return "/".join(part for part in (root, config.prefix, path) if part), options
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def validate_external_dataset_path(path: str) -> str:
|
|
190
|
+
if not path:
|
|
191
|
+
raise ValueError("external dataset path must not be empty")
|
|
192
|
+
return _Location.validate_prefix(path)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class StorageConnectionsResponse(_StorageModel):
|
|
196
|
+
connections: list[StorageConnection]
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from pydantic import ValidationError
|
|
3
|
+
|
|
4
|
+
from meshagent.api.storage_connections import CreateStorageConnection
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@pytest.mark.parametrize(
|
|
8
|
+
"prefix",
|
|
9
|
+
[
|
|
10
|
+
"../private",
|
|
11
|
+
"/absolute",
|
|
12
|
+
"a//b",
|
|
13
|
+
"a/../b",
|
|
14
|
+
"a%2fb",
|
|
15
|
+
"a?token=secret",
|
|
16
|
+
"gs://other-bucket",
|
|
17
|
+
],
|
|
18
|
+
)
|
|
19
|
+
def test_connection_rejects_unsafe_prefixes(prefix):
|
|
20
|
+
with pytest.raises(ValidationError):
|
|
21
|
+
CreateStorageConnection.model_validate(
|
|
22
|
+
{
|
|
23
|
+
"name": "analytics",
|
|
24
|
+
"config": {
|
|
25
|
+
"provider": "gcs",
|
|
26
|
+
"bucket": "test-bucket",
|
|
27
|
+
"prefix": prefix,
|
|
28
|
+
},
|
|
29
|
+
"credentials": {"provider": "gcs", "service_account_json": {}},
|
|
30
|
+
}
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_connection_rejects_mismatched_credentials():
|
|
35
|
+
with pytest.raises(ValidationError):
|
|
36
|
+
CreateStorageConnection.model_validate(
|
|
37
|
+
{
|
|
38
|
+
"name": "analytics",
|
|
39
|
+
"config": {"provider": "gcs", "bucket": "test-bucket"},
|
|
40
|
+
"credentials": {"provider": "azure", "account_key": "test-only"},
|
|
41
|
+
}
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@pytest.mark.parametrize(
|
|
46
|
+
"credentials", [{}, {"account_key": "test", "sas_token": "test"}]
|
|
47
|
+
)
|
|
48
|
+
def test_azure_requires_exactly_one_authentication_method(credentials):
|
|
49
|
+
with pytest.raises(ValidationError):
|
|
50
|
+
CreateStorageConnection.model_validate(
|
|
51
|
+
{
|
|
52
|
+
"name": "analytics",
|
|
53
|
+
"config": {
|
|
54
|
+
"provider": "azure",
|
|
55
|
+
"account": "testaccount",
|
|
56
|
+
"container": "test-container",
|
|
57
|
+
},
|
|
58
|
+
"credentials": {"provider": "azure", **credentials},
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_credential_repr_is_redacted():
|
|
64
|
+
connection = CreateStorageConnection.model_validate(
|
|
65
|
+
{
|
|
66
|
+
"name": "analytics",
|
|
67
|
+
"config": {
|
|
68
|
+
"provider": "s3",
|
|
69
|
+
"bucket": "test-bucket",
|
|
70
|
+
"region": "us-east-1",
|
|
71
|
+
},
|
|
72
|
+
"credentials": {
|
|
73
|
+
"provider": "s3",
|
|
74
|
+
"access_key_id": "test-key",
|
|
75
|
+
"secret_access_key": "never-display-this",
|
|
76
|
+
},
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
assert "never-display-this" not in repr(connection)
|
|
80
|
+
assert "never-display-this" not in repr(connection.credentials)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@pytest.mark.parametrize(
|
|
84
|
+
"config,expected_options",
|
|
85
|
+
[
|
|
86
|
+
(
|
|
87
|
+
{"provider": "gcs", "bucket": "test-bucket"},
|
|
88
|
+
{"google_skip_signature": "true"},
|
|
89
|
+
),
|
|
90
|
+
(
|
|
91
|
+
{"provider": "s3", "bucket": "test-bucket", "region": "us-east-1"},
|
|
92
|
+
{"aws_skip_signature": "true", "aws_region": "us-east-1"},
|
|
93
|
+
),
|
|
94
|
+
(
|
|
95
|
+
{
|
|
96
|
+
"provider": "azure",
|
|
97
|
+
"account": "testaccount",
|
|
98
|
+
"container": "test-container",
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"azure_skip_signature": "true",
|
|
102
|
+
"azure_storage_account_name": "testaccount",
|
|
103
|
+
},
|
|
104
|
+
),
|
|
105
|
+
],
|
|
106
|
+
)
|
|
107
|
+
def test_public_connections_explicitly_disable_signing(config, expected_options):
|
|
108
|
+
from uuid import uuid4
|
|
109
|
+
from meshagent.api.storage_connections import (
|
|
110
|
+
ResolvedStorageConnection,
|
|
111
|
+
UpdateStorageConnection,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
public = {**config, "anonymous": True}
|
|
115
|
+
connection = CreateStorageConnection(name="public", config=public)
|
|
116
|
+
assert connection.credentials is None
|
|
117
|
+
resolved = ResolvedStorageConnection(id=uuid4(), revision=1, config=public)
|
|
118
|
+
assert resolved.location_and_options("events.lance")[1] == expected_options
|
|
119
|
+
with pytest.raises(ValidationError):
|
|
120
|
+
CreateStorageConnection(name="private", config=config)
|
|
121
|
+
for model in (CreateStorageConnection, UpdateStorageConnection):
|
|
122
|
+
with pytest.raises(ValidationError):
|
|
123
|
+
model(
|
|
124
|
+
name="public",
|
|
125
|
+
config=public,
|
|
126
|
+
credentials={"provider": "gcs", "service_account_json": {}},
|
|
127
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.57.0"
|
|
@@ -61,6 +61,8 @@ meshagent/api/sqlite_batch_integration_test.py
|
|
|
61
61
|
meshagent/api/sqlite_replication_test.py
|
|
62
62
|
meshagent/api/sqlite_transfer.py
|
|
63
63
|
meshagent/api/sqlite_transfer_test.py
|
|
64
|
+
meshagent/api/storage_connections.py
|
|
65
|
+
meshagent/api/storage_connections_test.py
|
|
64
66
|
meshagent/api/storage_policy_test.py
|
|
65
67
|
meshagent/api/token_test.py
|
|
66
68
|
meshagent/api/urls.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.56.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{meshagent_api-0.56.1 → meshagent_api-0.57.0}/meshagent/api/sqlite_batch_integration_test.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|