pal-found-cli 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pal_found_cli/__init__.py +1 -0
- pal_found_cli/_version.py +24 -0
- pal_found_cli/admin/__init__.py +2 -0
- pal_found_cli/admin/scripts/__init__.py +2 -0
- pal_found_cli/admin/scripts/pal_found_admin_cli.py +600 -0
- pal_found_cli/aip_agents/__init__.py +1 -0
- pal_found_cli/aip_agents/metadata-allow-list.md +20 -0
- pal_found_cli/aip_agents/scripts/__init__.py +1 -0
- pal_found_cli/aip_agents/scripts/pal_found_aip_agents_cli.py +704 -0
- pal_found_cli/audit/__init__.py +1 -0
- pal_found_cli/audit/metadata-allow-list.md +4 -0
- pal_found_cli/audit/scripts/__init__.py +1 -0
- pal_found_cli/audit/scripts/pal_found_audit_cli.py +387 -0
- pal_found_cli/checkpoints/__init__.py +9 -0
- pal_found_cli/checkpoints/metadata-allow-list.md +7 -0
- pal_found_cli/checkpoints/scripts/__init__.py +1 -0
- pal_found_cli/checkpoints/scripts/pal_found_checkpoints_cli.py +527 -0
- pal_found_cli/common/__init__.py +56 -0
- pal_found_cli/common/access_control_guard.py +591 -0
- pal_found_cli/common/async_client_factory.py +161 -0
- pal_found_cli/common/auth_provider.py +82 -0
- pal_found_cli/common/binary_download_handler.py +288 -0
- pal_found_cli/common/config_loader.py +388 -0
- pal_found_cli/common/error_serializer.py +412 -0
- pal_found_cli/common/log_setup.py +241 -0
- pal_found_cli/common/metadata-allow-list.md +169 -0
- pal_found_cli/common/output_formatter.py +267 -0
- pal_found_cli/common/pagination_helper.py +212 -0
- pal_found_cli/common/retry.py +617 -0
- pal_found_cli/common/sdk_error_utils.py +112 -0
- pal_found_cli/common/session_manager.py +461 -0
- pal_found_cli/common/tracing_provider.py +107 -0
- pal_found_cli/connectivity/__init__.py +9 -0
- pal_found_cli/connectivity/metadata-allow-list.md +24 -0
- pal_found_cli/connectivity/scripts/__init__.py +1 -0
- pal_found_cli/connectivity/scripts/pal_found_connectivity_cli.py +720 -0
- pal_found_cli/data_health/__init__.py +9 -0
- pal_found_cli/data_health/metadata-allow-list.md +10 -0
- pal_found_cli/data_health/scripts/__init__.py +1 -0
- pal_found_cli/data_health/scripts/pal_found_data_health_cli.py +458 -0
- pal_found_cli/datasets/__init__.py +1 -0
- pal_found_cli/datasets/scripts/__init__.py +1 -0
- pal_found_cli/datasets/scripts/pal_found_datasets_cli.py +732 -0
- pal_found_cli/filesystem/__init__.py +2 -0
- pal_found_cli/filesystem/scripts/__init__.py +2 -0
- pal_found_cli/filesystem/scripts/pal_found_filesystem_cli.py +659 -0
- pal_found_cli/functions/__init__.py +2 -0
- pal_found_cli/functions/scripts/__init__.py +2 -0
- pal_found_cli/functions/scripts/pal_found_functions_cli.py +389 -0
- pal_found_cli/language_models/__init__.py +1 -0
- pal_found_cli/language_models/metadata-allow-list.md +6 -0
- pal_found_cli/language_models/scripts/__init__.py +1 -0
- pal_found_cli/language_models/scripts/pal_found_language_models_cli.py +342 -0
- pal_found_cli/media_sets/__init__.py +9 -0
- pal_found_cli/media_sets/metadata-allow-list.md +23 -0
- pal_found_cli/media_sets/scripts/__init__.py +1 -0
- pal_found_cli/media_sets/scripts/pal_found_media_sets_cli.py +642 -0
- pal_found_cli/models/__init__.py +5 -0
- pal_found_cli/models/metadata-allow-list.md +27 -0
- pal_found_cli/models/scripts/__init__.py +1 -0
- pal_found_cli/models/scripts/pal_found_models_cli.py +714 -0
- pal_found_cli/ontologies/__init__.py +1 -0
- pal_found_cli/ontologies/scripts/__init__.py +1 -0
- pal_found_cli/ontologies/scripts/pal_found_ontologies_cli.py +1225 -0
- pal_found_cli/orchestration/__init__.py +9 -0
- pal_found_cli/orchestration/metadata-allow-list.md +24 -0
- pal_found_cli/orchestration/scripts/__init__.py +1 -0
- pal_found_cli/orchestration/scripts/pal_found_orchestration_cli.py +640 -0
- pal_found_cli/sql_queries/__init__.py +9 -0
- pal_found_cli/sql_queries/metadata-allow-list.md +9 -0
- pal_found_cli/sql_queries/scripts/__init__.py +1 -0
- pal_found_cli/sql_queries/scripts/pal_found_sql_queries_cli.py +514 -0
- pal_found_cli/streams/__init__.py +9 -0
- pal_found_cli/streams/metadata-allow-list.md +19 -0
- pal_found_cli/streams/scripts/__init__.py +1 -0
- pal_found_cli/streams/scripts/pal_found_streams_cli.py +615 -0
- pal_found_cli/third_party_applications/__init__.py +9 -0
- pal_found_cli/third_party_applications/metadata-allow-list.md +13 -0
- pal_found_cli/third_party_applications/scripts/__init__.py +1 -0
- pal_found_cli/third_party_applications/scripts/pal_found_third_party_applications_cli.py +547 -0
- pal_found_cli/widgets/__init__.py +9 -0
- pal_found_cli/widgets/metadata-allow-list.md +12 -0
- pal_found_cli/widgets/scripts/__init__.py +1 -0
- pal_found_cli/widgets/scripts/pal_found_widgets_cli.py +571 -0
- pal_found_cli-0.1.2.dist-info/METADATA +74 -0
- pal_found_cli-0.1.2.dist-info/RECORD +89 -0
- pal_found_cli-0.1.2.dist-info/WHEEL +5 -0
- pal_found_cli-0.1.2.dist-info/entry_points.txt +19 -0
- pal_found_cli-0.1.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""AsyncClientFactory — creates stateless AsyncFoundryClient (SRS §3.4).
|
|
2
|
+
|
|
3
|
+
Creates an ``AsyncFoundryClient`` instance per invocation, configured with
|
|
4
|
+
``UserTokenAuth`` and the resolved hostname. When attribution is enabled,
|
|
5
|
+
the SDK attribution context variable is set for the current invocation.
|
|
6
|
+
|
|
7
|
+
Stateless per invocation (DCC-3): no client is cached across calls —
|
|
8
|
+
each ``create()`` returns a fresh client. This keeps tests isolated,
|
|
9
|
+
avoids shared mutable state, and matches the CLI's per-command lifecycle.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from collections.abc import Iterator
|
|
13
|
+
from contextlib import contextmanager
|
|
14
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
15
|
+
|
|
16
|
+
from pal_found_cli.common.config_loader import ConfigLoader, ConfigurationError
|
|
17
|
+
from pal_found_cli.common.tracing_provider import B3Context, TracingProvider
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from foundry_sdk import AsyncFoundryClient
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class AsyncClientFactory:
|
|
24
|
+
"""Creates stateless ``AsyncFoundryClient`` instances.
|
|
25
|
+
|
|
26
|
+
Usage
|
|
27
|
+
-----
|
|
28
|
+
>>> factory = AsyncClientFactory()
|
|
29
|
+
>>> client = factory.create(cfg)
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self) -> None:
|
|
33
|
+
# Per-instance state only — no class-level caching. Each factory
|
|
34
|
+
# instance is independent; ``create()`` never mutates shared state.
|
|
35
|
+
self._last_client: AsyncFoundryClient | None = None
|
|
36
|
+
|
|
37
|
+
def create(
|
|
38
|
+
self,
|
|
39
|
+
cfg: ConfigLoader,
|
|
40
|
+
*,
|
|
41
|
+
include_attribution: bool | None = None,
|
|
42
|
+
) -> "AsyncFoundryClient":
|
|
43
|
+
"""Create an ``AsyncFoundryClient`` from credentials in ``cfg``.
|
|
44
|
+
|
|
45
|
+
Validates token and hostname are present before constructing the
|
|
46
|
+
client. A new client is returned on every call — nothing is cached
|
|
47
|
+
on the class.
|
|
48
|
+
|
|
49
|
+
Parameters
|
|
50
|
+
----------
|
|
51
|
+
cfg : ConfigLoader
|
|
52
|
+
Configuration instance with loaded credentials.
|
|
53
|
+
include_attribution : bool or None, optional
|
|
54
|
+
``False`` clears SDK attribution for this client. ``None`` keeps
|
|
55
|
+
existing behavior and follows loaded configuration.
|
|
56
|
+
|
|
57
|
+
Returns
|
|
58
|
+
-------
|
|
59
|
+
AsyncFoundryClient
|
|
60
|
+
Freshly constructed SDK async client instance.
|
|
61
|
+
|
|
62
|
+
Raises
|
|
63
|
+
------
|
|
64
|
+
ConfigurationError
|
|
65
|
+
If credentials are missing or the SDK is not installed.
|
|
66
|
+
"""
|
|
67
|
+
token = cfg.token
|
|
68
|
+
hostname = cfg.hostname
|
|
69
|
+
|
|
70
|
+
# Strip before presence check so whitespace-only tokens are rejected
|
|
71
|
+
# (review finding A1 applied at the factory boundary as defense in depth).
|
|
72
|
+
if not token or not token.strip() or not hostname or not hostname.strip():
|
|
73
|
+
raise ConfigurationError(
|
|
74
|
+
"Missing FOUNDRY_TOKEN and/or FOUNDRY_HOSTNAME. "
|
|
75
|
+
"Set credentials in .env or shell environment."
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
from foundry_sdk import ATTRIBUTION_VAR, AsyncFoundryClient, UserTokenAuth
|
|
80
|
+
except ImportError as exc:
|
|
81
|
+
raise ConfigurationError(
|
|
82
|
+
"foundry-sdk not installed; run 'pip install foundry-platform-sdk'"
|
|
83
|
+
) from exc
|
|
84
|
+
|
|
85
|
+
auth: UserTokenAuth = UserTokenAuth(token)
|
|
86
|
+
|
|
87
|
+
client_kwargs: dict[str, Any] = {
|
|
88
|
+
"auth": auth,
|
|
89
|
+
"hostname": hostname,
|
|
90
|
+
"preview": True,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
# SDK reads attribution from a context variable when sending requests.
|
|
94
|
+
# Always set it here so disabled or empty config clears prior values.
|
|
95
|
+
if include_attribution is False:
|
|
96
|
+
ATTRIBUTION_VAR.set(None)
|
|
97
|
+
else:
|
|
98
|
+
rids: list[str] = []
|
|
99
|
+
if cfg.enable_attribution and cfg.attribution_rids:
|
|
100
|
+
rids = [r.strip() for r in cfg.attribution_rids.split(",")]
|
|
101
|
+
rids = [r for r in rids if r]
|
|
102
|
+
ATTRIBUTION_VAR.set(rids or None)
|
|
103
|
+
|
|
104
|
+
client: AsyncFoundryClient = AsyncFoundryClient(**client_kwargs)
|
|
105
|
+
self._last_client = client
|
|
106
|
+
return client
|
|
107
|
+
|
|
108
|
+
@contextmanager
|
|
109
|
+
def invocation_scope(
|
|
110
|
+
self,
|
|
111
|
+
cfg: ConfigLoader,
|
|
112
|
+
supplied: B3Context | None = None,
|
|
113
|
+
*,
|
|
114
|
+
include_attribution: bool | None = None,
|
|
115
|
+
) -> Iterator[B3Context | None]:
|
|
116
|
+
"""Keep one trace context active across client creation and SDK calls.
|
|
117
|
+
|
|
118
|
+
Callers enter this scope before :meth:`create` and leave it only after
|
|
119
|
+
retry handling and response processing finish.
|
|
120
|
+
|
|
121
|
+
When ``include_attribution`` is set, this scope restores the prior SDK
|
|
122
|
+
attribution context on every exit path.
|
|
123
|
+
"""
|
|
124
|
+
attribution_var: Any = None
|
|
125
|
+
attribution_token: Any = None
|
|
126
|
+
if include_attribution is not None:
|
|
127
|
+
try:
|
|
128
|
+
from foundry_sdk import ATTRIBUTION_VAR
|
|
129
|
+
except ImportError as exc:
|
|
130
|
+
raise ConfigurationError(
|
|
131
|
+
"foundry-sdk not installed; run 'pip install foundry-platform-sdk'"
|
|
132
|
+
) from exc
|
|
133
|
+
attribution_var = ATTRIBUTION_VAR
|
|
134
|
+
if include_attribution:
|
|
135
|
+
rids = [
|
|
136
|
+
rid.strip()
|
|
137
|
+
for rid in (cfg.attribution_rids or "").split(",")
|
|
138
|
+
if rid.strip()
|
|
139
|
+
]
|
|
140
|
+
attribution_token = attribution_var.set(
|
|
141
|
+
rids if cfg.enable_attribution and rids else None
|
|
142
|
+
)
|
|
143
|
+
else:
|
|
144
|
+
attribution_token = attribution_var.set(None)
|
|
145
|
+
|
|
146
|
+
try:
|
|
147
|
+
with TracingProvider(config=cfg).scope(supplied) as context:
|
|
148
|
+
yield context
|
|
149
|
+
finally:
|
|
150
|
+
if attribution_token is not None:
|
|
151
|
+
attribution_var.reset(attribution_token)
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def last_client(self) -> Optional["AsyncFoundryClient"]:
|
|
155
|
+
"""Most recently created client, or ``None``.
|
|
156
|
+
|
|
157
|
+
Convenience accessor for inspection/diagnostics only. Callers must
|
|
158
|
+
not rely on this for caching — ``create()`` always builds a new
|
|
159
|
+
client.
|
|
160
|
+
"""
|
|
161
|
+
return self._last_client
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""AuthProvider — constructs UserTokenAuth from environment (SRS §3.1).
|
|
2
|
+
|
|
3
|
+
Validates that FOUNDRY_TOKEN and FOUNDRY_HOSTNAME are present
|
|
4
|
+
and constructs a UserTokenAuth instance for the SDK.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from typing import TYPE_CHECKING
|
|
8
|
+
|
|
9
|
+
from pal_found_cli.common.config_loader import ConfigurationError
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from foundry_sdk import UserTokenAuth
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class AuthProvider:
|
|
16
|
+
"""Constructs and validates UserTokenAuth credentials.
|
|
17
|
+
|
|
18
|
+
Usage
|
|
19
|
+
-----
|
|
20
|
+
>>> auth = AuthProvider()
|
|
21
|
+
>>> auth_obj = auth.get_auth(token, hostname)
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
@staticmethod
|
|
25
|
+
def validate(token: str | None, hostname: str | None) -> tuple[bool, str | None]:
|
|
26
|
+
"""Validate that credentials are present and non-empty.
|
|
27
|
+
|
|
28
|
+
Whitespace-only values are rejected — a token consisting solely of
|
|
29
|
+
spaces is not a credential and must not pass validation (review
|
|
30
|
+
finding A1/A4, OWASP A07).
|
|
31
|
+
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
token : str or None
|
|
35
|
+
FOUNDRY_TOKEN value.
|
|
36
|
+
hostname : str or None
|
|
37
|
+
FOUNDRY_HOSTNAME value.
|
|
38
|
+
|
|
39
|
+
Returns
|
|
40
|
+
-------
|
|
41
|
+
Tuple[bool, str or None]
|
|
42
|
+
(is_valid, error_message). If valid, error_message is None.
|
|
43
|
+
Error messages reference the env var name only — never the
|
|
44
|
+
credential value — so they are safe to log.
|
|
45
|
+
"""
|
|
46
|
+
if token is None or token.strip() == "":
|
|
47
|
+
return False, "Missing FOUNDRY_TOKEN credential"
|
|
48
|
+
if hostname is None or hostname.strip() == "":
|
|
49
|
+
return False, "Missing FOUNDRY_HOSTNAME credential"
|
|
50
|
+
return True, None
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def get_auth(token: str) -> "UserTokenAuth":
|
|
54
|
+
"""Construct UserTokenAuth from a token string.
|
|
55
|
+
|
|
56
|
+
The SDK's ``UserTokenAuth`` is constructed from the token only;
|
|
57
|
+
hostname is consumed by the client factory, not by the auth object,
|
|
58
|
+
so it is intentionally absent from this signature (review finding A2).
|
|
59
|
+
|
|
60
|
+
Parameters
|
|
61
|
+
----------
|
|
62
|
+
token : str
|
|
63
|
+
FOUNDRY_TOKEN value. Caller is expected to have validated it
|
|
64
|
+
via :meth:`validate` first.
|
|
65
|
+
|
|
66
|
+
Returns
|
|
67
|
+
-------
|
|
68
|
+
UserTokenAuth
|
|
69
|
+
SDK auth instance.
|
|
70
|
+
|
|
71
|
+
Raises
|
|
72
|
+
------
|
|
73
|
+
ConfigurationError
|
|
74
|
+
If the ``foundry-platform-sdk`` package is not installed.
|
|
75
|
+
"""
|
|
76
|
+
try:
|
|
77
|
+
from foundry_sdk import UserTokenAuth
|
|
78
|
+
except ImportError as exc:
|
|
79
|
+
raise ConfigurationError(
|
|
80
|
+
"foundry-sdk not installed; run 'pip install foundry-platform-sdk'"
|
|
81
|
+
) from exc
|
|
82
|
+
return UserTokenAuth(token)
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""Bounded, atomic binary download persistence (DESIGN-005)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import inspect
|
|
7
|
+
import logging
|
|
8
|
+
import mimetypes
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import tempfile
|
|
12
|
+
import unicodedata
|
|
13
|
+
import uuid
|
|
14
|
+
from collections.abc import AsyncIterable
|
|
15
|
+
from dataclasses import asdict, dataclass
|
|
16
|
+
from datetime import UTC, datetime
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from pal_found_cli.common.config_loader import ConfigLoader
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
_SAFE_FILENAME = re.compile(r"[^A-Za-z0-9._-]+")
|
|
25
|
+
_DEFAULT_DOWNLOAD_ROOT = Path(".foundry-data/downloads")
|
|
26
|
+
_DEFAULT_MAX_DOWNLOAD_BYTES = 1_572_864
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DownloadError(Exception):
|
|
30
|
+
"""Base error for binary download persistence."""
|
|
31
|
+
|
|
32
|
+
exit_code = 6
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class InvalidDownloadError(DownloadError, ValueError):
|
|
36
|
+
"""Raised when download configuration or a filename is invalid."""
|
|
37
|
+
|
|
38
|
+
exit_code = 1
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class DownloadResult:
|
|
43
|
+
"""Metadata for a downloaded file prefix."""
|
|
44
|
+
|
|
45
|
+
file_path: str
|
|
46
|
+
file_size: int
|
|
47
|
+
checksum_md5: str
|
|
48
|
+
checksum_sha256: str
|
|
49
|
+
mime_type: str | None
|
|
50
|
+
truncated: bool
|
|
51
|
+
source_size: int | None
|
|
52
|
+
source_size_at_least: int | None
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> dict[str, Any]:
|
|
55
|
+
"""Return a JSON-serializable result envelope."""
|
|
56
|
+
return asdict(self)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class BinaryDownloadHandler:
|
|
60
|
+
"""Stream binary content into a contained UUID directory.
|
|
61
|
+
|
|
62
|
+
The handler reads at most ``max_download_bytes + 1`` bytes when source
|
|
63
|
+
length is unknown and publishes the stored prefix atomically.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
download_root: str | Path | None = None,
|
|
69
|
+
max_download_bytes: int | None = None,
|
|
70
|
+
*,
|
|
71
|
+
config: ConfigLoader | None = None,
|
|
72
|
+
) -> None:
|
|
73
|
+
cfg = config or ConfigLoader()
|
|
74
|
+
self.download_root = Path(
|
|
75
|
+
download_root
|
|
76
|
+
if download_root is not None
|
|
77
|
+
else getattr(cfg, "download_path", _DEFAULT_DOWNLOAD_ROOT)
|
|
78
|
+
).expanduser()
|
|
79
|
+
self.max_download_bytes = (
|
|
80
|
+
max_download_bytes
|
|
81
|
+
if max_download_bytes is not None
|
|
82
|
+
else getattr(cfg, "max_download_bytes", _DEFAULT_MAX_DOWNLOAD_BYTES)
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
async def save(
|
|
86
|
+
self,
|
|
87
|
+
chunks: AsyncIterable[bytes],
|
|
88
|
+
*,
|
|
89
|
+
original_filename: str | None,
|
|
90
|
+
namespace: str,
|
|
91
|
+
operation: str,
|
|
92
|
+
content_length: str | None = None,
|
|
93
|
+
content_encoding: str | None = None,
|
|
94
|
+
mime_type: str | None = None,
|
|
95
|
+
) -> DownloadResult:
|
|
96
|
+
"""Store a bounded stream and return checksums for persisted bytes."""
|
|
97
|
+
limit = self.max_download_bytes
|
|
98
|
+
if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
|
|
99
|
+
raise InvalidDownloadError("max_download_bytes must be a positive integer")
|
|
100
|
+
|
|
101
|
+
filename = self._safe_filename(
|
|
102
|
+
original_filename,
|
|
103
|
+
namespace=namespace,
|
|
104
|
+
operation=operation,
|
|
105
|
+
mime_type=mime_type,
|
|
106
|
+
)
|
|
107
|
+
root = self.download_root.resolve()
|
|
108
|
+
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
109
|
+
self._restrict_directory(root)
|
|
110
|
+
|
|
111
|
+
download_dir = root / str(uuid.uuid4())
|
|
112
|
+
download_dir.mkdir(mode=0o700)
|
|
113
|
+
# Path.mkdir applies the kernel umask to the supplied mode. Calling
|
|
114
|
+
# _restrict_directory invokes os.chmod directly, bypassing umask so
|
|
115
|
+
# the per-download UUID directory is owner-only (0o700) on POSIX,
|
|
116
|
+
# consistent with the protection applied to ``root`` above.
|
|
117
|
+
self._restrict_directory(download_dir)
|
|
118
|
+
final_path = (download_dir / filename).resolve()
|
|
119
|
+
if final_path.parent != download_dir.resolve() or not final_path.is_relative_to(
|
|
120
|
+
root
|
|
121
|
+
):
|
|
122
|
+
self._remove_empty_directory(download_dir)
|
|
123
|
+
raise InvalidDownloadError("Download filename escapes configured path")
|
|
124
|
+
|
|
125
|
+
known_size = self._applicable_content_length(content_length, content_encoding)
|
|
126
|
+
truncated = known_size is not None and known_size > limit
|
|
127
|
+
source_size_at_least: int | None = None
|
|
128
|
+
bytes_seen = 0
|
|
129
|
+
bytes_written = 0
|
|
130
|
+
md5 = hashlib.md5(usedforsecurity=False)
|
|
131
|
+
sha256 = hashlib.sha256()
|
|
132
|
+
temp_path: Path | None = None
|
|
133
|
+
stream_closed = False
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
fd, raw_temp_path = tempfile.mkstemp(
|
|
137
|
+
prefix=".download-", suffix=".tmp", dir=download_dir
|
|
138
|
+
)
|
|
139
|
+
temp_path = Path(raw_temp_path)
|
|
140
|
+
with os.fdopen(fd, "wb") as target:
|
|
141
|
+
async for chunk in chunks:
|
|
142
|
+
if not isinstance(chunk, bytes):
|
|
143
|
+
raise DownloadError("Download stream yielded a non-bytes chunk")
|
|
144
|
+
if not chunk:
|
|
145
|
+
continue
|
|
146
|
+
|
|
147
|
+
observation_limit = (
|
|
148
|
+
limit
|
|
149
|
+
if known_size is not None and known_size > limit
|
|
150
|
+
else limit + 1
|
|
151
|
+
)
|
|
152
|
+
remaining_probe = observation_limit - bytes_seen
|
|
153
|
+
observed = chunk[:remaining_probe]
|
|
154
|
+
bytes_seen += len(observed)
|
|
155
|
+
writable = observed[: max(0, limit - bytes_written)]
|
|
156
|
+
if writable:
|
|
157
|
+
target.write(writable)
|
|
158
|
+
md5.update(writable)
|
|
159
|
+
sha256.update(writable)
|
|
160
|
+
bytes_written += len(writable)
|
|
161
|
+
|
|
162
|
+
if bytes_seen > limit or (
|
|
163
|
+
known_size is not None
|
|
164
|
+
and known_size > limit
|
|
165
|
+
and bytes_written == limit
|
|
166
|
+
):
|
|
167
|
+
truncated = True
|
|
168
|
+
break
|
|
169
|
+
|
|
170
|
+
if (
|
|
171
|
+
known_size is not None
|
|
172
|
+
and known_size > limit
|
|
173
|
+
and bytes_written < limit
|
|
174
|
+
):
|
|
175
|
+
raise DownloadError(
|
|
176
|
+
"Download stream ended before the declared prefix was received"
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
await self._close_stream(chunks)
|
|
180
|
+
stream_closed = True
|
|
181
|
+
target.flush()
|
|
182
|
+
os.fsync(target.fileno())
|
|
183
|
+
|
|
184
|
+
os.chmod(temp_path, 0o600)
|
|
185
|
+
os.replace(temp_path, final_path)
|
|
186
|
+
temp_path = None
|
|
187
|
+
|
|
188
|
+
if known_size is None:
|
|
189
|
+
if bytes_seen > limit:
|
|
190
|
+
source_size = None
|
|
191
|
+
source_size_at_least = limit + 1
|
|
192
|
+
else:
|
|
193
|
+
source_size = bytes_seen
|
|
194
|
+
else:
|
|
195
|
+
source_size = known_size
|
|
196
|
+
|
|
197
|
+
if truncated:
|
|
198
|
+
warning: dict[str, Any] = {
|
|
199
|
+
"op": f"{namespace}.{operation}",
|
|
200
|
+
"file_size": bytes_written,
|
|
201
|
+
"max_download_bytes": limit,
|
|
202
|
+
"truncated": True,
|
|
203
|
+
"source_size": source_size,
|
|
204
|
+
"source_size_at_least": source_size_at_least,
|
|
205
|
+
}
|
|
206
|
+
logger.warning("Binary download truncated", extra=warning)
|
|
207
|
+
|
|
208
|
+
return DownloadResult(
|
|
209
|
+
file_path=str(final_path),
|
|
210
|
+
file_size=bytes_written,
|
|
211
|
+
checksum_md5=md5.hexdigest(),
|
|
212
|
+
checksum_sha256=sha256.hexdigest(),
|
|
213
|
+
mime_type=mime_type,
|
|
214
|
+
truncated=truncated,
|
|
215
|
+
source_size=source_size,
|
|
216
|
+
source_size_at_least=source_size_at_least,
|
|
217
|
+
)
|
|
218
|
+
except BaseException:
|
|
219
|
+
if temp_path is not None:
|
|
220
|
+
temp_path.unlink(missing_ok=True)
|
|
221
|
+
final_path.unlink(missing_ok=True)
|
|
222
|
+
self._remove_empty_directory(download_dir)
|
|
223
|
+
raise
|
|
224
|
+
finally:
|
|
225
|
+
if not stream_closed:
|
|
226
|
+
await self._close_stream(chunks)
|
|
227
|
+
|
|
228
|
+
@staticmethod
|
|
229
|
+
def _applicable_content_length(
|
|
230
|
+
content_length: str | None, content_encoding: str | None
|
|
231
|
+
) -> int | None:
|
|
232
|
+
encoding = (content_encoding or "identity").strip().casefold()
|
|
233
|
+
if encoding not in {"", "identity"} or content_length is None:
|
|
234
|
+
return None
|
|
235
|
+
try:
|
|
236
|
+
value = int(content_length.strip())
|
|
237
|
+
except (AttributeError, ValueError):
|
|
238
|
+
return None
|
|
239
|
+
return value if value >= 0 else None
|
|
240
|
+
|
|
241
|
+
@classmethod
|
|
242
|
+
def _safe_filename(
|
|
243
|
+
cls,
|
|
244
|
+
original_filename: str | None,
|
|
245
|
+
*,
|
|
246
|
+
namespace: str,
|
|
247
|
+
operation: str,
|
|
248
|
+
mime_type: str | None,
|
|
249
|
+
) -> str:
|
|
250
|
+
if original_filename is None or not original_filename.strip():
|
|
251
|
+
extension = mimetypes.guess_extension(mime_type or "") or ".bin"
|
|
252
|
+
timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
253
|
+
original_filename = f"{namespace}_{operation}_{timestamp}{extension}"
|
|
254
|
+
if "\x00" in original_filename or any(
|
|
255
|
+
separator in original_filename for separator in ("/", "\\")
|
|
256
|
+
):
|
|
257
|
+
raise InvalidDownloadError(
|
|
258
|
+
"Download filename contains a forbidden separator"
|
|
259
|
+
)
|
|
260
|
+
candidate = Path(original_filename)
|
|
261
|
+
if candidate.is_absolute() or original_filename in {".", ".."}:
|
|
262
|
+
raise InvalidDownloadError("Download filename must be a relative basename")
|
|
263
|
+
normalized = unicodedata.normalize("NFKC", original_filename).strip()
|
|
264
|
+
sanitized = _SAFE_FILENAME.sub("_", normalized).strip("._")
|
|
265
|
+
if not sanitized:
|
|
266
|
+
raise InvalidDownloadError("Download filename is empty after sanitization")
|
|
267
|
+
return sanitized[:255]
|
|
268
|
+
|
|
269
|
+
@staticmethod
|
|
270
|
+
def _restrict_directory(path: Path) -> None:
|
|
271
|
+
if os.name != "nt":
|
|
272
|
+
os.chmod(path, 0o700)
|
|
273
|
+
|
|
274
|
+
@staticmethod
|
|
275
|
+
def _remove_empty_directory(path: Path) -> None:
|
|
276
|
+
try:
|
|
277
|
+
path.rmdir()
|
|
278
|
+
except OSError:
|
|
279
|
+
pass
|
|
280
|
+
|
|
281
|
+
@staticmethod
|
|
282
|
+
async def _close_stream(chunks: AsyncIterable[bytes]) -> None:
|
|
283
|
+
close = getattr(chunks, "aclose", None)
|
|
284
|
+
if close is None:
|
|
285
|
+
return
|
|
286
|
+
result = close()
|
|
287
|
+
if inspect.isawaitable(result):
|
|
288
|
+
await result
|