deepctl-core 0.2.17__tar.gz → 0.2.19__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/PKG-INFO +1 -1
  2. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/pyproject.toml +1 -1
  3. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/auth.py +5 -2
  4. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/base_command.py +54 -12
  5. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/base_group_command.py +5 -2
  6. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/client.py +6 -2
  7. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/output.py +7 -5
  8. deepctl_core-0.2.19/src/deepctl_core/skill_bundle.py +405 -0
  9. deepctl_core-0.2.19/src/deepctl_core/skill_generator.py +1293 -0
  10. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/timing.py +5 -2
  11. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core.egg-info/PKG-INFO +1 -1
  12. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core.egg-info/SOURCES.txt +1 -0
  13. deepctl_core-0.2.17/src/deepctl_core/skill_generator.py +0 -1129
  14. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/README.md +0 -0
  15. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/setup.cfg +0 -0
  16. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/__init__.py +0 -0
  17. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/config.py +0 -0
  18. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/models.py +0 -0
  19. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/plugin_env.py +0 -0
  20. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/plugin_manager.py +0 -0
  21. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core/py.typed +0 -0
  22. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core.egg-info/dependency_links.txt +0 -0
  23. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core.egg-info/requires.txt +0 -0
  24. {deepctl_core-0.2.17 → deepctl_core-0.2.19}/src/deepctl_core.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: deepctl-core
3
- Version: 0.2.17
3
+ Version: 0.2.19
4
4
  Summary: Core components for deepctl
5
5
  Author-email: Deepgram <devrel@deepgram.com>
6
6
  Maintainer-email: Deepgram <devrel@deepgram.com>
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "deepctl-core"
7
- version = "0.2.17" # x-release-please-version
7
+ version = "0.2.19" # x-release-please-version
8
8
  description = "Core components for deepctl"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -9,14 +9,17 @@ from urllib.parse import urlencode
9
9
  import httpx
10
10
  import keyring
11
11
  from pydantic import BaseModel
12
- from rich.console import Console
13
12
  from rich.progress import Progress, SpinnerColumn, TextColumn
14
13
 
15
14
  from .client import _split_base_url
16
15
  from .config import Config
17
16
  from .models import ProfileInfo, ProfilesResult
17
+ from .output import stderr_console
18
18
 
19
- console = Console()
19
+ # Diagnostics only: everything printed here is status or error chrome, so
20
+ # it goes to stderr and can never corrupt the machine-readable payload a
21
+ # command writes to stdout. See get_status_console() in output.py.
22
+ console = stderr_console
20
23
 
21
24
  # Auth provider base URL (dx-id OIDC provider)
22
25
  AUTH_BASE_URL = os.getenv("DEEPGRAM_CLI_BASE_URL", "https://id.dx.deepgram.com")
@@ -4,15 +4,20 @@ from abc import ABC, abstractmethod
4
4
  from typing import Any, ClassVar
5
5
 
6
6
  import click
7
- from rich.console import Console
8
7
 
9
8
  from .auth import AuthManager
10
9
  from .client import DeepgramClient
11
10
  from .config import Config
12
- from .output import _agentic, print_error, print_info, print_warning, stderr_console
11
+ from .models import ErrorResult
12
+ from .output import _agentic, print_error, stderr_console
13
+ from .output import console as stdout_console
13
14
  from .timing import TimingContext
14
15
 
15
- console = Console()
16
+ # The PAYLOAD console -- stdout, deliberately. Tables, JSON and raw text
17
+ # written by _output_* are the machine-readable result and belong there.
18
+ # Shared instance rather than a bare Console() so it honours the agentic
19
+ # no-color/highlight settings; diagnostics use stderr_console instead.
20
+ console = stdout_console
16
21
 
17
22
 
18
23
  class BaseCommand(ABC):
@@ -99,16 +104,52 @@ class BaseCommand(ABC):
99
104
  "explicit flags",
100
105
  "environment variables",
101
106
  ]:
102
- print_info(f"Using credentials from {source}")
107
+ # Diagnostics, not the result. print_info /
108
+ # print_warning still write to stdout outside
109
+ # agentic mode, so with an explicit `-o json`
110
+ # these three lines landed in front of the
111
+ # payload and broke `| jq` (#104, success
112
+ # path). Route them straight to stderr; the
113
+ # prefixes mirror output.py so the rendering
114
+ # is unchanged in both modes.
115
+ info = "INFO:" if _agentic else "[blue]ℹ[/blue]"
116
+ stderr_console.print(
117
+ f"{info} Using credentials from {source}"
118
+ )
103
119
  if project_id:
104
- print_info(f"Affecting project: {project_id}")
120
+ stderr_console.print(
121
+ f"{info} Affecting project: {project_id}"
122
+ )
105
123
  else:
106
- print_warning("No project ID specified")
107
-
108
- except Exception:
109
- # guard() already printed helpful error messages;
110
- # exit without duplicating them.
111
- raise SystemExit(1)
124
+ warn = "WARN:" if _agentic else "[yellow]⚠[/yellow]"
125
+ stderr_console.print(
126
+ f"{warn} No project ID specified"
127
+ )
128
+
129
+ except Exception as auth_error:
130
+ # guard() already wrote the human-readable diagnosis to
131
+ # stderr, so don't duplicate it -- but stdout must still
132
+ # carry a parseable payload in a machine-readable
133
+ # format. A CI step doing `dg -o json ... > out.json`
134
+ # and parsing the result should get a structured error,
135
+ # not an empty file. output_result is a no-op in
136
+ # default mode, so this adds nothing for humans.
137
+ self._tag_telemetry_status("error")
138
+ try:
139
+ self.output_result(
140
+ ErrorResult(error=str(auth_error)), config
141
+ )
142
+ except OSError:
143
+ # Downstream stream closed. BrokenPipeError is an
144
+ # OSError, so one clause covers both.
145
+ pass
146
+ except ValueError as exc:
147
+ # Rich raises ValueError("I/O operation on closed
148
+ # file") when the stream went away mid-write; any
149
+ # other ValueError is a real bug.
150
+ if "closed file" not in str(exc):
151
+ raise
152
+ raise SystemExit(1) from auth_error
112
153
 
113
154
  # Check project ID if required
114
155
  if self.requires_project:
@@ -139,10 +180,11 @@ class BaseCommand(ABC):
139
180
  with TimingContext("output_processing"):
140
181
  try:
141
182
  self.output_result(result, config)
142
- except (BrokenPipeError, OSError):
183
+ except OSError:
143
184
  # Downstream stream closed (e.g. an MCP host disconnected
144
185
  # stdio after `dg mcp` finished). Nothing useful to log
145
186
  # here because the logger writes to the same closed stream.
187
+ # BrokenPipeError is an OSError, so one clause covers both.
146
188
  pass
147
189
  except ValueError as exc:
148
190
  if "closed file" not in str(exc):
@@ -3,14 +3,17 @@
3
3
  from typing import Any
4
4
 
5
5
  import click
6
- from rich.console import Console
7
6
 
8
7
  from .auth import AuthManager
9
8
  from .base_command import BaseCommand
10
9
  from .client import DeepgramClient
11
10
  from .config import Config
11
+ from .output import stderr_console
12
12
 
13
- console = Console()
13
+ # Diagnostics only: everything printed here is status or error chrome, so
14
+ # it goes to stderr and can never corrupt the machine-readable payload a
15
+ # command writes to stdout. See get_status_console() in output.py.
16
+ console = stderr_console
14
17
 
15
18
 
16
19
  class BaseGroupCommand(BaseCommand):
@@ -9,7 +9,8 @@ from typing import TYPE_CHECKING, Any, cast
9
9
  from deepgram import DeepgramClient as DGClient
10
10
  from deepgram import DeepgramClientEnvironment
11
11
  from deepgram.core.api_error import ApiError
12
- from rich.console import Console
12
+
13
+ from .output import stderr_console
13
14
 
14
15
  if TYPE_CHECKING:
15
16
  from collections.abc import Iterator
@@ -18,7 +19,10 @@ if TYPE_CHECKING:
18
19
  from .auth import AuthManager
19
20
  from .config import Config
20
21
 
21
- console = Console()
22
+ # Diagnostics only: everything printed here is status or error chrome, so
23
+ # it goes to stderr and can never corrupt the machine-readable payload a
24
+ # command writes to stdout. See get_status_console() in output.py.
25
+ console = stderr_console
22
26
 
23
27
 
24
28
  def _split_base_url(base_url: str) -> tuple[str, str, str]:
@@ -369,22 +369,24 @@ def print_error(message: str) -> None:
369
369
  stderr_console.print(f"[red]✗[/red] {message}")
370
370
 
371
371
 
372
- def print_warning(message: str) -> None:
373
- """Print warning message."""
372
+ def print_warning(message: str, *, stderr: bool = False) -> None:
373
+ """Print a warning; ``stderr=True`` keeps it off stdout outside agentic mode too."""
374
374
  if not _output_config["quiet"]:
375
375
  if _output_config["agentic"]:
376
376
  stderr_console.print(f"WARN: {message}")
377
377
  else:
378
- console.print(f"[yellow]⚠[/yellow] {message}")
378
+ (stderr_console if stderr else console).print(
379
+ f"[yellow]⚠[/yellow] {message}"
380
+ )
379
381
 
380
382
 
381
- def print_info(message: str) -> None:
383
+ def print_info(message: str, *, stderr: bool = False) -> None:
382
384
  """Print info message."""
383
385
  if not _output_config["quiet"]:
384
386
  if _output_config["agentic"]:
385
387
  stderr_console.print(f"INFO: {message}")
386
388
  else:
387
- console.print(f"[blue]ℹ[/blue] {message}")
389
+ (stderr_console if stderr else console).print(f"[blue]ℹ[/blue] {message}")
388
390
 
389
391
 
390
392
  def print_debug(message: str) -> None:
@@ -0,0 +1,405 @@
1
+ """Fetch the deepgram/skills bundle from an immutable pin and list its skills.
2
+
3
+ The repository is downloaded as one codeload tarball, unpacked with explicit
4
+ safety checks and published atomically into deepctl's cache. The skills come
5
+ from the upstream ``.claude-plugin/marketplace.json``, and each name is checked
6
+ to be one plain directory name before any path is built from it.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import hashlib
12
+ import io
13
+ import json
14
+ import os
15
+ import re
16
+ import shutil
17
+ import tarfile
18
+ import tempfile
19
+ from dataclasses import dataclass
20
+ from pathlib import Path
21
+ from typing import TYPE_CHECKING
22
+
23
+ import httpx
24
+
25
+ if TYPE_CHECKING:
26
+ from collections.abc import Callable
27
+
28
+ SKILLS_REPO = "deepgram/skills"
29
+ REF_ENV_VAR = "DEEPCTL_SKILLS_REF"
30
+
31
+ # The default bundle is pinned by full commit SHA plus the sha256 of its
32
+ # codeload tarball, because the release tag is a lightweight, movable tag.
33
+ DEFAULT_SKILLS_COMMIT = "0fc13fad726fb78e17fb1f05ba5942f0d022990f"
34
+ DEFAULT_SKILLS_SHA256 = (
35
+ "5b7f975378110372c8ae3a3c712b72ba2fa43b06f2d1d93497abf87262a23980"
36
+ )
37
+ #: The release that DEFAULT_SKILLS_COMMIT is. A label for people; never fetched.
38
+ DEFAULT_SKILLS_RELEASE = "deepgram-skills-v1.7.0"
39
+
40
+ _MANIFEST_PATH = ".claude-plugin/marketplace.json"
41
+ _PLUGIN_NAME = "deepgram"
42
+ _SKILL_ENTRY_FILE = "SKILL.md"
43
+
44
+ _DOWNLOAD_TIMEOUT = 30.0
45
+ # Caps on the download and what it may unpack to. The real bundle is about
46
+ # 160 KB with 65 members, so anything near these is not the bundle we expect.
47
+ # They bound member count, names and file data, but tarfile reads pax/GNU header
48
+ # metadata before they run. That is accepted: the pin is hash-checked first, and
49
+ # a user ref's tarball comes from codeload's git archive, not crafted pax data.
50
+ _MAX_BUNDLE_BYTES = 64 * 1024 * 1024
51
+ _MAX_EXTRACTED_BYTES = 256 * 1024 * 1024
52
+ _MAX_MEMBERS = 10_000
53
+ _MAX_MEMBER_NAME_BYTES = 512
54
+
55
+ _MAX_REF_LENGTH = 100
56
+ # The longest valid ref, "a/" * 49 + "aa", spells to 4 + 49 * 4 + 2 = 202 bytes.
57
+ # 120 leaves a typical Windows home, the cache root and the bundle's deepest
58
+ # file (46 characters) under MAX_PATH (260). "pinned-<sha>" is 47 bytes.
59
+ _MAX_CACHE_NAME_BYTES = 120
60
+ _REF_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*")
61
+ # A manifest entry must be exactly ``./skills/<name>`` or ``skills/<name>``,
62
+ # and ``<name>`` must be one plain directory name.
63
+ _ENTRY_PATTERN = re.compile(r"(?:\./)?skills/([A-Za-z0-9][A-Za-z0-9._-]*)")
64
+ # A path component Windows cannot store as written: a reserved device name,
65
+ # with or without an extension, or a name it would strip a trailing ' '/'.' from.
66
+ _WINDOWS_UNSAFE = re.compile(
67
+ r"(?i)(?:CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|(?:COM|LPT)[0-9\u00b9\u00b2\u00b3])"
68
+ r"(?:\..*)?|.*[ .]"
69
+ )
70
+
71
+ # Written into every cache directory this module publishes. A non-empty directory
72
+ # at a cache path without it is never moved or deleted. POSIX rename replaces an
73
+ # empty one, so one appearing in the rename window can go (no file data at risk).
74
+ _CACHE_MARKER = ".deepctl-skills-cache"
75
+ _STAGING_PREFIX = ".tmp-"
76
+
77
+
78
+ class SkillFetchError(Exception):
79
+ """The skills bundle could not be fetched, unpacked or trusted."""
80
+
81
+
82
+ class SkillRefNotFoundError(SkillFetchError):
83
+ """The skills repository has no such ref (HTTP 404)."""
84
+
85
+
86
+ class SkillRefInvalidError(SkillFetchError):
87
+ """The ref is not one this module will put in a URL or a directory name."""
88
+
89
+
90
+ @dataclass(frozen=True)
91
+ class RepoSkill:
92
+ """One validated skill: a plain directory name and its folder in the cache."""
93
+
94
+ name: str
95
+ path: Path
96
+
97
+
98
+ def _cache_name(ref: str) -> str:
99
+ """Return the cache directory name for a validated ``ref``."""
100
+ # The prefixes keep every user ref out of the hash-checked pin's directory,
101
+ # even on a case-insensitive filesystem. '%' is not a ref character, so
102
+ # spelling '/' as '%2F' cannot make two refs collide.
103
+ if ref == DEFAULT_SKILLS_COMMIT:
104
+ return f"pinned-{ref}"
105
+ return "ref-" + ref.replace("/", "%2F")
106
+
107
+
108
+ def portable_name(name: str) -> bool:
109
+ """True if ``name`` is one plain folder name safe on every OS (as entries)."""
110
+ match = _ENTRY_PATTERN.fullmatch(f"skills/{name}")
111
+ return bool(match) and not _WINDOWS_UNSAFE.fullmatch(name)
112
+
113
+
114
+ def validate_ref(ref: str) -> str:
115
+ """Return ``ref`` if it is safe in a URL path and a cache name, else raise."""
116
+ if len(ref) > _MAX_REF_LENGTH:
117
+ raise SkillRefInvalidError(
118
+ f"The skills ref is {len(ref)} characters, more than the "
119
+ f"{_MAX_REF_LENGTH} allowed."
120
+ )
121
+ if (
122
+ not _REF_PATTERN.fullmatch(ref)
123
+ or ".." in ref
124
+ or "//" in ref
125
+ or "/." in ref
126
+ or ref.endswith(("/", "."))
127
+ ):
128
+ raise SkillRefInvalidError(
129
+ f"The skills ref {ref!r} must start with a letter or digit, use only "
130
+ "letters, digits and '._/-', and not contain '..', empty segments or "
131
+ "segments starting with '.', or end in '.' or '/'."
132
+ )
133
+ if len(_cache_name(ref).encode("utf-8")) > _MAX_CACHE_NAME_BYTES:
134
+ raise SkillRefInvalidError(
135
+ f"The skills ref {ref!r} would need a cache directory name longer "
136
+ f"than {_MAX_CACHE_NAME_BYTES} bytes."
137
+ )
138
+ return ref
139
+
140
+
141
+ def resolve_skills_ref(ref: str | None = None) -> str:
142
+ """Pick the ref: ``ref``, then a non-blank ``DEEPCTL_SKILLS_REF``, then the pin.
143
+
144
+ Raises :class:`SkillRefInvalidError` if the chosen ref fails validation.
145
+ """
146
+ if ref is not None:
147
+ return validate_ref(ref)
148
+ from_env = os.environ.get(REF_ENV_VAR, "").strip()
149
+ if from_env:
150
+ return validate_ref(from_env)
151
+ return DEFAULT_SKILLS_COMMIT
152
+
153
+
154
+ def bundle_url(ref: str) -> str:
155
+ """Return the codeload tarball URL for a validated ``ref``."""
156
+ return f"https://codeload.github.com/{SKILLS_REPO}/tar.gz/{ref}"
157
+
158
+
159
+ def fetch_skill_bundle(
160
+ ref: str | None = None,
161
+ *,
162
+ cache_dir: Path | None = None,
163
+ force: bool = False,
164
+ download: Callable[[str], bytes] | None = None,
165
+ ) -> list[RepoSkill]:
166
+ """Fetch the skills bundle into the cache and return its skills.
167
+
168
+ A user ref (argument or ``DEEPCTL_SKILLS_REF``) has no known hash, so only
169
+ the pinned commit is checked against :data:`DEFAULT_SKILLS_SHA256`.
170
+
171
+ Only the pinned commit, whose content cannot change, is served from the
172
+ cache. Its directory is published only after the hash check and full
173
+ validation pass, and is validated again on every hit. Other refs may move,
174
+ so they are always downloaded. ``force`` skips the cache hit. ``cache_dir``
175
+ must be a directory deepctl owns. ``download`` returns a URL's bytes. The
176
+ returned paths are valid until the same ref is fetched again.
177
+
178
+ Raises :class:`SkillRefInvalidError`, :class:`SkillRefNotFoundError` or,
179
+ for any other failure, :class:`SkillFetchError`.
180
+ """
181
+ resolved = resolve_skills_ref(ref)
182
+ pinned = resolved == DEFAULT_SKILLS_COMMIT
183
+ root = cache_dir or Path.home() / ".deepctl" / "skills" / "repo_cache"
184
+ target = root / _cache_name(resolved)
185
+
186
+ if pinned and not force and _is_our_cache(target):
187
+ try:
188
+ return read_manifest_skills(target)
189
+ except SkillFetchError:
190
+ pass # Incomplete or damaged: download and replace it below.
191
+
192
+ data = (download or _download)(bundle_url(resolved))
193
+ if pinned and hashlib.sha256(data).hexdigest() != DEFAULT_SKILLS_SHA256:
194
+ raise SkillFetchError(
195
+ f"The downloaded {DEFAULT_SKILLS_RELEASE} bundle does not match its "
196
+ "pinned sha256, so it was not used."
197
+ )
198
+ return _publish(data, target)
199
+
200
+
201
+ def read_manifest_skills(root: Path) -> list[RepoSkill]:
202
+ """Return the skills ``root``'s manifest lists for the deepgram plugin.
203
+
204
+ Every entry is validated before any path is built from it. Other plugins
205
+ point at other repositories and are ignored. Raises
206
+ :class:`SkillFetchError` for a bad manifest, entry or skill folder.
207
+ """
208
+ try:
209
+ raw = json.loads((root / _MANIFEST_PATH).read_text(encoding="utf-8"))
210
+ except (OSError, ValueError) as exc:
211
+ raise SkillFetchError(f"Could not read the skills manifest: {exc}.")
212
+
213
+ plugins = raw.get("plugins") if isinstance(raw, dict) else None
214
+ if not isinstance(plugins, list):
215
+ plugins = []
216
+ matches = [
217
+ p for p in plugins if isinstance(p, dict) and p.get("name") == _PLUGIN_NAME
218
+ ]
219
+ entries = matches[0].get("skills") if len(matches) == 1 else None
220
+ if not isinstance(entries, list) or not entries:
221
+ raise SkillFetchError(
222
+ f"The skills manifest has no single {_PLUGIN_NAME!r} plugin listing skills."
223
+ )
224
+
225
+ names: list[str] = []
226
+ seen: set[str] = set()
227
+ for entry in entries:
228
+ match = _ENTRY_PATTERN.fullmatch(entry) if isinstance(entry, str) else None
229
+ # The pattern already refuses '/', '\\', ':', '.' and '..'.
230
+ if match is None or _WINDOWS_UNSAFE.fullmatch(match.group(1)):
231
+ raise SkillFetchError(
232
+ f"The skills manifest entry {entry!r} is not ./skills/<portable name>."
233
+ )
234
+ name = match.group(1)
235
+ if name.casefold() in seen:
236
+ raise SkillFetchError(f"The skills manifest lists {name!r} twice.")
237
+ seen.add(name.casefold())
238
+ names.append(name)
239
+
240
+ skills = [RepoSkill(name, root / "skills" / name) for name in names]
241
+ for skill in skills:
242
+ try:
243
+ found = (skill.path / _SKILL_ENTRY_FILE).is_file()
244
+ except OSError: # For example, a name too long for the filesystem.
245
+ found = False
246
+ if not found:
247
+ raise SkillFetchError(
248
+ f"The skill {skill.name!r} has no {_SKILL_ENTRY_FILE} in the bundle."
249
+ )
250
+ return skills
251
+
252
+
253
+ def _download(url: str, *, transport: httpx.BaseTransport | None = None) -> bytes:
254
+ """Return the bytes at ``url``, refusing more than the bundle size cap."""
255
+ try:
256
+ with (
257
+ httpx.Client(
258
+ transport=transport, timeout=_DOWNLOAD_TIMEOUT, follow_redirects=True
259
+ ) as client,
260
+ client.stream("GET", url) as resp,
261
+ ):
262
+ if resp.status_code == 404:
263
+ raise SkillRefNotFoundError(f"No skills bundle was found at {url}.")
264
+ if resp.status_code != 200:
265
+ raise SkillFetchError(
266
+ f"Downloading {url} failed with HTTP {resp.status_code}."
267
+ )
268
+ data = bytearray()
269
+ for chunk in resp.iter_bytes():
270
+ data += chunk
271
+ if len(data) > _MAX_BUNDLE_BYTES:
272
+ raise SkillFetchError(
273
+ f"The skills bundle is larger than {_MAX_BUNDLE_BYTES} bytes."
274
+ )
275
+ return bytes(data)
276
+ except httpx.HTTPError as exc:
277
+ raise SkillFetchError(f"Could not download {url}: {str(exc).rstrip('.')}.")
278
+
279
+
280
+ def _is_our_cache(path: Path) -> bool:
281
+ """True if ``path`` is a real directory that this module published."""
282
+ try:
283
+ return not path.is_symlink() and (path / _CACHE_MARKER).is_file()
284
+ except OSError:
285
+ return False
286
+
287
+
288
+ def _safe_members(tar: tarfile.TarFile) -> list[tuple[tarfile.TarInfo, str]]:
289
+ """Check every member, then return each with its top-level dir stripped.
290
+
291
+ Nothing is written until every member has passed.
292
+ """
293
+ checked: list[tuple[tarfile.TarInfo, str]] = []
294
+ tops: set[str] = set()
295
+ total = 0
296
+ for member in tar:
297
+ name = member.name
298
+ if len(checked) >= _MAX_MEMBERS:
299
+ raise SkillFetchError(f"The bundle has more than {_MAX_MEMBERS} members.")
300
+ if len(name.encode("utf-8", "surrogateescape")) > _MAX_MEMBER_NAME_BYTES:
301
+ raise SkillFetchError("The bundle has a member name that is too long.")
302
+ if not (member.isreg() or member.isdir()):
303
+ # Symlinks, hardlinks, devices and fifos never belong in a bundle.
304
+ raise SkillFetchError(
305
+ f"The bundle member {name!r} is not a regular file or directory."
306
+ )
307
+ parts = [p for p in name.split("/") if p not in ("", ".")]
308
+ # '\\' and ':' cover Windows separators, drives (C:\x, C:x) and streams.
309
+ windows = "\\" in name or ":" in name
310
+ windows = windows or any(_WINDOWS_UNSAFE.fullmatch(p) for p in parts)
311
+ if not parts or ".." in parts or name.startswith("/") or windows:
312
+ raise SkillFetchError(f"The bundle member {name!r} is not a safe path.")
313
+ total += member.size if member.isreg() else 0
314
+ if total > _MAX_EXTRACTED_BYTES:
315
+ raise SkillFetchError(
316
+ f"The bundle unpacks to more than {_MAX_EXTRACTED_BYTES} bytes."
317
+ )
318
+ tops.add(parts[0])
319
+ checked.append((member, "/".join(parts[1:])))
320
+ if len(tops) != 1:
321
+ raise SkillFetchError("The bundle does not have a single top-level directory.")
322
+ return checked
323
+
324
+
325
+ def _extract(data: bytes, dest: Path) -> None:
326
+ """Unpack the tarball ``data`` into the new directory ``dest``."""
327
+ try:
328
+ with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
329
+ members = _safe_members(tar)
330
+ dest.mkdir()
331
+ for member, rel in members:
332
+ path = dest.joinpath(*rel.split("/")) if rel else dest
333
+ if member.isdir():
334
+ path.mkdir(parents=True, exist_ok=True)
335
+ continue
336
+ path.parent.mkdir(parents=True, exist_ok=True)
337
+ src = tar.extractfile(member)
338
+ if src is None:
339
+ raise SkillFetchError(f"Could not read {member.name!r}.")
340
+ # 'xb' refuses to overwrite, so a duplicate member (or a
341
+ # case-insensitive clash) fails instead of replacing a file.
342
+ with src, path.open("xb") as out:
343
+ shutil.copyfileobj(src, out)
344
+ except (tarfile.TarError, OSError, EOFError) as exc:
345
+ raise SkillFetchError(f"Could not unpack the skills bundle: {exc}.")
346
+
347
+
348
+ def _publish(data: bytes, target: Path) -> list[RepoSkill]:
349
+ """Unpack, validate and atomically put the bundle at ``target``.
350
+
351
+ Work happens in a fresh ``mkdtemp`` directory beside ``target``, so every
352
+ rename stays on one filesystem. A previous cache is renamed into staging,
353
+ the new tree is renamed into place, and if that fails a directory is put
354
+ back; anything else is kept in staging, so nothing is lost or overwritten.
355
+ If another process publishes the same ref first, its copy is used.
356
+ """
357
+ try:
358
+ target.parent.mkdir(parents=True, exist_ok=True)
359
+ staging = Path(tempfile.mkdtemp(prefix=_STAGING_PREFIX, dir=target.parent))
360
+ except OSError as exc:
361
+ raise SkillFetchError(f"Could not prepare the skills cache: {exc}.")
362
+ new, previous, swapping = staging / "new", staging / "previous", False
363
+ try:
364
+ _extract(data, new)
365
+ skills = read_manifest_skills(new)
366
+ try:
367
+ (new / _CACHE_MARKER).write_text(target.name, encoding="utf-8")
368
+ if target.is_symlink() or target.exists():
369
+ if not _is_our_cache(target):
370
+ raise SkillFetchError(
371
+ f"{target} was not created by deepctl, so it was left alone"
372
+ )
373
+ os.replace(target, previous)
374
+ if not _is_our_cache(previous): # Swapped in after the check.
375
+ raise SkillFetchError(
376
+ f"{target} changed while publishing, so it was not replaced"
377
+ )
378
+ swapping = True
379
+ os.replace(new, target) # Can replace an empty dir; see _CACHE_MARKER.
380
+ except BaseException as exc: # Ctrl-C too: never rmtree the only copy.
381
+ if swapping and isinstance(exc, OSError) and _is_our_cache(target):
382
+ # Another process published this ref after ours moved aside.
383
+ return read_manifest_skills(target)
384
+ try: # Put back only a real dir: renaming a file or link clobbers.
385
+ if os.path.isdir(previous) and not os.path.islink(previous):
386
+ os.replace(previous, target) # Same empty-dir caveat.
387
+ except OSError:
388
+ pass # The finally sees that staging holds the only copy.
389
+ mine = _is_our_cache(target) and _is_our_cache(previous)
390
+ kept = os.path.lexists(previous) and not mine
391
+ where = f"; what was there is kept in {previous}" if kept else ""
392
+ if not isinstance(exc, (OSError, SkillFetchError)):
393
+ raise
394
+ raise SkillFetchError(
395
+ f"Could not publish the skills bundle to {target}: {exc}{where}."
396
+ )
397
+ return [RepoSkill(s.name, target / "skills" / s.name) for s in skills]
398
+ finally:
399
+ # Ownership: mkdtemp made ``staging`` in this call. Delete it only if
400
+ # ``previous`` is absent, or both it and ``target`` are proven ours, so
401
+ # a sole or foreign copy is kept. State, not a flag, survives a Ctrl-C.
402
+ if not os.path.lexists(previous) or (
403
+ _is_our_cache(target) and _is_our_cache(previous)
404
+ ):
405
+ shutil.rmtree(staging, ignore_errors=True)