syq 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syq-0.0.1/LICENSE +21 -0
- syq-0.0.1/PKG-INFO +85 -0
- syq-0.0.1/README.md +63 -0
- syq-0.0.1/pyproject.toml +44 -0
- syq-0.0.1/setup.cfg +4 -0
- syq-0.0.1/src/syq/__init__.py +19 -0
- syq-0.0.1/src/syq/bootstrap.py +331 -0
- syq-0.0.1/src/syq/client.py +114 -0
- syq-0.0.1/src/syq/py.typed +1 -0
- syq-0.0.1/src/syq/syq-release-manifest.json +69 -0
- syq-0.0.1/src/syq.egg-info/PKG-INFO +85 -0
- syq-0.0.1/src/syq.egg-info/SOURCES.txt +15 -0
- syq-0.0.1/src/syq.egg-info/dependency_links.txt +1 -0
- syq-0.0.1/src/syq.egg-info/top_level.txt +1 -0
- syq-0.0.1/tests/test_bootstrap.py +168 -0
- syq-0.0.1/tests/test_candidate.py +53 -0
- syq-0.0.1/tests/test_client.py +182 -0
syq-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Grant Reaber
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
syq-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: syq
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Version-pinned Python adapter for the syq parallel file copier
|
|
5
|
+
Author: Grant Reaber
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/greaber/syq
|
|
8
|
+
Project-URL: Repository, https://github.com/greaber/syq
|
|
9
|
+
Project-URL: Issues, https://github.com/greaber/syq/issues
|
|
10
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: MacOS
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Topic :: System :: Filesystems
|
|
17
|
+
Classifier: Topic :: System :: Networking
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# syq for Python
|
|
24
|
+
|
|
25
|
+
`syq` is the official preview Python adapter for the
|
|
26
|
+
[syq parallel file copier](https://github.com/greaber/syq). It invokes syq with
|
|
27
|
+
an argument array and never constructs a shell command.
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
python -m pip install syq
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Package installation does not download an executable. The first call that
|
|
34
|
+
needs syq downloads the exact release pinned by this SDK into the user cache.
|
|
35
|
+
For Python package `0.0.1`, that release is syq `0.1.5`.
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import syq
|
|
39
|
+
|
|
40
|
+
print(syq.__version__) # Python package version
|
|
41
|
+
print(syq.PINNED_SYQ_VERSION) # tested executable version
|
|
42
|
+
print(syq.managed_executable()) # downloads once, then returns the cached path
|
|
43
|
+
|
|
44
|
+
plan = syq.run([
|
|
45
|
+
"cp",
|
|
46
|
+
"project",
|
|
47
|
+
"--to",
|
|
48
|
+
"server",
|
|
49
|
+
"--into",
|
|
50
|
+
"/backup",
|
|
51
|
+
"--dry-run",
|
|
52
|
+
])
|
|
53
|
+
print(plan.stdout.decode())
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The managed executable is stored below
|
|
57
|
+
`$XDG_CACHE_HOME/syq/sdk/python/v0.1.5/` or, when `XDG_CACHE_HOME` is not an
|
|
58
|
+
absolute path, `~/.cache/syq/sdk/python/v0.1.5/`. The SDK checks the complete
|
|
59
|
+
cached binary against its embedded release manifest before every use. A corrupt
|
|
60
|
+
or missing cache entry is replaced atomically with a freshly downloaded,
|
|
61
|
+
verified binary.
|
|
62
|
+
|
|
63
|
+
`run()` raises `SyqProcessError` for a nonzero process status by default. The
|
|
64
|
+
exception retains the complete result, including stdout and stderr as bytes.
|
|
65
|
+
Pass `check=False` when the caller wants to interpret the status directly.
|
|
66
|
+
When `timeout` expires or the caller is interrupted, the SDK kills and reaps
|
|
67
|
+
syq's local process group, including child processes such as SSH transports,
|
|
68
|
+
before propagating the exception.
|
|
69
|
+
|
|
70
|
+
## Custom executable override
|
|
71
|
+
|
|
72
|
+
An explicit executable bypasses the managed version:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
result = syq.run(["--help"], executable="/opt/custom/bin/syq")
|
|
76
|
+
custom_version = syq.version(executable="syq") # intentional PATH lookup
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The SDK makes no compatibility or provenance guarantee for an override. Use it
|
|
80
|
+
for local development, controlled offline provisioning, or when deliberately
|
|
81
|
+
testing a different syq release.
|
|
82
|
+
|
|
83
|
+
The package targets Python 3.10 or newer on Linux and macOS and has no runtime
|
|
84
|
+
Python dependencies. See the [SDK compatibility policy](../README.md) for the
|
|
85
|
+
release mapping.
|
syq-0.0.1/README.md
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# syq for Python
|
|
2
|
+
|
|
3
|
+
`syq` is the official preview Python adapter for the
|
|
4
|
+
[syq parallel file copier](https://github.com/greaber/syq). It invokes syq with
|
|
5
|
+
an argument array and never constructs a shell command.
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
python -m pip install syq
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Package installation does not download an executable. The first call that
|
|
12
|
+
needs syq downloads the exact release pinned by this SDK into the user cache.
|
|
13
|
+
For Python package `0.0.1`, that release is syq `0.1.5`.
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
import syq
|
|
17
|
+
|
|
18
|
+
print(syq.__version__) # Python package version
|
|
19
|
+
print(syq.PINNED_SYQ_VERSION) # tested executable version
|
|
20
|
+
print(syq.managed_executable()) # downloads once, then returns the cached path
|
|
21
|
+
|
|
22
|
+
plan = syq.run([
|
|
23
|
+
"cp",
|
|
24
|
+
"project",
|
|
25
|
+
"--to",
|
|
26
|
+
"server",
|
|
27
|
+
"--into",
|
|
28
|
+
"/backup",
|
|
29
|
+
"--dry-run",
|
|
30
|
+
])
|
|
31
|
+
print(plan.stdout.decode())
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The managed executable is stored below
|
|
35
|
+
`$XDG_CACHE_HOME/syq/sdk/python/v0.1.5/` or, when `XDG_CACHE_HOME` is not an
|
|
36
|
+
absolute path, `~/.cache/syq/sdk/python/v0.1.5/`. The SDK checks the complete
|
|
37
|
+
cached binary against its embedded release manifest before every use. A corrupt
|
|
38
|
+
or missing cache entry is replaced atomically with a freshly downloaded,
|
|
39
|
+
verified binary.
|
|
40
|
+
|
|
41
|
+
`run()` raises `SyqProcessError` for a nonzero process status by default. The
|
|
42
|
+
exception retains the complete result, including stdout and stderr as bytes.
|
|
43
|
+
Pass `check=False` when the caller wants to interpret the status directly.
|
|
44
|
+
When `timeout` expires or the caller is interrupted, the SDK kills and reaps
|
|
45
|
+
syq's local process group, including child processes such as SSH transports,
|
|
46
|
+
before propagating the exception.
|
|
47
|
+
|
|
48
|
+
## Custom executable override
|
|
49
|
+
|
|
50
|
+
An explicit executable bypasses the managed version:
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
result = syq.run(["--help"], executable="/opt/custom/bin/syq")
|
|
54
|
+
custom_version = syq.version(executable="syq") # intentional PATH lookup
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The SDK makes no compatibility or provenance guarantee for an override. Use it
|
|
58
|
+
for local development, controlled offline provisioning, or when deliberately
|
|
59
|
+
testing a different syq release.
|
|
60
|
+
|
|
61
|
+
The package targets Python 3.10 or newer on Linux and macOS and has no runtime
|
|
62
|
+
Python dependencies. See the [SDK compatibility policy](../README.md) for the
|
|
63
|
+
release mapping.
|
syq-0.0.1/pyproject.toml
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools==80.9.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "syq"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
description = "Version-pinned Python adapter for the syq parallel file copier"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "Grant Reaber" },
|
|
15
|
+
]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: MacOS",
|
|
20
|
+
"Operating System :: POSIX :: Linux",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Topic :: System :: Filesystems",
|
|
24
|
+
"Topic :: System :: Networking",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/greaber/syq"
|
|
29
|
+
Repository = "https://github.com/greaber/syq"
|
|
30
|
+
Issues = "https://github.com/greaber/syq/issues"
|
|
31
|
+
|
|
32
|
+
[dependency-groups]
|
|
33
|
+
dev = [
|
|
34
|
+
"build==1.3.0",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[tool.setuptools.packages.find]
|
|
38
|
+
where = ["src"]
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.package-data]
|
|
41
|
+
syq = ["py.typed", "syq-release-manifest.json"]
|
|
42
|
+
|
|
43
|
+
[tool.uv]
|
|
44
|
+
required-version = "==0.11.6"
|
syq-0.0.1/setup.cfg
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Safe subprocess access to the syq executable."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version as distribution_version
|
|
4
|
+
|
|
5
|
+
from .bootstrap import PINNED_SYQ_VERSION, SyqInstallError, managed_executable
|
|
6
|
+
from .client import Result, SyqOutputError, SyqProcessError, run, version
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"PINNED_SYQ_VERSION",
|
|
10
|
+
"Result",
|
|
11
|
+
"SyqInstallError",
|
|
12
|
+
"SyqOutputError",
|
|
13
|
+
"SyqProcessError",
|
|
14
|
+
"managed_executable",
|
|
15
|
+
"run",
|
|
16
|
+
"version",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
__version__ = distribution_version("syq")
|
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
"""Install the syq release pinned by this Python package."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import gzip
|
|
6
|
+
import hashlib
|
|
7
|
+
import http.client
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import platform
|
|
11
|
+
import stat
|
|
12
|
+
import subprocess
|
|
13
|
+
import tempfile
|
|
14
|
+
import urllib.error
|
|
15
|
+
import urllib.parse
|
|
16
|
+
import urllib.request
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from functools import lru_cache
|
|
19
|
+
from importlib.resources import files
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any, BinaryIO
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
_DOWNLOAD_TIMEOUT_SECONDS = 30
|
|
25
|
+
_CHUNK_SIZE = 1024 * 1024
|
|
26
|
+
_EXPECTED_REPOSITORY = "https://github.com/greaber/syq"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SyqInstallError(RuntimeError):
|
|
30
|
+
"""The SDK could not install or validate its pinned syq executable."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True, slots=True)
|
|
34
|
+
class _Artifact:
|
|
35
|
+
target: str
|
|
36
|
+
archive_name: str
|
|
37
|
+
archive_sha256: str
|
|
38
|
+
archive_size: int
|
|
39
|
+
binary_sha256: str
|
|
40
|
+
binary_size: int
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@lru_cache(maxsize=1)
|
|
44
|
+
def _load_release_manifest() -> dict[str, Any]:
|
|
45
|
+
try:
|
|
46
|
+
raw = files("syq").joinpath("syq-release-manifest.json").read_bytes()
|
|
47
|
+
manifest = json.loads(raw)
|
|
48
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as error:
|
|
49
|
+
raise SyqInstallError(
|
|
50
|
+
"the packaged syq release manifest is invalid"
|
|
51
|
+
) from error
|
|
52
|
+
if not isinstance(manifest, dict):
|
|
53
|
+
raise SyqInstallError("the packaged syq release manifest is not an object")
|
|
54
|
+
if manifest.get("repository") != _EXPECTED_REPOSITORY:
|
|
55
|
+
raise SyqInstallError("the packaged syq release repository is unexpected")
|
|
56
|
+
version = manifest.get("version")
|
|
57
|
+
tag = manifest.get("tag")
|
|
58
|
+
if not isinstance(version, str) or tag != f"v{version}":
|
|
59
|
+
raise SyqInstallError("the packaged syq release version is invalid")
|
|
60
|
+
return manifest
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
PINNED_SYQ_VERSION = str(_load_release_manifest()["version"])
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _host_target() -> str:
|
|
67
|
+
system = platform.system().lower()
|
|
68
|
+
machine = platform.machine().lower()
|
|
69
|
+
if system == "linux" and machine in {"x86_64", "amd64"}:
|
|
70
|
+
return "linux-x86_64"
|
|
71
|
+
if system == "linux" and machine in {"aarch64", "arm64"}:
|
|
72
|
+
return "linux-aarch64"
|
|
73
|
+
if system == "darwin" and machine in {"arm64", "aarch64"}:
|
|
74
|
+
return "macos-arm64"
|
|
75
|
+
if system == "darwin" and machine in {"x86_64", "amd64"}:
|
|
76
|
+
return "macos-x86_64"
|
|
77
|
+
raise SyqInstallError(
|
|
78
|
+
f"syq {PINNED_SYQ_VERSION} has no binary for {system or 'unknown'} "
|
|
79
|
+
f"{machine or 'unknown'}"
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _positive_integer(value: Any, *, label: str) -> int:
|
|
84
|
+
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
|
85
|
+
raise SyqInstallError(f"the packaged {label} is invalid")
|
|
86
|
+
return value
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _digest(value: Any, *, label: str) -> str:
|
|
90
|
+
if (
|
|
91
|
+
not isinstance(value, str)
|
|
92
|
+
or len(value) != 64
|
|
93
|
+
or any(character not in "0123456789abcdef" for character in value)
|
|
94
|
+
):
|
|
95
|
+
raise SyqInstallError(f"the packaged {label} is invalid")
|
|
96
|
+
return value
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _artifact(manifest: dict[str, Any], target: str) -> _Artifact:
|
|
100
|
+
try:
|
|
101
|
+
entry = manifest["artifacts"][target]
|
|
102
|
+
archive = entry["archive"]
|
|
103
|
+
binary = entry["binary"]
|
|
104
|
+
except (KeyError, TypeError) as error:
|
|
105
|
+
raise SyqInstallError(f"the packaged metadata has no {target} artifact") from error
|
|
106
|
+
if not isinstance(archive, dict) or not isinstance(binary, dict):
|
|
107
|
+
raise SyqInstallError(f"the packaged metadata has an invalid {target} artifact")
|
|
108
|
+
archive_name = archive.get("name")
|
|
109
|
+
if (
|
|
110
|
+
not isinstance(archive_name, str)
|
|
111
|
+
or not archive_name.startswith("syq-")
|
|
112
|
+
or not archive_name.endswith(".gz")
|
|
113
|
+
or "/" in archive_name
|
|
114
|
+
or "\\" in archive_name
|
|
115
|
+
):
|
|
116
|
+
raise SyqInstallError("the packaged archive name is invalid")
|
|
117
|
+
return _Artifact(
|
|
118
|
+
target=target,
|
|
119
|
+
archive_name=archive_name,
|
|
120
|
+
archive_sha256=_digest(archive.get("sha256"), label="archive digest"),
|
|
121
|
+
archive_size=_positive_integer(archive.get("size"), label="archive size"),
|
|
122
|
+
binary_sha256=_digest(binary.get("sha256"), label="binary digest"),
|
|
123
|
+
binary_size=_positive_integer(binary.get("size"), label="binary size"),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _default_cache_root() -> Path:
|
|
128
|
+
configured = os.environ.get("XDG_CACHE_HOME")
|
|
129
|
+
if configured and os.path.isabs(configured):
|
|
130
|
+
return Path(configured)
|
|
131
|
+
try:
|
|
132
|
+
return Path.home() / ".cache"
|
|
133
|
+
except RuntimeError as error:
|
|
134
|
+
raise SyqInstallError("could not locate the user cache directory") from error
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _sha256_file(path: Path) -> str:
|
|
138
|
+
digest = hashlib.sha256()
|
|
139
|
+
with path.open("rb") as stream:
|
|
140
|
+
for chunk in iter(lambda: stream.read(_CHUNK_SIZE), b""):
|
|
141
|
+
digest.update(chunk)
|
|
142
|
+
return digest.hexdigest()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _matches(path: Path, *, size: int, sha256: str) -> bool:
|
|
146
|
+
try:
|
|
147
|
+
metadata = path.stat(follow_symlinks=False)
|
|
148
|
+
return (
|
|
149
|
+
stat.S_ISREG(metadata.st_mode)
|
|
150
|
+
and metadata.st_size == size
|
|
151
|
+
and _sha256_file(path) == sha256
|
|
152
|
+
)
|
|
153
|
+
except OSError:
|
|
154
|
+
return False
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _copy_bounded(
|
|
158
|
+
source: BinaryIO,
|
|
159
|
+
destination: BinaryIO,
|
|
160
|
+
*,
|
|
161
|
+
expected_size: int,
|
|
162
|
+
label: str,
|
|
163
|
+
) -> str:
|
|
164
|
+
digest = hashlib.sha256()
|
|
165
|
+
total = 0
|
|
166
|
+
while True:
|
|
167
|
+
chunk = source.read(min(_CHUNK_SIZE, expected_size + 1 - total))
|
|
168
|
+
if not chunk:
|
|
169
|
+
break
|
|
170
|
+
total += len(chunk)
|
|
171
|
+
if total > expected_size:
|
|
172
|
+
raise SyqInstallError(f"the downloaded {label} is larger than expected")
|
|
173
|
+
digest.update(chunk)
|
|
174
|
+
destination.write(chunk)
|
|
175
|
+
if total != expected_size:
|
|
176
|
+
raise SyqInstallError(
|
|
177
|
+
f"the downloaded {label} has size {total}, expected {expected_size}"
|
|
178
|
+
)
|
|
179
|
+
return digest.hexdigest()
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _download_archive(url: str, destination: Path, artifact: _Artifact) -> None:
|
|
183
|
+
request = urllib.request.Request(url, headers={"User-Agent": "syq-python-sdk"})
|
|
184
|
+
try:
|
|
185
|
+
with urllib.request.urlopen(
|
|
186
|
+
request, timeout=_DOWNLOAD_TIMEOUT_SECONDS
|
|
187
|
+
) as response, destination.open("wb") as output:
|
|
188
|
+
final_url = response.geturl()
|
|
189
|
+
if urllib.parse.urlparse(final_url).scheme != "https":
|
|
190
|
+
raise SyqInstallError("the syq download redirected away from HTTPS")
|
|
191
|
+
actual_sha256 = _copy_bounded(
|
|
192
|
+
response,
|
|
193
|
+
output,
|
|
194
|
+
expected_size=artifact.archive_size,
|
|
195
|
+
label="archive",
|
|
196
|
+
)
|
|
197
|
+
except SyqInstallError:
|
|
198
|
+
raise
|
|
199
|
+
except (OSError, http.client.HTTPException, urllib.error.URLError) as error:
|
|
200
|
+
raise SyqInstallError(f"could not download pinned syq from {url}") from error
|
|
201
|
+
if actual_sha256 != artifact.archive_sha256:
|
|
202
|
+
raise SyqInstallError("the downloaded syq archive failed SHA-256 verification")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _decompress_archive(
|
|
206
|
+
archive_path: Path, binary_path: Path, artifact: _Artifact
|
|
207
|
+
) -> None:
|
|
208
|
+
try:
|
|
209
|
+
with gzip.open(archive_path, "rb") as source, binary_path.open("wb") as output:
|
|
210
|
+
actual_sha256 = _copy_bounded(
|
|
211
|
+
source,
|
|
212
|
+
output,
|
|
213
|
+
expected_size=artifact.binary_size,
|
|
214
|
+
label="binary",
|
|
215
|
+
)
|
|
216
|
+
except SyqInstallError:
|
|
217
|
+
raise
|
|
218
|
+
except (EOFError, gzip.BadGzipFile, OSError) as error:
|
|
219
|
+
raise SyqInstallError("could not decompress the pinned syq archive") from error
|
|
220
|
+
if actual_sha256 != artifact.binary_sha256:
|
|
221
|
+
raise SyqInstallError("the downloaded syq binary failed SHA-256 verification")
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _query_binary(path: Path, argument: str) -> str:
|
|
225
|
+
try:
|
|
226
|
+
completed = subprocess.run(
|
|
227
|
+
[os.fspath(path), argument],
|
|
228
|
+
stdin=subprocess.DEVNULL,
|
|
229
|
+
stdout=subprocess.PIPE,
|
|
230
|
+
stderr=subprocess.PIPE,
|
|
231
|
+
timeout=10,
|
|
232
|
+
check=False,
|
|
233
|
+
shell=False,
|
|
234
|
+
)
|
|
235
|
+
except (OSError, subprocess.TimeoutExpired) as error:
|
|
236
|
+
raise SyqInstallError("the downloaded syq binary could not run") from error
|
|
237
|
+
try:
|
|
238
|
+
output = completed.stdout.decode("utf-8").strip()
|
|
239
|
+
except UnicodeDecodeError as error:
|
|
240
|
+
raise SyqInstallError(
|
|
241
|
+
"the downloaded syq binary returned invalid output"
|
|
242
|
+
) from error
|
|
243
|
+
if completed.returncode != 0:
|
|
244
|
+
raise SyqInstallError(
|
|
245
|
+
f"the downloaded syq binary rejected {argument} with status "
|
|
246
|
+
f"{completed.returncode}"
|
|
247
|
+
)
|
|
248
|
+
return output
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _validate_binary(path: Path, manifest: dict[str, Any]) -> None:
|
|
252
|
+
version = manifest["version"]
|
|
253
|
+
tag = manifest["tag"]
|
|
254
|
+
if _query_binary(path, "--version") != f"syq {version}":
|
|
255
|
+
raise SyqInstallError("the downloaded binary reports an unexpected version")
|
|
256
|
+
if _query_binary(path, "--build-identity") != tag:
|
|
257
|
+
raise SyqInstallError("the downloaded binary reports an unexpected build identity")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def managed_executable(
|
|
261
|
+
*, cache_dir: str | os.PathLike[str] | None = None
|
|
262
|
+
) -> Path:
|
|
263
|
+
"""Return the verified executable pinned by this SDK, downloading if needed.
|
|
264
|
+
|
|
265
|
+
The executable is cached by syq release and host target. Its complete bytes
|
|
266
|
+
are checked against the manifest packaged in this SDK before every use.
|
|
267
|
+
"""
|
|
268
|
+
|
|
269
|
+
manifest = _load_release_manifest()
|
|
270
|
+
target = _host_target()
|
|
271
|
+
artifact = _artifact(manifest, target)
|
|
272
|
+
cache_root = (
|
|
273
|
+
Path(cache_dir) if cache_dir is not None else _default_cache_root()
|
|
274
|
+
)
|
|
275
|
+
install_dir = (
|
|
276
|
+
cache_root
|
|
277
|
+
/ "syq"
|
|
278
|
+
/ "sdk"
|
|
279
|
+
/ "python"
|
|
280
|
+
/ f"v{manifest['version']}"
|
|
281
|
+
/ target
|
|
282
|
+
)
|
|
283
|
+
try:
|
|
284
|
+
install_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
285
|
+
except OSError as error:
|
|
286
|
+
raise SyqInstallError(f"could not prepare syq SDK cache at {install_dir}") from error
|
|
287
|
+
executable = install_dir / "syq"
|
|
288
|
+
if _matches(
|
|
289
|
+
executable, size=artifact.binary_size, sha256=artifact.binary_sha256
|
|
290
|
+
):
|
|
291
|
+
try:
|
|
292
|
+
executable.chmod(0o755)
|
|
293
|
+
except OSError as error:
|
|
294
|
+
raise SyqInstallError(
|
|
295
|
+
f"could not make cached syq executable: {executable}"
|
|
296
|
+
) from error
|
|
297
|
+
return executable
|
|
298
|
+
|
|
299
|
+
base_url = f"{manifest['repository']}/releases/download/{manifest['tag']}"
|
|
300
|
+
url = f"{base_url}/{artifact.archive_name}"
|
|
301
|
+
archive_path: Path | None = None
|
|
302
|
+
binary_path: Path | None = None
|
|
303
|
+
try:
|
|
304
|
+
archive_file = tempfile.NamedTemporaryFile(
|
|
305
|
+
prefix=".syq-download-", suffix=".gz", dir=install_dir, delete=False
|
|
306
|
+
)
|
|
307
|
+
archive_path = Path(archive_file.name)
|
|
308
|
+
archive_file.close()
|
|
309
|
+
binary_file = tempfile.NamedTemporaryFile(
|
|
310
|
+
prefix=".syq-install-", dir=install_dir, delete=False
|
|
311
|
+
)
|
|
312
|
+
binary_path = Path(binary_file.name)
|
|
313
|
+
binary_file.close()
|
|
314
|
+
_download_archive(url, archive_path, artifact)
|
|
315
|
+
_decompress_archive(archive_path, binary_path, artifact)
|
|
316
|
+
binary_path.chmod(0o755)
|
|
317
|
+
_validate_binary(binary_path, manifest)
|
|
318
|
+
os.replace(binary_path, executable)
|
|
319
|
+
binary_path = None
|
|
320
|
+
return executable
|
|
321
|
+
except SyqInstallError:
|
|
322
|
+
raise
|
|
323
|
+
except OSError as error:
|
|
324
|
+
raise SyqInstallError(f"could not install pinned syq at {executable}") from error
|
|
325
|
+
finally:
|
|
326
|
+
for temporary_path in (archive_path, binary_path):
|
|
327
|
+
if temporary_path is not None:
|
|
328
|
+
try:
|
|
329
|
+
temporary_path.unlink(missing_ok=True)
|
|
330
|
+
except OSError:
|
|
331
|
+
pass
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Minimal process adapter for syq's command-line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import signal
|
|
7
|
+
import subprocess
|
|
8
|
+
from collections.abc import Mapping, Sequence
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
|
|
11
|
+
from .bootstrap import managed_executable
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True, slots=True)
|
|
15
|
+
class Result:
|
|
16
|
+
"""The complete result of one syq process."""
|
|
17
|
+
|
|
18
|
+
argv: tuple[str, ...]
|
|
19
|
+
returncode: int
|
|
20
|
+
stdout: bytes
|
|
21
|
+
stderr: bytes
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SyqProcessError(RuntimeError):
|
|
25
|
+
"""A syq process completed with a nonzero status."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, result: Result) -> None:
|
|
28
|
+
self.result = result
|
|
29
|
+
super().__init__(f"syq exited with status {result.returncode}")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class SyqOutputError(ValueError):
|
|
33
|
+
"""syq returned output that the requested operation cannot interpret."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _text_arg(value: str | os.PathLike[str], *, label: str) -> str:
|
|
37
|
+
result = os.fspath(value)
|
|
38
|
+
if not isinstance(result, str):
|
|
39
|
+
raise TypeError(f"{label} must resolve to text, not bytes")
|
|
40
|
+
return result
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def run(
|
|
44
|
+
args: Sequence[str | os.PathLike[str]],
|
|
45
|
+
*,
|
|
46
|
+
executable: str | os.PathLike[str] | None = None,
|
|
47
|
+
check: bool = True,
|
|
48
|
+
cwd: str | os.PathLike[str] | None = None,
|
|
49
|
+
env: Mapping[str, str] | None = None,
|
|
50
|
+
timeout: float | None = None,
|
|
51
|
+
) -> Result:
|
|
52
|
+
"""Run syq without a shell and capture its complete byte output.
|
|
53
|
+
|
|
54
|
+
``args`` contains only arguments after the executable name. By default the
|
|
55
|
+
SDK downloads and uses its pinned syq release. Passing ``executable`` opts
|
|
56
|
+
into an untested custom binary. A missing custom executable, timeout, or
|
|
57
|
+
other spawn failure is reported by ``subprocess``. A completed nonzero
|
|
58
|
+
process raises :class:`SyqProcessError` unless ``check`` is false.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
if isinstance(args, (str, bytes, os.PathLike)):
|
|
62
|
+
raise TypeError("args must be a sequence of individual arguments")
|
|
63
|
+
executable_text = (
|
|
64
|
+
os.fspath(managed_executable())
|
|
65
|
+
if executable is None
|
|
66
|
+
else _text_arg(executable, label="executable")
|
|
67
|
+
)
|
|
68
|
+
argument_text = tuple(
|
|
69
|
+
_text_arg(argument, label=f"args[{index}]")
|
|
70
|
+
for index, argument in enumerate(args)
|
|
71
|
+
)
|
|
72
|
+
argv = (executable_text, *argument_text)
|
|
73
|
+
process = subprocess.Popen(
|
|
74
|
+
argv,
|
|
75
|
+
cwd=cwd,
|
|
76
|
+
env=env,
|
|
77
|
+
stdin=subprocess.DEVNULL,
|
|
78
|
+
stdout=subprocess.PIPE,
|
|
79
|
+
stderr=subprocess.PIPE,
|
|
80
|
+
shell=False,
|
|
81
|
+
start_new_session=True,
|
|
82
|
+
)
|
|
83
|
+
try:
|
|
84
|
+
stdout, stderr = process.communicate(timeout=timeout)
|
|
85
|
+
except BaseException:
|
|
86
|
+
try:
|
|
87
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
88
|
+
except ProcessLookupError:
|
|
89
|
+
pass
|
|
90
|
+
process.communicate()
|
|
91
|
+
raise
|
|
92
|
+
result = Result(
|
|
93
|
+
argv=argv,
|
|
94
|
+
returncode=process.returncode,
|
|
95
|
+
stdout=stdout,
|
|
96
|
+
stderr=stderr,
|
|
97
|
+
)
|
|
98
|
+
if check and result.returncode != 0:
|
|
99
|
+
raise SyqProcessError(result)
|
|
100
|
+
return result
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def version(*, executable: str | os.PathLike[str] | None = None) -> str:
|
|
104
|
+
"""Return the version of the pinned or explicitly overridden executable."""
|
|
105
|
+
|
|
106
|
+
result = run(["--version"], executable=executable)
|
|
107
|
+
try:
|
|
108
|
+
output = result.stdout.decode("utf-8").strip()
|
|
109
|
+
except UnicodeDecodeError as error:
|
|
110
|
+
raise SyqOutputError("syq --version did not return UTF-8") from error
|
|
111
|
+
prefix = "syq "
|
|
112
|
+
if not output.startswith(prefix) or len(output) == len(prefix):
|
|
113
|
+
raise SyqOutputError(f"unexpected syq --version output: {output!r}")
|
|
114
|
+
return output[len(prefix) :]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
{
|
|
2
|
+
"artifacts": {
|
|
3
|
+
"linux-aarch64": {
|
|
4
|
+
"archive": {
|
|
5
|
+
"name": "syq-linux-aarch64.gz",
|
|
6
|
+
"sha256": "949dc1d108ce9d659f9a35354e77afcc582170fc85d20fca20fd163c034e6c7e",
|
|
7
|
+
"size": 4232034
|
|
8
|
+
},
|
|
9
|
+
"binary": {
|
|
10
|
+
"name": "syq-linux-aarch64",
|
|
11
|
+
"sha256": "a8b6b0e7602d789668bdd433946144ffa67116f815ed1bf5bea8308f141d6b12",
|
|
12
|
+
"size": 8798640
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"linux-x86_64": {
|
|
16
|
+
"archive": {
|
|
17
|
+
"name": "syq-linux-x86_64.gz",
|
|
18
|
+
"sha256": "a77c091e8a3d85f6c8d534f66e456ef2095efc7aba64acf1a8a747e3f7937e67",
|
|
19
|
+
"size": 4650527
|
|
20
|
+
},
|
|
21
|
+
"binary": {
|
|
22
|
+
"name": "syq-linux-x86_64",
|
|
23
|
+
"sha256": "eaf06c96e851ecd573313832607439a0c8033b6f5f6cca9c3c8b4b081556066d",
|
|
24
|
+
"size": 10683104
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
"macos-arm64": {
|
|
28
|
+
"archive": {
|
|
29
|
+
"name": "syq-macos-arm64.gz",
|
|
30
|
+
"sha256": "99379e50f8787391267b03cd84a0f30b541af8da7fc3bb3dd6b2585b877cd722",
|
|
31
|
+
"size": 3610717
|
|
32
|
+
},
|
|
33
|
+
"binary": {
|
|
34
|
+
"name": "syq-macos-arm64",
|
|
35
|
+
"sha256": "acd13c45729625d4ec994f4fbdf0c3b37d1e31ebca0d2e1639a9fa6f78b8aef7",
|
|
36
|
+
"size": 7858272
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
"macos-x86_64": {
|
|
40
|
+
"archive": {
|
|
41
|
+
"name": "syq-macos-x86_64.gz",
|
|
42
|
+
"sha256": "709d9fbb172dc01e4348da44e0af281fe7d03b2e7f2cf9c7fecb7bb95fcec3d4",
|
|
43
|
+
"size": 3951723
|
|
44
|
+
},
|
|
45
|
+
"binary": {
|
|
46
|
+
"name": "syq-macos-x86_64",
|
|
47
|
+
"sha256": "1b0c20ae5e213e726c13129c67a598393c2c63af7433b5fd44a2daae91401f2c",
|
|
48
|
+
"size": 8718620
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
"helper_id": "v0.1.5-p0",
|
|
53
|
+
"homebrew_formula": {
|
|
54
|
+
"name": "syq.rb",
|
|
55
|
+
"sha256": "13e3f5dc3640ada43f5d6c8d335b6ebfce4cedaea8a13e6dc77376b36f2f49a0",
|
|
56
|
+
"size": 1166
|
|
57
|
+
},
|
|
58
|
+
"installer": {
|
|
59
|
+
"name": "install.sh",
|
|
60
|
+
"sha256": "e5f083fe911d7b1c0fad1760f4bd0f0971796709e71bc45cce19b17debcf851f",
|
|
61
|
+
"size": 4931
|
|
62
|
+
},
|
|
63
|
+
"repository": "https://github.com/greaber/syq",
|
|
64
|
+
"schema": 1,
|
|
65
|
+
"signature": "rf6gNEhMqL93o7bUKQEBJ4U+YKhh19Opvy9yRLq75zJNqBfantuT0Web5WQK/ZLiMmCJUwTNj+O2Z+OEUMtqBA==",
|
|
66
|
+
"signature_scheme": "ed25519-jcs-v1",
|
|
67
|
+
"tag": "v0.1.5",
|
|
68
|
+
"version": "0.1.5"
|
|
69
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: syq
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Version-pinned Python adapter for the syq parallel file copier
|
|
5
|
+
Author: Grant Reaber
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/greaber/syq
|
|
8
|
+
Project-URL: Repository, https://github.com/greaber/syq
|
|
9
|
+
Project-URL: Issues, https://github.com/greaber/syq/issues
|
|
10
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: MacOS
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Topic :: System :: Filesystems
|
|
17
|
+
Classifier: Topic :: System :: Networking
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# syq for Python
|
|
24
|
+
|
|
25
|
+
`syq` is the official preview Python adapter for the
|
|
26
|
+
[syq parallel file copier](https://github.com/greaber/syq). It invokes syq with
|
|
27
|
+
an argument array and never constructs a shell command.
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
python -m pip install syq
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Package installation does not download an executable. The first call that
|
|
34
|
+
needs syq downloads the exact release pinned by this SDK into the user cache.
|
|
35
|
+
For Python package `0.0.1`, that release is syq `0.1.5`.
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import syq
|
|
39
|
+
|
|
40
|
+
print(syq.__version__) # Python package version
|
|
41
|
+
print(syq.PINNED_SYQ_VERSION) # tested executable version
|
|
42
|
+
print(syq.managed_executable()) # downloads once, then returns the cached path
|
|
43
|
+
|
|
44
|
+
plan = syq.run([
|
|
45
|
+
"cp",
|
|
46
|
+
"project",
|
|
47
|
+
"--to",
|
|
48
|
+
"server",
|
|
49
|
+
"--into",
|
|
50
|
+
"/backup",
|
|
51
|
+
"--dry-run",
|
|
52
|
+
])
|
|
53
|
+
print(plan.stdout.decode())
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The managed executable is stored below
|
|
57
|
+
`$XDG_CACHE_HOME/syq/sdk/python/v0.1.5/` or, when `XDG_CACHE_HOME` is not an
|
|
58
|
+
absolute path, `~/.cache/syq/sdk/python/v0.1.5/`. The SDK checks the complete
|
|
59
|
+
cached binary against its embedded release manifest before every use. A corrupt
|
|
60
|
+
or missing cache entry is replaced atomically with a freshly downloaded,
|
|
61
|
+
verified binary.
|
|
62
|
+
|
|
63
|
+
`run()` raises `SyqProcessError` for a nonzero process status by default. The
|
|
64
|
+
exception retains the complete result, including stdout and stderr as bytes.
|
|
65
|
+
Pass `check=False` when the caller wants to interpret the status directly.
|
|
66
|
+
When `timeout` expires or the caller is interrupted, the SDK kills and reaps
|
|
67
|
+
syq's local process group, including child processes such as SSH transports,
|
|
68
|
+
before propagating the exception.
|
|
69
|
+
|
|
70
|
+
## Custom executable override
|
|
71
|
+
|
|
72
|
+
An explicit executable bypasses the managed version:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
result = syq.run(["--help"], executable="/opt/custom/bin/syq")
|
|
76
|
+
custom_version = syq.version(executable="syq") # intentional PATH lookup
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The SDK makes no compatibility or provenance guarantee for an override. Use it
|
|
80
|
+
for local development, controlled offline provisioning, or when deliberately
|
|
81
|
+
testing a different syq release.
|
|
82
|
+
|
|
83
|
+
The package targets Python 3.10 or newer on Linux and macOS and has no runtime
|
|
84
|
+
Python dependencies. See the [SDK compatibility policy](../README.md) for the
|
|
85
|
+
release mapping.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/syq/__init__.py
|
|
5
|
+
src/syq/bootstrap.py
|
|
6
|
+
src/syq/client.py
|
|
7
|
+
src/syq/py.typed
|
|
8
|
+
src/syq/syq-release-manifest.json
|
|
9
|
+
src/syq.egg-info/PKG-INFO
|
|
10
|
+
src/syq.egg-info/SOURCES.txt
|
|
11
|
+
src/syq.egg-info/dependency_links.txt
|
|
12
|
+
src/syq.egg-info/top_level.txt
|
|
13
|
+
tests/test_bootstrap.py
|
|
14
|
+
tests/test_candidate.py
|
|
15
|
+
tests/test_client.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
syq
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import gzip
|
|
4
|
+
import hashlib
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
import tempfile
|
|
8
|
+
import unittest
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from unittest import mock
|
|
11
|
+
|
|
12
|
+
import syq
|
|
13
|
+
from syq import bootstrap
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
FAKE_BINARY = b"""#!/bin/sh
|
|
17
|
+
case "$1" in
|
|
18
|
+
--version) printf 'syq 9.8.7\\n' ;;
|
|
19
|
+
--build-identity) printf 'v9.8.7\\n' ;;
|
|
20
|
+
*) exit 2 ;;
|
|
21
|
+
esac
|
|
22
|
+
"""
|
|
23
|
+
FAKE_ARCHIVE = gzip.compress(FAKE_BINARY, mtime=0)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _sha256(data: bytes) -> str:
|
|
27
|
+
return hashlib.sha256(data).hexdigest()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _release(binary_bytes: bytes = FAKE_BINARY) -> dict[str, object]:
|
|
31
|
+
archive_bytes = gzip.compress(binary_bytes, mtime=0)
|
|
32
|
+
return {
|
|
33
|
+
"repository": "https://github.com/greaber/syq",
|
|
34
|
+
"tag": "v9.8.7",
|
|
35
|
+
"version": "9.8.7",
|
|
36
|
+
"artifacts": {
|
|
37
|
+
"linux-x86_64": {
|
|
38
|
+
"archive": {
|
|
39
|
+
"name": "syq-linux-x86_64.gz",
|
|
40
|
+
"sha256": _sha256(archive_bytes),
|
|
41
|
+
"size": len(archive_bytes),
|
|
42
|
+
},
|
|
43
|
+
"binary": {
|
|
44
|
+
"name": "syq-linux-x86_64",
|
|
45
|
+
"sha256": _sha256(binary_bytes),
|
|
46
|
+
"size": len(binary_bytes),
|
|
47
|
+
},
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class _Response(io.BytesIO):
|
|
54
|
+
def geturl(self) -> str:
|
|
55
|
+
return "https://release-assets.githubusercontent.com/pinned"
|
|
56
|
+
|
|
57
|
+
def __enter__(self) -> _Response:
|
|
58
|
+
return self
|
|
59
|
+
|
|
60
|
+
def __exit__(self, *args: object) -> None:
|
|
61
|
+
self.close()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class BootstrapTests(unittest.TestCase):
|
|
65
|
+
def setUp(self) -> None:
|
|
66
|
+
self.temporary_directory = tempfile.TemporaryDirectory()
|
|
67
|
+
self.cache = Path(self.temporary_directory.name)
|
|
68
|
+
|
|
69
|
+
def tearDown(self) -> None:
|
|
70
|
+
self.temporary_directory.cleanup()
|
|
71
|
+
|
|
72
|
+
def _patch_release(self) -> tuple[mock._patch, mock._patch]:
|
|
73
|
+
return (
|
|
74
|
+
mock.patch.object(
|
|
75
|
+
bootstrap, "_load_release_manifest", return_value=_release()
|
|
76
|
+
),
|
|
77
|
+
mock.patch.object(bootstrap, "_host_target", return_value="linux-x86_64"),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def test_package_pin_matches_embedded_manifest(self) -> None:
|
|
81
|
+
manifest_path = Path(bootstrap.__file__).with_name("syq-release-manifest.json")
|
|
82
|
+
manifest = json.loads(manifest_path.read_bytes())
|
|
83
|
+
self.assertEqual(syq.PINNED_SYQ_VERSION, manifest["version"])
|
|
84
|
+
|
|
85
|
+
def test_downloads_validates_and_reuses_the_exact_binary(self) -> None:
|
|
86
|
+
manifest_patch, target_patch = self._patch_release()
|
|
87
|
+
download = mock.Mock(side_effect=lambda *args, **kwargs: _Response(FAKE_ARCHIVE))
|
|
88
|
+
with manifest_patch, target_patch, mock.patch.object(
|
|
89
|
+
bootstrap.urllib.request, "urlopen", download
|
|
90
|
+
):
|
|
91
|
+
first = bootstrap.managed_executable(cache_dir=self.cache)
|
|
92
|
+
second = bootstrap.managed_executable(cache_dir=self.cache)
|
|
93
|
+
|
|
94
|
+
self.assertEqual(first, second)
|
|
95
|
+
self.assertEqual(first.read_bytes(), FAKE_BINARY)
|
|
96
|
+
self.assertTrue(first.stat().st_mode & 0o100)
|
|
97
|
+
self.assertEqual(download.call_count, 1)
|
|
98
|
+
|
|
99
|
+
def test_corrupt_cached_binary_is_replaced(self) -> None:
|
|
100
|
+
manifest_patch, target_patch = self._patch_release()
|
|
101
|
+
download = mock.Mock(side_effect=lambda *args, **kwargs: _Response(FAKE_ARCHIVE))
|
|
102
|
+
with manifest_patch, target_patch, mock.patch.object(
|
|
103
|
+
bootstrap.urllib.request, "urlopen", download
|
|
104
|
+
):
|
|
105
|
+
executable = bootstrap.managed_executable(cache_dir=self.cache)
|
|
106
|
+
executable.write_bytes(b"tampered")
|
|
107
|
+
repaired = bootstrap.managed_executable(cache_dir=self.cache)
|
|
108
|
+
|
|
109
|
+
self.assertEqual(repaired.read_bytes(), FAKE_BINARY)
|
|
110
|
+
self.assertEqual(download.call_count, 2)
|
|
111
|
+
|
|
112
|
+
def test_corrupt_download_is_rejected_without_installing(self) -> None:
|
|
113
|
+
manifest_patch, target_patch = self._patch_release()
|
|
114
|
+
corrupted = bytes([FAKE_ARCHIVE[0] ^ 1]) + FAKE_ARCHIVE[1:]
|
|
115
|
+
with manifest_patch, target_patch, mock.patch.object(
|
|
116
|
+
bootstrap.urllib.request,
|
|
117
|
+
"urlopen",
|
|
118
|
+
return_value=_Response(corrupted),
|
|
119
|
+
):
|
|
120
|
+
with self.assertRaisesRegex(syq.SyqInstallError, "SHA-256"):
|
|
121
|
+
bootstrap.managed_executable(cache_dir=self.cache)
|
|
122
|
+
|
|
123
|
+
install_dir = (
|
|
124
|
+
self.cache / "syq" / "sdk" / "python" / "v9.8.7" / "linux-x86_64"
|
|
125
|
+
)
|
|
126
|
+
self.assertEqual(list(install_dir.iterdir()), [])
|
|
127
|
+
|
|
128
|
+
def test_hash_valid_binary_with_wrong_release_identity_is_rejected(self) -> None:
|
|
129
|
+
wrong_binary = FAKE_BINARY.replace(b"v9.8.7", b"source-build")
|
|
130
|
+
wrong_archive = gzip.compress(wrong_binary, mtime=0)
|
|
131
|
+
with mock.patch.object(
|
|
132
|
+
bootstrap, "_load_release_manifest", return_value=_release(wrong_binary)
|
|
133
|
+
), mock.patch.object(
|
|
134
|
+
bootstrap, "_host_target", return_value="linux-x86_64"
|
|
135
|
+
), mock.patch.object(
|
|
136
|
+
bootstrap.urllib.request,
|
|
137
|
+
"urlopen",
|
|
138
|
+
return_value=_Response(wrong_archive),
|
|
139
|
+
):
|
|
140
|
+
with self.assertRaisesRegex(syq.SyqInstallError, "build identity"):
|
|
141
|
+
bootstrap.managed_executable(cache_dir=self.cache)
|
|
142
|
+
|
|
143
|
+
def test_supported_host_aliases_select_release_targets(self) -> None:
|
|
144
|
+
cases = [
|
|
145
|
+
("Linux", "x86_64", "linux-x86_64"),
|
|
146
|
+
("Linux", "arm64", "linux-aarch64"),
|
|
147
|
+
("Darwin", "arm64", "macos-arm64"),
|
|
148
|
+
("Darwin", "amd64", "macos-x86_64"),
|
|
149
|
+
]
|
|
150
|
+
for system, machine, expected in cases:
|
|
151
|
+
with self.subTest(system=system, machine=machine), mock.patch.object(
|
|
152
|
+
bootstrap.platform, "system", return_value=system
|
|
153
|
+
), mock.patch.object(
|
|
154
|
+
bootstrap.platform, "machine", return_value=machine
|
|
155
|
+
):
|
|
156
|
+
self.assertEqual(bootstrap._host_target(), expected)
|
|
157
|
+
|
|
158
|
+
def test_unsupported_host_fails_before_downloading(self) -> None:
|
|
159
|
+
with mock.patch.object(
|
|
160
|
+
bootstrap.platform, "system", return_value="Windows"
|
|
161
|
+
), mock.patch.object(
|
|
162
|
+
bootstrap.platform, "machine", return_value="x86_64"
|
|
163
|
+
), self.assertRaisesRegex(syq.SyqInstallError, "no binary"):
|
|
164
|
+
bootstrap.managed_executable(cache_dir=self.cache)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
if __name__ == "__main__":
|
|
168
|
+
unittest.main()
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import tempfile
|
|
5
|
+
import unittest
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import syq
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
EXECUTABLE = os.environ.get("SYQ_CANDIDATE_EXECUTABLE")
|
|
12
|
+
EXPECTED_VERSION = os.environ.get("SYQ_CANDIDATE_VERSION")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@unittest.skipUnless(
|
|
16
|
+
EXECUTABLE and EXPECTED_VERSION,
|
|
17
|
+
"candidate compatibility requires SYQ_CANDIDATE_EXECUTABLE and version",
|
|
18
|
+
)
|
|
19
|
+
class CandidateCompatibilityTests(unittest.TestCase):
|
|
20
|
+
def test_candidate_version_and_local_copy(self) -> None:
|
|
21
|
+
assert EXECUTABLE is not None
|
|
22
|
+
assert EXPECTED_VERSION is not None
|
|
23
|
+
executable = Path(EXECUTABLE)
|
|
24
|
+
self.assertEqual(syq.version(executable=executable), EXPECTED_VERSION)
|
|
25
|
+
|
|
26
|
+
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
27
|
+
root = Path(temporary_directory)
|
|
28
|
+
source = root / "source; $(not-a-command)"
|
|
29
|
+
destination = root / "destination"
|
|
30
|
+
source.write_bytes(b"candidate compatibility\n")
|
|
31
|
+
|
|
32
|
+
result = syq.run(
|
|
33
|
+
["cp", source, "--as-new", destination, "--quiet"],
|
|
34
|
+
executable=executable,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
self.assertEqual(result.returncode, 0)
|
|
38
|
+
self.assertEqual(destination.read_bytes(), source.read_bytes())
|
|
39
|
+
|
|
40
|
+
def test_candidate_failure_is_retained(self) -> None:
|
|
41
|
+
assert EXECUTABLE is not None
|
|
42
|
+
result = syq.run(
|
|
43
|
+
["not-a-syq-command"],
|
|
44
|
+
executable=EXECUTABLE,
|
|
45
|
+
check=False,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
self.assertNotEqual(result.returncode, 0)
|
|
49
|
+
self.assertTrue(result.stderr)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__":
|
|
53
|
+
unittest.main()
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import signal
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
import tempfile
|
|
8
|
+
import time
|
|
9
|
+
import unittest
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from unittest import mock
|
|
12
|
+
|
|
13
|
+
import syq
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
FAKE_SYQ = """#!/bin/sh
|
|
17
|
+
case "$1" in
|
|
18
|
+
--version)
|
|
19
|
+
printf 'syq 9.8.7\\n'
|
|
20
|
+
;;
|
|
21
|
+
emit)
|
|
22
|
+
printf '%s' "$2"
|
|
23
|
+
printf 'diagnostic' >&2
|
|
24
|
+
;;
|
|
25
|
+
fail)
|
|
26
|
+
printf 'partial'
|
|
27
|
+
printf 'failed' >&2
|
|
28
|
+
exit 23
|
|
29
|
+
;;
|
|
30
|
+
spawn-descendant)
|
|
31
|
+
(
|
|
32
|
+
printf 'ready' > "$2.ready"
|
|
33
|
+
sleep 1
|
|
34
|
+
printf 'survived' > "$2"
|
|
35
|
+
) &
|
|
36
|
+
while [ ! -f "$2.ready" ]; do
|
|
37
|
+
sleep 0.01
|
|
38
|
+
done
|
|
39
|
+
sleep 30
|
|
40
|
+
;;
|
|
41
|
+
interrupt)
|
|
42
|
+
printf '%s' "$$" > "$2.pid"
|
|
43
|
+
(
|
|
44
|
+
printf 'ready' > "$2.ready"
|
|
45
|
+
sleep 1
|
|
46
|
+
printf 'survived' > "$2"
|
|
47
|
+
) &
|
|
48
|
+
while [ ! -f "$2.ready" ]; do
|
|
49
|
+
sleep 0.01
|
|
50
|
+
done
|
|
51
|
+
sleep 3
|
|
52
|
+
;;
|
|
53
|
+
*)
|
|
54
|
+
exit 2
|
|
55
|
+
;;
|
|
56
|
+
esac
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ClientTests(unittest.TestCase):
|
|
61
|
+
def setUp(self) -> None:
|
|
62
|
+
self.temporary_directory = tempfile.TemporaryDirectory()
|
|
63
|
+
self.executable = Path(self.temporary_directory.name) / "syq"
|
|
64
|
+
self.executable.write_text(FAKE_SYQ, encoding="utf-8")
|
|
65
|
+
self.executable.chmod(0o755)
|
|
66
|
+
|
|
67
|
+
def tearDown(self) -> None:
|
|
68
|
+
self.temporary_directory.cleanup()
|
|
69
|
+
|
|
70
|
+
def test_custom_executable_version_can_differ_from_package(self) -> None:
|
|
71
|
+
self.assertRegex(syq.__version__, r"^\d+\.\d+\.\d+$")
|
|
72
|
+
self.assertEqual(syq.version(executable=self.executable), "9.8.7")
|
|
73
|
+
|
|
74
|
+
def test_run_preserves_one_argument_with_shell_metacharacters(self) -> None:
|
|
75
|
+
argument = "a path; $(not-a-command)"
|
|
76
|
+
result = syq.run(["emit", argument], executable=self.executable)
|
|
77
|
+
self.assertEqual(result.argv, (os.fspath(self.executable), "emit", argument))
|
|
78
|
+
self.assertEqual(result.stdout, argument.encode())
|
|
79
|
+
self.assertEqual(result.stderr, b"diagnostic")
|
|
80
|
+
|
|
81
|
+
def test_default_run_uses_the_managed_executable(self) -> None:
|
|
82
|
+
with mock.patch(
|
|
83
|
+
"syq.client.managed_executable", return_value=self.executable
|
|
84
|
+
) as managed:
|
|
85
|
+
result = syq.run(["emit", "managed"])
|
|
86
|
+
|
|
87
|
+
managed.assert_called_once_with()
|
|
88
|
+
self.assertEqual(result.stdout, b"managed")
|
|
89
|
+
|
|
90
|
+
def test_explicit_executable_bypasses_the_managed_install(self) -> None:
|
|
91
|
+
with mock.patch(
|
|
92
|
+
"syq.client.managed_executable",
|
|
93
|
+
side_effect=AssertionError("managed install should not run"),
|
|
94
|
+
):
|
|
95
|
+
result = syq.run(["emit", "custom"], executable=self.executable)
|
|
96
|
+
|
|
97
|
+
self.assertEqual(result.stdout, b"custom")
|
|
98
|
+
|
|
99
|
+
def test_nonzero_result_is_retained(self) -> None:
|
|
100
|
+
with self.assertRaises(syq.SyqProcessError) as caught:
|
|
101
|
+
syq.run(["fail"], executable=self.executable)
|
|
102
|
+
|
|
103
|
+
self.assertEqual(caught.exception.result.returncode, 23)
|
|
104
|
+
self.assertEqual(caught.exception.result.stdout, b"partial")
|
|
105
|
+
self.assertEqual(caught.exception.result.stderr, b"failed")
|
|
106
|
+
|
|
107
|
+
def test_nonzero_result_can_be_returned(self) -> None:
|
|
108
|
+
result = syq.run(["fail"], executable=self.executable, check=False)
|
|
109
|
+
self.assertEqual(result.returncode, 23)
|
|
110
|
+
|
|
111
|
+
def test_timeout_stops_spawned_descendants(self) -> None:
|
|
112
|
+
marker = Path(self.temporary_directory.name) / "descendant-marker"
|
|
113
|
+
|
|
114
|
+
with self.assertRaises(subprocess.TimeoutExpired):
|
|
115
|
+
syq.run(
|
|
116
|
+
["spawn-descendant", marker],
|
|
117
|
+
executable=self.executable,
|
|
118
|
+
timeout=0.5,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
self.assertTrue(marker.with_suffix(".ready").exists())
|
|
122
|
+
time.sleep(0.75)
|
|
123
|
+
self.assertFalse(marker.exists())
|
|
124
|
+
|
|
125
|
+
def test_keyboard_interrupt_stops_spawned_descendants(self) -> None:
|
|
126
|
+
marker = Path(self.temporary_directory.name) / "interrupt-marker"
|
|
127
|
+
ready = marker.with_suffix(".ready")
|
|
128
|
+
pid_file = marker.with_suffix(".pid")
|
|
129
|
+
script = (
|
|
130
|
+
"from pathlib import Path\n"
|
|
131
|
+
"import sys\n"
|
|
132
|
+
"import syq\n"
|
|
133
|
+
"syq.run(['interrupt', Path(sys.argv[2])], "
|
|
134
|
+
"executable=Path(sys.argv[1]))\n"
|
|
135
|
+
)
|
|
136
|
+
wrapper = subprocess.Popen(
|
|
137
|
+
[
|
|
138
|
+
sys.executable,
|
|
139
|
+
"-c",
|
|
140
|
+
script,
|
|
141
|
+
os.fspath(self.executable),
|
|
142
|
+
os.fspath(marker),
|
|
143
|
+
],
|
|
144
|
+
stdin=subprocess.DEVNULL,
|
|
145
|
+
stdout=subprocess.PIPE,
|
|
146
|
+
stderr=subprocess.PIPE,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
try:
|
|
150
|
+
deadline = time.monotonic() + 3
|
|
151
|
+
while not ready.exists() and wrapper.poll() is None:
|
|
152
|
+
if time.monotonic() >= deadline:
|
|
153
|
+
self.fail(
|
|
154
|
+
"timed out waiting for the interrupt fixture; "
|
|
155
|
+
f"wrapper status is {wrapper.returncode}"
|
|
156
|
+
)
|
|
157
|
+
time.sleep(0.01)
|
|
158
|
+
self.assertIsNone(wrapper.poll())
|
|
159
|
+
|
|
160
|
+
wrapper.send_signal(signal.SIGINT)
|
|
161
|
+
wrapper.communicate(timeout=3)
|
|
162
|
+
|
|
163
|
+
self.assertNotEqual(wrapper.returncode, 0)
|
|
164
|
+
time.sleep(1.1)
|
|
165
|
+
self.assertFalse(marker.exists())
|
|
166
|
+
finally:
|
|
167
|
+
if wrapper.poll() is None:
|
|
168
|
+
wrapper.kill()
|
|
169
|
+
wrapper.communicate()
|
|
170
|
+
if pid_file.exists():
|
|
171
|
+
try:
|
|
172
|
+
os.killpg(int(pid_file.read_text(encoding="utf-8")), signal.SIGKILL)
|
|
173
|
+
except ProcessLookupError:
|
|
174
|
+
pass
|
|
175
|
+
|
|
176
|
+
def test_one_string_is_not_treated_as_an_argument_sequence(self) -> None:
|
|
177
|
+
with self.assertRaisesRegex(TypeError, "individual arguments"):
|
|
178
|
+
syq.run("emit", executable=self.executable)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
if __name__ == "__main__":
|
|
182
|
+
unittest.main()
|