orangehack 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
orangehack/transfer.py ADDED
@@ -0,0 +1,442 @@
1
+ """Moving bytes: the folder zip, SHA-256, the signed upload and the resumable download."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import contextlib
6
+ import fnmatch
7
+ import hashlib
8
+ import os
9
+ import re
10
+ import shutil
11
+ import tempfile
12
+ import time
13
+ import zipfile
14
+ from collections.abc import Callable, Iterator
15
+ from dataclasses import dataclass, field
16
+ from pathlib import Path
17
+ from urllib.parse import unquote, urlparse
18
+
19
+ import httpx
20
+
21
+ from .errors import CliError, Exit
22
+
23
+ CHUNK = 1024 * 1024
24
+ ALWAYS_SKIPPED = (".git", "__pycache__", ".venv", "venv", ".ipynb_checkpoints", ".DS_Store")
25
+ # The one list of secrets (README, the spec and /docs/cli describe it and point here): files and
26
+ # folders that hold keys, tokens and passwords far more often than an answer. Never packed, whatever
27
+ # the ignore file says, and named in the output so the person knows what stayed behind.
28
+ SECRET_NAMES = (
29
+ # environment files
30
+ ".env",
31
+ ".env.*",
32
+ "*.env",
33
+ ".envrc",
34
+ # private keys and certificates
35
+ "*.pem",
36
+ "*.key",
37
+ "*.p12",
38
+ "*.pfx",
39
+ "id_rsa*",
40
+ "id_ed25519*",
41
+ "id_ecdsa*",
42
+ "id_dsa*",
43
+ # tool credentials
44
+ ".netrc",
45
+ ".npmrc",
46
+ ".pypirc",
47
+ "credentials*",
48
+ "kaggle.json",
49
+ )
50
+ SECRET_DIRS = (".ssh", ".aws", ".kube", ".gnupg")
51
+ IGNORE_FILE = ".orangehackignore"
52
+ # A zip entry's time: fixed, so the same folder packs to the same bytes.
53
+ ZIP_TIME = (2020, 1, 1, 0, 0, 0)
54
+
55
+ Progress = Callable[[int, int], None]
56
+
57
+
58
+ def sha256_of(path: Path) -> str:
59
+ digest = hashlib.sha256()
60
+ with path.open("rb") as handle:
61
+ for block in iter(lambda: handle.read(CHUNK), b""):
62
+ digest.update(block)
63
+ return digest.hexdigest()
64
+
65
+
66
+ def same_hash(local: str, remote: str) -> bool:
67
+ remote = remote.lower().removeprefix("sha256:")
68
+ return bool(remote) and remote == local.lower()
69
+
70
+
71
+ # --- the folder ----------------------------------------------------------------------------------
72
+
73
+
74
+ def _patterns(root: Path) -> list[str]:
75
+ path = root / IGNORE_FILE
76
+ if not path.is_file():
77
+ return []
78
+ try:
79
+ lines = path.read_text(encoding="utf-8").splitlines()
80
+ except (OSError, UnicodeDecodeError) as exc:
81
+ raise _unreadable(path, exc) from exc
82
+ return [line.strip() for line in lines if line.strip() and not line.startswith("#")]
83
+
84
+
85
+ def _secret(name: str) -> bool:
86
+ return any(fnmatch.fnmatch(name, pattern) for pattern in SECRET_NAMES)
87
+
88
+
89
+ def _unreadable(path: Path | str, exc: Exception) -> CliError:
90
+ """A file or folder we cannot read: refused, so a partial answer is never handed in."""
91
+ reason = getattr(exc, "strerror", None) or str(exc)
92
+ return CliError(
93
+ Exit.USAGE,
94
+ f"Не удалось прочитать {path}: {reason}.",
95
+ "Дайте права на чтение или исключите этот путь в .orangehackignore; "
96
+ "неполный архив не сдаём.",
97
+ code="unreadable",
98
+ )
99
+
100
+
101
+ def _walk_error(exc: OSError) -> None:
102
+ raise _unreadable(exc.filename or "папку", exc) from exc
103
+
104
+
105
+ def _ignored(relative: str, patterns: list[str]) -> bool:
106
+ """`.orangehackignore`: a name without `/` matches at any depth, a leading `/` anchors it to
107
+ the folder, a trailing `/` is optional, `*` `?` `[...]` are wildcards; `!` is not supported."""
108
+ parts = relative.split("/")
109
+ if any(part in ALWAYS_SKIPPED for part in parts) or relative == IGNORE_FILE:
110
+ return True
111
+ for pattern in patterns:
112
+ bare = pattern.rstrip("/")
113
+ anchored = bare.startswith("/") or "/" in bare
114
+ bare = bare.lstrip("/")
115
+ if fnmatch.fnmatch(relative, bare) or fnmatch.fnmatch(relative, f"{bare}/*"):
116
+ return True
117
+ if not anchored and any(fnmatch.fnmatch(part, bare) for part in parts):
118
+ return True
119
+ return False
120
+
121
+
122
+ @dataclass
123
+ class Found:
124
+ """What a folder walk gives: the files to pack and what it left out on purpose."""
125
+
126
+ files: list[tuple[Path, str]] = field(default_factory=list)
127
+ links: list[str] = field(default_factory=list)
128
+ secrets: list[str] = field(default_factory=list)
129
+
130
+
131
+ def files_of(paths: list[Path]) -> Found:
132
+ """Every file to pack with its name in the archive, in a stable order.
133
+
134
+ With several paths each one keeps its own name as the first part of its entries, so two folders
135
+ with a `main.py` each stay apart. Symbolic links inside a folder are never followed, file or
136
+ folder alike: a link can point out of the folder (a key in the home directory).
137
+ """
138
+ found = Found()
139
+ prefixed = len(paths) > 1
140
+ for path in paths:
141
+ if path.is_file():
142
+ if prefixed and _secret(path.name):
143
+ found.secrets.append(path.name)
144
+ else:
145
+ found.files.append((path, path.name))
146
+ continue
147
+ prefix = f"{path.resolve().name}/" if prefixed else ""
148
+ patterns = _patterns(path)
149
+ entries = []
150
+ for current, dirs, names in os.walk(path, onerror=_walk_error):
151
+ base = Path(current)
152
+ kept = []
153
+ for d in sorted(dirs):
154
+ relative = (base / d).relative_to(path).as_posix()
155
+ if _ignored(relative, patterns):
156
+ continue
157
+ if (base / d).is_symlink():
158
+ found.links.append(prefix + relative)
159
+ continue
160
+ if d in SECRET_DIRS:
161
+ found.secrets.append(f"{prefix}{relative}/")
162
+ continue
163
+ kept.append(d)
164
+ dirs[:] = kept
165
+ for name in names:
166
+ relative = (base / name).relative_to(path).as_posix()
167
+ if _ignored(relative, patterns):
168
+ continue
169
+ if (base / name).is_symlink():
170
+ found.links.append(prefix + relative)
171
+ elif _secret(name):
172
+ found.secrets.append(prefix + relative)
173
+ else:
174
+ entries.append((base / name, prefix + relative))
175
+ found.files.extend(sorted(entries, key=lambda item: item[1]))
176
+ found.links.sort()
177
+ found.secrets.sort()
178
+ return found
179
+
180
+
181
+ @dataclass
182
+ class Packed:
183
+ path: Path
184
+ names: list[str]
185
+ links: list[str]
186
+ secrets: list[str]
187
+
188
+
189
+ @contextlib.contextmanager
190
+ def pack(paths: list[Path], name: str) -> Iterator[Packed]:
191
+ """Zip folders and several files into one temporary archive with deterministic bytes.
192
+
193
+ The temporary folder goes away when the block ends, however it ends.
194
+ """
195
+ found = files_of(paths)
196
+ if not found.files:
197
+ raise CliError(Exit.USAGE, "Нечего упаковать: в папке нет файлов (или все исключены).")
198
+ seen: set[str] = set()
199
+ for _, arcname in found.files:
200
+ if arcname in seen:
201
+ raise CliError(
202
+ Exit.USAGE,
203
+ f"В архиве два файла с одним именем: {arcname}.",
204
+ "Переименуйте один из них или сдайте их общую папку.",
205
+ code="duplicate_entry",
206
+ )
207
+ seen.add(arcname)
208
+ directory = Path(tempfile.mkdtemp(prefix="orangehack-"))
209
+ try:
210
+ target = directory / name
211
+ with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
212
+ for source, arcname in found.files:
213
+ info = zipfile.ZipInfo(arcname, date_time=ZIP_TIME)
214
+ info.compress_type = zipfile.ZIP_DEFLATED
215
+ info.external_attr = 0o644 << 16
216
+ try:
217
+ reader = source.open("rb")
218
+ except OSError as exc:
219
+ raise _unreadable(source, exc) from exc
220
+ with reader, archive.open(info, "w") as writer:
221
+ try:
222
+ for block in iter(lambda r=reader: r.read(CHUNK), b""): # type: ignore[misc]
223
+ writer.write(block)
224
+ except OSError as exc:
225
+ raise _unreadable(source, exc) from exc
226
+ yield Packed(target, [arcname for _, arcname in found.files], found.links, found.secrets)
227
+ finally:
228
+ shutil.rmtree(directory, ignore_errors=True)
229
+
230
+
231
+ # --- upload --------------------------------------------------------------------------------------
232
+
233
+
234
+ @dataclass
235
+ class Ticket:
236
+ file_id: str
237
+ url: str
238
+ method: str
239
+ headers: dict[str, str]
240
+
241
+
242
+ def _stream(path: Path, progress: Progress | None) -> Iterator[bytes]:
243
+ total = path.stat().st_size
244
+ sent = 0
245
+ with path.open("rb") as handle:
246
+ for block in iter(lambda: handle.read(CHUNK), b""):
247
+ sent += len(block)
248
+ if progress:
249
+ progress(sent, total)
250
+ yield block
251
+
252
+
253
+ def put(
254
+ storage: httpx.Client,
255
+ ticket: Ticket,
256
+ path: Path,
257
+ *,
258
+ progress: Progress | None = None,
259
+ sleep: Callable[[float], None] = time.sleep,
260
+ renew: Callable[[], Ticket] | None = None,
261
+ ) -> Ticket:
262
+ """Send the whole file to the signed address; retried whole up to three times.
263
+
264
+ Returns the ticket that succeeded: a ticket that expired is renewed through `renew`, which
265
+ gives a new `file_id`.
266
+ """
267
+ size = path.stat().st_size
268
+ for attempt in range(4):
269
+ headers = {**ticket.headers, "Content-Length": str(size)}
270
+ try:
271
+ response = storage.request(
272
+ ticket.method, ticket.url, content=_stream(path, progress), headers=headers
273
+ )
274
+ except httpx.TransportError as exc:
275
+ if attempt == 3:
276
+ raise CliError(
277
+ Exit.NETWORK, "Файл не удалось отправить: обрывается связь."
278
+ ) from exc
279
+ sleep(2.0**attempt)
280
+ continue
281
+ if response.status_code < 300:
282
+ return ticket
283
+ if response.status_code in (400, 403) and renew is not None and attempt < 3:
284
+ ticket = renew() # an expired signature answers 403 (S3) or 400
285
+ continue
286
+ if response.status_code >= 500 and attempt < 3:
287
+ sleep(2.0**attempt)
288
+ continue
289
+ raise CliError(
290
+ Exit.NETWORK,
291
+ f"Хранилище не приняло файл (HTTP {response.status_code}).",
292
+ "Повторите команду; если повторяется, напишите организаторам.",
293
+ )
294
+ raise CliError(Exit.NETWORK, "Файл не удалось отправить.")
295
+
296
+
297
+ # --- download ------------------------------------------------------------------------------------
298
+
299
+ _FILENAME_STAR = re.compile(r"filename\*\s*=\s*[^']*'[^']*'([^;]+)", re.I)
300
+ _FILENAME = re.compile(r'filename\s*=\s*"?([^";]+)"?', re.I)
301
+
302
+
303
+ # Device names Windows will not create a file under, with any extension.
304
+ _RESERVED = {"CON", "PRN", "AUX", "NUL"} | {f"{p}{i}" for p in ("COM", "LPT") for i in range(1, 10)}
305
+
306
+
307
+ def safe_name(name: str, fallback: str = "file") -> str:
308
+ """A file name every OS accepts: no slashes, no control or reserved characters, no `CON`."""
309
+ name = re.sub(r'[\\/:*?"<>|\x00-\x1f\x7f]', "_", name).strip(" .")
310
+ if not name:
311
+ return fallback
312
+ if name.split(".")[0].upper() in _RESERVED:
313
+ name = f"_{name}"
314
+ return name
315
+
316
+
317
+ def _disposition_name(response: httpx.Response) -> str:
318
+ disposition = response.headers.get("Content-Disposition", "")
319
+ match = _FILENAME_STAR.search(disposition)
320
+ name = unquote(match.group(1)) if match else ""
321
+ if not name:
322
+ match = _FILENAME.search(disposition)
323
+ name = match.group(1) if match else ""
324
+ return name
325
+
326
+
327
+ def _last_part(name: str) -> str:
328
+ """The part after the last slash or backslash, split as text: `Path` on Windows would read
329
+ `a:b.csv` as a drive and drop `a:`."""
330
+ return name.replace("\\", "/").rsplit("/", 1)[-1]
331
+
332
+
333
+ def filename_of(response: httpx.Response, fallback: str) -> str:
334
+ name = _disposition_name(response) or unquote(Path(urlparse(str(response.url)).path).name)
335
+ return safe_name(_last_part(name), fallback)
336
+
337
+
338
+ @dataclass
339
+ class Opened:
340
+ url: str
341
+
342
+
343
+ def download(
344
+ storage: httpx.Client,
345
+ reopen: Callable[[], Opened],
346
+ directory: Path,
347
+ key: str,
348
+ fallback_name: str,
349
+ *,
350
+ progress: Progress | None = None,
351
+ sleep: Callable[[float], None] = time.sleep,
352
+ taken: set[Path] | None = None,
353
+ overwrite: bool = False,
354
+ kept_aside: Callable[[Path, Path], None] | None = None,
355
+ ) -> Path:
356
+ """Download into `directory`, resuming `.<key>.part` with a range request when it exists.
357
+
358
+ The server answering 206 continues the file; 200 restarts it; 416 finishes it only when its
359
+ `Content-Range: bytes */N` equals the part's size, else the part is dropped and the file
360
+ restarts; 403 means the signed address expired and is opened again. A dropped connection
361
+ retries up to five times. A name already in `taken` (saved earlier in the same run), or a local
362
+ file with other bytes, gets a `-2`, `-3` suffix instead of being replaced; `overwrite` replaces
363
+ the local file, and `kept_aside(wanted, saved)` hears about a file saved beside one.
364
+ """
365
+ directory.mkdir(parents=True, exist_ok=True)
366
+ part = directory / f".{key}.part"
367
+ opened = reopen()
368
+ for attempt in range(6):
369
+ have = part.stat().st_size if part.exists() else 0
370
+ headers = {"Range": f"bytes={have}-"} if have else {}
371
+ try:
372
+ with storage.stream("GET", opened.url, headers=headers) as response:
373
+ if response.status_code in (400, 403) and attempt < 5:
374
+ opened = reopen()
375
+ continue
376
+ if response.status_code == 416 and have:
377
+ span = response.headers.get("Content-Range", "")
378
+ if span.strip() != f"bytes */{have}":
379
+ part.unlink() # not the whole file: start it again
380
+ continue
381
+ # Storage's 416 has no Content-Disposition, and the URL path is its own key.
382
+ name = safe_name(_last_part(_disposition_name(response)), fallback_name)
383
+ return _finish(part, directory / name, taken, overwrite, kept_aside)
384
+ if response.status_code not in (200, 206):
385
+ raise CliError(Exit.NETWORK, f"Файл не скачался (HTTP {response.status_code}).")
386
+ name = filename_of(response, fallback_name)
387
+ if response.status_code == 200:
388
+ have = 0
389
+ total = int(response.headers.get("Content-Length") or 0)
390
+ mode = "wb"
391
+ else:
392
+ span = response.headers.get("Content-Range", "")
393
+ total = int(span.rsplit("/", 1)[-1]) if "/" in span and "*" not in span else 0
394
+ mode = "ab"
395
+ written = have
396
+ with part.open(mode) as handle:
397
+ # No chunk size: a sized chunker would lose the bytes it holds when the connection drops.
398
+ for block in response.iter_bytes():
399
+ handle.write(block)
400
+ written += len(block)
401
+ if progress:
402
+ progress(written, total)
403
+ if total and written != total:
404
+ raise httpx.ReadError("short read")
405
+ return _finish(part, directory / name, taken, overwrite, kept_aside)
406
+ except httpx.TransportError as exc:
407
+ if attempt == 5:
408
+ raise CliError(
409
+ Exit.NETWORK,
410
+ "Скачивание оборвалось. Запустите команду ещё раз: оно продолжится с места обрыва.",
411
+ ) from exc
412
+ sleep(min(2.0**attempt, 8.0))
413
+ raise CliError(Exit.NETWORK, "Файл не скачался.")
414
+
415
+
416
+ def _same_bytes(a: Path, b: Path) -> bool:
417
+ return b.is_file() and a.stat().st_size == b.stat().st_size and sha256_of(a) == sha256_of(b)
418
+
419
+
420
+ def _finish(
421
+ part: Path,
422
+ wanted: Path,
423
+ taken: set[Path] | None,
424
+ overwrite: bool,
425
+ kept_aside: Callable[[Path, Path], None] | None,
426
+ ) -> Path:
427
+ target, number, beside = wanted, 1, False
428
+ while True:
429
+ if taken is not None and target in taken:
430
+ pass
431
+ elif not overwrite and target.exists() and not _same_bytes(part, target):
432
+ beside = True
433
+ else:
434
+ break
435
+ number += 1
436
+ target = wanted.with_name(f"{wanted.stem}-{number}{wanted.suffix}")
437
+ if taken is not None:
438
+ taken.add(target)
439
+ os.replace(part, target)
440
+ if beside and kept_aside is not None:
441
+ kept_aside(wanted, target)
442
+ return target
@@ -0,0 +1,104 @@
1
+ """At most once a day: is there a newer CLI, and is this one still supported?
2
+
3
+ `GET /api/v1/system/cli` is public and is asked without the token (a token is sent only to the
4
+ endpoints it is allowed on). Off with `ORANGEHACK_NO_UPDATE_CHECK=1` or `CI=true`. A failed check is
5
+ silent; the minimum learned from the last good answer still applies.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import os
12
+ import re
13
+ import time
14
+ from typing import Any
15
+
16
+ from . import __version__
17
+ from .client import Client
18
+ from .config import cache_dir
19
+ from .errors import CliError, Exit
20
+ from .output import Output
21
+
22
+ DAY = 24 * 60 * 60
23
+ # One try with a short wait: the check is a courtesy and must never hold a command up.
24
+ CHECK_TIMEOUT = 3.0
25
+
26
+
27
+ def parse(version: str) -> tuple[int, ...]:
28
+ """`1.2.3`, `1.2.3rc1`, `v1.2` -> numeric parts; anything unreadable is (0,)."""
29
+ match = re.match(r"^v?(\d+(?:\.\d+)*)", version.strip())
30
+ if not match:
31
+ return (0,)
32
+ numbers = match.group(1).split(".")
33
+ parts = [int(n) for n in numbers]
34
+ while len(parts) > 1 and parts[-1] == 0:
35
+ parts.pop()
36
+ return tuple(parts)
37
+
38
+
39
+ def disabled() -> bool:
40
+ return (
41
+ os.environ.get("ORANGEHACK_NO_UPDATE_CHECK", "") in ("1", "true", "yes")
42
+ or os.environ.get("CI", "").lower() == "true"
43
+ )
44
+
45
+
46
+ def _state_path() -> Any:
47
+ return cache_dir() / "version.json"
48
+
49
+
50
+ def _load() -> dict[str, Any]:
51
+ try:
52
+ data = json.loads(_state_path().read_text(encoding="utf-8"))
53
+ return data if isinstance(data, dict) else {}
54
+ except (OSError, ValueError):
55
+ return {}
56
+
57
+
58
+ def _store(data: dict[str, Any]) -> None:
59
+ try:
60
+ path = _state_path()
61
+ path.parent.mkdir(parents=True, exist_ok=True)
62
+ path.write_text(json.dumps(data), encoding="utf-8")
63
+ except OSError:
64
+ pass
65
+
66
+
67
+ def install_command(answer: dict[str, Any]) -> str:
68
+ return str(answer.get("install") or "pipx upgrade orangehack")
69
+
70
+
71
+ def check(client: Client, output: Output, *, now: float | None = None) -> None:
72
+ """Exit 8 when below the minimum; one stderr line when a newer version exists."""
73
+ if disabled():
74
+ return
75
+ now = time.time() if now is None else now
76
+ state = _load()
77
+ answer = state.get("answer") if state.get("url") == client.base_url else None
78
+ if not isinstance(answer, dict) or now - float(state.get("checked_at", 0)) >= DAY:
79
+ try:
80
+ fresh = client.get(
81
+ "/api/v1/system/cli", auth=False, retries=False, timeout=CHECK_TIMEOUT
82
+ )
83
+ except Exception:
84
+ fresh = None
85
+ if isinstance(fresh, dict):
86
+ answer = fresh
87
+ # Asked today, whatever the answer: a platform that is down is not asked on every command.
88
+ _store({"checked_at": now, "url": client.base_url, "answer": answer or {}})
89
+ if not isinstance(answer, dict):
90
+ return
91
+ installed = parse(__version__)
92
+ minimum = str(answer.get("minimum") or "")
93
+ if minimum and installed < parse(minimum):
94
+ raise CliError(
95
+ Exit.TOO_OLD,
96
+ f"Эта версия orangehack ({__version__}) больше не поддерживается: нужна {minimum} или новее.",
97
+ f"Обновите: {install_command(answer)}",
98
+ code="cli_too_old",
99
+ )
100
+ latest = str(answer.get("latest") or "")
101
+ if latest and installed < parse(latest):
102
+ output.note(
103
+ f"Вышла orangehack {latest} (у вас {__version__}). Обновить: {install_command(answer)}"
104
+ )