genblaze-cli 0.3.5__tar.gz → 0.3.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -79,3 +79,7 @@ CLAUDE.local.md
79
79
  *.tmp
80
80
  *.bak
81
81
  *.log
82
+ .pre-pr-check/
83
+ pre-pr-check-findings-batch-*.md
84
+ # Node (libs/spec type-generation toolchain)
85
+ node_modules/
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: genblaze-cli
3
- Version: 0.3.5
3
+ Version: 0.3.7
4
4
  Summary: CLI for genblaze manifest operations
5
5
  Project-URL: Homepage, https://github.com/backblaze-labs/genblaze
6
6
  Project-URL: Documentation, https://github.com/backblaze-labs/genblaze#readme
@@ -19,7 +19,7 @@ Classifier: Topic :: Multimedia
19
19
  Classifier: Topic :: Software Development :: Libraries
20
20
  Requires-Python: >=3.11
21
21
  Requires-Dist: click<9,>=8.0
22
- Requires-Dist: genblaze-core<0.4,>=0.3.7
22
+ Requires-Dist: genblaze-core<0.4,>=0.3.9
23
23
  Provides-Extra: dev
24
24
  Requires-Dist: pillow>=10.0; extra == 'dev'
25
25
  Requires-Dist: pytest>=7.0; extra == 'dev'
@@ -19,20 +19,28 @@ if TYPE_CHECKING:
19
19
  # 256 KB chunks; matches the transfer layer's streaming granularity.
20
20
  _FETCH_CHUNK = 256 * 1024
21
21
 
22
- # A presigned URL's query string is a bearer credential for that object (#75);
23
- # it must never be echoed in CLI output or error messages.
24
- _URL_QUERY_RE = re.compile(r"https?://\S+?\?\S+")
22
+ # A presigned URL's query string, or basic-auth userinfo (user:pass@host), is
23
+ # a bearer credential for that object (#75, #201); it must never be echoed in
24
+ # CLI output or error messages. Matches any http(s) URL — not just ones with a
25
+ # query string — so a userinfo-only URL embedded in a transfer-layer exception
26
+ # message is also caught; _redact_url no-ops on a URL with neither.
27
+ _URL_RE = re.compile(r"https?://\S+")
25
28
 
26
29
 
27
30
  def _redact_url(url: str) -> str:
28
31
  parsed = urlsplit(url)
29
- if parsed.query:
30
- return urlunsplit(parsed._replace(query="REDACTED"))
32
+ # Strip userinfo by editing the netloc string directly rather than
33
+ # rebuilding it from parsed.hostname/.port — hostname strips the []
34
+ # brackets off an IPv6 literal, which would otherwise corrupt the URL.
35
+ netloc = parsed.netloc.rsplit("@", 1)[-1] if "@" in parsed.netloc else parsed.netloc
36
+ if parsed.query or netloc != parsed.netloc:
37
+ query = "REDACTED" if parsed.query else parsed.query
38
+ return urlunsplit(parsed._replace(netloc=netloc, query=query))
31
39
  return url
32
40
 
33
41
 
34
42
  def _redact_text(text: str) -> str:
35
- return _URL_QUERY_RE.sub(lambda m: _redact_url(m.group(0)), text)
43
+ return _URL_RE.sub(lambda m: _redact_url(m.group(0)), text)
36
44
 
37
45
 
38
46
  def _hash_url_bytes(url: str, extra_roots: tuple[Path, ...] = ()) -> tuple[str, int]:
@@ -48,36 +56,42 @@ def _hash_url_bytes(url: str, extra_roots: tuple[Path, ...] = ()) -> tuple[str,
48
56
  temp-dir allowlist). Bytes are hashed as they arrive; a whole asset is
49
57
  never held in memory.
50
58
  """
51
- scheme = urlsplit(url).scheme
59
+ parsed = urlsplit(url)
60
+ scheme = parsed.scheme
52
61
  if scheme == "https":
53
- from genblaze_core.exceptions import StorageError
62
+ # TODO(#202): these are genblaze_core underscore-private symbols;
63
+ # the private-import cleanup (promoting a public streaming helper)
64
+ # is a deferred follow-up, not part of this fix.
54
65
  from genblaze_core.storage.transfer import (
55
66
  _DEFAULT_DOWNLOAD_TIMEOUT,
56
67
  _DEFAULT_MAX_DOWNLOAD_BYTES,
68
+ _HashingStreamReader,
57
69
  _http_get_stream,
58
70
  )
59
71
 
60
- digest = hashlib.sha256()
61
- size = 0
62
72
  resp = _http_get_stream(url, timeout=_DEFAULT_DOWNLOAD_TIMEOUT)
73
+ # Reuse the transfer layer's own chunk-read/incremental-sha256/cap
74
+ # loop instead of duplicating it (#202) — same exception type and
75
+ # message shape by construction, so callers see one error taxonomy.
76
+ reader = _HashingStreamReader(resp, max_bytes=_DEFAULT_MAX_DOWNLOAD_BYTES)
63
77
  try:
64
- while chunk := resp.read(_FETCH_CHUNK):
65
- digest.update(chunk)
66
- size += len(chunk)
67
- if size > _DEFAULT_MAX_DOWNLOAD_BYTES:
68
- # Same exception type and message shape as the transfer
69
- # layer's own cap, so callers see one error taxonomy.
70
- raise StorageError(
71
- f"Download exceeds {_DEFAULT_MAX_DOWNLOAD_BYTES} byte limit"
72
- )
78
+ while reader.read(_FETCH_CHUNK):
79
+ pass
73
80
  finally:
74
81
  resp.release_conn()
75
- return digest.hexdigest(), size
82
+ return reader.sha256_hex, reader.size
76
83
  if scheme == "file":
77
84
  from genblaze_core._utils import ALLOWED_FILE_ROOTS
78
85
 
86
+ # RFC 8089: empty or 'localhost' netloc only; anything else is a
87
+ # remote-authority reference and must not be silently resolved as a
88
+ # local path (#201) — matches core's validate_chain_input_url.
89
+ if parsed.netloc and parsed.netloc != "localhost":
90
+ raise ValueError(
91
+ f"file:// URL must have empty or 'localhost' netloc; got {parsed.netloc!r}"
92
+ )
79
93
  allowed = (*ALLOWED_FILE_ROOTS, *(r.resolve() for r in extra_roots))
80
- resolved = Path(url2pathname(urlsplit(url).path)).resolve()
94
+ resolved = Path(url2pathname(parsed.path)).resolve()
81
95
  if not any(resolved.is_relative_to(root) for root in allowed):
82
96
  raise ValueError(f"file:// path outside allowed roots: {resolved}")
83
97
  digest = hashlib.sha256()
@@ -130,12 +144,18 @@ def _fetch_and_compare(
130
144
  @click.option(
131
145
  "--hash-only",
132
146
  is_flag=True,
133
- help="Only verify canonical_hash; skip output sha256 and asset-metadata checks.",
147
+ help=(
148
+ "Only verify canonical_hash; skip output sha256 and asset-metadata checks. "
149
+ "Mutually exclusive with --fetch."
150
+ ),
134
151
  )
135
152
  @click.option(
136
153
  "--fetch",
137
154
  is_flag=True,
138
- help="Also download each output asset and compare its bytes to the declared sha256.",
155
+ help=(
156
+ "Also download each output asset and compare its bytes to the declared "
157
+ "sha256. Mutually exclusive with --hash-only."
158
+ ),
139
159
  )
140
160
  @click.option(
141
161
  "--allowed-root",
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "genblaze-cli"
7
- version = "0.3.5"
7
+ version = "0.3.7"
8
8
  description = "CLI for genblaze manifest operations"
9
9
  authors = [{name = "Jeronimo De Leon", email = "jdeleon@backblaze.com"}]
10
10
  readme = "README.md"
@@ -22,7 +22,7 @@ classifiers = [
22
22
  keywords = ["genblaze", "ai", "media", "manifest", "provenance", "c2pa-ready", "genai", "pipeline", "cli", "command-line"]
23
23
 
24
24
  dependencies = [
25
- "genblaze-core>=0.3.7,<0.4",
25
+ "genblaze-core>=0.3.9,<0.4",
26
26
  # <9: bounded-major convention (#61) — caps below the next major so a
27
27
  # future click release can't silently break the CLI at resolve time.
28
28
  "click>=8.0,<9",
@@ -663,3 +663,38 @@ def test_version_label_shows_cli_package() -> None:
663
663
  result = runner.invoke(cli, ["--version"])
664
664
  assert result.exit_code == 0
665
665
  assert "genblaze-cli" in result.output
666
+
667
+
668
+ # --- GHSA-x99g-wcm2-r2v8: hostile identity fields in an untrusted manifest ---
669
+
670
+
671
+ @pytest.mark.parametrize("absolute", [False, True])
672
+ def test_index_rejects_traversing_run_id(tmp_path: Path, absolute: bool) -> None:
673
+ # Both forms land inside tmp_path if unfixed, so a regression is caught
674
+ # without the test itself writing outside its sandbox. Seven ".." climb
675
+ # from runs/<4-level partition> under work/index_out back to tmp_path.
676
+ run_id = str(tmp_path / "escaped") if absolute else "../" * 7 + "escaped"
677
+ manifest_path = _create_manifest_json(tmp_path)
678
+ data = json.loads(manifest_path.read_text(encoding="utf-8"))
679
+ data["run"]["run_id"] = run_id
680
+ manifest_path.write_text(json.dumps(data), encoding="utf-8")
681
+ out_dir = tmp_path / "work" / "index_out"
682
+
683
+ result = CliRunner().invoke(cli, ["index", str(manifest_path), "-o", str(out_dir)])
684
+
685
+ assert result.exit_code != 0
686
+ assert "run_id" in _combined_output(result)
687
+ assert not out_dir.exists()
688
+ assert not list(tmp_path.rglob("escaped*"))
689
+
690
+
691
+ def test_verify_rejects_traversing_step_id(tmp_path: Path) -> None:
692
+ manifest_path = _create_manifest_json(tmp_path)
693
+ data = json.loads(manifest_path.read_text(encoding="utf-8"))
694
+ data["run"]["steps"][0]["step_id"] = "../evil"
695
+ manifest_path.write_text(json.dumps(data), encoding="utf-8")
696
+
697
+ result = CliRunner().invoke(cli, ["verify", str(manifest_path)])
698
+
699
+ assert result.exit_code != 0
700
+ assert "step_id" in _combined_output(result)
@@ -293,6 +293,173 @@ def test_verify_fetch_no_output_assets_reports_nothing_to_fetch(tmp_path: Path)
293
293
  assert "no output assets to fetch" in result.output
294
294
 
295
295
 
296
+ def test_verify_fetch_rejects_file_url_with_remote_netloc(tmp_path: Path) -> None:
297
+ """file://remote-host/... must fail, not silently resolve to a local path
298
+ (#201) — mirrors core's validate_chain_input_url netloc contract."""
299
+ step = StepBuilder("test", "test-model").prompt("hello").build()
300
+ step.status = StepStatus.SUCCEEDED
301
+ asset = Asset(url="file://remote-host/tmp/a", media_type="image/png")
302
+ asset.sha256 = "f" * 64
303
+ step.assets = [asset]
304
+ run = RunBuilder("fetch-remote-netloc").add_step(step).build()
305
+ manifest_path = tmp_path / "manifest.json"
306
+ manifest_path.write_text(Manifest.from_run(run).to_canonical_json(), encoding="utf-8")
307
+
308
+ runner = CliRunner()
309
+ result = runner.invoke(cli, ["verify", "--fetch", str(manifest_path)])
310
+ combined = result.output + getattr(result, "stderr", "")
311
+
312
+ assert result.exit_code != 0
313
+ assert "netloc" in combined
314
+ assert "outside allowed roots" not in combined
315
+
316
+
317
+ def test_verify_fetch_allows_file_url_with_localhost_netloc(tmp_path: Path) -> None:
318
+ """RFC 8089 permits an explicit 'localhost' authority; only non-empty,
319
+ non-localhost netlocs are rejected."""
320
+ manifest_path, asset_path = _create_fetch_manifest(tmp_path)
321
+ manifest_json = manifest_path.read_text(encoding="utf-8")
322
+ localhost_url = f"file://localhost{asset_path.resolve()}"
323
+ manifest_path.write_text(manifest_json.replace(asset_path.as_uri(), localhost_url))
324
+
325
+ runner = CliRunner()
326
+ result = runner.invoke(cli, ["verify", "--fetch", str(manifest_path)])
327
+
328
+ assert result.exit_code == 0, result.output
329
+ assert "fetched and matched" in result.output
330
+
331
+
332
+ def test_redact_url_blanks_userinfo() -> None:
333
+ """A presigned/basic-auth userinfo (user:pass@) is a credential; it must
334
+ not survive redaction any more than the query string does (#201)."""
335
+ from genblaze_cli.commands.verify import _redact_url
336
+
337
+ redacted = _redact_url("https://user:pass@cdn.example.com/output.png")
338
+
339
+ assert "user" not in redacted
340
+ assert "pass" not in redacted
341
+ assert "cdn.example.com/output.png" in redacted
342
+
343
+
344
+ def test_verify_fetch_redacts_userinfo_in_label(tmp_path: Path) -> None:
345
+ """A userinfo-bearing asset URL must not leak credentials into the
346
+ per-asset label used in both OK and failure output."""
347
+ step = StepBuilder("test", "test-model").prompt("hello").build()
348
+ step.status = StepStatus.SUCCEEDED
349
+ asset = Asset(url="https://user:s3cr3t@cdn.example.com/output.png", media_type="image/png")
350
+ asset.sha256 = "f" * 64 # deliberately wrong; forces a failure line with the label
351
+ step.assets = [asset]
352
+ run = RunBuilder("fetch-userinfo").add_step(step).build()
353
+ manifest_path = tmp_path / "manifest.json"
354
+ manifest_path.write_text(Manifest.from_run(run).to_canonical_json(), encoding="utf-8")
355
+
356
+ runner = CliRunner()
357
+ result = runner.invoke(cli, ["verify", "--fetch", str(manifest_path)])
358
+ combined = result.output + getattr(result, "stderr", "")
359
+
360
+ assert result.exit_code != 0
361
+ assert "s3cr3t" not in combined
362
+
363
+
364
+ def test_redact_url_preserves_ipv6_brackets() -> None:
365
+ """Stripping userinfo must not corrupt an IPv6 host literal's brackets —
366
+ rebuilding netloc from parsed.hostname (which unwraps them) would."""
367
+ from genblaze_cli.commands.verify import _redact_url
368
+
369
+ redacted = _redact_url("https://user:pass@[::1]:8443/output.png")
370
+
371
+ assert "user" not in redacted
372
+ assert "pass" not in redacted
373
+ assert "[::1]:8443" in redacted
374
+
375
+
376
+ def test_verify_fetch_redacts_userinfo_without_query_in_failure_text(
377
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
378
+ ) -> None:
379
+ """A userinfo-bearing URL with no query string must still be redacted out
380
+ of freeform exception text (e.g. a transfer-layer error echoing the URL),
381
+ not just out of the per-asset label — _redact_text's matcher must not
382
+ require a literal '?' to catch a credential (#201)."""
383
+ from genblaze_core.exceptions import StorageError
384
+
385
+ url = "https://user:s3cr3t@cdn.example.com/output.png"
386
+ step = StepBuilder("test", "test-model").prompt("hello").build()
387
+ step.status = StepStatus.SUCCEEDED
388
+ asset = Asset(url=url, media_type="image/png")
389
+ asset.set_hash(b"whatever")
390
+ step.assets = [asset]
391
+ run = RunBuilder("fetch-userinfo-noquery").add_step(step).build()
392
+ manifest_path = tmp_path / "manifest.json"
393
+ manifest_path.write_text(Manifest.from_run(run).to_canonical_json(), encoding="utf-8")
394
+
395
+ def _raise(url: str, *, timeout: float) -> None:
396
+ # Mirrors the transfer layer echoing the raw URL into its own
397
+ # exception message on a download failure.
398
+ raise StorageError(f"Download failed for {url}: connection refused")
399
+
400
+ monkeypatch.setattr("genblaze_core.storage.transfer._http_get_stream", _raise)
401
+
402
+ runner = CliRunner()
403
+ result = runner.invoke(cli, ["verify", "--fetch", str(manifest_path)])
404
+ combined = result.output + getattr(result, "stderr", "")
405
+
406
+ assert result.exit_code != 0
407
+ assert "s3cr3t" not in combined
408
+
409
+
410
+ def test_verify_fetch_enforces_max_download_bytes_cap(
411
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
412
+ ) -> None:
413
+ """The max-bytes cap must still be enforced through the
414
+ _HashingStreamReader wiring used to fetch https:// assets (#202)."""
415
+ step = StepBuilder("test", "test-model").prompt("hello").build()
416
+ step.status = StepStatus.SUCCEEDED
417
+ asset = Asset(url="https://cdn.example.com/output.png", media_type="image/png")
418
+ asset.sha256 = "f" * 64
419
+ step.assets = [asset]
420
+ run = RunBuilder("fetch-cap").add_step(step).build()
421
+ manifest_path = tmp_path / "manifest.json"
422
+ manifest_path.write_text(Manifest.from_run(run).to_canonical_json(), encoding="utf-8")
423
+
424
+ class _FakeResp:
425
+ def __init__(self) -> None:
426
+ self._buf = b"x" * 20
427
+ self.released = False
428
+
429
+ def read(self, size: int) -> bytes:
430
+ chunk, self._buf = self._buf[:size], self._buf[size:]
431
+ return chunk
432
+
433
+ def release_conn(self) -> None:
434
+ self.released = True
435
+
436
+ monkeypatch.setattr(
437
+ "genblaze_core.storage.transfer._http_get_stream", lambda url, *, timeout: _FakeResp()
438
+ )
439
+ monkeypatch.setattr("genblaze_core.storage.transfer._DEFAULT_MAX_DOWNLOAD_BYTES", 10)
440
+
441
+ runner = CliRunner()
442
+ result = runner.invoke(cli, ["verify", "--fetch", str(manifest_path)])
443
+ combined = result.output + getattr(result, "stderr", "")
444
+
445
+ assert result.exit_code != 0
446
+ assert "byte limit" in combined
447
+
448
+
449
+ def test_verify_help_mentions_mutual_exclusion() -> None:
450
+ """--fetch and --hash-only mutual exclusion must be discoverable from
451
+ --help, not just from triggering the UsageError (#201)."""
452
+ runner = CliRunner()
453
+ result = runner.invoke(cli, ["verify", "--help"])
454
+
455
+ assert result.exit_code == 0
456
+ # Click wraps long help strings across lines, so assert against the
457
+ # normalized full block rather than a single line.
458
+ normalized = " ".join(result.output.split())
459
+ assert "mutually exclusive with --hash-only" in normalized.lower()
460
+ assert "mutually exclusive with --fetch" in normalized.lower()
461
+
462
+
296
463
  def test_verify_fetch_rejects_http_scheme(tmp_path: Path) -> None:
297
464
  """Plain http:// is rejected; https-only, same policy as the transfer layer."""
298
465
  step = StepBuilder("test", "test-model").prompt("hello").build()
File without changes