patchcraft 0.5.1__cp312-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchcraft/__init__.py +56 -0
- patchcraft/_accel.py +116 -0
- patchcraft/_accel_native.pyd +0 -0
- patchcraft/_foldgeom.py +87 -0
- patchcraft/_version.py +24 -0
- patchcraft/cache.py +254 -0
- patchcraft/extract.py +147 -0
- patchcraft/geometry.py +298 -0
- patchcraft/metrics.py +167 -0
- patchcraft/pair.py +166 -0
- patchcraft/py.typed +0 -0
- patchcraft/reconstruct.py +137 -0
- patchcraft/resize.py +205 -0
- patchcraft/stitch.py +229 -0
- patchcraft-0.5.1.dist-info/METADATA +188 -0
- patchcraft-0.5.1.dist-info/RECORD +19 -0
- patchcraft-0.5.1.dist-info/WHEEL +5 -0
- patchcraft-0.5.1.dist-info/licenses/LICENSE +21 -0
- patchcraft-0.5.1.dist-info/top_level.txt +1 -0
patchcraft/__init__.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""PatchCraft: image patch extraction, pairing and reconstruction utilities."""
|
|
2
|
+
|
|
3
|
+
from patchcraft._accel import accel_available
|
|
4
|
+
from patchcraft.cache import Cache
|
|
5
|
+
from patchcraft.extract import Patchify, extract
|
|
6
|
+
from patchcraft.geometry import (
|
|
7
|
+
PairedTilingSpec,
|
|
8
|
+
TilingSpec,
|
|
9
|
+
num_patches,
|
|
10
|
+
paired_tilings,
|
|
11
|
+
scale_factor,
|
|
12
|
+
tilings,
|
|
13
|
+
)
|
|
14
|
+
from patchcraft.metrics import patch_metrics, per_patch_mse, per_patch_psnr
|
|
15
|
+
from patchcraft.pair import PatchMeta, PatchPair, pair
|
|
16
|
+
from patchcraft.reconstruct import reconstruct
|
|
17
|
+
from patchcraft.resize import resize
|
|
18
|
+
from patchcraft.stitch import WeightKind, stitch
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
# The redundant alias is the PEP 484 form for an explicit re-export, so
|
|
22
|
+
# `mypy --strict` in a consuming project accepts `patchcraft.__version__`.
|
|
23
|
+
from patchcraft._version import __version__ as __version__
|
|
24
|
+
except ImportError: # pragma: no cover - source tree that was never built
|
|
25
|
+
# importlib.metadata is the slow path and is only reached in a checkout
|
|
26
|
+
# with no build behind it, so the import cost never lands on real users.
|
|
27
|
+
from importlib.metadata import PackageNotFoundError
|
|
28
|
+
from importlib.metadata import version as _installed_version
|
|
29
|
+
|
|
30
|
+
try:
|
|
31
|
+
__version__ = _installed_version("patchcraft")
|
|
32
|
+
except PackageNotFoundError:
|
|
33
|
+
__version__ = "0+unknown"
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"Cache",
|
|
37
|
+
"PairedTilingSpec",
|
|
38
|
+
"PatchMeta",
|
|
39
|
+
"PatchPair",
|
|
40
|
+
"Patchify",
|
|
41
|
+
"TilingSpec",
|
|
42
|
+
"WeightKind",
|
|
43
|
+
"accel_available",
|
|
44
|
+
"extract",
|
|
45
|
+
"num_patches",
|
|
46
|
+
"pair",
|
|
47
|
+
"paired_tilings",
|
|
48
|
+
"patch_metrics",
|
|
49
|
+
"per_patch_mse",
|
|
50
|
+
"per_patch_psnr",
|
|
51
|
+
"reconstruct",
|
|
52
|
+
"resize",
|
|
53
|
+
"scale_factor",
|
|
54
|
+
"stitch",
|
|
55
|
+
"tilings",
|
|
56
|
+
]
|
patchcraft/_accel.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Bridge to the optional Rust accelerator (``patchcraft._accel_native``).
|
|
2
|
+
|
|
3
|
+
The accelerator is a pyo3/Rust extension module compiled into this wheel on
|
|
4
|
+
the platforms that have one. Where it is absent the wheel is pure Python and
|
|
5
|
+
every path here reports unavailable. This module detects it at runtime and
|
|
6
|
+
exposes a single primitive, :func:`fold_weighted`, for the overlap paths of
|
|
7
|
+
``reconstruct`` and ``stitch``. Any failure at all, whether the extension was
|
|
8
|
+
never built, the ABI mismatches, ``PATCHCRAFT_ACCEL=0`` is set in the
|
|
9
|
+
environment or the tensor is ineligible, yields ``None``/``False`` and the
|
|
10
|
+
caller falls back to the pure-torch path. Nothing here raises for an
|
|
11
|
+
unavailable accelerator.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import importlib
|
|
16
|
+
import os
|
|
17
|
+
from types import ModuleType
|
|
18
|
+
|
|
19
|
+
import torch
|
|
20
|
+
|
|
21
|
+
__all__ = ["accel_available", "fold_weighted"]
|
|
22
|
+
|
|
23
|
+
_ABI_VERSION_REQUIRED = 1
|
|
24
|
+
|
|
25
|
+
_module: ModuleType | None = None
|
|
26
|
+
_checked = False
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _load() -> ModuleType | None:
|
|
30
|
+
"""Import the native module once, cache the outcome, return it or None."""
|
|
31
|
+
global _checked, _module
|
|
32
|
+
if not _checked:
|
|
33
|
+
_checked = True
|
|
34
|
+
try:
|
|
35
|
+
mod = importlib.import_module("patchcraft._accel_native")
|
|
36
|
+
except ImportError:
|
|
37
|
+
pass
|
|
38
|
+
else:
|
|
39
|
+
if getattr(mod, "_ABI_VERSION", None) == _ABI_VERSION_REQUIRED:
|
|
40
|
+
_module = mod
|
|
41
|
+
return _module
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _env_disabled() -> bool:
|
|
45
|
+
"""Evaluated per call so tests/debugging can toggle without cache resets."""
|
|
46
|
+
return os.environ.get("PATCHCRAFT_ACCEL", "1") == "0"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def accel_available() -> bool:
|
|
50
|
+
"""Whether the overlap paths will use the native accelerator."""
|
|
51
|
+
return not _env_disabled() and _load() is not None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def fold_weighted(
|
|
55
|
+
patches: torch.Tensor,
|
|
56
|
+
image_shape: tuple[int, int, int],
|
|
57
|
+
stride: tuple[int, int],
|
|
58
|
+
kernel: torch.Tensor | None,
|
|
59
|
+
) -> torch.Tensor | None:
|
|
60
|
+
"""Sum ``patches`` into a ``(C, H, W)`` image, optionally kernel-weighted.
|
|
61
|
+
|
|
62
|
+
``out[c, y, x] = sum_p patches[p, c, y - row_p, x - col_p] * kernel[...]``
|
|
63
|
+
over the patches covering ``(y, x)``, which is the numerator of the overlap fold,
|
|
64
|
+
before the count/denominator division the caller performs. Summation per
|
|
65
|
+
pixel is in descending patch index (= ascending kernel offset), matching
|
|
66
|
+
ATen col2im's per-pixel order, the one order that is bit-exact against
|
|
67
|
+
``F.fold``.
|
|
68
|
+
|
|
69
|
+
Returns ``None`` when the accelerator is unavailable or the input is
|
|
70
|
+
ineligible (non-CPU device, dtype outside float32/float64, mismatched
|
|
71
|
+
kernel); the caller then uses its pure-torch path. Geometry is NOT
|
|
72
|
+
validated here, because callers validate with ``check_fold_geometry`` first.
|
|
73
|
+
"""
|
|
74
|
+
mod = None if _env_disabled() else _load()
|
|
75
|
+
if mod is None:
|
|
76
|
+
return None
|
|
77
|
+
if patches.device.type != "cpu":
|
|
78
|
+
return None
|
|
79
|
+
if patches.dtype not in (torch.float32, torch.float64):
|
|
80
|
+
return None
|
|
81
|
+
if patches.requires_grad:
|
|
82
|
+
# The native kernel writes a leaf tensor through raw pointers, so no
|
|
83
|
+
# autograd graph. Fall back to the differentiable torch path.
|
|
84
|
+
return None
|
|
85
|
+
if kernel is not None and (
|
|
86
|
+
kernel.dtype != patches.dtype or kernel.device != patches.device
|
|
87
|
+
):
|
|
88
|
+
return None
|
|
89
|
+
|
|
90
|
+
c, h, w = image_shape
|
|
91
|
+
n, _c, ph, pw = patches.shape
|
|
92
|
+
sh, sw = stride
|
|
93
|
+
|
|
94
|
+
work = patches.contiguous()
|
|
95
|
+
kern = kernel.contiguous() if kernel is not None else None
|
|
96
|
+
out = torch.empty((c, h, w), dtype=work.dtype, device=work.device)
|
|
97
|
+
if work.numel() == 0 or out.numel() == 0:
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
mod.fold_add(
|
|
101
|
+
work.data_ptr(),
|
|
102
|
+
out.data_ptr(),
|
|
103
|
+
n,
|
|
104
|
+
c,
|
|
105
|
+
ph,
|
|
106
|
+
pw,
|
|
107
|
+
h,
|
|
108
|
+
w,
|
|
109
|
+
sh,
|
|
110
|
+
sw,
|
|
111
|
+
kern.data_ptr() if kern is not None else None,
|
|
112
|
+
"f32" if work.dtype == torch.float32 else "f64",
|
|
113
|
+
)
|
|
114
|
+
# `work` and `kern` must stay alive until the native call returns; it is
|
|
115
|
+
# synchronous and both are still local references here, so this holds.
|
|
116
|
+
return out
|
|
Binary file
|
patchcraft/_foldgeom.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Shared fold-geometry validation for reconstruct and stitch (THEORY §9.2/§9.9).
|
|
2
|
+
|
|
3
|
+
The two functions enforced identical rules on the image/patch-grid
|
|
4
|
+
relationship with ~60 lines of verbatim-duplicated validation; this module
|
|
5
|
+
centralizes them. Error messages are byte-identical to the pre-refactor
|
|
6
|
+
ones, with ``op`` filling the function-name slot.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import torch
|
|
11
|
+
|
|
12
|
+
from patchcraft.extract import _as_pair
|
|
13
|
+
|
|
14
|
+
__all__ = ["check_fold_geometry"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def check_fold_geometry(
|
|
18
|
+
patches: torch.Tensor,
|
|
19
|
+
image_shape: tuple[int, int, int],
|
|
20
|
+
stride: int | tuple[int, int],
|
|
21
|
+
dilation: int | tuple[int, int],
|
|
22
|
+
*,
|
|
23
|
+
op: str,
|
|
24
|
+
) -> tuple[int, int, int, int]:
|
|
25
|
+
"""Validate the fold geometry; return ``(h, w, num_h, num_w)``.
|
|
26
|
+
|
|
27
|
+
Caller owns the dtype/ndim checks (their messages differ between
|
|
28
|
+
``reconstruct`` and ``stitch``); this owns everything from
|
|
29
|
+
``image_shape`` shape-checking through grid/L consistency.
|
|
30
|
+
"""
|
|
31
|
+
n_patches, c, ph, pw = patches.shape
|
|
32
|
+
|
|
33
|
+
if not (isinstance(image_shape, tuple) and len(image_shape) == 3):
|
|
34
|
+
raise ValueError(
|
|
35
|
+
f"image_shape must be a 3-tuple (C, H, W), got {image_shape!r}"
|
|
36
|
+
)
|
|
37
|
+
for axis_name, val in zip(("C", "H", "W"), image_shape, strict=True):
|
|
38
|
+
if not isinstance(val, int) or isinstance(val, bool) or val <= 0:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
f"image_shape[{axis_name}] must be a positive int, got {val!r}"
|
|
41
|
+
)
|
|
42
|
+
target_c, h, w = image_shape
|
|
43
|
+
if target_c != c:
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"image_shape channels={target_c} does not match patches channel count {c}"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
sh, sw = _as_pair(stride, "stride")
|
|
49
|
+
dh, dw = _as_pair(dilation, "dilation")
|
|
50
|
+
|
|
51
|
+
if dh != 1 or dw != 1:
|
|
52
|
+
raise ValueError(
|
|
53
|
+
f"{op} requires dilation==1, got dilation=({dh}, {dw}). "
|
|
54
|
+
"Patches extracted with dilation > 1 cannot round-trip; consume them as features."
|
|
55
|
+
)
|
|
56
|
+
if sh > ph or sw > pw:
|
|
57
|
+
raise ValueError(
|
|
58
|
+
f"{op} forbids stride > patch_size (partial coverage forbidden), "
|
|
59
|
+
f"got stride=({sh}, {sw}) and patch_size=({ph}, {pw})."
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
num_h = (h - ph) // sh + 1
|
|
63
|
+
num_w = (w - pw) // sw + 1
|
|
64
|
+
if num_h <= 0 or num_w <= 0:
|
|
65
|
+
raise ValueError(
|
|
66
|
+
f"image_shape={image_shape} too small for patch_size=({ph}, {pw}) "
|
|
67
|
+
f"and stride=({sh}, {sw})"
|
|
68
|
+
)
|
|
69
|
+
covered_h = (num_h - 1) * sh + ph
|
|
70
|
+
covered_w = (num_w - 1) * sw + pw
|
|
71
|
+
if covered_h != h or covered_w != w:
|
|
72
|
+
raise ValueError(
|
|
73
|
+
f"patch grid leaves pixels uncovered (partial coverage forbidden): "
|
|
74
|
+
f"image_shape={image_shape}, patch_size=({ph}, {pw}), "
|
|
75
|
+
f"stride=({sh}, {sw}) covers ({covered_h}, {covered_w}) of "
|
|
76
|
+
f"({h}, {w}). Choose a geometry with exact coverage "
|
|
77
|
+
f"(see patchcraft.tilings)."
|
|
78
|
+
)
|
|
79
|
+
expected_n_patches = num_h * num_w
|
|
80
|
+
if n_patches != expected_n_patches:
|
|
81
|
+
raise ValueError(
|
|
82
|
+
f"patches.shape[0]={n_patches} inconsistent with grid implied by "
|
|
83
|
+
f"image_shape={image_shape}, patch_size=({ph}, {pw}), "
|
|
84
|
+
f"stride=({sh}, {sw}): expected L={expected_n_patches} "
|
|
85
|
+
f"(num_h={num_h}, num_w={num_w})."
|
|
86
|
+
)
|
|
87
|
+
return (h, w, num_h, num_w)
|
patchcraft/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.5.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 5, 1)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'g375362d3b'
|
patchcraft/cache.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""Content-addressed disk cache.
|
|
2
|
+
|
|
3
|
+
Bytes in, bytes out. Optional ``zstandard`` compression (transparent fallback
|
|
4
|
+
when not installed). Retry on transient ``PermissionError`` from OneDrive,
|
|
5
|
+
antivirus, or the Windows Search indexer. Atomic write via ``*.tmp`` plus
|
|
6
|
+
``os.replace``. Sidecar JSON carries the full key, version, content checksum.
|
|
7
|
+
|
|
8
|
+
Contract: docs/THEORY.md §4 and §9.5.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import time
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
__all__ = ["Cache"]
|
|
21
|
+
|
|
22
|
+
_WRITE_BACKOFF: tuple[float, ...] = (0.25, 0.5, 1.0, 2.0, 4.0)
|
|
23
|
+
_READ_BACKOFF: tuple[float, ...] = (0.25,)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _retry[T](
|
|
27
|
+
op: Callable[[], T],
|
|
28
|
+
backoff: tuple[float, ...],
|
|
29
|
+
) -> T:
|
|
30
|
+
"""Retry ``op`` on ``PermissionError`` with the given backoff schedule.
|
|
31
|
+
|
|
32
|
+
First attempt is immediate; subsequent attempts sleep for
|
|
33
|
+
``backoff[i-1]`` seconds. After exhausting the schedule the last
|
|
34
|
+
``PermissionError`` is re-raised. All other exceptions propagate
|
|
35
|
+
immediately.
|
|
36
|
+
"""
|
|
37
|
+
last: PermissionError | None = None
|
|
38
|
+
for attempt in range(len(backoff) + 1):
|
|
39
|
+
try:
|
|
40
|
+
return op()
|
|
41
|
+
except PermissionError as exc:
|
|
42
|
+
last = exc
|
|
43
|
+
if attempt < len(backoff):
|
|
44
|
+
time.sleep(backoff[attempt])
|
|
45
|
+
assert last is not None # invariant from loop above
|
|
46
|
+
raise last
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _try_zstandard() -> Any | None:
|
|
50
|
+
try:
|
|
51
|
+
import zstandard
|
|
52
|
+
except ImportError:
|
|
53
|
+
return None
|
|
54
|
+
return zstandard
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _normalize_part(part: Any) -> Any:
|
|
58
|
+
"""Coerce a key-part into something JSON-stable.
|
|
59
|
+
|
|
60
|
+
Lists and tuples both serialize as JSON arrays, so order matters. Dicts
|
|
61
|
+
are sorted by key. Bytes are hashed (avoids embedding binary blobs in
|
|
62
|
+
the key). Anything else falls back to ``repr``, which is explicit but uglier."""
|
|
63
|
+
if isinstance(part, (str, int, float, bool)) or part is None:
|
|
64
|
+
return part
|
|
65
|
+
if isinstance(part, (list, tuple)):
|
|
66
|
+
return [_normalize_part(p) for p in part]
|
|
67
|
+
if isinstance(part, dict):
|
|
68
|
+
return {str(k): _normalize_part(v) for k, v in sorted(part.items())}
|
|
69
|
+
if isinstance(part, bytes):
|
|
70
|
+
return {"__bytes_sha256__": hashlib.sha256(part).hexdigest()}
|
|
71
|
+
return {"__repr__": repr(part)}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Cache:
|
|
75
|
+
"""Single-namespace content-addressed cache on disk.
|
|
76
|
+
|
|
77
|
+
``root`` is created on construction if missing. ``namespace`` is used
|
|
78
|
+
as a subdirectory and as part of the key (so two namespaces never
|
|
79
|
+
collide even if a caller produces identical key parts). ``version``
|
|
80
|
+
is the invalidation lever: bump it, and old entries become
|
|
81
|
+
unreadable by construction without any delete.
|
|
82
|
+
|
|
83
|
+
Filenames address entries by the first 16 hex chars (64 bits) of the
|
|
84
|
+
full key. Two distinct keys sharing that prefix overwrite each other
|
|
85
|
+
on ``put``; ``get`` detects the mismatch via the sidecar and reports
|
|
86
|
+
a transparent miss, never wrong bytes. The birthday bound puts the
|
|
87
|
+
collision probability at ~50% only around 5e9 entries, so the prefix
|
|
88
|
+
stays at 16 chars; widen ``_paths`` if that ever stops being true.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
root: str | os.PathLike[str],
|
|
94
|
+
namespace: str,
|
|
95
|
+
version: int = 1,
|
|
96
|
+
) -> None:
|
|
97
|
+
if not isinstance(namespace, str) or not namespace:
|
|
98
|
+
raise ValueError(
|
|
99
|
+
f"namespace must be a non-empty str, got {namespace!r}"
|
|
100
|
+
)
|
|
101
|
+
if not isinstance(version, int) or isinstance(version, bool) or version <= 0:
|
|
102
|
+
raise ValueError(f"version must be a positive int, got {version!r}")
|
|
103
|
+
|
|
104
|
+
self._namespace = namespace
|
|
105
|
+
self._version = version
|
|
106
|
+
self._root = Path(root) / namespace
|
|
107
|
+
self._root.mkdir(parents=True, exist_ok=True)
|
|
108
|
+
self._zstd = _try_zstandard()
|
|
109
|
+
|
|
110
|
+
@property
|
|
111
|
+
def root(self) -> Path:
|
|
112
|
+
return self._root
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def namespace(self) -> str:
|
|
116
|
+
return self._namespace
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def version(self) -> int:
|
|
120
|
+
return self._version
|
|
121
|
+
|
|
122
|
+
def key_for(self, *parts: Any) -> str:
|
|
123
|
+
"""SHA-256 over a canonical JSON of ``parts``, namespace, version."""
|
|
124
|
+
canonical = {
|
|
125
|
+
"namespace": self._namespace,
|
|
126
|
+
"version": self._version,
|
|
127
|
+
"parts": [_normalize_part(p) for p in parts],
|
|
128
|
+
}
|
|
129
|
+
blob = json.dumps(canonical, sort_keys=True, separators=(",", ":")).encode()
|
|
130
|
+
return hashlib.sha256(blob).hexdigest()
|
|
131
|
+
|
|
132
|
+
def _paths(self, key: str) -> tuple[Path, Path]:
|
|
133
|
+
short = key[:16]
|
|
134
|
+
return self._root / f"{short}.bin", self._root / f"{short}.json"
|
|
135
|
+
|
|
136
|
+
def put(self, key: str, data: bytes) -> None:
|
|
137
|
+
"""Store ``data`` under ``key``. Atomic; retries on transient races."""
|
|
138
|
+
if not isinstance(key, str):
|
|
139
|
+
raise TypeError(f"key must be str, got {type(key).__name__}")
|
|
140
|
+
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
141
|
+
raise TypeError(f"data must be bytes-like, got {type(data).__name__}")
|
|
142
|
+
data_bytes = bytes(data)
|
|
143
|
+
|
|
144
|
+
bin_path, sidecar_path = self._paths(key)
|
|
145
|
+
|
|
146
|
+
if self._zstd is not None:
|
|
147
|
+
cctx = self._zstd.ZstdCompressor(level=3)
|
|
148
|
+
payload = cctx.compress(data_bytes)
|
|
149
|
+
compressed = True
|
|
150
|
+
else:
|
|
151
|
+
payload = data_bytes
|
|
152
|
+
compressed = False
|
|
153
|
+
|
|
154
|
+
checksum = hashlib.sha256(data_bytes).hexdigest()
|
|
155
|
+
sidecar: dict[str, Any] = {
|
|
156
|
+
"key": key,
|
|
157
|
+
"namespace": self._namespace,
|
|
158
|
+
"version": self._version,
|
|
159
|
+
"checksum": checksum,
|
|
160
|
+
"compressed": compressed,
|
|
161
|
+
"size": len(data_bytes),
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
tmp_bin = bin_path.with_suffix(bin_path.suffix + ".tmp")
|
|
165
|
+
tmp_side = sidecar_path.with_suffix(sidecar_path.suffix + ".tmp")
|
|
166
|
+
|
|
167
|
+
def _write_payload() -> None:
|
|
168
|
+
with open(tmp_bin, "wb") as f:
|
|
169
|
+
f.write(payload)
|
|
170
|
+
os.replace(tmp_bin, bin_path)
|
|
171
|
+
|
|
172
|
+
def _write_sidecar() -> None:
|
|
173
|
+
with open(tmp_side, "w", encoding="utf-8") as f:
|
|
174
|
+
json.dump(sidecar, f)
|
|
175
|
+
os.replace(tmp_side, sidecar_path)
|
|
176
|
+
|
|
177
|
+
_retry(_write_payload, _WRITE_BACKOFF)
|
|
178
|
+
_retry(_write_sidecar, _WRITE_BACKOFF)
|
|
179
|
+
|
|
180
|
+
def get(self, key: str) -> bytes | None:
|
|
181
|
+
"""Return cached bytes, or ``None`` if absent / version-mismatched.
|
|
182
|
+
|
|
183
|
+
Raises ``IOError`` on sidecar/payload mismatch (corrupt entry).
|
|
184
|
+
"""
|
|
185
|
+
if not isinstance(key, str):
|
|
186
|
+
raise TypeError(f"key must be str, got {type(key).__name__}")
|
|
187
|
+
|
|
188
|
+
bin_path, sidecar_path = self._paths(key)
|
|
189
|
+
if not (bin_path.exists() and sidecar_path.exists()):
|
|
190
|
+
return None
|
|
191
|
+
|
|
192
|
+
def _read_sidecar() -> dict[str, Any]:
|
|
193
|
+
with open(sidecar_path, "rb") as f:
|
|
194
|
+
parsed = json.loads(f.read())
|
|
195
|
+
if not isinstance(parsed, dict):
|
|
196
|
+
raise OSError(
|
|
197
|
+
f"sidecar at {sidecar_path} is not a JSON object"
|
|
198
|
+
)
|
|
199
|
+
return parsed
|
|
200
|
+
|
|
201
|
+
try:
|
|
202
|
+
sidecar = _retry(_read_sidecar, _READ_BACKOFF)
|
|
203
|
+
except FileNotFoundError:
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
# Different key collided into the same 16-hex prefix? Treat as miss.
|
|
207
|
+
if sidecar.get("key") != key:
|
|
208
|
+
return None
|
|
209
|
+
# Stale version (cache invalidation lever). Transparent miss.
|
|
210
|
+
if sidecar.get("version") != self._version:
|
|
211
|
+
return None
|
|
212
|
+
if sidecar.get("namespace") != self._namespace:
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
def _read_payload() -> bytes:
|
|
216
|
+
with open(bin_path, "rb") as f:
|
|
217
|
+
return f.read()
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
payload = _retry(_read_payload, _READ_BACKOFF)
|
|
221
|
+
except FileNotFoundError:
|
|
222
|
+
return None
|
|
223
|
+
|
|
224
|
+
if sidecar.get("compressed"):
|
|
225
|
+
if self._zstd is None:
|
|
226
|
+
raise OSError(
|
|
227
|
+
f"cache entry {key[:16]!r} is zstd-compressed but "
|
|
228
|
+
"zstandard is not installed in this environment"
|
|
229
|
+
)
|
|
230
|
+
dctx = self._zstd.ZstdDecompressor()
|
|
231
|
+
try:
|
|
232
|
+
data: bytes = dctx.decompress(payload)
|
|
233
|
+
except self._zstd.ZstdError as exc:
|
|
234
|
+
# Compressed-payload corruption manifests here before checksum.
|
|
235
|
+
raise OSError(
|
|
236
|
+
f"cache entry {key[:16]!r} payload corrupt (zstd decode "
|
|
237
|
+
f"failed); remove it from {self._root} to invalidate"
|
|
238
|
+
) from exc
|
|
239
|
+
else:
|
|
240
|
+
data = payload
|
|
241
|
+
|
|
242
|
+
if hashlib.sha256(data).hexdigest() != sidecar.get("checksum"):
|
|
243
|
+
raise OSError(
|
|
244
|
+
f"cache entry {key[:16]!r} checksum mismatch, corrupt; "
|
|
245
|
+
f"remove {bin_path.name} and {sidecar_path.name} from "
|
|
246
|
+
f"{self._root} to invalidate"
|
|
247
|
+
)
|
|
248
|
+
return data
|
|
249
|
+
|
|
250
|
+
def __repr__(self) -> str:
|
|
251
|
+
return (
|
|
252
|
+
f"Cache(root={str(self._root)!r}, namespace={self._namespace!r}, "
|
|
253
|
+
f"version={self._version})"
|
|
254
|
+
)
|
patchcraft/extract.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Patch extraction: strided-window fast path, F.unfold for dilation > 1.
|
|
2
|
+
|
|
3
|
+
Contract: docs/THEORY.md §1 and §9.1, docs/ADR/0001-patch-extraction-api.md,
|
|
4
|
+
docs/ADR/0002-patchify-transform.md.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
import torch.nn.functional as F # noqa: N812 (torch convention)
|
|
10
|
+
|
|
11
|
+
__all__ = ["Patchify", "extract"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Dtypes that F.unfold (im2col) supports on CPU in torch 2.x. The dilation == 1
|
|
15
|
+
# fast path is restricted to these to keep the accept/reject contract unchanged:
|
|
16
|
+
# Tensor.unfold views accept any dtype, but integer dtypes must still raise
|
|
17
|
+
# NotImplementedError from im2col, so they fall through to F.unfold as before.
|
|
18
|
+
_FAST_PATH_DTYPES = frozenset(
|
|
19
|
+
{
|
|
20
|
+
torch.bool,
|
|
21
|
+
torch.float16,
|
|
22
|
+
torch.bfloat16,
|
|
23
|
+
torch.float32,
|
|
24
|
+
torch.float64,
|
|
25
|
+
torch.complex64,
|
|
26
|
+
torch.complex128,
|
|
27
|
+
}
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _as_pair(value: int | tuple[int, int], name: str) -> tuple[int, int]:
|
|
32
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
33
|
+
if value <= 0:
|
|
34
|
+
raise ValueError(f"{name} must be positive, got {value}")
|
|
35
|
+
return (value, value)
|
|
36
|
+
if isinstance(value, tuple) and len(value) == 2:
|
|
37
|
+
h, w = value
|
|
38
|
+
h_ok = isinstance(h, int) and not isinstance(h, bool)
|
|
39
|
+
w_ok = isinstance(w, int) and not isinstance(w, bool)
|
|
40
|
+
if not (h_ok and w_ok):
|
|
41
|
+
raise ValueError(f"{name} must contain ints, got {value!r}")
|
|
42
|
+
if h <= 0 or w <= 0:
|
|
43
|
+
raise ValueError(f"{name} must be positive, got {value!r}")
|
|
44
|
+
return (h, w)
|
|
45
|
+
raise ValueError(f"{name} must be int or (int, int), got {value!r}")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def extract(
|
|
49
|
+
image: torch.Tensor,
|
|
50
|
+
patch_size: int | tuple[int, int],
|
|
51
|
+
stride: int | tuple[int, int],
|
|
52
|
+
dilation: int | tuple[int, int] = 1,
|
|
53
|
+
) -> torch.Tensor:
|
|
54
|
+
"""Extract rectangular patches from a `(C, H, W)` image.
|
|
55
|
+
|
|
56
|
+
Returns `Tensor[L, C, ph, pw]` in row-major order. Patch `k` has its
|
|
57
|
+
top-left at `(k // num_w * sh, k % num_w * sw)`. Truncation is the only
|
|
58
|
+
boundary policy: if the geometry fits no patch, returns `Tensor[0, C, ph, pw]`.
|
|
59
|
+
Dtype and device of `image` are preserved.
|
|
60
|
+
"""
|
|
61
|
+
if not isinstance(image, torch.Tensor):
|
|
62
|
+
raise TypeError(f"image must be torch.Tensor, got {type(image).__name__}")
|
|
63
|
+
if image.ndim != 3:
|
|
64
|
+
raise ValueError(f"image must have ndim==3 (C, H, W), got ndim={image.ndim}")
|
|
65
|
+
|
|
66
|
+
ph, pw = _as_pair(patch_size, "patch_size")
|
|
67
|
+
sh, sw = _as_pair(stride, "stride")
|
|
68
|
+
dh, dw = _as_pair(dilation, "dilation")
|
|
69
|
+
|
|
70
|
+
c, h, w = image.shape
|
|
71
|
+
eff_h = dh * (ph - 1) + 1
|
|
72
|
+
eff_w = dw * (pw - 1) + 1
|
|
73
|
+
|
|
74
|
+
if h < eff_h or w < eff_w:
|
|
75
|
+
return torch.empty(0, c, ph, pw, dtype=image.dtype, device=image.device)
|
|
76
|
+
|
|
77
|
+
if dh == 1 and dw == 1 and image.dtype in _FAST_PATH_DTYPES:
|
|
78
|
+
# Fast path: strided window view + one copy. Measured 13-21x faster
|
|
79
|
+
# than F.unfold (im2col) on CPU, bit-exact (same pixels, same order).
|
|
80
|
+
nh = (h - ph) // sh + 1
|
|
81
|
+
nw = (w - pw) // sw + 1
|
|
82
|
+
windows = image.unfold(1, ph, sh).unfold(2, pw, sw) # (C, nh, nw, ph, pw)
|
|
83
|
+
out = windows.permute(1, 2, 0, 3, 4).reshape(nh * nw, c, ph, pw)
|
|
84
|
+
if out.untyped_storage().data_ptr() == image.untyped_storage().data_ptr():
|
|
85
|
+
# Degenerate geometry (e.g. ph == pw == 1): the permuted window is
|
|
86
|
+
# contiguous, reshape returned a view aliasing the image, and the
|
|
87
|
+
# contract promises independent memory.
|
|
88
|
+
out = out.clone()
|
|
89
|
+
return out
|
|
90
|
+
|
|
91
|
+
unfolded = F.unfold(
|
|
92
|
+
image.unsqueeze(0),
|
|
93
|
+
kernel_size=(ph, pw),
|
|
94
|
+
dilation=(dh, dw),
|
|
95
|
+
stride=(sh, sw),
|
|
96
|
+
)
|
|
97
|
+
return unfolded[0].view(c, ph, pw, -1).permute(3, 0, 1, 2).contiguous()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class Patchify:
|
|
101
|
+
"""Callable that extracts patches with a frozen geometry.
|
|
102
|
+
|
|
103
|
+
Drop-in for `torchvision.transforms.Compose([...])`:
|
|
104
|
+
|
|
105
|
+
transform = Compose([
|
|
106
|
+
ToTensor(),
|
|
107
|
+
GaussianBlur(kernel_size=3),
|
|
108
|
+
Patchify(patch_size=4, stride=2),
|
|
109
|
+
])
|
|
110
|
+
patches = transform(pil_image) # Tensor[L, C, 4, 4]
|
|
111
|
+
|
|
112
|
+
Equivalent to `lambda img: extract(img, patch_size, stride, dilation)`,
|
|
113
|
+
but composable, repr-friendly, and validates the geometry at construction
|
|
114
|
+
instead of at first call. Holds no state beyond the geometry: no cache,
|
|
115
|
+
no fixed image size, no device. See ADR 0002.
|
|
116
|
+
|
|
117
|
+
Output shape is `(L, C, ph, pw)`, the same as `extract`. Subsequent
|
|
118
|
+
transforms in the Compose chain receive the patch stack, not a single
|
|
119
|
+
patch; they must accept `(N, C, H, W)` or be wrapped.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
__slots__ = ("_dh", "_dw", "_ph", "_pw", "_sh", "_sw")
|
|
123
|
+
|
|
124
|
+
def __init__(
|
|
125
|
+
self,
|
|
126
|
+
patch_size: int | tuple[int, int],
|
|
127
|
+
stride: int | tuple[int, int],
|
|
128
|
+
dilation: int | tuple[int, int] = 1,
|
|
129
|
+
) -> None:
|
|
130
|
+
self._ph, self._pw = _as_pair(patch_size, "patch_size")
|
|
131
|
+
self._sh, self._sw = _as_pair(stride, "stride")
|
|
132
|
+
self._dh, self._dw = _as_pair(dilation, "dilation")
|
|
133
|
+
|
|
134
|
+
def __call__(self, image: torch.Tensor) -> torch.Tensor:
|
|
135
|
+
return extract(
|
|
136
|
+
image,
|
|
137
|
+
patch_size=(self._ph, self._pw),
|
|
138
|
+
stride=(self._sh, self._sw),
|
|
139
|
+
dilation=(self._dh, self._dw),
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
def __repr__(self) -> str:
|
|
143
|
+
return (
|
|
144
|
+
f"Patchify(patch_size=({self._ph}, {self._pw}), "
|
|
145
|
+
f"stride=({self._sh}, {self._sw}), "
|
|
146
|
+
f"dilation=({self._dh}, {self._dw}))"
|
|
147
|
+
)
|