gdsdiff 0.1.0__py3-none-any.whl

gdsdiff/tiling.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .config import LayerSpec
+from .geometry import PolygonBuffer
+
+
+OLD_SIDE = 1
+NEW_SIDE = 2
+BOTH_SIDES = OLD_SIDE | NEW_SIDE
+
+
+@dataclass(frozen=True, order=True)
+class TileKey:
+    layer: LayerSpec
+    tx: int
+    ty: int
+
+    def to_json(self) -> dict[str, int]:
+        return {
+            "layer": self.layer.layer,
+            "datatype": self.layer.datatype,
+            "tx": self.tx,
+            "ty": self.ty,
+        }
+
+
+@dataclass(frozen=True)
+class TileConfig:
+    tile_size: int
+    max_tiles_per_polygon: int = 4096
+
+    def __post_init__(self) -> None:
+        if self.tile_size <= 0:
+            raise ValueError("tile_size must be positive")
+        if self.max_tiles_per_polygon <= 0:
+            raise ValueError("max_tiles_per_polygon must be positive")
+
+
+@dataclass(frozen=True)
+class OversizedPolygon:
+    layer: LayerSpec
+    canonical_id: int
+    side_mask: int
+    tile_bbox: tuple[int, int, int, int]
+
+    def to_json(self) -> dict[str, object]:
+        return {
+            "layer": self.layer.to_json(),
+            "canonical_id": self.canonical_id,
+            "side_mask": self.side_mask,
+            "tile_bbox": list(self.tile_bbox),
+        }
+
+
+@dataclass(frozen=True)
+class CpuTilePrefilterResult:
+    candidate_tiles: tuple[TileKey, ...]
+    oversized_candidates: tuple[OversizedPolygon, ...] = field(default_factory=tuple)
+    assignment_count: int = 0
+    collapsed_pair_count: int = 0
+    oversized_matched_count: int = 0
+
+    @property
+    def candidate_tile_count(self) -> int:
+        return len(self.candidate_tiles)
+
+    def to_json(self) -> dict[str, object]:
+        return {
+            "candidate_tiles": [tile.to_json() for tile in self.candidate_tiles],
+            "candidate_tile_count": len(self.candidate_tiles),
+            "oversized_candidates": [candidate.to_json() for candidate in self.oversized_candidates],
+            "assignment_count": self.assignment_count,
+            "collapsed_pair_count": self.collapsed_pair_count,
+            "oversized_matched_count": self.oversized_matched_count,
+        }
+
+
+def floor_div(value: int, divisor: int) -> int:
+    if divisor <= 0:
+        raise ValueError("divisor must be positive")
+    return value // divisor
+
+
+def bbox_tile_range(bbox: tuple[int, int, int, int], tile_size: int) -> tuple[int, int, int, int]:
+    if tile_size <= 0:
+        raise ValueError("tile_size must be positive")
+    x0, y0, x1, y1 = bbox
+    return (
+        floor_div(x0, tile_size),
+        floor_div(y0, tile_size),
+        floor_div(x1, tile_size),
+        floor_div(y1, tile_size),
+    )
+
+
+def run_cpu_tile_prefilter(
+    old_buffer: PolygonBuffer,
+    old_canonical_ids,
+    new_buffer: PolygonBuffer,
+    new_canonical_ids,
+    config: TileConfig,
+) -> CpuTilePrefilterResult:
+    pair_side_masks: dict[tuple[TileKey, int], int] = {}
+    oversized_side_masks: dict[tuple[LayerSpec, int, tuple[int, int, int, int]], int] = {}
+    assignment_count = 0
+    oversized_matched_count = 0
+
+    for buffer, ids, side in ((old_buffer, old_canonical_ids, OLD_SIDE), (new_buffer, new_canonical_ids, NEW_SIDE)):
+        if len(ids) != buffer.polygon_count:
+            raise ValueError("canonical id vector length must match polygon count")
+        for index in range(buffer.polygon_count):
+            layer = buffer.layer_table.layers[int(buffer.layer_ids[index])]
+            canonical_id = int(ids[index])
+            tile_bbox = bbox_tile_range(tuple(int(value) for value in buffer.bboxes[index]), config.tile_size)
+            tx0, ty0, tx1, ty1 = tile_bbox
+            span_x = tx1 - tx0 + 1
+            span_y = ty1 - ty0 + 1
+            if span_x <= 0 or span_y <= 0:
+                raise ValueError("invalid tile span")
+            count = span_x * span_y
+            if count > config.max_tiles_per_polygon:
+                key = (layer, canonical_id, tile_bbox)
+                oversized_side_masks[key] = oversized_side_masks.get(key, 0) | side
+                continue
+            for tx in range(tx0, tx1 + 1):
+                for ty in range(ty0, ty1 + 1):
+                    key = (TileKey(layer, tx, ty), canonical_id)
+                    pair_side_masks[key] = pair_side_masks.get(key, 0) | side
+                    assignment_count += 1
+
+    candidate_tiles = {
+        tile_key
+        for (tile_key, _canonical_id), side_mask in pair_side_masks.items()
+        if side_mask != BOTH_SIDES
+    }
+    oversized_candidates = []
+    for (layer, canonical_id, tile_bbox), side_mask in oversized_side_masks.items():
+        if side_mask == BOTH_SIDES:
+            oversized_matched_count += 1
+        else:
+            oversized_candidates.append(
+                OversizedPolygon(layer=layer, canonical_id=canonical_id, side_mask=side_mask, tile_bbox=tile_bbox)
+            )
+
+    return CpuTilePrefilterResult(
+        candidate_tiles=tuple(sorted(candidate_tiles)),
+        oversized_candidates=tuple(sorted(oversized_candidates, key=lambda item: (item.layer, item.canonical_id, item.tile_bbox))),
+        assignment_count=assignment_count,
+        collapsed_pair_count=len(pair_side_masks),
+        oversized_matched_count=oversized_matched_count,
+    )

gdsdiff/windowed_exact.py

@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+
+import gdstk
+
+from .config import LayerSpec
+from .exact_oracle import DiffFragment, ExactOracleResult, LayerExactDiff, run_full_exact_oracle
+from .geometry import PolygonBuffer, polygon_bbox, polygon_twice_area
+from .grid import snap_to_grid
+from .multiprocessing_support import fork_context
+from .spatial import BBox, SpatialIndex
+from .windows import CandidateWindow
+
+_WORKER_OLD_BUFFER = None
+_WORKER_NEW_BUFFER = None
+_WORKER_PRECISION = 1.0
+
+
+@dataclass(frozen=True)
+class WindowedExactFailure:
+    window: CandidateWindow
+    reason: str
+
+    def to_json(self) -> dict[str, object]:
+        return {"window": self.window.to_json(), "reason": self.reason}
+
+
+@dataclass(frozen=True)
+class WindowedExactResult:
+    layers: tuple[LayerExactDiff, ...]
+    escalated_layers: tuple[LayerSpec, ...] = field(default_factory=tuple)
+    failures: tuple[WindowedExactFailure, ...] = field(default_factory=tuple)
+
+    @property
+    def equivalent(self) -> bool:
+        return all(layer.xor_twice_area == 0 for layer in self.layers)
+
+    @property
+    def exact_result(self) -> ExactOracleResult:
+        return ExactOracleResult(self.layers)
+
+
+def run_windowed_exact_oracle(
+    old_buffer: PolygonBuffer,
+    new_buffer: PolygonBuffer,
+    windows: tuple[CandidateWindow, ...] | list[CandidateWindow],
+    *,
+    old_index: SpatialIndex | None = None,
+    new_index: SpatialIndex | None = None,
+    precision: float = 1.0,
+    max_polygons_per_window: int = 200_000,
+) -> WindowedExactResult:
+    old_index = old_index or SpatialIndex.from_buffer(old_buffer)
+    new_index = new_index or SpatialIndex.from_buffer(new_buffer)
+
+    layers_from_windows = tuple(sorted({window.layer for window in windows}))
+    direction_points: dict[LayerSpec, dict[str, list[tuple[tuple[int, int], ...]]]] = {
+        layer: {"old_minus_new": [], "new_minus_old": []} for layer in layers_from_windows
+    }
+    escalated_layers: set[LayerSpec] = set()
+    failures: list[WindowedExactFailure] = []
+    window_tasks = []
+
+    for window in windows:
+        old_indices = old_index.query(window.layer, window.verification_bbox)
+        new_indices = new_index.query(window.layer, window.verification_bbox)
+        window_tasks.append((window, old_indices, new_indices, max_polygons_per_window))
+
+    task_results = _run_window_tasks(old_buffer, new_buffer, window_tasks, precision=precision)
+    for window, old_minus, new_minus, failure_reason in task_results:
+        if failure_reason is not None:
+            escalated_layers.add(window.layer)
+            failures.append(WindowedExactFailure(window, failure_reason))
+
+    for window, old_minus, new_minus, failure_reason in task_results:
+        if failure_reason is not None or window.layer in escalated_layers:
+            continue
+        direction_points[window.layer]["old_minus_new"].extend(old_minus)
+        direction_points[window.layer]["new_minus_old"].extend(new_minus)
+
+    full_layer_diffs = _full_diffs_by_layer(old_buffer, new_buffer, precision=precision) if escalated_layers else {}
+    layer_diffs: list[LayerExactDiff] = []
+    for layer in layers_from_windows:
+        if layer in escalated_layers:
+            layer_diffs.append(
+                full_layer_diffs.get(
+                    layer,
+                    LayerExactDiff(layer=layer),
+                )
+            )
+            continue
+        old_minus = _union_fragments(direction_points[layer]["old_minus_new"], layer, precision=precision)
+        new_minus = _union_fragments(direction_points[layer]["new_minus_old"], layer, precision=precision)
+        xor = _union_fragments([fragment.points for fragment in old_minus] + [fragment.points for fragment in new_minus], layer, precision=precision)
+        layer_diffs.append(
+            LayerExactDiff(
+                layer=layer,
+                old_minus_new=old_minus,
+                new_minus_old=new_minus,
+                xor=xor,
+            )
+        )
+
+    return WindowedExactResult(
+        layers=tuple(sorted(layer_diffs, key=lambda diff: diff.layer)),
+        escalated_layers=tuple(sorted(escalated_layers)),
+        failures=tuple(failures),
+    )
+
+
+def _run_window_tasks(
+    old_buffer: PolygonBuffer,
+    new_buffer: PolygonBuffer,
+    window_tasks,
+    *,
+    precision: float,
+):
+    if not window_tasks:
+        return ()
+    worker_count = _resolve_window_worker_count(len(window_tasks))
+    if worker_count <= 1:
+        return tuple(_compute_window_task((old_buffer, new_buffer, precision, *task)) for task in window_tasks)
+    context = fork_context()
+    if context is None:
+        return tuple(_compute_window_task((old_buffer, new_buffer, precision, *task)) for task in window_tasks)
+
+    global _WORKER_OLD_BUFFER, _WORKER_NEW_BUFFER, _WORKER_PRECISION
+    _WORKER_OLD_BUFFER = old_buffer
+    _WORKER_NEW_BUFFER = new_buffer
+    _WORKER_PRECISION = precision
+    try:
+        try:
+            with context.Pool(processes=worker_count) as pool:
+                return tuple(pool.map(_compute_window_task_worker, window_tasks, chunksize=1))
+        except OSError:
+            return tuple(_compute_window_task((old_buffer, new_buffer, precision, *task)) for task in window_tasks)
+    finally:
+        _WORKER_OLD_BUFFER = None
+        _WORKER_NEW_BUFFER = None
+
+
+def _resolve_window_worker_count(window_count: int) -> int:
+    if window_count <= 1:
+        return 1
+    return max(1, min(os.cpu_count() or 1, window_count))
+
+
+def _compute_window_task_worker(task):
+    if _WORKER_OLD_BUFFER is None or _WORKER_NEW_BUFFER is None:
+        raise RuntimeError("windowed exact worker globals are not initialized")
+    return _compute_window_task((_WORKER_OLD_BUFFER, _WORKER_NEW_BUFFER, _WORKER_PRECISION, *task))
+
+
+def _compute_window_task(task):
+    old_buffer, new_buffer, precision, window, old_indices, new_indices, max_polygons_per_window = task
+    if len(old_indices) + len(new_indices) > max_polygons_per_window:
+        return window, (), (), "window polygon threshold exceeded"
+    try:
+        old_minus, new_minus = _compute_window_directional_fragments(
+            old_buffer,
+            old_indices,
+            new_buffer,
+            new_indices,
+            window,
+            precision=precision,
+        )
+    except (RuntimeError, ValueError, TypeError) as exc:
+        return window, (), (), f"gdstk boolean failed: {exc}"
+    return window, old_minus, new_minus, None
+
+
+def _compute_window_directional_fragments(
+    old_buffer: PolygonBuffer,
+    old_indices: tuple[int, ...],
+    new_buffer: PolygonBuffer,
+    new_indices: tuple[int, ...],
+    window: CandidateWindow,
+    *,
+    precision: float,
+) -> tuple[tuple[tuple[tuple[int, int], ...], ...], tuple[tuple[tuple[int, int], ...], ...]]:
+    origin = window.verification_bbox[0], window.verification_bbox[1]
+    old_polys = _buffer_indices_to_gdstk(old_buffer, old_indices, window.layer, origin)
+    new_polys = _buffer_indices_to_gdstk(new_buffer, new_indices, window.layer, origin)
+    verification_rect = _bbox_rectangle(window.verification_bbox, window.layer, origin)
+    core_rect = _bbox_rectangle(window.core_bbox, window.layer, origin)
+
+    old_clipped = gdstk.boolean(old_polys, [verification_rect], "and", precision=precision) or []
+    new_clipped = gdstk.boolean(new_polys, [verification_rect], "and", precision=precision) or []
+    old_minus = gdstk.boolean(old_clipped, new_clipped, "not", precision=precision) or []
+    new_minus = gdstk.boolean(new_clipped, old_clipped, "not", precision=precision) or []
+    old_core = gdstk.boolean(old_minus, [core_rect], "and", precision=precision) or []
+    new_core = gdstk.boolean(new_minus, [core_rect], "and", precision=precision) or []
+    return _global_points_from_gdstk(old_core, origin), _global_points_from_gdstk(new_core, origin)
+
+
+def _buffer_indices_to_gdstk(
+    buffer: PolygonBuffer,
+    indices: tuple[int, ...],
+    layer: LayerSpec,
+    origin: tuple[int, int],
+) -> list[gdstk.Polygon]:
+    ox, oy = origin
+    polygons = []
+    for index in indices:
+        points = tuple((int(x) - ox, int(y) - oy) for x, y in buffer.polygon_vertices(index))
+        if len(points) >= 3:
+            polygons.append(gdstk.Polygon(points, layer=layer.layer, datatype=layer.datatype))
+    return polygons
+
+
+def _bbox_rectangle(bbox: BBox, layer: LayerSpec, origin: tuple[int, int]) -> gdstk.Polygon:
+    ox, oy = origin
+    x0, y0, x1, y1 = bbox
+    return gdstk.rectangle(
+        (x0 - ox, y0 - oy),
+        (x1 + 1 - ox, y1 + 1 - oy),
+        layer=layer.layer,
+        datatype=layer.datatype,
+    )
+
+
+def _global_points_from_gdstk(polygons: list[gdstk.Polygon], origin: tuple[int, int]) -> tuple[tuple[tuple[int, int], ...], ...]:
+    ox, oy = origin
+    result = []
+    for polygon in polygons:
+        points = tuple((snap_to_grid(point[0], 1) + ox, snap_to_grid(point[1], 1) + oy) for point in polygon.points)
+        if len(points) >= 3:
+            result.append(points)
+    return tuple(result)
+
+
+def _union_fragments(
+    polygons: list[tuple[tuple[int, int], ...]],
+    layer: LayerSpec,
+    *,
+    precision: float,
+) -> tuple[DiffFragment, ...]:
+    if not polygons:
+        return ()
+    origin = _points_origin(polygons)
+    ox, oy = origin
+    local = [gdstk.Polygon([(x - ox, y - oy) for x, y in points], layer=layer.layer, datatype=layer.datatype) for points in polygons]
+    merged = gdstk.boolean(local, [], "or", precision=precision) or []
+    return _fragments_from_gdstk(merged, origin)
+
+
+def _fragments_from_gdstk(polygons: list[gdstk.Polygon], origin: tuple[int, int]) -> tuple[DiffFragment, ...]:
+    ox, oy = origin
+    fragments = []
+    for polygon in polygons:
+        points = tuple((snap_to_grid(point[0], 1) + ox, snap_to_grid(point[1], 1) + oy) for point in polygon.points)
+        if len(points) < 3:
+            continue
+        fragments.append(DiffFragment(points=points, bbox=polygon_bbox(points), twice_area=polygon_twice_area(points)))
+    return tuple(sorted(fragments, key=lambda fragment: (fragment.bbox, fragment.points)))
+
+
+def _points_origin(polygons: list[tuple[tuple[int, int], ...]]) -> tuple[int, int]:
+    return min(point[0] for polygon in polygons for point in polygon), min(point[1] for polygon in polygons for point in polygon)
+
+
+def _full_diffs_by_layer(
+    old_buffer: PolygonBuffer,
+    new_buffer: PolygonBuffer,
+    *,
+    precision: float,
+) -> dict[LayerSpec, LayerExactDiff]:
+    return {layer_diff.layer: layer_diff for layer_diff in run_full_exact_oracle(old_buffer, new_buffer, precision=precision).layers}

gdsdiff/windows.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .config import LayerSpec
+from .spatial import BBox, SpatialIndex, expand_bbox
+from .tiling import CpuTilePrefilterResult, TileConfig, TileKey
+
+
+@dataclass(frozen=True)
+class CandidateWindow:
+    layer: LayerSpec
+    core_bbox: BBox
+    verification_bbox: BBox
+    source_candidate_count: int
+    estimated_old_polygons: int
+    estimated_new_polygons: int
+    reason: str = "tile mismatch"
+
+    def to_json(self) -> dict[str, object]:
+        return {
+            "layer": self.layer.to_json(),
+            "core_bbox": list(self.core_bbox),
+            "verification_bbox": list(self.verification_bbox),
+            "source_candidate_count": self.source_candidate_count,
+            "estimated_old_polygons": self.estimated_old_polygons,
+            "estimated_new_polygons": self.estimated_new_polygons,
+            "reason": self.reason,
+        }
+
+
+@dataclass(frozen=True)
+class WindowConfig:
+    halo_dbu: int = 2
+    max_tiles_per_window: int = 4096
+
+    def __post_init__(self) -> None:
+        if self.halo_dbu < 0:
+            raise ValueError("halo_dbu must be nonnegative")
+        if self.max_tiles_per_window <= 0:
+            raise ValueError("max_tiles_per_window must be positive")
+
+
+def build_candidate_windows(
+    prefilter: CpuTilePrefilterResult,
+    tile_config: TileConfig,
+    old_index: SpatialIndex,
+    new_index: SpatialIndex,
+    window_config: WindowConfig | None = None,
+) -> tuple[CandidateWindow, ...]:
+    window_config = window_config or WindowConfig()
+    windows: list[CandidateWindow] = []
+    windows.extend(
+        _windows_from_tiles(
+            prefilter.candidate_tiles,
+            tile_config,
+            old_index,
+            new_index,
+            window_config,
+        )
+    )
+    for oversized in prefilter.oversized_candidates:
+        tx0, ty0, tx1, ty1 = oversized.tile_bbox
+        core = _tile_rect_to_bbox(tx0, ty0, tx1, ty1, tile_config.tile_size)
+        windows.append(
+            _make_window(
+                oversized.layer,
+                core,
+                source_count=(tx1 - tx0 + 1) * (ty1 - ty0 + 1),
+                old_index=old_index,
+                new_index=new_index,
+                window_config=window_config,
+                reason="oversized polygon",
+            )
+        )
+    return tuple(sorted(windows, key=lambda window: (window.layer, window.core_bbox, window.reason)))
+
+
+def _windows_from_tiles(
+    tiles: tuple[TileKey, ...],
+    tile_config: TileConfig,
+    old_index: SpatialIndex,
+    new_index: SpatialIndex,
+    window_config: WindowConfig,
+) -> list[CandidateWindow]:
+    grouped: dict[LayerSpec, list[TileKey]] = {}
+    for tile in tiles:
+        grouped.setdefault(tile.layer, []).append(tile)
+    windows: list[CandidateWindow] = []
+    for layer, layer_tiles in grouped.items():
+        rects = _merge_tiles_to_rectangles(layer_tiles, window_config.max_tiles_per_window)
+        for tx0, ty0, tx1, ty1, count in rects:
+            core = _tile_rect_to_bbox(tx0, ty0, tx1, ty1, tile_config.tile_size)
+            windows.append(
+                _make_window(
+                    layer,
+                    core,
+                    source_count=count,
+                    old_index=old_index,
+                    new_index=new_index,
+                    window_config=window_config,
+                    reason="tile mismatch",
+                )
+            )
+    return windows
+
+
+def _merge_tiles_to_rectangles(tiles: list[TileKey], max_tiles_per_window: int) -> list[tuple[int, int, int, int, int]]:
+    rows: dict[int, list[int]] = {}
+    for tile in tiles:
+        rows.setdefault(tile.ty, []).append(tile.tx)
+    row_runs: list[tuple[int, int, int]] = []
+    for ty, txs in rows.items():
+        for x0, x1 in _runs(sorted(set(txs))):
+            for split_x0, split_x1 in _split_run(x0, x1, max_tiles_per_window):
+                row_runs.append((ty, split_x0, split_x1))
+    row_runs.sort()
+    rectangles: list[tuple[int, int, int, int, int]] = []
+    active: dict[tuple[int, int], tuple[int, int, int, int, int]] = {}
+    for ty, x0, x1 in row_runs:
+        key = (x0, x1)
+        prev = active.pop(key, None)
+        width = x1 - x0 + 1
+        if prev is not None and prev[3] + 1 == ty and prev[4] + width <= max_tiles_per_window:
+            active[key] = (x0, prev[1], x1, ty, prev[4] + width)
+        else:
+            if prev is not None:
+                rectangles.append(prev)
+            active[key] = (x0, ty, x1, ty, width)
+    rectangles.extend(active.values())
+    return sorted(rectangles)
+
+
+def _runs(values: list[int]) -> list[tuple[int, int]]:
+    if not values:
+        return []
+    result = []
+    start = previous = values[0]
+    for value in values[1:]:
+        if value == previous + 1:
+            previous = value
+            continue
+        result.append((start, previous))
+        start = previous = value
+    result.append((start, previous))
+    return result
+
+
+def _split_run(x0: int, x1: int, max_width: int) -> list[tuple[int, int]]:
+    if max_width <= 0:
+        raise ValueError("max_width must be positive")
+    result = []
+    current = x0
+    while current <= x1:
+        end = min(x1, current + max_width - 1)
+        result.append((current, end))
+        current = end + 1
+    return result
+
+
+def _tile_rect_to_bbox(tx0: int, ty0: int, tx1: int, ty1: int, tile_size: int) -> BBox:
+    return tx0 * tile_size, ty0 * tile_size, (tx1 + 1) * tile_size - 1, (ty1 + 1) * tile_size - 1
+
+
+def _make_window(
+    layer: LayerSpec,
+    core: BBox,
+    *,
+    source_count: int,
+    old_index: SpatialIndex,
+    new_index: SpatialIndex,
+    window_config: WindowConfig,
+    reason: str,
+) -> CandidateWindow:
+    verification = expand_bbox(core, window_config.halo_dbu)
+    return CandidateWindow(
+        layer=layer,
+        core_bbox=core,
+        verification_bbox=verification,
+        source_candidate_count=source_count,
+        estimated_old_polygons=len(old_index.query(layer, verification)),
+        estimated_new_polygons=len(new_index.query(layer, verification)),
+        reason=reason,
+    )