PyPatchMatch 2.1.1__tar.gz → 2.2.0b1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/CLAUDE.md +12 -21
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/PKG-INFO +5 -3
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/README.md +3 -1
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/_lib.py +3 -2
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/_version.py +3 -3
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/csrc/inpaint.cpp +138 -26
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/csrc/inpaint.h +9 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/csrc/masked_image.cpp +21 -9
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/csrc/masked_image.h +12 -10
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/csrc/nnf.cpp +9 -16
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/csrc/nnf.h +21 -16
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/csrc/pyinterface.cpp +35 -42
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/pyproject.toml +1 -1
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/scripts/evaluate_inpainting.py +12 -11
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/tests/test_e2e.py +27 -26
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/tests/test_patch_match.py +67 -25
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/CMakeLists.txt +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/LICENSE +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/SECURITY.md +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/examples/images/chelsea.png +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/examples/images/forest_pruned.bmp +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/__init__.py +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/csrc/pyinterface.h +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/patch_match.py +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/patchmatch/py.typed +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/scripts/build_opencv.py +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/scripts/render_readme_example.py +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/tests/conftest.py +0 -0
- {pypatchmatch-2.1.1 → pypatchmatch-2.2.0b1}/tests/test_properties.py +0 -0
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
# CLAUDE.md
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
coverage and fill-quality runs are skills in `.claude/skills/`.
|
|
3
|
+
The details of the native code, the C interface, packaging and CI are path-scoped rules
|
|
4
|
+
in `.claude/rules/`, which load when a matching file is read or edited; read the rule
|
|
5
|
+
before changing such a file through Bash. The sanitizer, C++ coverage and fill-quality
|
|
6
|
+
runs are skills in `.claude/skills/`.
|
|
8
7
|
|
|
9
8
|
## Project
|
|
10
9
|
|
|
@@ -27,24 +26,18 @@ uv run pre-commit install
|
|
|
27
26
|
uv run pytest # all tests, with the coverage gate
|
|
28
27
|
uv run pytest -m "not e2e" # fast unit tests only
|
|
29
28
|
uv run pytest -m e2e --no-cov # end-to-end tests only
|
|
30
|
-
uv run pytest tests/test_patch_match.py::test_inpaint_global_mask --no-cov
|
|
31
|
-
uv run pytest -k regularity --no-cov
|
|
32
29
|
uv run mypy # strict, also a pre-commit hook
|
|
33
30
|
uv run pre-commit run --all-files # what the CI lint job runs
|
|
34
|
-
uv run ruff check --fix . && uv run ruff format .
|
|
35
31
|
uv build # sdist and wheel into dist/
|
|
36
32
|
```
|
|
37
33
|
|
|
38
34
|
- Pass `--no-cov` for any partial run: `addopts` always enables coverage, and
|
|
39
35
|
`fail_under = 95` (branch coverage, subprocesses included through
|
|
40
36
|
`patch = ["subprocess"]`) fails the run otherwise.
|
|
41
|
-
-
|
|
42
|
-
`
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
- `--cov=patchmatch` measures the source tree, so coverage only works with the editable
|
|
46
|
-
install that `uv sync` creates. With a non-editable install (`uv sync --no-editable`,
|
|
47
|
-
`uv pip install .`) it reports 0 % and the gate fails.
|
|
37
|
+
- Keep the editable install that `uv sync` creates. With a non-editable one
|
|
38
|
+
(`uv sync --no-editable`, `uv pip install .`), `--cov=patchmatch` measures the unused
|
|
39
|
+
source tree and reports 0 %, and `python -m pytest` imports the source tree from the
|
|
40
|
+
repository root, without the compiled library.
|
|
48
41
|
- `uv sync` rebuilds the library only when a `tool.uv.cache-keys` entry changes
|
|
49
42
|
(`CMakeLists.txt`, `patchmatch/csrc/*`, `pyproject.toml`, git commit or tags). Force it
|
|
50
43
|
with `uv sync --reinstall-package pypatchmatch`.
|
|
@@ -69,11 +62,8 @@ Other entry points:
|
|
|
69
62
|
- `uv run python scripts/render_readme_example.py` runs `py_example.py` and renders the
|
|
70
63
|
picture at the top of the README, `examples/images/readme_example.jpg`; rerun it and
|
|
71
64
|
commit the picture after changes that affect the fills.
|
|
72
|
-
- CMake options: `
|
|
73
|
-
|
|
74
|
-
`float-cast-overflow`, every finding aborts; turns off fast math, which lets the
|
|
75
|
-
compiler drop checks; GCC and Clang only), `PATCHMATCH_COVERAGE` (OFF, gcov counters
|
|
76
|
-
at `-Og`; GCC and Clang only).
|
|
65
|
+
- CMake options: the `option()` lines of `CMakeLists.txt`. `PATCHMATCH_SANITIZE` turns
|
|
66
|
+
off fast math, which lets the compiler drop checks.
|
|
77
67
|
|
|
78
68
|
## Architecture
|
|
79
69
|
|
|
@@ -114,7 +104,8 @@ CMake installs the shared library into the package directory.
|
|
|
114
104
|
goes through a test parameter instead of a `# type: ignore`.
|
|
115
105
|
- Backwards compatibility: `from patchmatch import patch_match` and
|
|
116
106
|
`patch_match.CShapeT`/`CMatT` keep working (tested).
|
|
117
|
-
- Comments explain why
|
|
107
|
+
- Comments explain why, in one short line where the code would otherwise be misread or
|
|
108
|
+
wrongly simplified; details and measurements go into the commit message.
|
|
118
109
|
- No `NOSONAR` markers: change the code so that the SonarCloud rule is satisfied, and ask
|
|
119
110
|
when that seems impossible. The only exception is `nnf.cpp`: cpp:S2245 flags every
|
|
120
111
|
declaration of `std::mt19937`, which only drives the randomized search.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: PyPatchMatch
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0b1
|
|
4
4
|
Summary: This library implements the PatchMatch based inpainting algorithm.
|
|
5
5
|
Keywords: inpainting,patchmatch,image-processing,opencv
|
|
6
6
|
Author-Email: The InvokeAI Project <lincoln.stein@gmail.com>, Jiayuan Mao <maojiayuan@gmail.com>, Kyle Schouviller <kyle0654@hotmail.com>, Matthias Wild <mauwii@outlook.de>, Younesse ANDAM <younesse.andam@gmail.com>
|
|
@@ -43,7 +43,7 @@ and Python interfaces. This implementation is heavily based on the implementatio
|
|
|
43
43
|
Younesse ANDAM: [younesse-cv/PatchMatch](https://github.com/younesse-cv/PatchMatch),
|
|
44
44
|
with some bug fixes, and updates.
|
|
45
45
|
|
|
46
|
-

|
|
47
47
|
|
|
48
48
|
From left to right: the photo, the input of
|
|
49
49
|
[examples/py_example.py](https://github.com/mauwii/PyPatchMatch/blob/main/examples/py_example.py)
|
|
@@ -98,7 +98,9 @@ The previous import path `from patchmatch import patch_match` keeps working.
|
|
|
98
98
|
|
|
99
99
|
`patch_size` is the radius of the compared patches, which span `2 * patch_size + 1`
|
|
100
100
|
pixels in each direction. Larger patches follow larger structures but are much slower;
|
|
101
|
-
the default of 15 compares 31x31 patches, the examples use 3.
|
|
101
|
+
the default of 15 compares 31x31 patches, the examples use 3. Below 5, the coarse levels
|
|
102
|
+
that lay out the fill are also tried with a radius of 5, which is kept where it fits
|
|
103
|
+
the surroundings better, so that repeating patterns continue across larger holes.
|
|
102
104
|
|
|
103
105
|
`patchmatch.set_random_seed(seed)` sets the seed of the randomized search, modulo
|
|
104
106
|
`2**32`, and `patchmatch.set_verbose(True)` prints the progress of the native code to
|
|
@@ -68,7 +68,9 @@ The previous import path `from patchmatch import patch_match` keeps working.
|
|
|
68
68
|
|
|
69
69
|
`patch_size` is the radius of the compared patches, which span `2 * patch_size + 1`
|
|
70
70
|
pixels in each direction. Larger patches follow larger structures but are much slower;
|
|
71
|
-
the default of 15 compares 31x31 patches, the examples use 3.
|
|
71
|
+
the default of 15 compares 31x31 patches, the examples use 3. Below 5, the coarse levels
|
|
72
|
+
that lay out the fill are also tried with a radius of 5, which is kept where it fits
|
|
73
|
+
the surroundings better, so that repeating patterns continue across larger holes.
|
|
72
74
|
|
|
73
75
|
`patchmatch.set_random_seed(seed)` sets the seed of the randomized search, modulo
|
|
74
76
|
`2**32`, and `patchmatch.set_verbose(True)` prints the progress of the native code to
|
|
@@ -82,8 +82,9 @@ def find_library() -> Path:
|
|
|
82
82
|
)
|
|
83
83
|
|
|
84
84
|
|
|
85
|
-
def load_library() -> ctypes.CDLL:
|
|
86
|
-
|
|
85
|
+
def load_library(path: Path | None = None) -> ctypes.CDLL:
|
|
86
|
+
"""Load the installed library, or the build at ``path``."""
|
|
87
|
+
lib = ctypes.CDLL(str(path or find_library()))
|
|
87
88
|
|
|
88
89
|
lib.PM_set_random_seed.argtypes = [ctypes.c_uint]
|
|
89
90
|
lib.PM_set_random_seed.restype = None
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '2.
|
|
22
|
-
__version_tuple__ = version_tuple = (2,
|
|
21
|
+
__version__ = version = '2.2.0b1'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 2, 0, 'b1')
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'gb66a4a1d8'
|
|
@@ -58,6 +58,17 @@ namespace
|
|
|
58
58
|
return count > 0 ? sum / count : sum;
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
// The pixel loops read three channels and index the masks with the image coordinates.
|
|
62
|
+
void check_inputs(const MaskedImage &image)
|
|
63
|
+
{
|
|
64
|
+
if (image.image().empty())
|
|
65
|
+
return;
|
|
66
|
+
CV_Assert(image.image().type() == CV_8UC3);
|
|
67
|
+
CV_Assert(image.mask().type() == CV_8UC1 && image.mask().size() == image.size());
|
|
68
|
+
if (image.has_global_mask())
|
|
69
|
+
CV_Assert(image.global_mask().type() == CV_8UC1 && image.global_mask().size() == image.size());
|
|
70
|
+
}
|
|
71
|
+
|
|
61
72
|
bool is_known(const MaskedImage &image, int y, int x)
|
|
62
73
|
{
|
|
63
74
|
return !image.is_masked(y, x) && !image.is_globally_masked(y, x);
|
|
@@ -68,6 +79,28 @@ namespace
|
|
|
68
79
|
return image.is_masked(y, x) && !image.is_globally_masked(y, x);
|
|
69
80
|
}
|
|
70
81
|
|
|
82
|
+
bool has_hole(const MaskedImage &image)
|
|
83
|
+
{
|
|
84
|
+
for (int y = 0; y < image.size().height; ++y)
|
|
85
|
+
{
|
|
86
|
+
for (int x = 0; x < image.size().width; ++x)
|
|
87
|
+
{
|
|
88
|
+
if (is_hole(image, y, x))
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// A patch smaller than a structure cannot tell where it continues: on the coarsest level
|
|
96
|
+
// with a hole, holes wider than the patches were filled from several shifted copies.
|
|
97
|
+
constexpr int kCoarsePatchSize = 5;
|
|
98
|
+
|
|
99
|
+
int nnf_iterations(int level)
|
|
100
|
+
{
|
|
101
|
+
return std::min(7, 1 + level);
|
|
102
|
+
}
|
|
103
|
+
|
|
71
104
|
// Chessboard distance of the hole pixels to the nearest known pixel, 0 beyond radius.
|
|
72
105
|
cv::Mat hole_depths(const MaskedImage &image, int radius)
|
|
73
106
|
{
|
|
@@ -328,6 +361,8 @@ Inpainting::Inpainting(
|
|
|
328
361
|
|
|
329
362
|
void Inpainting::_initialize_pyramid()
|
|
330
363
|
{
|
|
364
|
+
check_inputs(m_initial);
|
|
365
|
+
|
|
331
366
|
// The colors under the holes would reach the fill through the gradients of their
|
|
332
367
|
// neighbors and the target of a single-level pyramid. Black, like on coarser levels.
|
|
333
368
|
auto source = m_initial.clone();
|
|
@@ -347,22 +382,62 @@ void Inpainting::_initialize_pyramid()
|
|
|
347
382
|
source = source.downsample();
|
|
348
383
|
m_pyramid.push_back(source);
|
|
349
384
|
}
|
|
385
|
+
|
|
386
|
+
for (int level = 0; level < static_cast<int>(m_pyramid.size()); ++level)
|
|
387
|
+
{
|
|
388
|
+
if (has_hole(m_pyramid[level]))
|
|
389
|
+
m_coarsest_hole_level = level;
|
|
390
|
+
}
|
|
391
|
+
if (m_distance_metric->patch_size() < kCoarsePatchSize)
|
|
392
|
+
m_coarse_metric = m_distance_metric->with_patch_size(kCoarsePatchSize);
|
|
350
393
|
}
|
|
351
394
|
|
|
352
395
|
cv::Mat Inpainting::run(bool verbose, unsigned int random_seed)
|
|
353
396
|
{
|
|
354
397
|
NearestNeighborField::seed_random(random_seed);
|
|
355
|
-
const auto nr_levels = static_cast<int>(m_pyramid.size());
|
|
356
398
|
|
|
357
|
-
|
|
399
|
+
int level = static_cast<int>(m_pyramid.size()) - 1;
|
|
358
400
|
MaskedImage target;
|
|
359
|
-
|
|
401
|
+
// Below level 2 no level is left to compare the fills on.
|
|
402
|
+
if (m_coarse_metric && m_coarsest_hole_level >= 2)
|
|
360
403
|
{
|
|
361
|
-
|
|
404
|
+
target = _run_levels(level, m_coarsest_hole_level + 1, target, m_distance_metric, verbose);
|
|
405
|
+
target = _fill_coarse_levels(target, verbose);
|
|
406
|
+
level = m_coarsest_hole_level - 2;
|
|
407
|
+
}
|
|
408
|
+
target = _run_levels(level, 0, target, m_distance_metric, verbose);
|
|
409
|
+
|
|
410
|
+
// Only the holes are filled. The maximization step of level 0 keeps the known pixels,
|
|
411
|
+
// but the pyramid leaves globally masked pixels black, which are neither filled nor
|
|
412
|
+
// used as a source. Copy every pixel outside the holes from the input.
|
|
413
|
+
// A plain loop instead of OpenCV matrix expressions: those pull OpenCV's whole
|
|
414
|
+
// expression module into the statically linked library, 40 % more on disk.
|
|
415
|
+
cv::Mat result = target.image();
|
|
416
|
+
for (int y = 0; y < result.rows; ++y)
|
|
417
|
+
{
|
|
418
|
+
for (int x = 0; x < result.cols; ++x)
|
|
419
|
+
{
|
|
420
|
+
if (m_initial.is_masked(y, x) && !m_initial.is_globally_masked(y, x))
|
|
421
|
+
continue;
|
|
422
|
+
const unsigned char *input = m_initial.get_image(y, x);
|
|
423
|
+
std::copy(input, input + 3, result.ptr<unsigned char>(y, x));
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
blend_hole_borders(result, m_initial);
|
|
427
|
+
return result;
|
|
428
|
+
}
|
|
362
429
|
|
|
363
|
-
|
|
430
|
+
// Runs the levels from `from` down to `to`; target is empty on the coarsest level.
|
|
431
|
+
MaskedImage Inpainting::_run_levels(
|
|
432
|
+
int from, int to, MaskedImage target, const PatchDistanceMetric *metric, bool verbose)
|
|
433
|
+
{
|
|
434
|
+
m_level_metric = metric;
|
|
435
|
+
for (int level = from; level >= to; --level)
|
|
436
|
+
{
|
|
437
|
+
trace(verbose, "Inpainting level: ", level);
|
|
364
438
|
|
|
365
|
-
const
|
|
439
|
+
const MaskedImage &source = m_pyramid[level];
|
|
440
|
+
const bool coarsest = level == static_cast<int>(m_pyramid.size()) - 1;
|
|
366
441
|
if (coarsest)
|
|
367
442
|
{
|
|
368
443
|
target = source.clone();
|
|
@@ -373,34 +448,70 @@ cv::Mat Inpainting::run(bool verbose, unsigned int random_seed)
|
|
|
373
448
|
if (level < 2)
|
|
374
449
|
m_source2target.reset();
|
|
375
450
|
else
|
|
376
|
-
m_source2target = coarsest ? NearestNeighborField(source, target,
|
|
377
|
-
: NearestNeighborField(source, target,
|
|
378
|
-
m_target2source = coarsest ? NearestNeighborField(target, source,
|
|
379
|
-
: NearestNeighborField(target, source,
|
|
451
|
+
m_source2target = coarsest ? NearestNeighborField(source, target, metric)
|
|
452
|
+
: NearestNeighborField(source, target, metric, *m_source2target);
|
|
453
|
+
m_target2source = coarsest ? NearestNeighborField(target, source, metric)
|
|
454
|
+
: NearestNeighborField(target, source, metric, m_target2source);
|
|
380
455
|
|
|
381
456
|
trace(verbose, "Initialization done.");
|
|
382
457
|
|
|
383
458
|
target = _expectation_maximization(source, target, level, verbose);
|
|
384
459
|
}
|
|
460
|
+
return target;
|
|
461
|
+
}
|
|
385
462
|
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
463
|
+
// Fills the coarsest level with a hole and the next one with patch_size and with the
|
|
464
|
+
// larger patches, and keeps the fill with the lower energy on the following level. The
|
|
465
|
+
// larger patches lock onto repeating structures but pick flat sources elsewhere, which
|
|
466
|
+
// made fills of foliage gray.
|
|
467
|
+
MaskedImage Inpainting::_fill_coarse_levels(const MaskedImage &target, bool verbose)
|
|
468
|
+
{
|
|
469
|
+
const int level = m_coarsest_hole_level;
|
|
470
|
+
const auto source2target = m_source2target;
|
|
471
|
+
const auto target2source = m_target2source;
|
|
472
|
+
|
|
473
|
+
const MaskedImage plain = _run_levels(level, level - 1, target, m_distance_metric, verbose);
|
|
474
|
+
const double plain_energy = _hole_energy(level - 2, plain);
|
|
475
|
+
const auto plain_source2target = m_source2target;
|
|
476
|
+
const auto plain_target2source = m_target2source;
|
|
477
|
+
|
|
478
|
+
m_source2target = source2target;
|
|
479
|
+
m_target2source = target2source;
|
|
480
|
+
MaskedImage coarse = _run_levels(level, level - 1, target, m_coarse_metric.get(), verbose);
|
|
481
|
+
const double coarse_energy = _hole_energy(level - 2, coarse);
|
|
482
|
+
trace(verbose, "Energy with patch_size: ", plain_energy, ", with larger patches: ", coarse_energy);
|
|
483
|
+
m_level_metric = m_distance_metric;
|
|
484
|
+
if (coarse_energy < plain_energy)
|
|
485
|
+
return coarse;
|
|
486
|
+
m_source2target = plain_source2target;
|
|
487
|
+
m_target2source = plain_target2source;
|
|
488
|
+
return plain;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// The mean distance of the patches with holes after a search with patch_size on the level,
|
|
492
|
+
// started from the field of the level above, like the first iteration of that level.
|
|
493
|
+
double Inpainting::_hole_energy(int level, const MaskedImage &target) const
|
|
494
|
+
{
|
|
495
|
+
const MaskedImage &source = m_pyramid[level];
|
|
496
|
+
NearestNeighborField field(target, source, m_distance_metric, m_target2source);
|
|
497
|
+
field.minimize(nnf_iterations(level));
|
|
498
|
+
|
|
499
|
+
const int patch_size = m_distance_metric->patch_size();
|
|
500
|
+
const auto size = source.size();
|
|
501
|
+
double sum = 0;
|
|
502
|
+
int count = 0;
|
|
503
|
+
for (int i = 0; i < size.height; ++i)
|
|
393
504
|
{
|
|
394
|
-
for (int
|
|
505
|
+
for (int j = 0; j < size.width; ++j)
|
|
395
506
|
{
|
|
396
|
-
if (
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
507
|
+
if (source.contains_mask(i, j, patch_size) && !source.is_globally_masked(i, j))
|
|
508
|
+
{
|
|
509
|
+
sum += field.at(i, j, 2);
|
|
510
|
+
++count;
|
|
511
|
+
}
|
|
400
512
|
}
|
|
401
513
|
}
|
|
402
|
-
|
|
403
|
-
return result;
|
|
514
|
+
return count > 0 ? sum / count : 0;
|
|
404
515
|
}
|
|
405
516
|
|
|
406
517
|
// EM-Like algorithm (see "PatchMatch" - page 6).
|
|
@@ -409,7 +520,7 @@ MaskedImage Inpainting::_expectation_maximization(
|
|
|
409
520
|
const MaskedImage &source, MaskedImage target, int level, bool verbose)
|
|
410
521
|
{
|
|
411
522
|
const int nr_iters_em = 1 + 2 * level;
|
|
412
|
-
const int nr_iters_nnf =
|
|
523
|
+
const int nr_iters_nnf = nnf_iterations(level);
|
|
413
524
|
|
|
414
525
|
MaskedImage new_source;
|
|
415
526
|
MaskedImage new_target;
|
|
@@ -491,7 +602,7 @@ void Inpainting::_update_links(const MaskedImage &source, bool image_changed)
|
|
|
491
602
|
update(*m_source2target);
|
|
492
603
|
update(m_target2source);
|
|
493
604
|
};
|
|
494
|
-
const int patch_size =
|
|
605
|
+
const int patch_size = m_level_metric->patch_size();
|
|
495
606
|
const auto size = source.size();
|
|
496
607
|
for (int i = 0; i < size.height; ++i)
|
|
497
608
|
{
|
|
@@ -516,6 +627,7 @@ void Inpainting::_expectation_step(
|
|
|
516
627
|
holes.emplace(source);
|
|
517
628
|
const VoteCaster caster(
|
|
518
629
|
nnf, source2target, vote, source, upscaled, best_only, holes.has_value() ? &holes.value() : nullptr);
|
|
630
|
+
// The larger patches of m_coarse_metric would average more votes, which blurred the fills.
|
|
519
631
|
const int patch_size = m_distance_metric->patch_size();
|
|
520
632
|
const auto &kDistance2Similarity = distance2similarity();
|
|
521
633
|
const auto source_size = nnf.source_size();
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#pragma once
|
|
2
2
|
|
|
3
|
+
#include <memory>
|
|
3
4
|
#include <optional>
|
|
4
5
|
#include <vector>
|
|
5
6
|
|
|
@@ -16,6 +17,9 @@ public:
|
|
|
16
17
|
|
|
17
18
|
private:
|
|
18
19
|
void _initialize_pyramid(void);
|
|
20
|
+
MaskedImage _run_levels(int from, int to, MaskedImage target, const PatchDistanceMetric *metric, bool verbose);
|
|
21
|
+
MaskedImage _fill_coarse_levels(const MaskedImage &target, bool verbose);
|
|
22
|
+
double _hole_energy(int level, const MaskedImage &target) const;
|
|
19
23
|
MaskedImage _expectation_maximization(const MaskedImage &source, MaskedImage target, int level, bool verbose);
|
|
20
24
|
void _update_links(const MaskedImage &source, bool image_changed);
|
|
21
25
|
void _expectation_step(
|
|
@@ -29,4 +33,9 @@ private:
|
|
|
29
33
|
std::optional<NearestNeighborField> m_source2target;
|
|
30
34
|
NearestNeighborField m_target2source;
|
|
31
35
|
const PatchDistanceMetric *m_distance_metric;
|
|
36
|
+
// Searches on the levels that lay out the fill, if patch_size is smaller than kCoarsePatchSize.
|
|
37
|
+
std::unique_ptr<PatchDistanceMetric> m_coarse_metric;
|
|
38
|
+
int m_coarsest_hole_level = -1;
|
|
39
|
+
// Searches on the current level; the votes use patch_size.
|
|
40
|
+
const PatchDistanceMetric *m_level_metric = nullptr;
|
|
32
41
|
};
|
|
@@ -52,19 +52,31 @@ namespace
|
|
|
52
52
|
|
|
53
53
|
bool MaskedImage::contains_mask(int y, int x, int patch_size) const
|
|
54
54
|
{
|
|
55
|
-
auto
|
|
56
|
-
|
|
55
|
+
const auto size = this->size();
|
|
56
|
+
if (!m_hole_sums_computed)
|
|
57
57
|
{
|
|
58
|
-
|
|
58
|
+
m_hole_sums = cv::Mat(size.height + 1, size.width + 1, CV_32S, cv::Scalar(0));
|
|
59
|
+
for (int yy = 0; yy < size.height; ++yy)
|
|
59
60
|
{
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
for (int xx = 0; xx < size.width; ++xx)
|
|
62
|
+
{
|
|
63
|
+
const int hole = is_masked(yy, xx) && !is_globally_masked(yy, xx) ? 1 : 0;
|
|
64
|
+
m_hole_sums.at<int>(yy + 1, xx + 1) = hole + m_hole_sums.at<int>(yy, xx + 1) +
|
|
65
|
+
m_hole_sums.at<int>(yy + 1, xx) - m_hole_sums.at<int>(yy, xx);
|
|
66
|
+
}
|
|
65
67
|
}
|
|
68
|
+
m_hole_sums_computed = true;
|
|
66
69
|
}
|
|
67
|
-
|
|
70
|
+
// In 64 bits: patch_size may be close to the largest int.
|
|
71
|
+
const auto clip = [](std::int64_t value, int limit) {
|
|
72
|
+
return static_cast<int>(std::clamp<std::int64_t>(value, 0, limit));
|
|
73
|
+
};
|
|
74
|
+
const int top = clip(std::int64_t{y} - patch_size, size.height);
|
|
75
|
+
const int bottom = clip(std::int64_t{y} + patch_size + 1, size.height);
|
|
76
|
+
const int left = clip(std::int64_t{x} - patch_size, size.width);
|
|
77
|
+
const int right = clip(std::int64_t{x} + patch_size + 1, size.width);
|
|
78
|
+
const auto sum = [this](int yy, int xx) { return m_hole_sums.at<int>(yy, xx); };
|
|
79
|
+
return sum(bottom, right) - sum(top, right) - sum(bottom, left) + sum(top, left) > 0;
|
|
68
80
|
}
|
|
69
81
|
|
|
70
82
|
MaskedImage MaskedImage::downsample() const
|
|
@@ -12,13 +12,6 @@ public:
|
|
|
12
12
|
: m_image(image), m_mask(mask), m_global_mask(global_mask), m_has_global_mask(!global_mask.empty())
|
|
13
13
|
{
|
|
14
14
|
}
|
|
15
|
-
MaskedImage(
|
|
16
|
-
const cv::Mat &image, const cv::Mat &mask, const cv::Mat &global_mask, const cv::Mat &grady,
|
|
17
|
-
const cv::Mat &gradx, bool grad_computed)
|
|
18
|
-
: m_image(image), m_mask(mask), m_global_mask(global_mask), m_has_global_mask(!global_mask.empty()),
|
|
19
|
-
m_image_grady(grady), m_image_gradx(gradx), m_image_grad_computed(grad_computed)
|
|
20
|
-
{
|
|
21
|
-
}
|
|
22
15
|
MaskedImage(int width, int height)
|
|
23
16
|
: m_image(cv::Size(width, height), CV_8UC3, cv::Scalar::all(0)),
|
|
24
17
|
m_mask(cv::Size(width, height), CV_8U, cv::Scalar::all(0))
|
|
@@ -31,11 +24,10 @@ public:
|
|
|
31
24
|
MaskedImage &operator=(MaskedImage &&) noexcept = default;
|
|
32
25
|
~MaskedImage() = default;
|
|
33
26
|
|
|
27
|
+
// Without the gradients, which would be outdated once the clone is changed.
|
|
34
28
|
MaskedImage clone() const
|
|
35
29
|
{
|
|
36
|
-
return MaskedImage(
|
|
37
|
-
m_image.clone(), m_mask.clone(), m_global_mask.clone(), m_image_grady.clone(), m_image_gradx.clone(),
|
|
38
|
-
m_image_grad_computed);
|
|
30
|
+
return MaskedImage(m_image.clone(), m_mask.clone(), m_global_mask.clone());
|
|
39
31
|
}
|
|
40
32
|
|
|
41
33
|
cv::Size size() const
|
|
@@ -74,11 +66,13 @@ public:
|
|
|
74
66
|
m_global_mask = cv::Mat(m_mask.size(), CV_8U);
|
|
75
67
|
m_global_mask.setTo(cv::Scalar(0));
|
|
76
68
|
m_has_global_mask = !m_global_mask.empty();
|
|
69
|
+
m_hole_sums_computed = false;
|
|
77
70
|
}
|
|
78
71
|
void set_global_mask_mat(const cv::Mat &other)
|
|
79
72
|
{
|
|
80
73
|
m_global_mask = other;
|
|
81
74
|
m_has_global_mask = !other.empty();
|
|
75
|
+
m_hole_sums_computed = false;
|
|
82
76
|
}
|
|
83
77
|
|
|
84
78
|
bool is_masked(int y, int x) const
|
|
@@ -92,14 +86,17 @@ public:
|
|
|
92
86
|
void set_mask(int y, int x, bool value)
|
|
93
87
|
{
|
|
94
88
|
m_mask.at<unsigned char>(y, x) = static_cast<unsigned char>(value);
|
|
89
|
+
m_hole_sums_computed = false;
|
|
95
90
|
}
|
|
96
91
|
void set_global_mask(int y, int x, bool value)
|
|
97
92
|
{
|
|
98
93
|
m_global_mask.at<unsigned char>(y, x) = static_cast<unsigned char>(value);
|
|
94
|
+
m_hole_sums_computed = false;
|
|
99
95
|
}
|
|
100
96
|
void clear_mask()
|
|
101
97
|
{
|
|
102
98
|
m_mask.setTo(cv::Scalar(0));
|
|
99
|
+
m_hole_sums_computed = false;
|
|
103
100
|
}
|
|
104
101
|
|
|
105
102
|
const unsigned char *get_image(int y, int x) const
|
|
@@ -127,4 +124,9 @@ private:
|
|
|
127
124
|
mutable cv::Mat m_image_grady;
|
|
128
125
|
mutable cv::Mat m_image_gradx;
|
|
129
126
|
mutable bool m_image_grad_computed = false;
|
|
127
|
+
// computed on first use by contains_mask: the masked pixels outside the global mask
|
|
128
|
+
// above and left of each pixel. Copies share them with the masks, so change a mask only
|
|
129
|
+
// through an image that nothing has copied yet, as downsample and upsample do.
|
|
130
|
+
mutable cv::Mat m_hole_sums;
|
|
131
|
+
mutable bool m_hole_sums_computed = false;
|
|
130
132
|
};
|
|
@@ -200,13 +200,7 @@ void NearestNeighborField::_allocate_sums()
|
|
|
200
200
|
NearestNeighborField::Measured NearestNeighborField::_measure(int y, int x, int y_target, int x_target, int bound) const
|
|
201
201
|
{
|
|
202
202
|
if (m_sums.empty())
|
|
203
|
-
|
|
204
|
-
const int distance =
|
|
205
|
-
bound < PatchDistanceMetric::kDistanceScale
|
|
206
|
-
? m_distance_metric->distance_below(m_source, y, x, m_target, y_target, x_target, bound)
|
|
207
|
-
: (*m_distance_metric)(m_source, y, x, m_target, y_target, x_target);
|
|
208
|
-
return {distance, -1};
|
|
209
|
-
}
|
|
203
|
+
return {(*m_distance_metric)(m_source, y, x, m_target, y_target, x_target), -1};
|
|
210
204
|
const int patch_size = m_distance_metric->patch_size();
|
|
211
205
|
const std::int64_t sum = patch_sum(m_source, {y, x}, m_target, {y_target, x_target}, patch_size, bound);
|
|
212
206
|
if (sum < 0)
|
|
@@ -259,6 +253,7 @@ void NearestNeighborField::_initialize_field_from(const NearestNeighborField &ot
|
|
|
259
253
|
const auto &other_size = other.source_size();
|
|
260
254
|
double fi = static_cast<double>(this_size.height) / other_size.height;
|
|
261
255
|
double fj = static_cast<double>(this_size.width) / other_size.width;
|
|
256
|
+
const int patch_size = m_distance_metric->patch_size();
|
|
262
257
|
|
|
263
258
|
for (int i = 0; i < this_size.height; ++i)
|
|
264
259
|
{
|
|
@@ -266,6 +261,12 @@ void NearestNeighborField::_initialize_field_from(const NearestNeighborField &ot
|
|
|
266
261
|
{
|
|
267
262
|
if (m_source.is_globally_masked(i, j))
|
|
268
263
|
continue;
|
|
264
|
+
// The inpainting links them to themselves (_update_links); measuring them first took half its time.
|
|
265
|
+
if (!m_source.contains_mask(i, j, patch_size) && !m_target.contains_mask(i, j, patch_size))
|
|
266
|
+
{
|
|
267
|
+
set_identity(i, j);
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
269
270
|
|
|
270
271
|
auto ilow = static_cast<int>(std::min(i / fi, static_cast<double>(other_size.height - 1)));
|
|
271
272
|
auto jlow = static_cast<int>(std::min(j / fj, static_cast<double>(other_size.width - 1)));
|
|
@@ -283,8 +284,7 @@ void NearestNeighborField::_initialize_field_from(const NearestNeighborField &ot
|
|
|
283
284
|
|
|
284
285
|
// A hole that is new on this level: search near the patch before a neighbor's link replaces its own.
|
|
285
286
|
const bool linked_to_itself = other_value[0] == ilow && other_value[1] == jlow;
|
|
286
|
-
if (linked_to_itself && measured.distance > 0 &&
|
|
287
|
-
m_target.contains_mask(y_target, x_target, m_distance_metric->patch_size()))
|
|
287
|
+
if (linked_to_itself && measured.distance > 0 && m_target.contains_mask(y_target, x_target, patch_size))
|
|
288
288
|
_random_search(i, j);
|
|
289
289
|
}
|
|
290
290
|
}
|
|
@@ -400,13 +400,6 @@ int PatchSSDDistanceMetric::operator()(
|
|
|
400
400
|
source, {source_y, source_x}, target, {target_y, target_x}, patch_size(), PatchDistanceMetric::kDistanceScale);
|
|
401
401
|
}
|
|
402
402
|
|
|
403
|
-
int PatchSSDDistanceMetric::distance_below(
|
|
404
|
-
const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x,
|
|
405
|
-
int bound) const
|
|
406
|
-
{
|
|
407
|
-
return distance_masked_images(source, {source_y, source_x}, target, {target_y, target_x}, patch_size(), bound);
|
|
408
|
-
}
|
|
409
|
-
|
|
410
403
|
int RegularityGuidedPatchDistanceMetricV2::operator()(
|
|
411
404
|
const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const
|
|
412
405
|
{
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
#include <algorithm>
|
|
4
4
|
#include <cstdint>
|
|
5
|
+
#include <limits>
|
|
6
|
+
#include <memory>
|
|
5
7
|
#include <opencv2/core.hpp>
|
|
6
8
|
#include <stdexcept>
|
|
7
9
|
#include <vector>
|
|
@@ -11,7 +13,11 @@
|
|
|
11
13
|
class PatchDistanceMetric
|
|
12
14
|
{
|
|
13
15
|
public:
|
|
14
|
-
explicit PatchDistanceMetric(int patch_size) : m_patch_size(patch_size)
|
|
16
|
+
explicit PatchDistanceMetric(int patch_size) : m_patch_size(patch_size)
|
|
17
|
+
{
|
|
18
|
+
// 0 crashes, negative sizes never terminate, and 2 * patch_size + 1 has to fit an int.
|
|
19
|
+
CV_Assert(patch_size >= 1 && patch_size <= (std::numeric_limits<int>::max() - 1) / 2);
|
|
20
|
+
}
|
|
15
21
|
virtual ~PatchDistanceMetric() = default;
|
|
16
22
|
|
|
17
23
|
int patch_size() const
|
|
@@ -21,12 +27,10 @@ public:
|
|
|
21
27
|
virtual int operator()(
|
|
22
28
|
const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y,
|
|
23
29
|
int target_x) const = 0;
|
|
24
|
-
// The
|
|
25
|
-
virtual int
|
|
26
|
-
const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x,
|
|
27
|
-
int /* bound */) const
|
|
30
|
+
// The same metric with another patch size, or nullptr if it has none.
|
|
31
|
+
virtual std::unique_ptr<PatchDistanceMetric> with_patch_size(int) const
|
|
28
32
|
{
|
|
29
|
-
return
|
|
33
|
+
return nullptr;
|
|
30
34
|
}
|
|
31
35
|
static const int kDistanceScale;
|
|
32
36
|
|
|
@@ -110,12 +114,6 @@ public:
|
|
|
110
114
|
static void seed_random(unsigned int seed);
|
|
111
115
|
|
|
112
116
|
private:
|
|
113
|
-
// Private: a link written from outside would not update its sum.
|
|
114
|
-
int *mutable_ptr(int y, int x)
|
|
115
|
-
{
|
|
116
|
-
return m_field.ptr<int>(y, x);
|
|
117
|
-
}
|
|
118
|
-
|
|
119
117
|
// A distance and the sum of the pixel costs behind it, -1 if unknown.
|
|
120
118
|
struct Measured
|
|
121
119
|
{
|
|
@@ -137,7 +135,7 @@ private:
|
|
|
137
135
|
}
|
|
138
136
|
void _set(int y, int x, int y_target, int x_target, Measured measured)
|
|
139
137
|
{
|
|
140
|
-
auto ptr =
|
|
138
|
+
auto ptr = m_field.ptr<int>(y, x);
|
|
141
139
|
ptr[0] = y_target;
|
|
142
140
|
ptr[1] = x_target;
|
|
143
141
|
ptr[2] = measured.distance;
|
|
@@ -173,9 +171,10 @@ public:
|
|
|
173
171
|
int operator()(
|
|
174
172
|
const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y,
|
|
175
173
|
int target_x) const override;
|
|
176
|
-
int
|
|
177
|
-
|
|
178
|
-
|
|
174
|
+
std::unique_ptr<PatchDistanceMetric> with_patch_size(int patch_size) const override
|
|
175
|
+
{
|
|
176
|
+
return std::make_unique<PatchSSDDistanceMetric>(patch_size);
|
|
177
|
+
}
|
|
179
178
|
static const int kSSDScale;
|
|
180
179
|
};
|
|
181
180
|
|
|
@@ -188,10 +187,16 @@ public:
|
|
|
188
187
|
// The distance is divided by 1 + weight, so a negative weight flips its sign.
|
|
189
188
|
if (!(weight >= 0))
|
|
190
189
|
throw std::invalid_argument("the guide weight must be >= 0");
|
|
190
|
+
// operator() reads the first two channels at coordinates scaled to the map.
|
|
191
|
+
CV_Assert(!ijmap.empty() && ijmap.depth() == CV_32F && ijmap.channels() >= 2);
|
|
191
192
|
}
|
|
192
193
|
int operator()(
|
|
193
194
|
const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y,
|
|
194
195
|
int target_x) const override;
|
|
196
|
+
std::unique_ptr<PatchDistanceMetric> with_patch_size(int patch_size) const override
|
|
197
|
+
{
|
|
198
|
+
return std::make_unique<RegularityGuidedPatchDistanceMetricV2>(patch_size, m_ijmap, m_weight);
|
|
199
|
+
}
|
|
195
200
|
|
|
196
201
|
private:
|
|
197
202
|
cv::Mat m_ijmap;
|
|
@@ -9,10 +9,6 @@
|
|
|
9
9
|
|
|
10
10
|
#include "inpaint.h"
|
|
11
11
|
|
|
12
|
-
int _dtype_py_to_cv(PM_dtype_e dtype_py);
|
|
13
|
-
cv::Mat _py_to_cv2(PM_mat_t pymat);
|
|
14
|
-
PM_mat_t _cv2_to_py(cv::Mat cvmat);
|
|
15
|
-
|
|
16
12
|
namespace
|
|
17
13
|
{
|
|
18
14
|
// The settings are set by one thread while others may be inpainting, since ctypes
|
|
@@ -68,6 +64,41 @@ namespace
|
|
|
68
64
|
return PM_mat_t{nullptr, {0, 0, 0}, PM_dtype_e::PM_UINT8};
|
|
69
65
|
}
|
|
70
66
|
|
|
67
|
+
int _dtype_py_to_cv(PM_dtype_e dtype_py)
|
|
68
|
+
{
|
|
69
|
+
switch (dtype_py)
|
|
70
|
+
{
|
|
71
|
+
case PM_dtype_e::PM_UINT8:
|
|
72
|
+
return CV_8U;
|
|
73
|
+
case PM_dtype_e::PM_FLOAT32:
|
|
74
|
+
return CV_32F;
|
|
75
|
+
default:
|
|
76
|
+
throw std::invalid_argument("unsupported dtype");
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
cv::Mat _py_to_cv2(PM_mat_t pymat)
|
|
81
|
+
{
|
|
82
|
+
int dtype = CV_MAKETYPE(_dtype_py_to_cv(pymat.dtype), pymat.shape.channels);
|
|
83
|
+
return cv::Mat(cv::Size(pymat.shape.width, pymat.shape.height), dtype, pymat.data_ptr).clone();
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
PM_mat_t _cv2_to_py(const cv::Mat &cvmat)
|
|
87
|
+
{
|
|
88
|
+
CV_Assert(cvmat.depth() == CV_8U);
|
|
89
|
+
PM_shape_t shape = {cvmat.size().width, cvmat.size().height, cvmat.channels()};
|
|
90
|
+
size_t dsize = cvmat.total() * cvmat.elemSize();
|
|
91
|
+
|
|
92
|
+
// A null pointer reports an error to Python, so allocate at least one byte. A failed
|
|
93
|
+
// allocation throws std::bad_alloc, which guarded() reports.
|
|
94
|
+
auto data = std::make_unique<unsigned char[]>(dsize > 0 ? dsize : 1);
|
|
95
|
+
if (dsize > 0)
|
|
96
|
+
std::memcpy(data.get(), cvmat.data, dsize);
|
|
97
|
+
|
|
98
|
+
// Python passes the buffer back to PM_free_pymat, which frees it.
|
|
99
|
+
return PM_mat_t{data.release(), shape, PM_dtype_e::PM_UINT8};
|
|
100
|
+
}
|
|
101
|
+
|
|
71
102
|
PM_mat_t inpaint(
|
|
72
103
|
PM_mat_t source_py, PM_mat_t mask_py, const PM_mat_t *global_mask_py, const PatchDistanceMetric &metric)
|
|
73
104
|
{
|
|
@@ -134,41 +165,3 @@ PM_mat_t PM_inpaint2_regularity(
|
|
|
134
165
|
RegularityGuidedPatchDistanceMetricV2(patch_size, _py_to_cv2(ijmap_py), guide_weight));
|
|
135
166
|
});
|
|
136
167
|
}
|
|
137
|
-
|
|
138
|
-
int _dtype_py_to_cv(PM_dtype_e dtype_py)
|
|
139
|
-
{
|
|
140
|
-
switch (dtype_py)
|
|
141
|
-
{
|
|
142
|
-
case PM_dtype_e::PM_UINT8:
|
|
143
|
-
return CV_8U;
|
|
144
|
-
case PM_dtype_e::PM_FLOAT32:
|
|
145
|
-
return CV_32F;
|
|
146
|
-
default:
|
|
147
|
-
throw std::invalid_argument("unsupported dtype");
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
cv::Mat _py_to_cv2(PM_mat_t pymat)
|
|
152
|
-
{
|
|
153
|
-
int dtype = CV_MAKETYPE(_dtype_py_to_cv(pymat.dtype), pymat.shape.channels);
|
|
154
|
-
return cv::Mat(cv::Size(pymat.shape.width, pymat.shape.height), dtype, pymat.data_ptr).clone();
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
PM_mat_t _cv2_to_py(cv::Mat cvmat)
|
|
158
|
-
{
|
|
159
|
-
if (!cvmat.isContinuous())
|
|
160
|
-
cvmat = cvmat.clone();
|
|
161
|
-
|
|
162
|
-
CV_Assert(cvmat.depth() == CV_8U);
|
|
163
|
-
PM_shape_t shape = {cvmat.size().width, cvmat.size().height, cvmat.channels()};
|
|
164
|
-
size_t dsize = cvmat.total() * cvmat.elemSize();
|
|
165
|
-
|
|
166
|
-
// A null pointer reports an error to Python, so allocate at least one byte. A failed
|
|
167
|
-
// allocation throws std::bad_alloc, which guarded() reports.
|
|
168
|
-
auto data = std::make_unique<unsigned char[]>(dsize > 0 ? dsize : 1);
|
|
169
|
-
if (dsize > 0)
|
|
170
|
-
std::memcpy(data.get(), cvmat.data, dsize);
|
|
171
|
-
|
|
172
|
-
// Python passes the buffer back to PM_free_pymat, which frees it.
|
|
173
|
-
return PM_mat_t{data.release(), shape, PM_dtype_e::PM_UINT8};
|
|
174
|
-
}
|
|
@@ -92,7 +92,7 @@ fallback_version = "0.0.0"
|
|
|
92
92
|
|
|
93
93
|
[tool.cibuildwheel]
|
|
94
94
|
# The wheels are Python-version independent (py3-none-<platform>), so each platform
|
|
95
|
-
# is built and tested once; the CI workflow covers the
|
|
95
|
+
# is built and tested once; the CI workflow covers the newest Python version.
|
|
96
96
|
build = "cp311-*"
|
|
97
97
|
skip = "*-musllinux_*"
|
|
98
98
|
# OpenCV is built once per job in before-all, only for the runner's native arch
|
|
@@ -5,9 +5,11 @@ in examples/images/SOURCES.md. forest_pruned.bmp and every ``*_pruned.*`` image
|
|
|
5
5
|
git-ignored examples/images/local/ are added as object removals without a ground
|
|
6
6
|
truth; their pure white pixels are the holes. grass-edge cuts a band at the right
|
|
7
7
|
border, as in outpainting; brick-large and grass-large cut a circle of radius 242 into
|
|
8
|
-
the center, 70 % of the image.
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
the center, 70 % of the image. pattern repeats almost exactly, so its error tells a fill
|
|
9
|
+
that continues it (2 to 6) from a broken one; pattern-wide widens the hole to a radius
|
|
10
|
+
of 60, where patch size 3 mixed the rings before 2.2 (error 37). forest-global and
|
|
11
|
+
brick-global add a global mask, darkened in the report: the plant of
|
|
12
|
+
examples/py_example_global_mask.py and the left third of brick.png.
|
|
11
13
|
|
|
12
14
|
Usage:
|
|
13
15
|
uv run python scripts/evaluate_inpainting.py
|
|
@@ -66,6 +68,7 @@ HOLES = {
|
|
|
66
68
|
"gravel": (256, 256, 60, 60),
|
|
67
69
|
"coffee": (260, 85, 55, 60),
|
|
68
70
|
"chelsea": (52, 215, 24, 40),
|
|
71
|
+
"pattern": (320, 320, 40, 40),
|
|
69
72
|
}
|
|
70
73
|
COLUMNS = ("seam", "detail", "error")
|
|
71
74
|
|
|
@@ -127,6 +130,8 @@ def cases() -> Iterator[Case]:
|
|
|
127
130
|
for name in ("brick", "grass"):
|
|
128
131
|
truth = load(IMAGES / f"{name}.png")
|
|
129
132
|
yield cut(f"{name}-large", truth, ellipse(truth.shape, 256, 256, 242, 242))
|
|
133
|
+
pattern = load(IMAGES / "pattern.png")
|
|
134
|
+
yield cut("pattern-wide", pattern, ellipse(pattern.shape, 320, 320, 60, 60))
|
|
130
135
|
plant = np.zeros(pruned.shape[:2], dtype=bool)
|
|
131
136
|
plant[290:, 100:180] = True
|
|
132
137
|
yield Case("forest-global", pruned, white(pruned), None, plant)
|
|
@@ -194,12 +199,6 @@ def measure(
|
|
|
194
199
|
return values
|
|
195
200
|
|
|
196
201
|
|
|
197
|
-
def load_library(path: Path) -> ctypes.CDLL:
|
|
198
|
-
"""Load another build of the library than the installed one."""
|
|
199
|
-
_lib.find_library = lambda: path
|
|
200
|
-
return _lib.load_library()
|
|
201
|
-
|
|
202
|
-
|
|
203
202
|
def evaluate(library: ctypes.CDLL, case: Case, patch_size: int, seeds: int) -> Result:
|
|
204
203
|
patch_match._lib = library
|
|
205
204
|
mask = case.holes.astype(np.uint8)
|
|
@@ -523,12 +522,14 @@ def main() -> None:
|
|
|
523
522
|
if args.seeds < 1:
|
|
524
523
|
parser.error("--seeds must be at least 1")
|
|
525
524
|
|
|
526
|
-
current =
|
|
525
|
+
current = (
|
|
526
|
+
_lib.load_library(args.library.resolve()) if args.library else patch_match._lib
|
|
527
|
+
)
|
|
527
528
|
if current is None:
|
|
528
529
|
parser.error("the installed package has no native library, pass --library")
|
|
529
530
|
libraries, labels = [current], ["fill"]
|
|
530
531
|
if args.baseline:
|
|
531
|
-
libraries.insert(0, load_library(args.baseline.resolve()))
|
|
532
|
+
libraries.insert(0, _lib.load_library(args.baseline.resolve()))
|
|
532
533
|
labels = ["baseline", "current"]
|
|
533
534
|
selected = set(args.cases.split(",")) if args.cases else None
|
|
534
535
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""End-to-end tests of the user workflows shown in examples/ and the README."""
|
|
2
2
|
|
|
3
|
+
import functools
|
|
3
4
|
import importlib.util
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
|
|
@@ -59,6 +60,21 @@ def assert_plausible_fill(
|
|
|
59
60
|
assert color_diff.max() < 10, color_diff
|
|
60
61
|
|
|
61
62
|
|
|
63
|
+
@functools.cache
|
|
64
|
+
def fill(
|
|
65
|
+
image: str, hole: tuple[int, int, int, int] | None, patch_size: int
|
|
66
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
67
|
+
"""``image`` with white holes and its fill with seed 0, shared by several tests."""
|
|
68
|
+
source = evaluation.load(IMAGES / image)
|
|
69
|
+
if hole is not None:
|
|
70
|
+
source[evaluation.ellipse(source.shape, *hole)] = 255
|
|
71
|
+
patchmatch.set_random_seed(0)
|
|
72
|
+
result = patchmatch.inpaint(source, patch_size=patch_size)
|
|
73
|
+
source.setflags(write=False)
|
|
74
|
+
result.setflags(write=False)
|
|
75
|
+
return source, result
|
|
76
|
+
|
|
77
|
+
|
|
62
78
|
@pytest.fixture
|
|
63
79
|
def pruned_image() -> Image.Image:
|
|
64
80
|
return Image.open(IMAGES / "forest_pruned.bmp")
|
|
@@ -69,18 +85,15 @@ def seed():
|
|
|
69
85
|
patchmatch.set_random_seed(0)
|
|
70
86
|
|
|
71
87
|
|
|
72
|
-
|
|
73
|
-
def test_remove_white_regions(pruned_image, tmp_path, as_array):
|
|
88
|
+
def test_remove_white_regions(tmp_path):
|
|
74
89
|
"""examples/py_example.py: white pixels are the holes, the result is saved."""
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
result = patchmatch.inpaint(image, patch_size=3)
|
|
90
|
+
source, result = fill("forest_pruned.bmp", None, 3)
|
|
78
91
|
|
|
79
92
|
output = tmp_path / "forest_recovered.bmp"
|
|
80
93
|
Image.fromarray(result).save(output)
|
|
81
94
|
saved = np.array(Image.open(output))
|
|
82
95
|
np.testing.assert_array_equal(saved, result)
|
|
83
|
-
assert_plausible_fill(
|
|
96
|
+
assert_plausible_fill(source, saved)
|
|
84
97
|
|
|
85
98
|
|
|
86
99
|
def test_global_mask(pruned_image):
|
|
@@ -117,14 +130,10 @@ def test_global_mask_is_not_used_as_source(pruned_image):
|
|
|
117
130
|
np.testing.assert_array_equal(result[excluded], source[excluded])
|
|
118
131
|
|
|
119
132
|
|
|
120
|
-
def test_fill_is_as_detailed_as_its_surroundings(
|
|
133
|
+
def test_fill_is_as_detailed_as_its_surroundings():
|
|
121
134
|
"""The weighted mean of all overlapping patches blurred the fill (detail 0.69)."""
|
|
122
|
-
source =
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
result = patchmatch.inpaint(source, patch_size=3)
|
|
126
|
-
|
|
127
|
-
assert evaluation.measure(result, holes)["detail"] > 0.8
|
|
135
|
+
source, result = fill("forest_pruned.bmp", None, 3)
|
|
136
|
+
assert evaluation.measure(result, evaluation.white(source))["detail"] > 0.8
|
|
128
137
|
|
|
129
138
|
|
|
130
139
|
@pytest.mark.parametrize(
|
|
@@ -142,14 +151,8 @@ def test_no_seam_at_the_hole_border(image, hole, patch_size, limit):
|
|
|
142
151
|
side of it (seam 2.03). Keeping them still left the colors of the fill off along
|
|
143
152
|
the border: seam 1.73 for forest with patch size 3, 3.24 for chelsea with 15.
|
|
144
153
|
"""
|
|
145
|
-
source =
|
|
146
|
-
|
|
147
|
-
source[evaluation.ellipse(source.shape, *hole)] = 255
|
|
148
|
-
holes = evaluation.white(source)
|
|
149
|
-
|
|
150
|
-
result = patchmatch.inpaint(source, patch_size=patch_size)
|
|
151
|
-
|
|
152
|
-
assert evaluation.measure(result, holes)["seam"] < limit
|
|
154
|
+
source, result = fill(image, hole, patch_size)
|
|
155
|
+
assert evaluation.measure(result, evaluation.white(source))["seam"] < limit
|
|
153
156
|
|
|
154
157
|
|
|
155
158
|
@pytest.mark.parametrize(
|
|
@@ -176,16 +179,14 @@ def test_stem_is_filled_with_meadow(pruned_image, seed, use_global_mask):
|
|
|
176
179
|
assert result[stem].mean() > 0.85 * source[beside].mean()
|
|
177
180
|
|
|
178
181
|
|
|
179
|
-
|
|
180
|
-
def test_explicit_mask_matches_implicit_white_mask(pruned_image, hole_value):
|
|
182
|
+
def test_explicit_mask_matches_implicit_white_mask(pruned_image):
|
|
181
183
|
"""README: an explicit mask behaves like the default mask of white pixels."""
|
|
182
184
|
holes = evaluation.white(np.array(pruned_image))
|
|
183
|
-
mask = Image.fromarray(holes.astype(np.uint8) *
|
|
185
|
+
mask = Image.fromarray(holes.astype(np.uint8) * 255)
|
|
184
186
|
|
|
185
187
|
explicit = patchmatch.inpaint(pruned_image, mask, patch_size=3)
|
|
186
|
-
implicit = patchmatch.inpaint(pruned_image, patch_size=3)
|
|
187
188
|
|
|
188
|
-
np.testing.assert_array_equal(explicit,
|
|
189
|
+
np.testing.assert_array_equal(explicit, fill("forest_pruned.bmp", None, 3)[1])
|
|
189
190
|
|
|
190
191
|
|
|
191
192
|
# Regions of pure background in forest_pruned.bmp as (y, x, height, width).
|
|
@@ -112,23 +112,9 @@ def test_inpaint_explicit_mask(image, hole_mask):
|
|
|
112
112
|
def test_inpaint_global_mask(image, hole_mask):
|
|
113
113
|
global_mask = np.zeros_like(hole_mask)
|
|
114
114
|
global_mask[:8] = 1
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
np.testing.assert_array_equal(result[:8], image[:8])
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
def test_inpaint_ignores_colors_under_global_mask(image, hole_mask):
|
|
122
|
-
global_mask = np.zeros_like(hole_mask)
|
|
123
|
-
global_mask[16:32, :24] = 1 # touches the hole
|
|
124
|
-
results = []
|
|
125
|
-
for color in (0, 255):
|
|
126
|
-
image[global_mask == 1] = color
|
|
127
|
-
patchmatch.set_random_seed(0)
|
|
128
|
-
results.append(
|
|
129
|
-
patchmatch.inpaint(image, hole_mask, global_mask=global_mask, patch_size=3)
|
|
130
|
-
)
|
|
131
|
-
np.testing.assert_array_equal(results[0][HOLE], results[1][HOLE])
|
|
115
|
+
assert_filled(
|
|
116
|
+
patchmatch.inpaint(image, global_mask=global_mask, patch_size=3), image
|
|
117
|
+
)
|
|
132
118
|
|
|
133
119
|
|
|
134
120
|
@pytest.mark.parametrize("shape", [(0, 0, 3), (0, 10, 3), (10, 0, 3)])
|
|
@@ -224,6 +210,24 @@ def test_inpaint_textures_the_image_border(patch_size, transpose):
|
|
|
224
210
|
assert outermost > 0.5 * known, outermost / known
|
|
225
211
|
|
|
226
212
|
|
|
213
|
+
def test_inpaint_continues_a_repeating_pattern():
|
|
214
|
+
"""A hole over the center of concentric rings that repeat every 64 pixels.
|
|
215
|
+
|
|
216
|
+
The 7x7 patches of patch_size 3 were smaller than the hole on its coarsest
|
|
217
|
+
level, so they filled it from several shifted copies (error 56 to 60).
|
|
218
|
+
"""
|
|
219
|
+
palette = np.array([[230, 60, 90], [40, 70, 140], [250, 240, 190], [90, 200, 190]])
|
|
220
|
+
y, x = np.mgrid[0:128, 0:128]
|
|
221
|
+
rings = np.hypot(y % 64 - 32, x % 64 - 32) // 8
|
|
222
|
+
image = palette[rings.astype(int) % 4].astype(np.uint8)
|
|
223
|
+
mask = (np.hypot(y - 96, x - 96) < 20).astype(np.uint8)
|
|
224
|
+
|
|
225
|
+
result = patchmatch.inpaint(image, mask, patch_size=3)
|
|
226
|
+
|
|
227
|
+
error = np.abs(result.astype(int) - image)[mask == 1].mean()
|
|
228
|
+
assert error < 5, error
|
|
229
|
+
|
|
230
|
+
|
|
227
231
|
@pytest.mark.parametrize("regularity", [False, True], ids=["inpaint", "regularity"])
|
|
228
232
|
@pytest.mark.parametrize("tall", [True, False], ids=["tall", "wide"])
|
|
229
233
|
def test_inpaint_very_large_image(tall, regularity):
|
|
@@ -255,14 +259,6 @@ def test_inpaint_does_not_modify_inputs(image, hole_mask):
|
|
|
255
259
|
np.testing.assert_array_equal(hole_mask, mask_before)
|
|
256
260
|
|
|
257
261
|
|
|
258
|
-
def test_inpaint_is_deterministic_with_seed(image):
|
|
259
|
-
patchmatch.set_random_seed(42)
|
|
260
|
-
first = patchmatch.inpaint(image, patch_size=3)
|
|
261
|
-
patchmatch.set_random_seed(42)
|
|
262
|
-
second = patchmatch.inpaint(image, patch_size=3)
|
|
263
|
-
np.testing.assert_array_equal(first, second)
|
|
264
|
-
|
|
265
|
-
|
|
266
262
|
def test_concurrent_inpaint_is_deterministic(image):
|
|
267
263
|
# the native code runs without the GIL, so the threads really overlap
|
|
268
264
|
expected = patchmatch.inpaint(image, patch_size=3)
|
|
@@ -475,6 +471,52 @@ def test_native_rejects_negative_guide_weight(image, hole_mask, ijmap):
|
|
|
475
471
|
patch_match._call(lib.PM_inpaint_regularity, *args)
|
|
476
472
|
|
|
477
473
|
|
|
474
|
+
@pytest.mark.parametrize(
|
|
475
|
+
("argument", "value"),
|
|
476
|
+
[
|
|
477
|
+
("image", np.zeros((HEIGHT, WIDTH, 3), dtype=np.float32)),
|
|
478
|
+
("image", np.zeros((HEIGHT, WIDTH, 4), dtype=np.uint8)),
|
|
479
|
+
("mask", np.zeros((HEIGHT, WIDTH, 3), dtype=np.uint8)),
|
|
480
|
+
("mask", np.zeros((HEIGHT // 2, WIDTH, 1), dtype=np.uint8)),
|
|
481
|
+
("global_mask", np.zeros((HEIGHT, WIDTH, 1), dtype=np.float32)),
|
|
482
|
+
("global_mask", np.zeros((HEIGHT, WIDTH // 2, 1), dtype=np.uint8)),
|
|
483
|
+
("ijmap", np.zeros((0, WIDTH, 3), dtype=np.float32)),
|
|
484
|
+
("ijmap", np.zeros((HEIGHT, WIDTH, 3), dtype=np.uint8)),
|
|
485
|
+
("ijmap", np.zeros((HEIGHT, WIDTH, 1), dtype=np.float32)),
|
|
486
|
+
("patch_size", ctypes.c_int(0)),
|
|
487
|
+
("patch_size", ctypes.c_int(2**30)),
|
|
488
|
+
],
|
|
489
|
+
ids=[
|
|
490
|
+
"image-float",
|
|
491
|
+
"image-rgba",
|
|
492
|
+
"mask-3-channel",
|
|
493
|
+
"mask-size",
|
|
494
|
+
"global-mask-float",
|
|
495
|
+
"global-mask-size",
|
|
496
|
+
"ijmap-empty",
|
|
497
|
+
"ijmap-uint8",
|
|
498
|
+
"ijmap-1-channel",
|
|
499
|
+
"patch-size-0",
|
|
500
|
+
"patch-size-2**30",
|
|
501
|
+
],
|
|
502
|
+
)
|
|
503
|
+
def test_native_rejects_invalid_inputs(image, hole_mask, ijmap, argument, value):
|
|
504
|
+
"""The C++ code checks what it indexes, for callers that bypass Python."""
|
|
505
|
+
args = {
|
|
506
|
+
"image": image,
|
|
507
|
+
"mask": hole_mask[..., np.newaxis],
|
|
508
|
+
"global_mask": np.zeros_like(hole_mask)[..., np.newaxis],
|
|
509
|
+
"ijmap": ijmap,
|
|
510
|
+
"patch_size": ctypes.c_int(3),
|
|
511
|
+
"guide_weight": ctypes.c_float(0.25),
|
|
512
|
+
}
|
|
513
|
+
args[argument] = value
|
|
514
|
+
func = patch_match._get_lib().PM_inpaint2_regularity
|
|
515
|
+
values = list(args.values())
|
|
516
|
+
with pytest.raises(RuntimeError, match="Assertion failed"):
|
|
517
|
+
patch_match._call(func, *values)
|
|
518
|
+
|
|
519
|
+
|
|
478
520
|
def test_native_accepts_empty_image():
|
|
479
521
|
# Python returns empty images before the native call; C callers still get an
|
|
480
522
|
# empty result instead of a failure. OpenCV 4.6 returns it with shape (0, 0, 1).
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|