shrinkray 0.0.0__py3-none-any.whl → 25.12.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shrinkray/__main__.py +130 -960
- shrinkray/cli.py +70 -0
- shrinkray/display.py +75 -0
- shrinkray/formatting.py +108 -0
- shrinkray/passes/bytes.py +217 -10
- shrinkray/passes/clangdelta.py +47 -17
- shrinkray/passes/definitions.py +84 -4
- shrinkray/passes/genericlanguages.py +61 -7
- shrinkray/passes/json.py +6 -0
- shrinkray/passes/patching.py +65 -57
- shrinkray/passes/python.py +66 -23
- shrinkray/passes/sat.py +505 -91
- shrinkray/passes/sequences.py +26 -6
- shrinkray/problem.py +206 -27
- shrinkray/process.py +49 -0
- shrinkray/reducer.py +187 -25
- shrinkray/state.py +599 -0
- shrinkray/subprocess/__init__.py +24 -0
- shrinkray/subprocess/client.py +253 -0
- shrinkray/subprocess/protocol.py +190 -0
- shrinkray/subprocess/worker.py +491 -0
- shrinkray/tui.py +915 -0
- shrinkray/ui.py +72 -0
- shrinkray/work.py +34 -6
- {shrinkray-0.0.0.dist-info → shrinkray-25.12.26.0.dist-info}/METADATA +44 -27
- shrinkray-25.12.26.0.dist-info/RECORD +33 -0
- {shrinkray-0.0.0.dist-info → shrinkray-25.12.26.0.dist-info}/WHEEL +2 -1
- shrinkray-25.12.26.0.dist-info/entry_points.txt +3 -0
- shrinkray-25.12.26.0.dist-info/top_level.txt +1 -0
- shrinkray/learning.py +0 -221
- shrinkray-0.0.0.dist-info/RECORD +0 -22
- shrinkray-0.0.0.dist-info/entry_points.txt +0 -3
- {shrinkray-0.0.0.dist-info → shrinkray-25.12.26.0.dist-info/licenses}/LICENSE +0 -0
shrinkray/ui.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""UI abstractions for shrink ray."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
import humanize
|
|
7
|
+
import trio
|
|
8
|
+
from attrs import define
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from shrinkray.problem import BasicReductionProblem
|
|
13
|
+
from shrinkray.state import ShrinkRayState
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@define(slots=False)
|
|
17
|
+
class ShrinkRayUI[TestCase](ABC):
|
|
18
|
+
"""Base class for shrink ray UI implementations."""
|
|
19
|
+
|
|
20
|
+
state: "ShrinkRayState[TestCase]"
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def reducer(self):
|
|
24
|
+
return self.state.reducer
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def problem(self) -> "BasicReductionProblem":
|
|
28
|
+
return self.reducer.target # type: ignore
|
|
29
|
+
|
|
30
|
+
def install_into_nursery(self, nursery: trio.Nursery): # noqa: B027
|
|
31
|
+
"""Install any background tasks into the nursery.
|
|
32
|
+
|
|
33
|
+
Subclasses may override this to add tasks that run alongside reduction.
|
|
34
|
+
The default implementation does nothing.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
async def run(self, nursery: trio.Nursery): # noqa: B027
|
|
38
|
+
"""Run the UI update loop.
|
|
39
|
+
|
|
40
|
+
Subclasses may override this to display progress or status updates.
|
|
41
|
+
The default implementation does nothing.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class BasicUI[TestCase](ShrinkRayUI[TestCase]):
|
|
46
|
+
"""Simple text-based UI for non-interactive use."""
|
|
47
|
+
|
|
48
|
+
async def run(self, nursery: trio.Nursery):
|
|
49
|
+
initial = self.state.initial
|
|
50
|
+
size = self.state.problem.size
|
|
51
|
+
print(
|
|
52
|
+
f"Starting reduction. Initial test case size: "
|
|
53
|
+
f"{humanize.naturalsize(size(initial))}",
|
|
54
|
+
flush=True,
|
|
55
|
+
)
|
|
56
|
+
prev_reduction = 0
|
|
57
|
+
while True:
|
|
58
|
+
initial = self.state.initial
|
|
59
|
+
current = self.state.problem.current_test_case
|
|
60
|
+
size = self.state.problem.size
|
|
61
|
+
reduction = size(initial) - size(current)
|
|
62
|
+
if reduction > prev_reduction:
|
|
63
|
+
print(
|
|
64
|
+
f"Reduced test case to {humanize.naturalsize(size(current))} "
|
|
65
|
+
f"(deleted {humanize.naturalsize(reduction)}, "
|
|
66
|
+
f"{humanize.naturalsize(reduction - prev_reduction)} since last time)",
|
|
67
|
+
flush=True,
|
|
68
|
+
)
|
|
69
|
+
prev_reduction = reduction
|
|
70
|
+
await trio.sleep(5)
|
|
71
|
+
else:
|
|
72
|
+
await trio.sleep(0.1)
|
shrinkray/work.py
CHANGED
|
@@ -1,14 +1,30 @@
|
|
|
1
|
+
"""Parallelism coordination and work management.
|
|
2
|
+
|
|
3
|
+
This module provides WorkContext, which manages parallel execution of
|
|
4
|
+
reduction work using Trio. The key insight is that test-case reduction
|
|
5
|
+
benefits from parallelism even though results are sequential: we can
|
|
6
|
+
speculatively test multiple candidates and keep the first success.
|
|
7
|
+
|
|
8
|
+
Key concepts:
|
|
9
|
+
- Lazy evaluation: Don't compute results until needed
|
|
10
|
+
- Backpressure: Limit in-flight work to avoid memory exhaustion
|
|
11
|
+
- Order preservation: Results come out in input order for reproducibility
|
|
12
|
+
"""
|
|
13
|
+
|
|
1
14
|
import heapq
|
|
15
|
+
from collections.abc import Awaitable, Callable, Sequence
|
|
2
16
|
from contextlib import asynccontextmanager
|
|
3
17
|
from enum import IntEnum
|
|
4
18
|
from itertools import islice
|
|
5
19
|
from random import Random
|
|
6
|
-
from typing import
|
|
20
|
+
from typing import TypeVar
|
|
7
21
|
|
|
8
22
|
import trio
|
|
9
23
|
|
|
10
24
|
|
|
11
25
|
class Volume(IntEnum):
|
|
26
|
+
"""Logging verbosity levels."""
|
|
27
|
+
|
|
12
28
|
quiet = 0
|
|
13
29
|
normal = 1
|
|
14
30
|
verbose = 2
|
|
@@ -19,16 +35,26 @@ S = TypeVar("S")
|
|
|
19
35
|
T = TypeVar("T")
|
|
20
36
|
|
|
21
37
|
|
|
22
|
-
|
|
38
|
+
class WorkContext:
|
|
39
|
+
"""Coordinates parallel work execution for reduction.
|
|
23
40
|
|
|
41
|
+
WorkContext provides methods for parallel map, filter, and search
|
|
42
|
+
operations that are tailored for test-case reduction:
|
|
24
43
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
44
|
+
- map(): Lazy parallel map with backpressure
|
|
45
|
+
- filter(): Parallel filter, yielding matching items
|
|
46
|
+
- find_first_value(): Find first item satisfying a predicate
|
|
47
|
+
- find_large_integer(): Binary search for largest valid integer
|
|
48
|
+
|
|
49
|
+
The parallelism is speculative: multiple candidates are tested
|
|
50
|
+
concurrently, but only one result "wins". This trades CPU for
|
|
51
|
+
wall-clock time, which is usually worthwhile when interestingness
|
|
52
|
+
tests are slow.
|
|
53
|
+
"""
|
|
28
54
|
|
|
29
55
|
def __init__(
|
|
30
56
|
self,
|
|
31
|
-
random:
|
|
57
|
+
random: Random | None = None,
|
|
32
58
|
parallelism: int = 1,
|
|
33
59
|
volume: Volume = Volume.normal,
|
|
34
60
|
):
|
|
@@ -61,6 +87,7 @@ class WorkContext:
|
|
|
61
87
|
await send.send(await f(x))
|
|
62
88
|
break
|
|
63
89
|
else:
|
|
90
|
+
send.close()
|
|
64
91
|
return
|
|
65
92
|
|
|
66
93
|
n = 2
|
|
@@ -98,6 +125,7 @@ class WorkContext:
|
|
|
98
125
|
async for x, v in results:
|
|
99
126
|
if v:
|
|
100
127
|
await send.send(x)
|
|
128
|
+
send.close()
|
|
101
129
|
|
|
102
130
|
yield receive
|
|
103
131
|
nursery.cancel_scope.cancel()
|
|
@@ -1,29 +1,41 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: shrinkray
|
|
3
|
-
Version:
|
|
3
|
+
Version: 25.12.26.0
|
|
4
4
|
Summary: Shrink Ray
|
|
5
|
-
|
|
5
|
+
Author-email: "David R. MacIver" <david@drmaciver.com>
|
|
6
6
|
License: MIT
|
|
7
|
-
|
|
8
|
-
Author-email: david@drmaciver.com
|
|
9
|
-
Requires-Python: >=3.12,<4.0
|
|
10
|
-
Classifier: Development Status :: 3 - Alpha
|
|
11
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
-
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
-
Requires-Dist: binaryornot (>=0.4.4,<0.5.0)
|
|
16
|
-
Requires-Dist: chardet (>=5.2.0,<6.0.0)
|
|
17
|
-
Requires-Dist: click (>=8.0.1)
|
|
18
|
-
Requires-Dist: exceptiongroup (>=1.2.0,<2.0.0)
|
|
19
|
-
Requires-Dist: humanize (>=4.9.0,<5.0.0)
|
|
20
|
-
Requires-Dist: libcst (>=1.1.0,<2.0.0)
|
|
21
|
-
Requires-Dist: trio (>=0.28.0,<0.29.0)
|
|
22
|
-
Requires-Dist: urwid (>=2.2.3,<3.0.0)
|
|
23
|
-
Project-URL: Changelog, https://github.com/DRMacIver/shrinkray/releases
|
|
24
|
-
Project-URL: Documentation, https://shrinkray.readthedocs.io
|
|
7
|
+
Project-URL: Homepage, https://github.com/DRMacIver/shrinkray
|
|
25
8
|
Project-URL: Repository, https://github.com/DRMacIver/shrinkray
|
|
9
|
+
Project-URL: Documentation, https://shrinkray.readthedocs.io
|
|
10
|
+
Project-URL: Changelog, https://github.com/DRMacIver/shrinkray/releases
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Requires-Python: >=3.12
|
|
26
13
|
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: click>=8.0.1
|
|
16
|
+
Requires-Dist: chardet>=5.2.0
|
|
17
|
+
Requires-Dist: trio>=0.28.0
|
|
18
|
+
Requires-Dist: textual>=0.47.0
|
|
19
|
+
Requires-Dist: humanize>=4.9.0
|
|
20
|
+
Requires-Dist: libcst>=1.1.0
|
|
21
|
+
Requires-Dist: exceptiongroup>=1.2.0
|
|
22
|
+
Requires-Dist: binaryornot>=0.4.4
|
|
23
|
+
Requires-Dist: black
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: coverage>=7.13.0; extra == "dev"
|
|
26
|
+
Requires-Dist: hypothesis>=6.92.1; extra == "dev"
|
|
27
|
+
Requires-Dist: hypothesmith>=0.3.1; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest-trio; extra == "dev"
|
|
30
|
+
Requires-Dist: pytest-asyncio; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest-textual-snapshot; extra == "dev"
|
|
32
|
+
Requires-Dist: coverage[toml]; extra == "dev"
|
|
33
|
+
Requires-Dist: pygments; extra == "dev"
|
|
34
|
+
Requires-Dist: basedpyright; extra == "dev"
|
|
35
|
+
Requires-Dist: ruff; extra == "dev"
|
|
36
|
+
Requires-Dist: pexpect>=4.9.0; extra == "dev"
|
|
37
|
+
Requires-Dist: pyte>=0.8.2; extra == "dev"
|
|
38
|
+
Dynamic: license-file
|
|
27
39
|
|
|
28
40
|
# Shrink Ray
|
|
29
41
|
|
|
@@ -77,21 +89,27 @@ Most test-case reducers only work well on a few formats. Shrink Ray is designed
|
|
|
77
89
|
|
|
78
90
|
It's designed to be highly parallel, and work with a very wide variety of formats, through a mix of good generic algorithms and format-specific reduction passes.
|
|
79
91
|
|
|
80
|
-
|
|
92
|
+
## Versioning and Releases
|
|
81
93
|
|
|
82
|
-
|
|
94
|
+
Shrink Ray uses calendar versioning (calver) in the format YY.M.D.N (e.g., 25.12.26.0 for the first release on December 26, 2025, 25.12.26.1 for the second, etc.).
|
|
95
|
+
|
|
96
|
+
New releases are published automatically when changes are pushed to main if there are any changes to the source code or pyproject.toml since the previous release.
|
|
97
|
+
|
|
98
|
+
Shrinkray makes no particularly strong backwards compatibility guarantees. I aim to keep its behaviour relatively stable between releases, but for example will not be particularly shy about dropping old versions of Python or adding new dependencies. The basic workflow of running a simple reduction will rarely, if ever, change, but the UI is likely to be continuously evolving for some time.
|
|
83
99
|
|
|
84
100
|
## Installation
|
|
85
101
|
|
|
86
|
-
Shrink Ray requires Python 3.12 or later, and can be installed using pip.
|
|
102
|
+
Shrink Ray requires Python 3.12 or later, and can be installed using pip or uv like any other python package.
|
|
87
103
|
|
|
88
|
-
|
|
104
|
+
You can install the latest release from PyPI or run directly from the main branch:
|
|
89
105
|
|
|
90
106
|
```
|
|
107
|
+
pipx install shrinkray
|
|
108
|
+
# or
|
|
91
109
|
pipx install git+https://github.com/DRMacIver/shrinkray.git
|
|
92
110
|
```
|
|
93
111
|
|
|
94
|
-
(if you don't have or want [pipx](https://pypa.github.io/pipx/) you could also do this with pip and it would work fine)
|
|
112
|
+
(if you don't have or want [pipx](https://pypa.github.io/pipx/) you could also do this with pip or `uv pip` and it would work fine)
|
|
95
113
|
|
|
96
114
|
Shrink Ray requires Python 3.12 or later and won't work on earlier versions. If everything is working correctly, it should refuse to install
|
|
97
115
|
on versions it's incompatible with. If you do not have Python 3.12 installed, I recommend [pyenv](https://github.com/pyenv/pyenv) for managing
|
|
@@ -167,4 +185,3 @@ Shrink Ray is something of a labour of love - I wanted to have a tool that actua
|
|
|
167
185
|
|
|
168
186
|
That being said, it is first and foremost designed to be a useful tool for practical engineering problems.
|
|
169
187
|
If you find it useful as such, please [consider sponsoring my development of it](https://github.com/sponsors/DRMacIver).
|
|
170
|
-
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
shrinkray/__init__.py,sha256=b5MvcvhsEGYya3GRXNbCJAlAL5IZHSsETLK_vtfmXRY,18
|
|
2
|
+
shrinkray/__main__.py,sha256=sRYLrG-7FMa-y067JyYmLZMOkO2FJ2V-BenxqtBwQj0,10887
|
|
3
|
+
shrinkray/cli.py,sha256=1-qjaIchyCDd-YCdGWtK7q9j9qr6uX6AqtwW8m5QCQg,1697
|
|
4
|
+
shrinkray/display.py,sha256=WYN05uqmUVpZhwi2pxr1U-wLHWZ9KiL0RUlTCBJ1N3E,2430
|
|
5
|
+
shrinkray/formatting.py,sha256=tXCGnhJn-WJGpHMaLHRCAXK8aKJBbrOdiW9QGERrQEk,3121
|
|
6
|
+
shrinkray/problem.py,sha256=Kp7QN10E4tzjdpoqJve8_RT26VpywzQwY0gX2VkBGCo,17277
|
|
7
|
+
shrinkray/process.py,sha256=-eP8h5X0ESbkcTic8FFEzkd4-vwaZ0YI5tLxUR25L8U,1599
|
|
8
|
+
shrinkray/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
shrinkray/reducer.py,sha256=66Q5BjTLKamO2M04i2CSrbThp7PyGTRu63_ueQnjc7g,19849
|
|
10
|
+
shrinkray/state.py,sha256=_-gyAkUm0vEdF1U0Fz_Deykj-kY2u3nGn3X6BfH3viA,22371
|
|
11
|
+
shrinkray/tui.py,sha256=HtvqimSr1r7IX_fukRsCsVxyhEdyj2W-HLqjVATt1eM,31235
|
|
12
|
+
shrinkray/ui.py,sha256=xuDUwU-MM3AetvwUB7bfzav0P_drUsBrKFPhON_Nr-k,2251
|
|
13
|
+
shrinkray/work.py,sha256=DXeqJTB_G8r7e8vrsMW2J56CJ-nhgymKBH55DT8SXs8,7901
|
|
14
|
+
shrinkray/passes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
+
shrinkray/passes/bytes.py,sha256=pX2kBeH38SM4j55f8bNF_2PVBCDo6XdUj73EvOFH9f8,23904
|
|
16
|
+
shrinkray/passes/clangdelta.py,sha256=t9EQ_kc159HRs48JwB5JvlJCsiCscrZgf2nhHCZRZX0,8419
|
|
17
|
+
shrinkray/passes/definitions.py,sha256=WEYtcWOBv7ghZRUtJWBxHRU3SnO000EFohfAD4le7KM,4267
|
|
18
|
+
shrinkray/passes/genericlanguages.py,sha256=qbTuJgUieHqWtf7cly2tm0qdbrVXzVoWcAnFo7fny94,10400
|
|
19
|
+
shrinkray/passes/json.py,sha256=AcmroHgb38Aa3aApwcuYNzmyya_vnCi2RFSVqomiDg8,2586
|
|
20
|
+
shrinkray/passes/patching.py,sha256=1uOTir3IbywKmsg6IIhSnxHFovZTdUCS-8PSwzgza00,8936
|
|
21
|
+
shrinkray/passes/python.py,sha256=3WN1lZTf5oVL8FCTGomhrCuE04wIX9ocKcmFV86NMZA,6875
|
|
22
|
+
shrinkray/passes/sat.py,sha256=2FGMM4rh4AX1BVEbry082C4aLCOEOXlfr3exHbxYgSQ,19514
|
|
23
|
+
shrinkray/passes/sequences.py,sha256=jCK1fWBxCz79u7JWSps9wf7Yru7W_FAsJwdgg--CLxU,3040
|
|
24
|
+
shrinkray/subprocess/__init__.py,sha256=FyV2y05uwQ1RTZGwREI0aAVaLX1jiwRcWsdsksFmdbM,451
|
|
25
|
+
shrinkray/subprocess/client.py,sha256=xSFqm5UyQT0WJ5aBVVkuiWDsHjZYv7RqjgrjjyX0rK0,9269
|
|
26
|
+
shrinkray/subprocess/protocol.py,sha256=LuHl0IkKpDzYhAGZz_EiTHNqDNq_v1ozg5aUSl7UzE4,6203
|
|
27
|
+
shrinkray/subprocess/worker.py,sha256=MbubmnuXNFxD_SRKdiFDkGhdkCEgRxdn0tPN0HoJpyk,18998
|
|
28
|
+
shrinkray-25.12.26.0.dist-info/licenses/LICENSE,sha256=iMKX79AuokJfIZUnGUARdUp30vVAoIPOJ7ek8TY63kk,1072
|
|
29
|
+
shrinkray-25.12.26.0.dist-info/METADATA,sha256=ywZp_Jeav917ikM3SPZ3MlchAgF0udIIffGzIJf9pmU,9628
|
|
30
|
+
shrinkray-25.12.26.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
31
|
+
shrinkray-25.12.26.0.dist-info/entry_points.txt,sha256=wIZvnGyOdVeaLTiv2klnSyTe-EKkkwn4SwHh9bmJ7qk,104
|
|
32
|
+
shrinkray-25.12.26.0.dist-info/top_level.txt,sha256=fLif8-rFoFOnf5h8-vs3ECkKNWQopTQh3xvl1s7pchQ,10
|
|
33
|
+
shrinkray-25.12.26.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
shrinkray
|
shrinkray/learning.py
DELETED
|
@@ -1,221 +0,0 @@
|
|
|
1
|
-
import hashlib
|
|
2
|
-
import os
|
|
3
|
-
import random
|
|
4
|
-
import re
|
|
5
|
-
import site
|
|
6
|
-
import subprocess
|
|
7
|
-
import sys
|
|
8
|
-
from glob import glob
|
|
9
|
-
from typing import Awaitable, Callable
|
|
10
|
-
|
|
11
|
-
import trio
|
|
12
|
-
from attrs import define
|
|
13
|
-
|
|
14
|
-
from shrinkray.problem import BasicReductionProblem
|
|
15
|
-
from shrinkray.reducer import ShrinkRay
|
|
16
|
-
from shrinkray.work import WorkContext
|
|
17
|
-
|
|
18
|
-
WHITESPACE = re.compile(rb"\s+")
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def whitespace_normalize(s):
|
|
22
|
-
return WHITESPACE.sub(b" ", s).strip()
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def threshold_value(seed: bytes, test_case: bytes) -> float:
|
|
26
|
-
return random.Random(hashlib.sha1(seed + b":" + test_case).digest()).random()
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
@define
|
|
30
|
-
class RandomInterestingnessTest:
|
|
31
|
-
seed: bytes
|
|
32
|
-
threshold: float
|
|
33
|
-
key_substring: bytes
|
|
34
|
-
base_interestingness: Callable[[bytes], Awaitable[bool]]
|
|
35
|
-
|
|
36
|
-
async def __call__(self, test_case: bytes) -> bool:
|
|
37
|
-
if threshold_value(self.seed, test_case) > self.threshold:
|
|
38
|
-
await trio.lowlevel.checkpoint()
|
|
39
|
-
return False
|
|
40
|
-
normalized = whitespace_normalize(test_case)
|
|
41
|
-
if self.key_substring not in normalized:
|
|
42
|
-
await trio.lowlevel.checkpoint()
|
|
43
|
-
return False
|
|
44
|
-
return await self.base_interestingness(test_case)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
class CouldNotLearn(Exception):
|
|
48
|
-
pass
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
class CorpusExplorer:
|
|
52
|
-
def __init__(
|
|
53
|
-
self,
|
|
54
|
-
is_interesting: Callable[[bytes], Awaitable[bool]],
|
|
55
|
-
corpus: list[str],
|
|
56
|
-
parallelism=os.cpu_count(),
|
|
57
|
-
):
|
|
58
|
-
self.__is_interesting = is_interesting
|
|
59
|
-
self.__is_interesting_cache = {}
|
|
60
|
-
self.__corpus_files = [t for t in corpus if os.stat(t).st_size > 0]
|
|
61
|
-
self.__corpus_data = []
|
|
62
|
-
self.__limit = trio.CapacityLimiter(parallelism)
|
|
63
|
-
self.__wait_for_me = object()
|
|
64
|
-
self.__random = random.Random()
|
|
65
|
-
self.__parallelism = parallelism
|
|
66
|
-
|
|
67
|
-
async def is_interesting(self, tc):
|
|
68
|
-
try:
|
|
69
|
-
result = self.__is_interesting_cache[tc]
|
|
70
|
-
if result != self.__wait_for_me:
|
|
71
|
-
await trio.lowlevel.checkpoint()
|
|
72
|
-
return result
|
|
73
|
-
while result == self.__wait_for_me:
|
|
74
|
-
await trio.sleep(0.01)
|
|
75
|
-
result = self.__is_interesting_cache[tc]
|
|
76
|
-
assert result is not self.__wait_for_me
|
|
77
|
-
return result
|
|
78
|
-
except KeyError:
|
|
79
|
-
self.__is_interesting_cache[tc] = self.__wait_for_me
|
|
80
|
-
async with self.__limit:
|
|
81
|
-
result = await self.__is_interesting(tc)
|
|
82
|
-
self.__is_interesting_cache[tc] = result
|
|
83
|
-
return result
|
|
84
|
-
|
|
85
|
-
async def random_corpus_member(self):
|
|
86
|
-
return list(await self.random_corpus_sample(1))[0]
|
|
87
|
-
|
|
88
|
-
async def random_corpus_sample(self, max_elements=100):
|
|
89
|
-
result = []
|
|
90
|
-
max_from_data = len(self.__corpus_data)
|
|
91
|
-
|
|
92
|
-
async def run():
|
|
93
|
-
nonlocal max_from_data
|
|
94
|
-
while len(result) < max_elements and (
|
|
95
|
-
max_from_data > 0 or self.__corpus_files
|
|
96
|
-
):
|
|
97
|
-
i = self.__random.randrange(0, len(self.__corpus_files) + max_from_data)
|
|
98
|
-
if i < len(self.__corpus_files):
|
|
99
|
-
j = len(self.__corpus_files) - 1
|
|
100
|
-
self.__corpus_files[i], self.__corpus_files[j] = (
|
|
101
|
-
self.__corpus_files[j],
|
|
102
|
-
self.__corpus_files[i],
|
|
103
|
-
)
|
|
104
|
-
f = self.__corpus_files.pop()
|
|
105
|
-
with open(f, "rb") as input:
|
|
106
|
-
data = input.read()
|
|
107
|
-
|
|
108
|
-
if await self.is_interesting(data):
|
|
109
|
-
self.__corpus_data.append(data)
|
|
110
|
-
result.append(data)
|
|
111
|
-
else:
|
|
112
|
-
i -= len(self.__corpus_files)
|
|
113
|
-
j = max_from_data - 1
|
|
114
|
-
self.__corpus_data[i], self.__corpus_data[j] = (
|
|
115
|
-
self.__corpus_data[j],
|
|
116
|
-
self.__corpus_data[i],
|
|
117
|
-
)
|
|
118
|
-
result.append(self.__corpus_data[j])
|
|
119
|
-
max_from_data -= 1
|
|
120
|
-
await trio.lowlevel.checkpoint()
|
|
121
|
-
|
|
122
|
-
if self.__parallelism > 1:
|
|
123
|
-
async with trio.open_nursery() as nursery:
|
|
124
|
-
for _ in range(self.__parallelism):
|
|
125
|
-
nursery.start_soon(run)
|
|
126
|
-
else:
|
|
127
|
-
await run()
|
|
128
|
-
return result[:max_elements]
|
|
129
|
-
|
|
130
|
-
async def random_reduction_problem(self):
|
|
131
|
-
while True:
|
|
132
|
-
sample = await self.random_corpus_sample(400)
|
|
133
|
-
|
|
134
|
-
index = defaultdict(list)
|
|
135
|
-
|
|
136
|
-
for i, s in enumerate(sample):
|
|
137
|
-
for j, c in enumerate(whitespace_normalize(s)):
|
|
138
|
-
index[c].append((i, j))
|
|
139
|
-
|
|
140
|
-
queue = dequeue()
|
|
141
|
-
|
|
142
|
-
strings = [b""]
|
|
143
|
-
|
|
144
|
-
if not normalized:
|
|
145
|
-
continue
|
|
146
|
-
while True:
|
|
147
|
-
start = self.__random.randrange(0, len(normalized))
|
|
148
|
-
if not WHITESPACE.match(normalized[start : start + 1]):
|
|
149
|
-
break
|
|
150
|
-
while True:
|
|
151
|
-
end = self.__random.randint(start, min(start + 20, len(normalized) - 1))
|
|
152
|
-
if not WHITESPACE.match(normalized[end : end + 1]):
|
|
153
|
-
break
|
|
154
|
-
substring = normalized[start:end]
|
|
155
|
-
while True:
|
|
156
|
-
seed = str(self.__random.randint(0, 10**6)).encode("ascii")
|
|
157
|
-
value = threshold_value(seed=seed, test_case=initial)
|
|
158
|
-
if value >= 0.5:
|
|
159
|
-
threshold = self.__random.random() * (1.0 - value) + value
|
|
160
|
-
break
|
|
161
|
-
|
|
162
|
-
return initial, RandomInterestingnessTest(
|
|
163
|
-
seed=seed,
|
|
164
|
-
threshold=threshold,
|
|
165
|
-
key_substring=substring,
|
|
166
|
-
base_interestingness=self.is_interesting,
|
|
167
|
-
)
|
|
168
|
-
|
|
169
|
-
async def full_shrink(self, initial, condition) -> bytes:
|
|
170
|
-
problem: BasicReductionProblem[bytes] = BasicReductionProblem(
|
|
171
|
-
initial=initial,
|
|
172
|
-
is_interesting=condition,
|
|
173
|
-
work=WorkContext(parallelism=self.__parallelism, random=self.__random),
|
|
174
|
-
)
|
|
175
|
-
|
|
176
|
-
reducer = ShrinkRay(
|
|
177
|
-
target=problem,
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
await reducer.run()
|
|
181
|
-
|
|
182
|
-
return problem.current_test_case
|
|
183
|
-
|
|
184
|
-
async def all_shrinks(self, sample, condition):
|
|
185
|
-
shrinks = set()
|
|
186
|
-
|
|
187
|
-
async def run_and_add(c, condition):
|
|
188
|
-
shrinks.add(await self.full_shrink(c, condition))
|
|
189
|
-
|
|
190
|
-
async with trio.open_nursery() as nursery:
|
|
191
|
-
for d in sample:
|
|
192
|
-
nursery.start_soon(run_and_add, d, condition)
|
|
193
|
-
|
|
194
|
-
return shrinks
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
def python_files() -> list[str]:
|
|
198
|
-
return [
|
|
199
|
-
f
|
|
200
|
-
for d in site.getsitepackages()
|
|
201
|
-
for f in glob(os.path.join(d, "**", "*.py"), recursive=True)
|
|
202
|
-
]
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
206
|
-
|
|
207
|
-
IS_PYTHON_SCRIPT = os.path.join(ROOT, "scripts", "ispython.py")
|
|
208
|
-
|
|
209
|
-
assert os.path.exists(IS_PYTHON_SCRIPT), ROOT
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
async def is_python(data):
|
|
213
|
-
return (
|
|
214
|
-
await trio.run_process(
|
|
215
|
-
[sys.executable, IS_PYTHON_SCRIPT],
|
|
216
|
-
stdin=data,
|
|
217
|
-
stdout=subprocess.DEVNULL,
|
|
218
|
-
stderr=subprocess.DEVNULL,
|
|
219
|
-
check=False,
|
|
220
|
-
)
|
|
221
|
-
).returncode == 0
|
shrinkray-0.0.0.dist-info/RECORD
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
shrinkray/__init__.py,sha256=b5MvcvhsEGYya3GRXNbCJAlAL5IZHSsETLK_vtfmXRY,18
|
|
2
|
-
shrinkray/__main__.py,sha256=DZDim25VQpQbKWNlK6EHS8Skz_IiK_Ykzutq0LldbXk,39128
|
|
3
|
-
shrinkray/learning.py,sha256=K1FIkWRoue_auW4DmOrAOpiXumTfLRsBhH12a08lInA,7133
|
|
4
|
-
shrinkray/passes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
-
shrinkray/passes/bytes.py,sha256=9W-EoLt2Ateis3GMydkdVTSzx81YdEIt3f8FdTg-yXI,15846
|
|
6
|
-
shrinkray/passes/clangdelta.py,sha256=lwhsrf9S75ULZ6OhIg_xlERFz7igLGjN9e9oDtzMFXw,7440
|
|
7
|
-
shrinkray/passes/definitions.py,sha256=5EbxLg2LOgLrTiMUrlhMIC8hW6TyDhi5tcVBT0MA2hc,1208
|
|
8
|
-
shrinkray/passes/genericlanguages.py,sha256=xxnPvV_ibW6e3SFIzgu5qxCujpQ7L_Drg1Bttvhj5xk,8214
|
|
9
|
-
shrinkray/passes/json.py,sha256=iw-BdMCoOgebyZtlKibbj8CGE75RmlQZLghsG_VQXQ4,2348
|
|
10
|
-
shrinkray/passes/patching.py,sha256=wfbYLajcav26w1HFXzyisEBOSfSTD8KC2qZ_EYLCN8E,8618
|
|
11
|
-
shrinkray/passes/python.py,sha256=3ku9lzecA5UupzfHZOydDAV5Hqg83z9NTd6SfU-fQ3g,4933
|
|
12
|
-
shrinkray/passes/sat.py,sha256=T70lcvNY1qJdiV1zlGRARUyqNb9lAE19LsrQa4i9wM0,5192
|
|
13
|
-
shrinkray/passes/sequences.py,sha256=Q8Ldj23FjsLsfLzJS3_CU7hrU6jNuyG9BpvlBHO9lmU,2291
|
|
14
|
-
shrinkray/problem.py,sha256=_LRPy9GRZ2nHohAQWkH6lOaVjV0h7SlrY_KTwjfKL68,10208
|
|
15
|
-
shrinkray/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
-
shrinkray/reducer.py,sha256=Wm3e1g8HjJZ_qnkMYv9SnLbicBnmBr2O1vkBi6lgpmc,14274
|
|
17
|
-
shrinkray/work.py,sha256=1Cmen3juWUIt5CFnUBhgR3qpIdYOnm8ykLv7EdJkgyQ,6707
|
|
18
|
-
shrinkray-0.0.0.dist-info/LICENSE,sha256=iMKX79AuokJfIZUnGUARdUp30vVAoIPOJ7ek8TY63kk,1072
|
|
19
|
-
shrinkray-0.0.0.dist-info/METADATA,sha256=qAQK5h4szOvKtj1saZPW5wFb70uEY9Y9zoJbbtpYQg8,8994
|
|
20
|
-
shrinkray-0.0.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
21
|
-
shrinkray-0.0.0.dist-info/entry_points.txt,sha256=u5D_5iY3G1LeHVljUDlmmHmQamz-Me5vLccZ9P7Lwm4,53
|
|
22
|
-
shrinkray-0.0.0.dist-info/RECORD,,
|
|
File without changes
|