shrinkray 0.0.0__py3-none-any.whl

shrinkray/passes/clangdelta.py

@@ -0,0 +1,230 @@
+import os
+import subprocess
+from glob import glob
+from shutil import which
+from tempfile import NamedTemporaryFile
+
+import trio
+
+from shrinkray.passes.definitions import ReductionPump
+from shrinkray.problem import ReductionProblem
+from shrinkray.work import NotFound
+
+C_FILE_EXTENSIONS = (".c", ".cpp", ".h", ".hpp", ".cxx", ".cc")
+
+
+def find_clang_delta():
+    clang_delta = which("clang_delta") or ""
+    if not clang_delta:
+        possible_locations = glob(
+            "/opt/homebrew//Cellar/creduce/*/libexec/clang_delta"
+        ) + glob("/usr/libexec/clang_delta")
+        if possible_locations:
+            clang_delta = max(possible_locations)
+    return clang_delta
+
+
+TRANSFORMATIONS: list[str] = [
+    "aggregate-to-scalar",
+    "binop-simplification",
+    "callexpr-to-value",
+    "class-template-to-class",
+    "combine-global-var",
+    "combine-local-var",
+    "copy-propagation",
+    "empty-struct-to-int",
+    "expression-detector",
+    "instantiate-template-param",
+    "instantiate-template-type-param-to-int",
+    "lift-assignment-expr",
+    "local-to-global",
+    "move-function-body",
+    "move-global-var",
+    "param-to-global",
+    "param-to-local",
+    "reduce-array-dim",
+    "reduce-array-size",
+    "reduce-class-template-param",
+    "reduce-pointer-level",
+    "reduce-pointer-pairs",
+    "remove-addr-taken",
+    "remove-array",
+    "remove-base-class",
+    "remove-ctor-initializer",
+    "remove-enum-member-value",
+    "remove-namespace",
+    "remove-nested-function",
+    "remove-pointer",
+    "remove-trivial-base-template",
+    "remove-unresolved-base",
+    "remove-unused-enum-member",
+    "remove-unused-field",
+    "remove-unused-function",
+    "remove-unused-outer-class",
+    "remove-unused-var",
+    "rename-class",
+    "rename-cxx-method",
+    "rename-fun",
+    "rename-param",
+    "rename-var",
+    "replace-array-access-with-index",
+    "replace-array-index-var",
+    "replace-callexpr",
+    "replace-class-with-base-template-spec",
+    "replace-dependent-name",
+    "replace-dependent-typedef",
+    "replace-derived-class",
+    "replace-function-def-with-decl",
+    "replace-one-level-typedef-type",
+    "replace-simple-typedef",
+    "replace-undefined-function",
+    "return-void",
+    "simple-inliner",
+    "simplify-callexpr",
+    "simplify-comma-expr",
+    "simplify-dependent-typedef",
+    "simplify-if",
+    "simplify-nested-class",
+    "simplify-recursive-template-instantiation",
+    "simplify-struct",
+    "simplify-struct-union-decl",
+    "template-arg-to-int",
+    "template-non-type-arg-to-int",
+    "unify-function-decl",
+    "union-to-struct",
+    "vector-to-array",
+]
+
+
+class ClangDelta:
+    def __init__(self, path: str):
+        self.path_to_exec = path
+
+        self.transformations: list[str] = TRANSFORMATIONS
+
+    def __validate_transformation(self, transformation: str) -> None:
+        if transformation not in self.transformations:
+            raise ValueError(f"Invalid transformation {transformation}")
+
+    async def query_instances(self, transformation: str, data: bytes) -> int:
+        self.__validate_transformation(transformation)
+        with NamedTemporaryFile(suffix=".cpp", delete_on_close=False) as tmp:
+            tmp.write(data)
+            tmp.close()
+
+            try:
+                results = (
+                    await trio.run_process(
+                        [
+                            self.path_to_exec,
+                            f"--query-instances={transformation}",
+                            tmp.name,
+                        ],
+                        capture_stdout=True,
+                        capture_stderr=True,
+                    )
+                ).stdout
+            except subprocess.CalledProcessError as e:
+                msg = (e.stdout + e.stderr).strip()
+                if msg == b"Error: Unsupported file type!":
+                    raise ValueError("Not a C or C++ test case")
+                elif b"Assertion failed" in msg:
+                    return 0
+                else:
+                    raise ClangDeltaError(msg)
+            finally:
+                os.unlink(tmp.name)
+
+            prefix = b"Available transformation instances:"
+            assert results.startswith(prefix)
+            return int(results[len(prefix) :].strip().decode("ascii"))
+
+    async def apply_transformation(
+        self, transformation: str, counter: int, data: bytes
+    ) -> bytes:
+        self.__validate_transformation(transformation)
+        with NamedTemporaryFile(suffix=".cpp", delete_on_close=False) as tmp:
+            tmp.write(data)
+            tmp.close()
+
+            try:
+                return (
+                    await trio.run_process(
+                        [
+                            self.path_to_exec,
+                            f"--transformation={transformation}",
+                            f"--counter={int(counter)}",
+                            tmp.name,
+                        ],
+                        capture_stdout=True,
+                        capture_stderr=True,
+                    )
+                ).stdout
+            except subprocess.CalledProcessError as e:
+                if e.stdout.strip() == b"Error: Unsupported file type!":
+                    raise ValueError("Not a C or C++ test case")
+                elif (
+                    e.stdout.strip()
+                    == b"Error: No modification to the transformed program!"
+                ):
+                    return data
+                else:
+                    raise ClangDeltaError(e.stdout + e.stderr)
+            finally:
+                os.unlink(tmp.name)
+
+
+class ClangDeltaError(Exception):
+    pass
+
+
+def clang_delta_pump(
+    clang_delta: ClangDelta, transformation: str
+) -> ReductionPump[bytes]:
+    async def apply(problem: ReductionProblem[bytes]) -> bytes:
+        target = problem.current_test_case
+        assert target is not None
+        try:
+            n = await clang_delta.query_instances(transformation, target)
+        except ValueError:
+            import traceback
+
+            traceback.print_exc()
+            return target
+        i = 1
+        while i <= n:
+
+            async def can_apply(j: int) -> bool:
+                attempt = await clang_delta.apply_transformation(
+                    transformation, j, target
+                )
+                assert attempt is not None
+                if attempt == target:
+                    return False
+                return await problem.is_interesting(attempt)
+
+            try:
+                i = await problem.work.find_first_value(range(i, n + 1), can_apply)
+            except NotFound:
+                break
+            except ClangDeltaError as e:
+                # Clang delta has a large number of internal assertions that you can trigger
+                # if you feed it bad enough C++. We solve this problem by ignoring it.
+                if b"Assertion failed" in e.args[0]:
+                    return target
+
+            target = await clang_delta.apply_transformation(transformation, i, target)
+            assert target is not None
+            n = await clang_delta.query_instances(transformation, target)
+        return target
+
+    apply.__name__ = f"clang_delta({transformation})"
+
+    return apply
+
+
+def clang_delta_pumps(clang_delta: ClangDelta) -> list[ReductionPump[bytes]]:
+    return [
+        clang_delta_pump(clang_delta, transformation)
+        for transformation in clang_delta.transformations
+    ]

shrinkray/passes/definitions.py

@@ -0,0 +1,52 @@
+from abc import ABC, abstractmethod
+from functools import wraps
+from typing import Awaitable, Callable, Generic, TypeVar
+
+from shrinkray.problem import ReductionProblem
+
+S = TypeVar("S")
+T = TypeVar("T")
+
+
+ReductionPass = Callable[[ReductionProblem[T]], Awaitable[None]]
+ReductionPump = Callable[[ReductionProblem[T]], Awaitable[T]]
+
+
+class ParseError(Exception):
+    pass
+
+
+class Format(Generic[S, T], ABC):
+    @property
+    def name(self) -> str:
+        return repr(self)
+
+    @abstractmethod
+    def parse(self, input: S) -> T: ...
+
+    def is_valid(self, input: S) -> bool:
+        try:
+            self.parse(input)
+            return True
+        except ParseError:
+            return False
+
+    @abstractmethod
+    def dumps(self, input: T) -> S: ...
+
+
+def compose(format: Format[S, T], reduction_pass: ReductionPass[T]) -> ReductionPass[S]:
+    @wraps(reduction_pass)
+    async def wrapped_pass(problem: ReductionProblem[S]) -> None:
+        view = problem.view(format)
+
+        try:
+            view.current_test_case
+        except ParseError:
+            return
+
+        await reduction_pass(view)
+
+    wrapped_pass.__name__ = f"{format.name}/{reduction_pass.__name__}"
+
+    return wrapped_pass

shrinkray/passes/genericlanguages.py

@@ -0,0 +1,277 @@
+"""
+Module of reduction passes designed for "things that look like programming languages".
+"""
+
+import re
+from functools import wraps
+from string import ascii_lowercase, ascii_uppercase
+from typing import AnyStr, Callable
+
+import trio
+from attr import define
+
+from shrinkray.passes.bytes import ByteReplacement, delete_intervals
+from shrinkray.passes.definitions import Format, ParseError, ReductionPass
+from shrinkray.passes.patching import PatchApplier, Patches, apply_patches
+from shrinkray.problem import BasicReductionProblem, ReductionProblem
+from shrinkray.work import NotFound
+
+
+@define(frozen=True)
+class Substring(Format[AnyStr, AnyStr]):
+    prefix: AnyStr
+    suffix: AnyStr
+
+    @property
+    def name(self) -> str:
+        return f"Substring({len(self.prefix)}, {len(self.suffix)})"
+
+    def parse(self, input: AnyStr) -> AnyStr:
+        if input.startswith(self.prefix) and input.endswith(self.suffix):
+            return input[len(self.prefix) : len(input) - len(self.suffix)]
+        else:
+            raise ParseError()
+
+    def dumps(self, input: AnyStr) -> AnyStr:
+        return self.prefix + input + self.suffix
+
+
+class RegionReplacingPatches(Patches[dict[int, AnyStr], AnyStr]):
+    def __init__(self, regions):
+        assert regions
+        for (_, v), (u, _) in zip(regions, regions[1:]):
+            assert v <= u
+        self.regions = regions
+
+    @property
+    def empty(self):
+        return {}
+
+    def combine(self, *patches):
+        result = {}
+        for p in patches:
+            result.update(p)
+        return result
+
+    def apply(self, patch, target):
+        empty = target[:0]
+        parts = []
+        prev = 0
+        for j, (u, v) in enumerate(self.regions):
+            assert v <= len(target)
+            parts.append(target[prev:u])
+            try:
+                parts.append(patch[j])
+            except KeyError:
+                parts.append(target[u:v])
+            prev = v
+        parts.append(target[prev:])
+        return empty.join(parts)
+
+    def size(self, patch):
+        total = 0
+        for i, s in patch.items():
+            u, v = self.regions[i]
+            return v - u - len(s)
+
+
+def regex_pass(
+    pattern: AnyStr | re.Pattern[AnyStr],
+    flags: re.RegexFlag = 0,
+) -> Callable[[ReductionPass[AnyStr]], ReductionPass[AnyStr]]:
+    if not isinstance(pattern, re.Pattern):
+        pattern = re.compile(pattern, flags=flags)
+
+    def inner(fn: ReductionPass[AnyStr]) -> ReductionPass[AnyStr]:
+        @wraps(fn)
+        async def reduction_pass(problem: ReductionProblem[AnyStr]) -> None:
+            matching_regions = []
+            initial_values_for_regions = []
+
+            i = 0
+            while i < len(problem.current_test_case):
+                search = pattern.search(problem.current_test_case, i)
+                if search is None:
+                    break
+
+                u, v = search.span()
+                matching_regions.append((u, v))
+                initial_values_for_regions.append(problem.current_test_case[u:v])
+
+                i = v
+
+            if not matching_regions:
+                return
+
+            patches = RegionReplacingPatches(matching_regions)
+
+            patch_applier = PatchApplier(patches, problem)
+
+            async with trio.open_nursery() as nursery:
+
+                async def reduce_region(i: int) -> None:
+                    async def is_interesting(s):
+                        return await patch_applier.try_apply_patch({i: s})
+
+                    subproblem = BasicReductionProblem(
+                        initial_values_for_regions[i],
+                        is_interesting,
+                        work=problem.work,
+                    )
+                    nursery.start_soon(fn, subproblem)
+
+                for i in range(len(matching_regions)):
+                    await reduce_region(i)
+
+        return reduction_pass
+
+    return inner
+
+
+async def reduce_integer(problem: ReductionProblem[int]) -> None:
+    assert problem.current_test_case >= 0
+
+    if await problem.is_interesting(0):
+        return
+
+    lo = 0
+    hi = problem.current_test_case
+
+    while lo + 1 < hi:
+        mid = (lo + hi) // 2
+        if await problem.is_interesting(mid):
+            hi = mid
+        else:
+            lo = mid
+
+        if await problem.is_interesting(hi - 1):
+            hi -= 1
+
+        if await problem.is_interesting(lo + 1):
+            return
+        else:
+            lo += 1
+
+
+class IntegerFormat(Format[bytes, int]):
+    def parse(self, input: bytes) -> int:
+        try:
+            return int(input.decode("ascii"))
+        except (ValueError, UnicodeDecodeError):
+            raise ParseError()
+
+    def dumps(self, input: int) -> bytes:
+        return str(input).encode("ascii")
+
+
+@regex_pass(b"[0-9]+")
+async def reduce_integer_literals(problem: ReductionProblem[bytes]) -> None:
+    await reduce_integer(problem.view(IntegerFormat()))
+
+
+@regex_pass(rb"[0-9]+ [*+-/] [0-9]+")
+async def combine_expressions(problem: ReductionProblem[bytes]) -> None:
+    try:
+        # NB: Use of eval is safe, as everything passed to this is a simple
+        # arithmetic expression. Would ideally replace with a guaranteed
+        # safe version though.
+        await problem.is_interesting(
+            str(eval(problem.current_test_case)).encode("ascii")
+        )
+    except ArithmeticError:
+        pass
+
+
+@regex_pass(rb'([\'"])\s*\1')
+async def merge_adjacent_strings(problem: ReductionProblem[bytes]) -> None:
+    await problem.is_interesting(b"")
+
+
+@regex_pass(rb"''|\"\"|false|\(\)|\[\]", re.IGNORECASE)
+async def replace_falsey_with_zero(problem: ReductionProblem[bytes]) -> None:
+    await problem.is_interesting(b"0")
+
+
+async def simplify_brackets(problem: ReductionProblem[bytes]) -> None:
+    bracket_types = [b"[]", b"{}", b"()"]
+
+    patches = [dict(zip(u, v)) for u in bracket_types for v in bracket_types if u > v]
+
+    await apply_patches(problem, ByteReplacement(), patches)
+
+
+IDENTIFIER = re.compile(rb"(\b[A-Za-z][A-Za-z0-9_]*\b)|([0-9]+)")
+
+
+def shortlex(s):
+    return (len(s), s)
+
+
+async def normalize_identifiers(problem: ReductionProblem[bytes]) -> None:
+    identifiers = {m.group(0) for m in IDENTIFIER.finditer(problem.current_test_case)}
+    replacements = set(identifiers)
+
+    for char_type in [ascii_lowercase, ascii_uppercase]:
+        for cc in char_type.encode("ascii"):
+            c = bytes([cc])
+            if c not in replacements:
+                replacements.add(c)
+                break
+
+    replacements = sorted(replacements, key=shortlex)
+    targets = sorted(identifiers, key=shortlex, reverse=True)
+
+    # TODO: This could use better parallelisation.
+    for t in targets:
+        pattern = re.compile(rb"\b" + t + rb"\b")
+        source = problem.current_test_case
+        if not pattern.search(source):
+            continue
+
+        async def can_replace(r):
+            if shortlex(r) >= shortlex(t):
+                return False
+            attempt = pattern.sub(r, source)
+            assert attempt != source
+            return await problem.is_interesting(attempt)
+
+        try:
+            await problem.work.find_first_value(replacements, can_replace)
+        except NotFound:
+            pass
+
+
+def iter_indices(s, substring):
+    try:
+        i = s.index(substring)
+        yield i
+        while True:
+            i = s.index(substring, i + 1)
+            yield i
+    except ValueError:
+        return
+
+
+async def cut_comments(problem: ReductionProblem[bytes], start, end, include_end=True):
+    cuts = []
+    target = problem.current_test_case
+    # python comments
+    for i in iter_indices(target, start):
+        try:
+            j = target.index(end, i + 1)
+        except ValueError:
+            if include_end:
+                continue
+            j = len(target)
+        if include_end:
+            cuts.append((i, j + len(end)))
+        else:
+            cuts.append((i, j))
+    await delete_intervals(problem, cuts)
+
+
+async def cut_comment_like_things(problem: ReductionProblem[bytes]):
+    await cut_comments(problem, b"#", b"\n", include_end=False)
+    await cut_comments(problem, b"//", b"\n", include_end=False)
+    await cut_comments(problem, b'"""', b'"""')
+    await cut_comments(problem, b"/*", b"*/")

shrinkray/passes/json.py

@@ -0,0 +1,91 @@
+import json
+from copy import deepcopy
+from typing import Any
+
+from attrs import define
+
+from shrinkray.passes.definitions import Format, ParseError, ReductionPass
+from shrinkray.passes.patching import Patches, apply_patches
+from shrinkray.problem import ReductionProblem
+
+
+def is_json(s: bytes) -> bool:
+    try:
+        json.loads(s)
+        return True
+    except ValueError:
+        return False
+
+
+@define(frozen=True)
+class _JSON(Format[bytes, Any]):
+    def __repr__(self) -> str:
+        return "JSON"
+
+    @property
+    def name(self) -> str:
+        return "JSON"
+
+    def parse(self, input: bytes) -> Any:
+        try:
+            return json.loads(input)
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            raise ParseError(*e.args)
+
+    def dumps(self, input: Any) -> bytes:
+        return json.dumps(input).encode("utf-8")
+
+
+JSON = _JSON()
+
+
+def gather_identifiers(value: Any) -> set[str]:
+    result = set()
+    stack = [value]
+    while stack:
+        target = stack.pop()
+        if isinstance(target, dict):
+            result.update(target.keys())
+            stack.extend(target.values())
+        elif isinstance(target, list):
+            stack.extend(target)
+    return result
+
+
+class DeleteIdentifiers(Patches[frozenset[str], Any]):
+    @property
+    def empty(self) -> frozenset[str]:
+        return frozenset()
+
+    def combine(self, *patches: frozenset[str]) -> frozenset[str]:
+        result = set()
+        for p in patches:
+            result.update(p)
+        return frozenset(result)
+
+    def apply(self, patch: frozenset[str], target: Any) -> Any:
+        target = deepcopy(target)
+        stack = [target]
+        while stack:
+            value = stack.pop()
+            if isinstance(value, dict):
+                for k in patch:
+                    value.pop(k, None)
+                stack.extend(value.values())
+            elif isinstance(value, list):
+                stack.extend(value)
+        return target
+
+    def size(self, patch: frozenset[str]) -> int:
+        return len(patch)
+
+
+async def delete_identifiers(problem: ReductionProblem[Any]):
+    identifiers = gather_identifiers(problem.current_test_case)
+
+    await apply_patches(
+        problem, DeleteIdentifiers(), [frozenset({id}) for id in identifiers]
+    )
+
+
+JSON_PASSES: list[ReductionPass[Any]] = [delete_identifiers]