crosshair-tool 0.0.99__cp312-cp312-macosx_10_13_x86_64.whl

crosshair/path_cover.py

@@ -0,0 +1,209 @@
+import builtins
+import enum
+import re
+import traceback
+from dataclasses import dataclass
+from inspect import BoundArguments
+from typing import Callable, List, Optional, Set, TextIO, Tuple, Type
+
+from crosshair.condition_parser import condition_parser
+from crosshair.core import (
+    ExceptionFilter,
+    LazyCreationRepr,
+    deep_realize,
+    explore_paths,
+    realize,
+)
+from crosshair.fnutil import FunctionInfo
+from crosshair.options import AnalysisOptions
+from crosshair.statespace import RootNode, StateSpace, context_statespace
+from crosshair.tracers import (
+    COMPOSITE_TRACER,
+    CoverageResult,
+    CoverageTracingModule,
+    NoTracing,
+    PushedModule,
+)
+from crosshair.util import (
+    ReferencedIdentifier,
+    ch_stack,
+    debug,
+    format_boundargs,
+    name_of_type,
+)
+
+
+class CoverageType(enum.Enum):
+    OPCODE = "OPCODE"
+    PATH = "PATH"
+
+
+@dataclass
+class PathSummary:
+    args: BoundArguments
+    formatted_args: str
+    result: str
+    exc: Optional[Type[BaseException]]
+    exc_message: Optional[str]
+    post_args: BoundArguments
+    coverage: CoverageResult
+    references: Set[ReferencedIdentifier]
+
+
+def path_cover(
+    ctxfn: FunctionInfo,
+    options: AnalysisOptions,
+    coverage_type: CoverageType,
+    arg_formatter: Callable[[BoundArguments], str] = format_boundargs,
+) -> List[PathSummary]:
+    fn, sig = ctxfn.callable()
+    while getattr(fn, "__wrapped__", None):
+        # Usually we don't want to run decorator code. (and we certainly don't want
+        # to measure coverage on the decorator rather than the real body) Unwrap:
+        fn = fn.__wrapped__  # type: ignore
+    search_root = RootNode()
+
+    paths: List[PathSummary] = []
+    coverage: CoverageTracingModule = CoverageTracingModule(fn)
+
+    def run_path(args: BoundArguments):
+        nonlocal coverage
+        with NoTracing():
+            coverage = CoverageTracingModule(fn)
+        with PushedModule(coverage):
+            return fn(*args.args, **args.kwargs)
+
+    def on_path_complete(
+        space: StateSpace,
+        pre_args: BoundArguments,
+        post_args: BoundArguments,
+        ret,
+        exc: Optional[BaseException],
+        exc_stack: Optional[traceback.StackSummary],
+    ) -> bool:
+        with ExceptionFilter() as efilter:
+            space.detach_path()
+
+            reprer = context_statespace().extra(LazyCreationRepr)
+            formatted_pre_args = reprer.eval_friendly_format(pre_args, arg_formatter)
+
+            pre_args = deep_realize(pre_args)
+            post_args = deep_realize(post_args)
+            ret = reprer.eval_friendly_format(ret, lambda x: builtins.repr(x))
+            references = reprer.repr_references
+
+            cov = coverage.get_results(fn)
+            debug("Realized args:", formatted_pre_args)
+            if exc is not None:
+                debug("user-level exception found", type(exc), exc, ch_stack(exc_stack))
+                exc_message = realize(str(exc)) if len(exc.args) > 0 else None
+                paths.append(
+                    PathSummary(
+                        pre_args,
+                        formatted_pre_args,
+                        ret,
+                        type(exc),
+                        exc_message,
+                        post_args,
+                        cov,
+                        references,
+                    )
+                )
+            else:
+                debug("Realized return:", ret)
+                paths.append(
+                    PathSummary(
+                        pre_args,
+                        formatted_pre_args,
+                        ret,
+                        None,
+                        None,
+                        post_args,
+                        cov,
+                        references,
+                    )
+                )
+            return False
+        debug("Skipping path (failed to realize values)", efilter.user_exc)
+        return False
+
+    explore_paths(run_path, sig, options, search_root, on_path_complete)
+
+    opcodes_found: Set[int] = set()
+    selected: List[PathSummary] = []
+    while paths:
+        next_best = max(
+            paths,
+            key=lambda p: (
+                len(p.coverage.offsets_covered - opcodes_found),  # high coverage
+                -len(p.formatted_args),  # with small input size
+            ),
+        )
+        cur_offsets = next_best.coverage.offsets_covered
+        if coverage_type == CoverageType.OPCODE:
+            debug("Next best path covers these opcode offsets:", cur_offsets)
+            if len(cur_offsets - opcodes_found) == 0:
+                break
+        selected.append(next_best)
+        opcodes_found |= cur_offsets
+        paths = [p for p in paths if p is not next_best]
+    return selected
+
+
+def output_argument_dictionary_paths(
+    fn: Callable, paths: List[PathSummary], stdout: TextIO, stderr: TextIO
+):
+    for path in paths:
+        stdout.write(path.formatted_args + "\n")
+    stdout.flush()
+
+
+def output_eval_exression_paths(
+    fn: Callable, paths: List[PathSummary], stdout: TextIO, stderr: TextIO
+):
+    for path in paths:
+        stdout.write(fn.__name__ + "(" + path.formatted_args + ")\n")
+    stdout.flush()
+
+
+def import_statements_for_references(references: Set[ReferencedIdentifier]) -> Set[str]:
+    imports: Set[str] = set()
+    for ref in references:
+        if ref.modulename == "builtins":
+            continue
+        if "." in ref.qualname:
+            class_name, _ = ref.qualname.split(".", 1)
+            imports.add(f"from {ref.modulename} import {class_name}")
+        else:
+            imports.add(f"from {ref.modulename} import {ref.qualname}")
+    return imports
+
+
+def output_pytest_paths(
+    fn: Callable, paths: List[PathSummary]
+) -> Tuple[Set[str], List[str]]:
+    fn_name = fn.__qualname__
+    lines: List[str] = []
+    references = {ReferencedIdentifier(fn.__module__, fn_name)}
+    imports: Set[str] = set()
+
+    name_with_underscores = fn_name.replace(".", "_")
+    for idx, path in enumerate(paths):
+        test_name_suffix = "" if idx == 0 else "_" + str(idx + 1)
+        exec_fn = f"{fn_name}({path.formatted_args})"
+        lines.append(f"def test_{name_with_underscores}{test_name_suffix}():")
+        if path.exc is None:
+            lines.append(f"    assert {exec_fn} == {path.result}")
+        else:
+            imports.add("import pytest")
+            if path.exc_message is not None:
+                lines.append(
+                    f"    with pytest.raises({name_of_type(path.exc)}, match={repr(re.escape(path.exc_message))}):"
+                )
+            else:
+                lines.append(f"    with pytest.raises({name_of_type(path.exc)}):")
+            lines.append(f"        {exec_fn}")
+        lines.append("")
+        references |= path.references
+    imports |= import_statements_for_references(references)
+    return (imports, lines)

crosshair/path_cover_test.py

@@ -0,0 +1,138 @@
+import functools
+import re
+import textwrap
+from dataclasses import dataclass
+from enum import Enum
+from io import StringIO
+from typing import Callable, Optional
+
+from crosshair.fnutil import FunctionInfo
+from crosshair.options import DEFAULT_OPTIONS
+from crosshair.path_cover import (
+    CoverageType,
+    output_eval_exression_paths,
+    output_pytest_paths,
+    path_cover,
+)
+from crosshair.statespace import context_statespace
+from crosshair.tracers import NoTracing
+
+
+def _foo(x: int) -> int:
+    if x > 100:
+        return 100
+    return x
+
+
+def _regex(x: str) -> bool:
+    compiled = re.compile("f(o)+")
+    return bool(compiled.fullmatch(x))
+
+
+def _exceptionex(x: int) -> int:
+    if x == 42:
+        raise ValueError
+    return x
+
+
+def _symbolic_exception_example(x: str) -> str:
+    if x == "foo'bar\"baz":
+        raise ValueError(x)
+    return x
+
+
+def _has_no_successful_paths(x: int) -> None:
+    with NoTracing():
+        context_statespace().defer_assumption("fail", lambda: False)
+
+
+@dataclass
+class Train:
+    class Color(Enum):
+        RED = 0
+
+    color: Color
+
+
+def _paint_train(train: Train, color: Train.Color) -> Train:
+    return Train(color=color)
+
+
+OPTS = DEFAULT_OPTIONS.overlay(max_iterations=10)
+foo = FunctionInfo.from_fn(_foo)
+decorated_foo = FunctionInfo.from_fn(functools.lru_cache()(_foo))
+regex = FunctionInfo.from_fn(_regex)
+exceptionex = FunctionInfo.from_fn(_exceptionex)
+symbolic_exception_example = FunctionInfo.from_fn(_symbolic_exception_example)
+has_no_successful_paths = FunctionInfo.from_fn(_has_no_successful_paths)
+paint_train = FunctionInfo.from_fn(_paint_train)
+
+
+def test_path_cover_foo() -> None:
+    paths = list(path_cover(foo, OPTS, CoverageType.OPCODE))
+    assert len(paths) == 2
+    small, large = sorted(paths, key=lambda p: p.result)  # type: ignore
+    assert large.result == "100"
+    assert large.args.arguments["x"] > 100
+    assert small.result == repr(small.args.arguments["x"])
+
+
+def test_path_cover_decorated_foo() -> None:
+    paths = list(path_cover(decorated_foo, OPTS, CoverageType.OPCODE))
+    assert len(paths) == 2
+
+
+def test_path_cover_regex() -> None:
+    paths = list(path_cover(regex, OPTS, CoverageType.OPCODE))
+    assert len(paths) == 1
+    paths = list(path_cover(regex, OPTS, CoverageType.PATH))
+    input_output = set((p.args.arguments["x"], p.result) for p in paths)
+    assert ("foo", "True") in input_output
+
+
+def test_path_cover_exception_example() -> None:
+    paths = list(path_cover(exceptionex, OPTS, CoverageType.OPCODE))
+    out, err = StringIO(), StringIO()
+    output_eval_exression_paths(_exceptionex, paths, out, err)
+    assert "_exceptionex(42)" in out.getvalue()
+
+
+def test_path_cover_symbolic_exception_message() -> None:
+    paths = list(path_cover(symbolic_exception_example, OPTS, CoverageType.OPCODE))
+    _imports, lines = output_pytest_paths(_symbolic_exception_example, paths)
+    expected = textwrap.dedent(
+        """\
+        def test__symbolic_exception_example():
+            with pytest.raises(ValueError, match='foo\\'bar"baz'):
+                _symbolic_exception_example('foo\\'bar"baz')"""
+    )
+    assert expected in "\n".join(lines)
+
+
+def test_has_no_successful_paths() -> None:
+    assert list(path_cover(has_no_successful_paths, OPTS, CoverageType.OPCODE)) == []
+
+
+def test_path_cover_lambda() -> None:
+    def lambdaFn(a: Optional[Callable[[int], int]]):
+        if a:
+            return a(2) + 4
+        else:
+            return "hello"
+
+    assert path_cover(FunctionInfo.from_fn(lambdaFn), OPTS, CoverageType.OPCODE)
+    # TODO: more detailed assert?
+
+
+def test_path_cover_pytest_output() -> None:
+    paths = list(path_cover(paint_train, OPTS, CoverageType.OPCODE))
+    imports, lines = output_pytest_paths(_paint_train, paths)
+    assert lines == [
+        "def test__paint_train():",
+        "    assert _paint_train(Train(Train.Color.RED), Train.Color.RED) == Train(color=Train.Color.RED)",
+        "",
+    ]
+    assert imports == {
+        "from crosshair.path_cover_test import _paint_train",
+        "from crosshair.path_cover_test import Train",
+    }

crosshair/path_search.py

@@ -0,0 +1,161 @@
+import enum
+import traceback
+from dataclasses import dataclass
+from inspect import BoundArguments
+from typing import Callable, Optional, Type
+
+from crosshair.copyext import CopyMode, deepcopyext
+from crosshair.core import ExceptionFilter, LazyCreationRepr, explore_paths
+from crosshair.fnutil import FunctionInfo
+from crosshair.libimpl.builtinslib import SymbolicInt
+from crosshair.options import AnalysisOptions
+from crosshair.statespace import RootNode, StateSpace, context_statespace
+from crosshair.tracers import CoverageResult, NoTracing, ResumedTracing
+from crosshair.util import (
+    CrossHairInternal,
+    ch_stack,
+    debug,
+    format_boundargs_as_dictionary,
+)
+
+
+class OptimizationKind(enum.Enum):
+    SIMPLIFY = "SIMPLIFY"
+    # TODO: simplify mode is quite dumb atm; partly because eval_friendly_format
+    # realizes prior to generating the string.
+    NONE = "NONE"
+    MINIMIZE_INT = "MINIMIZE_INT"
+
+
+@dataclass
+class PathSummary:
+    args: BoundArguments
+    result: object
+    exc: Optional[Type[BaseException]]
+    post_args: BoundArguments
+    coverage: CoverageResult
+
+
+def path_search(
+    ctxfn: FunctionInfo,
+    options: AnalysisOptions,
+    argument_formatter: Optional[Callable[[BoundArguments], str]],
+    optimization_kind: OptimizationKind,
+    optimize_fn: Optional[Callable],
+    on_example: Callable[[str], None],
+) -> None:
+
+    if argument_formatter is None:
+        checked_format = (
+            lambda args: context_statespace()
+            .extra(LazyCreationRepr)
+            .eval_friendly_format(args, format_boundargs_as_dictionary)
+        )
+    else:
+
+        def checked_format(args: BoundArguments) -> str:
+            assert argument_formatter is not None
+            args = deepcopyext(args, CopyMode.REALIZE, {})
+            try:
+                return argument_formatter(args)
+            except Exception as exc:
+                raise CrossHairInternal(str(exc)) from exc
+
+    if optimization_kind == OptimizationKind.SIMPLIFY:
+        assert optimize_fn is None
+
+        def scorechar(codepoint: int):
+            if codepoint >= ord("a"):
+                return codepoint - ord("a")
+            elif codepoint >= ord("0"):
+                return 30 + codepoint - ord("0")
+            else:
+                return 100 + codepoint
+
+        def shrinkscore(ret, args: BoundArguments):
+            reprstr = checked_format(args)
+            return len(reprstr) * 1000 + sum(scorechar(ord(ch)) for ch in reprstr)
+
+        optimization_kind = OptimizationKind.MINIMIZE_INT
+        optimize_fn = shrinkscore
+
+    fn, sig = ctxfn.callable()
+    search_root = RootNode()
+
+    best_input: Optional[str] = None
+    best_score: Optional[int] = None
+    _optimize_fn = optimize_fn or (lambda _ret, args: fn(*args.args, **args.kwargs))
+
+    def on_path_complete(
+        space: StateSpace,
+        pre_args: BoundArguments,
+        post_args: BoundArguments,
+        ret,
+        exc: Optional[BaseException],
+        exc_stack: Optional[traceback.StackSummary],
+    ) -> bool:
+        nonlocal best_input, best_score
+        with NoTracing():
+            if exc is not None:
+                debug(
+                    "Aborting path, hit exception",
+                    type(exc),
+                    exc,
+                    ch_stack(exc_stack),
+                )
+                return False
+            debug("Path succeeded")
+            if optimization_kind == OptimizationKind.NONE:
+                with ResumedTracing():
+                    best_input = checked_format(pre_args)
+                debug("Found input:", best_input)
+                on_example(best_input)
+                return True
+        with NoTracing(), ExceptionFilter() as efilter:
+            with ResumedTracing():
+                score = _optimize_fn(ret, pre_args)
+                space.detach_path()
+            smt_score = SymbolicInt._coerce_to_smt_sort(score)
+            if smt_score is None:
+                debug("non integer score returned; ignoring.")
+                return False
+            if space.smt_fork(smt_score < 0, probability_true=0.0):
+                debug("Score was leass than zero; ignoring.")
+                return False
+            if best_score is not None and space.smt_fork(
+                smt_score >= best_score, probability_true=0.0
+            ):
+                debug("Does not beat the best score of", best_score)
+                return False
+            known_min = 0
+            known_max = best_score
+            test = 1000 if known_max is None else (known_max // 2)
+            while True:
+                debug(known_min, test, known_max)
+                if space.smt_fork(smt_score < test, probability_true=1.0):
+                    known_max = test - 1
+                else:
+                    known_min = test
+                    if known_max is None:
+                        test = known_min * 100
+                        continue
+                if known_min == known_max:
+                    best_score = known_min
+                    with ResumedTracing():
+                        best_input = checked_format(pre_args)
+                    break
+                test = (known_min + known_max + 1) // 2
+            debug("Minimized score to", best_score)
+            debug("For input:", best_input)
+            on_example(best_input)
+            return best_score == 0
+        debug("Skipping path (failure during scoring)", efilter.user_exc)
+        return False
+
+    explore_paths(
+        lambda ba: fn(*ba.args, **ba.kwargs),
+        sig,
+        options,
+        search_root,
+        on_path_complete,
+    )

crosshair/path_search_test.py

@@ -0,0 +1,52 @@
+import ast
+from inspect import BoundArguments
+from typing import Callable, Optional
+
+from crosshair.fnutil import FunctionInfo
+from crosshair.options import DEFAULT_OPTIONS, AnalysisOptions, AnalysisOptionSet
+from crosshair.path_search import OptimizationKind, path_search
+
+
+def ten_over_difference(x: int, y: int) -> int:
+    if x != 42:
+        return 10 // (x - y)
+    return 100
+
+
+def do_path_search(
+    fn: Callable,
+    options: AnalysisOptions,
+    argument_formatter: Optional[Callable[[BoundArguments], str]],
+    optimization_kind: OptimizationKind,
+    optimize_fn: Optional[Callable] = None,
+) -> Optional[str]:
+    fninfo = FunctionInfo.from_fn(fn)
+    final_example: Optional[str] = None
+
+    def on_example(example: str) -> None:
+        nonlocal final_example
+        final_example = example
+
+    path_search(
+        fninfo, options, argument_formatter, optimization_kind, optimize_fn, on_example
+    )
+    return final_example
+
+
+def test_optimize_options() -> None:
+    opts = DEFAULT_OPTIONS.overlay(AnalysisOptionSet(max_uninteresting_iterations=10))
+    ret = do_path_search(
+        ten_over_difference, opts, None, optimization_kind=OptimizationKind.SIMPLIFY
+    )
+    assert ret in ('{"x": 1, "y": 0}', '{"x": 0, "y": 1}')
+    ret = do_path_search(
+        ten_over_difference, opts, None, optimization_kind=OptimizationKind.MINIMIZE_INT
+    )
+    assert ret is not None
+    parsed_ret = ast.literal_eval(ret)
+    assert parsed_ret["x"] - parsed_ret["y"] > 10
+    ret = do_path_search(
+        ten_over_difference, opts, None, optimization_kind=OptimizationKind.NONE
+    )
+    assert ret is not None
+    ast.literal_eval(ret)  # (just ensure the result is parseable)