pine2pyne 0.2.4__py3-none-any.whl

pine2pyne/compat/__init__.py

@@ -0,0 +1,19 @@
+"""Byte-compatibility check against the closed PyneComp compiler's outputs."""
+from .comparison import Comparison, ModuleComparator, Verdict
+from .corpus import AssetsCorpus, Corpus, CorpusCase, default_assets_root
+from .report import CompatReport
+from .runner import CaseResult, CompatRunner, run_parallel
+
+__all__ = [
+    'AssetsCorpus',
+    'CaseResult',
+    'CompatReport',
+    'CompatRunner',
+    'Comparison',
+    'Corpus',
+    'CorpusCase',
+    'ModuleComparator',
+    'Verdict',
+    'default_assets_root',
+    'run_parallel',
+]

pine2pyne/compat/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == '__main__':
+    sys.exit(main())

pine2pyne/compat/cli.py

@@ -0,0 +1,114 @@
+"""``python -m pine2pyne.compat`` / ``pine2pyne-compat``: byte compatibility with PyneComp.
+
+Transpiles every Pine input of the reference corpus and compares the result
+with what the closed PyneComp compiler produced for it::
+
+    python -m pine2pyne.compat                     # pynecomp profile, summary
+    python -m pine2pyne.compat -j 8 --worst 20     # parallel, closest mismatches
+    python -m pine2pyne.compat --json report.json  # every case, machine readable
+    python -m pine2pyne.compat --diff indicator/heiken_ashi_candles_mdWsHY8m
+
+Exit status: 0 when every compared case is byte-exact, 1 otherwise.
+"""
+from __future__ import annotations
+
+import argparse
+import difflib
+import sys
+import warnings
+from pathlib import Path
+from typing import Sequence
+
+from dishka import Provider, Scope, from_context, make_container, provide
+
+from ..core.contracts import OutputProfile, Transpiler, TranspilerSettings
+from ..di import build_providers
+from .comparison import ModuleComparator, Verdict
+from .corpus import AssetsCorpus, Corpus, default_assets_root
+from .report import CompatReport
+from .runner import CompatRunner, run_parallel
+
+
+class CompatProvider(Provider):
+    """The compatibility checker, on top of the transpiler providers."""
+
+    scope = Scope.APP
+
+    assets_root = from_context(provides=Path, scope=Scope.APP)
+
+    @provide
+    def corpus(self, assets_root: Path) -> Corpus:
+        return AssetsCorpus(assets_root)
+
+    @provide
+    def comparator(self) -> ModuleComparator:
+        return ModuleComparator()
+
+    runner = provide(CompatRunner)
+
+
+def make_compat_container(profile: OutputProfile, assets_root: Path):
+    return make_container(
+        *build_providers(profile), CompatProvider(),
+        context={TranspilerSettings: TranspilerSettings(profile=profile), Path: assets_root},
+    )
+
+
+def parse_args(argv: Sequence[str] | None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(prog='pine2pyne-compat', description=__doc__.split('\n')[0],
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--assets', type=Path, default=default_assets_root(),
+                        help='corpus root holding pine/ and pyne/ (default: %(default)s)')
+    parser.add_argument('--profile', default=OutputProfile.PYNECOMP.value,
+                        choices=[profile.value for profile in OutputProfile], help='output profile to check')
+    parser.add_argument('-j', '--jobs', type=int, default=1, help='worker processes')
+    parser.add_argument('-k', '--filter', default='', help='only cases whose name contains this text')
+    parser.add_argument('--worst', type=int, default=0, help='list the N closest mismatches')
+    parser.add_argument('--json', type=Path, help='write every case to this JSON file')
+    parser.add_argument('--diff', metavar='CASE', help='print a unified diff for one case (name or part of it)')
+    return parser.parse_args(argv)
+
+
+def _diff(container, needle: str) -> int:
+    corpus = container.get(Corpus)
+    matches = [case for case in corpus.cases() if needle in case.name]
+    if not matches:
+        print(f'no case matches {needle!r}', file=sys.stderr)
+        return 2
+    case = matches[0]
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore')
+        got = container.get(Transpiler).transpile(case.read_source())
+    expected = case.read_expected() or ''
+    sys.stdout.writelines(difflib.unified_diff(
+        got.splitlines(True), expected.splitlines(True), f'pine2pyne/{case.name}', f'pynecomp/{case.name}'))
+    return 0 if got == expected else 1
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = parse_args(argv)
+    profile = OutputProfile.parse(args.profile)
+    container = make_compat_container(profile, args.assets)
+    try:
+        if args.diff:
+            return _diff(container, args.diff)
+        corpus = container.get(Corpus)
+        if args.filter:
+            corpus_cases = [case for case in corpus.cases() if args.filter in case.name]
+        else:
+            corpus_cases = list(corpus.cases())
+        if args.jobs > 1:
+            results = run_parallel(corpus_cases, profile, args.jobs)
+        else:
+            results = container.get(CompatRunner).run(corpus_cases)
+        report = CompatReport(profile.value, results)
+        print(report.to_text(worst=args.worst))
+        if args.json:
+            args.json.write_text(report.to_json(), encoding='utf-8')
+        return 0 if report.count(Verdict.EXACT) == report.compared else 1
+    finally:
+        container.close()
+
+
+if __name__ == '__main__':
+    sys.exit(main())

pine2pyne/compat/comparison.py

@@ -0,0 +1,105 @@
+"""Comparing one generated module with the reference module."""
+from __future__ import annotations
+
+import ast
+import difflib
+from dataclasses import dataclass
+from enum import Enum
+
+
+class Verdict(str, Enum):
+    #: byte-for-byte identical
+    EXACT = 'exact'
+    #: identical but for the newline at the end of the file (161 references have
+    #: none, at random: no property of the Pine source predicts it)
+    FINAL_NEWLINE = 'final-newline'
+    #: the same Python program, laid out differently
+    LAYOUT = 'layout'
+    #: a different program
+    DIFFERENT = 'different'
+    #: pine2pyne refused or crashed
+    ERROR = 'error'
+    #: no reference output to compare with
+    NO_REFERENCE = 'no-reference'
+
+
+@dataclass(frozen=True)
+class Comparison:
+    verdict: Verdict
+    #: share of equal lines, 0..1 (difflib ratio over lines)
+    similarity: float = 0.0
+    #: 1-based line of the first difference in the reference module
+    first_difference_line: int = 0
+    #: region of the reference module the first difference is in
+    region: str = ''
+    got_line: str = ''
+    expected_line: str = ''
+    error: str = ''
+
+
+def _program(source: str) -> str | None:
+    """The module's AST without its docstring, or None when it does not parse."""
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        return None
+    body = tree.body
+    if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) \
+            and isinstance(body[0].value.value, str):
+        body = body[1:]
+    return ast.dump(ast.Module(body=body, type_ignores=[]))
+
+
+def region_of(lines: list[str], index: int) -> str:
+    """Name the part of a PyneComp module that 0-based line ``index`` is in."""
+    region = 'header'
+    in_signature = False
+    for position, line in enumerate(lines[:index + 1]):
+        stripped = line.strip()
+        if region == 'header':
+            if position > 0 and stripped == '"""':
+                region = 'imports'
+            continue
+        if in_signature:
+            if line.startswith(')'):
+                in_signature = False
+                region = 'body'
+            continue
+        if line.startswith('if __name__'):
+            region = 'footer'
+        elif line.startswith(('from ', 'import ')) and region in ('imports', 'header'):
+            region = 'imports'
+        elif line.startswith('@script.'):
+            region = 'decorator'
+        elif line.startswith('def main('):
+            region = 'signature'
+            in_signature = not line.rstrip().endswith(':')
+            if not in_signature:
+                region = 'body'
+        elif line and not line.startswith((' ', ')')) and region in ('imports', 'module'):
+            region = 'module'
+    return region
+
+
+class ModuleComparator:
+    """Byte comparison first; on a mismatch, says whether only layout differs."""
+
+    def compare(self, got: str, expected: str) -> Comparison:
+        if got == expected:
+            return Comparison(Verdict.EXACT, similarity=1.0)
+        if got.rstrip('\n') == expected.rstrip('\n'):
+            return Comparison(Verdict.FINAL_NEWLINE, similarity=1.0, region='footer')
+        got_lines, expected_lines = got.split('\n'), expected.split('\n')
+        matcher = difflib.SequenceMatcher(None, got_lines, expected_lines, autojunk=False)
+        first = next((opcode for opcode in matcher.get_opcodes() if opcode[0] != 'equal'), None)
+        index = first[3] if first else min(len(got_lines), len(expected_lines))
+        got_index = first[1] if first else index
+        same_program = _program(got) is not None and _program(got) == _program(expected)
+        return Comparison(
+            Verdict.LAYOUT if same_program else Verdict.DIFFERENT,
+            similarity=matcher.ratio(),
+            first_difference_line=index + 1,
+            region=region_of(expected_lines, min(index, len(expected_lines) - 1)),
+            got_line=got_lines[got_index] if got_index < len(got_lines) else '<end of file>',
+            expected_line=expected_lines[index] if index < len(expected_lines) else '<end of file>',
+        )

pine2pyne/compat/corpus.py

@@ -0,0 +1,73 @@
+"""The reference corpus: Pine inputs and the closed compiler's outputs.
+
+Layout of ``assets/``::
+
+    pine/<kind>/<bucket>/<stem>.pine        kind: indicator | strategy
+    pyne/<kind>_<stem>.py
+    pine/lib/<author>/<name>/v<N>.pine
+    pyne/lib/<author>/<name>/v<N>.py
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator, Protocol
+
+SCRIPT_KINDS = ('indicator', 'strategy')
+
+
+@dataclass(frozen=True)
+class CorpusCase:
+    """One Pine input and the reference output for it."""
+
+    name: str
+    kind: str  # indicator | strategy | library
+    bucket: str  # bit-exact | verified | lib
+    pine_path: Path
+    expected_path: Path
+
+    def read_source(self) -> str:
+        return self.pine_path.read_text(encoding='utf-8')
+
+    def read_expected(self) -> str | None:
+        if not self.expected_path.exists():
+            return None
+        return self.expected_path.read_text(encoding='utf-8')
+
+
+class Corpus(Protocol):
+    def cases(self) -> Iterator[CorpusCase]: ...
+
+
+class AssetsCorpus:
+    """The ``assets/pine`` -> ``assets/pyne`` pairs of this repository."""
+
+    def __init__(self, root: Path) -> None:
+        self.root = Path(root)
+
+    @property
+    def pine_root(self) -> Path:
+        return self.root / 'pine'
+
+    @property
+    def pyne_root(self) -> Path:
+        return self.root / 'pyne'
+
+    def cases(self) -> Iterator[CorpusCase]:
+        for kind in SCRIPT_KINDS:
+            for pine_path in sorted((self.pine_root / kind).rglob('*.pine')):
+                bucket = pine_path.parent.name
+                yield CorpusCase(
+                    name=f'{kind}/{pine_path.stem}', kind=kind, bucket=bucket, pine_path=pine_path,
+                    expected_path=self.pyne_root / f'{kind}_{pine_path.stem}.py')
+        library_root = self.pine_root / 'lib'
+        for pine_path in sorted(library_root.rglob('*.pine')):
+            relative = pine_path.relative_to(library_root)
+            yield CorpusCase(
+                name=f'lib/{relative.with_suffix("").as_posix()}', kind='library', bucket='lib',
+                pine_path=pine_path, expected_path=self.pyne_root / 'lib' / relative.with_suffix('.py'))
+
+
+def default_assets_root() -> Path:
+    """``assets/`` next to the package (a source checkout)."""
+    return Path(__file__).resolve().parents[2] / 'assets'

pine2pyne/compat/report.py

@@ -0,0 +1,95 @@
+"""Summaries of a compatibility run."""
+from __future__ import annotations
+
+import json
+from collections import Counter
+from dataclasses import dataclass
+from statistics import mean, median
+from typing import Sequence
+
+from .comparison import Verdict
+from .runner import CaseResult
+
+
+@dataclass(frozen=True)
+class CompatReport:
+    profile: str
+    results: Sequence[CaseResult]
+
+    def count(self, verdict: Verdict) -> int:
+        return sum(1 for result in self.results if result.comparison.verdict is verdict)
+
+    @property
+    def compared(self) -> int:
+        return sum(1 for result in self.results if result.comparison.verdict is not Verdict.NO_REFERENCE)
+
+    def by_bucket(self) -> dict[str, Counter]:
+        buckets: dict[str, Counter] = {}
+        for result in self.results:
+            buckets.setdefault(f'{result.case.kind}/{result.case.bucket}', Counter())[result.comparison.verdict] += 1
+        return buckets
+
+    def regions(self) -> Counter:
+        return Counter(result.comparison.region for result in self.results
+                       if result.comparison.verdict in (Verdict.DIFFERENT, Verdict.LAYOUT))
+
+    def errors(self) -> Counter:
+        return Counter(result.comparison.error.split(':')[0] for result in self.results
+                       if result.comparison.verdict is Verdict.ERROR)
+
+    def similarities(self) -> list[float]:
+        return [result.comparison.similarity for result in self.results
+                if result.comparison.verdict not in (Verdict.ERROR, Verdict.NO_REFERENCE)]
+
+    def to_text(self, worst: int = 0) -> str:
+        compared = self.compared or 1
+        lines = [f'pine2pyne vs PyneComp reference outputs (profile: {self.profile})', '']
+        for verdict in Verdict:
+            number = self.count(verdict)
+            lines.append(f'  {verdict.value:<13} {number:>5}  ({100 * number / compared:5.1f}%)')
+        similarities = self.similarities()
+        if similarities:
+            lines.append(f'  line similarity: mean {mean(similarities):.3f}, median {median(similarities):.3f}')
+        lines += ['', 'By bucket:']
+        for bucket, counter in sorted(self.by_bucket().items()):
+            parts = ', '.join(f'{verdict.value}={counter[verdict]}' for verdict in Verdict if counter[verdict])
+            lines.append(f'  {bucket:<22} {parts}')
+        regions = self.regions()
+        if regions:
+            lines += ['', 'First difference by region:']
+            lines += [f'  {region or "?":<10} {number:>5}' for region, number in regions.most_common()]
+        errors = self.errors()
+        if errors:
+            lines += ['', 'Errors:']
+            lines += [f'  {name:<28} {number:>5}' for name, number in errors.most_common()]
+        if worst:
+            ranked = sorted((result for result in self.results
+                             if result.comparison.verdict in (Verdict.DIFFERENT, Verdict.LAYOUT)),
+                            key=lambda result: -result.comparison.similarity)[:worst]
+            if ranked:
+                lines += ['', f'Closest {len(ranked)} mismatches (first difference):']
+                for result in ranked:
+                    comparison = result.comparison
+                    lines.append(f'  {result.case.name}  [{comparison.similarity:.3f}, line '
+                                 f'{comparison.first_difference_line}, {comparison.region}]')
+                    lines.append(f'    - {comparison.got_line.strip()[:150]}')
+                    lines.append(f'    + {comparison.expected_line.strip()[:150]}')
+        return '\n'.join(lines)
+
+    def to_json(self) -> str:
+        return json.dumps({
+            'profile': self.profile,
+            'summary': {verdict.value: self.count(verdict) for verdict in Verdict},
+            'cases': [{
+                'name': result.case.name,
+                'pine': str(result.case.pine_path),
+                'expected': str(result.case.expected_path),
+                'verdict': result.comparison.verdict.value,
+                'similarity': round(result.comparison.similarity, 4),
+                'first_difference_line': result.comparison.first_difference_line,
+                'region': result.comparison.region,
+                'got_line': result.comparison.got_line,
+                'expected_line': result.comparison.expected_line,
+                'error': result.comparison.error,
+            } for result in self.results],
+        }, ensure_ascii=False, indent=1)

pine2pyne/compat/runner.py

@@ -0,0 +1,63 @@
+"""Running the corpus through a transpiler and collecting comparisons."""
+from __future__ import annotations
+
+import warnings
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import dataclass
+from typing import Callable, Iterable, Sequence
+
+from ..core.contracts import OutputProfile, Transpiler
+from .comparison import Comparison, ModuleComparator, Verdict
+from .corpus import CorpusCase
+
+
+@dataclass(frozen=True)
+class CaseResult:
+    case: CorpusCase
+    comparison: Comparison
+
+
+def _describe(error: BaseException) -> str:
+    text = str(error).strip().split('\n')[0]
+    return f'{type(error).__name__}: {text}'[:300]
+
+
+class CompatRunner:
+    """Transpiles each case and compares the result with its reference."""
+
+    def __init__(self, transpiler: Transpiler, comparator: ModuleComparator) -> None:
+        self._transpiler = transpiler
+        self._comparator = comparator
+
+    def run_case(self, case: CorpusCase) -> CaseResult:
+        expected = case.read_expected()
+        if expected is None:
+            return CaseResult(case, Comparison(Verdict.NO_REFERENCE))
+        try:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                got = self._transpiler.transpile(case.read_source())
+        except Exception as error:  # a refusal or a crash is a result, not a harness failure
+            return CaseResult(case, Comparison(Verdict.ERROR, error=_describe(error)))
+        return CaseResult(case, self._comparator.compare(got, expected))
+
+    def run(self, cases: Iterable[CorpusCase], progress: Callable[[CaseResult], None] | None = None) -> list[CaseResult]:
+        results = []
+        for case in cases:
+            result = self.run_case(case)
+            if progress:
+                progress(result)
+            results.append(result)
+        return results
+
+
+def _run_in_worker(arguments: tuple[str, CorpusCase]) -> CaseResult:
+    profile, case = arguments
+    from ..di import get_transpiler
+    return CompatRunner(get_transpiler(profile), ModuleComparator()).run_case(case)
+
+
+def run_parallel(cases: Sequence[CorpusCase], profile: OutputProfile, jobs: int) -> list[CaseResult]:
+    """``cases`` in ``jobs`` worker processes, each with its own container."""
+    with ProcessPoolExecutor(max_workers=jobs) as pool:
+        return list(pool.map(_run_in_worker, [(profile.value, case) for case in cases], chunksize=8))