sincewhen 0.1.0__py3-none-any.whl

sincewhen/__init__.py

@@ -0,0 +1,16 @@
+"""Find out which Python version added each feature your code uses."""
+
+from .detect import Detection, detect, minimum_version
+from .features import DatasetError, Feature, load_features, lookup
+from .versions import Version
+
+__all__ = [
+    "DatasetError",
+    "Detection",
+    "Feature",
+    "Version",
+    "detect",
+    "load_features",
+    "lookup",
+    "minimum_version",
+]

sincewhen/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

sincewhen/cli.py

@@ -0,0 +1,222 @@
+"""Command-line interface for sincewhen."""
+
+import argparse
+import json
+import sys
+from importlib.metadata import version
+from pathlib import Path
+
+from .detect import Detection, detect
+from .features import Feature, enclosing_module, lookup
+from .versions import Version
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="sincewhen",
+        description="Find out which Python version added each feature your code uses.",
+    )
+    parser.add_argument(
+        "paths",
+        nargs="*",
+        type=Path,
+        help="Python files to analyze (use - for standard input)",
+    )
+    parser.add_argument(
+        "-s",
+        "--search",
+        metavar="TERM",
+        help="look a feature up by name instead of analyzing code",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        help="report every use, not just the first of each feature",
+    )
+    parser.add_argument(
+        "--since",
+        metavar="VERSION",
+        type=Version.parse,
+        help="hide features older than this, for when the ancient ones are noise",
+    )
+    parser.add_argument("--json", action="store_true", help="emit JSON")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {version('sincewhen')}",
+        help="show the version of sincewhen itself and exit",
+    )
+    return parser
+
+
+def _read(path: Path) -> tuple[str, str]:
+    if str(path) == "-":
+        return sys.stdin.read(), "<stdin>"
+    return path.read_text(encoding="utf-8"), str(path)
+
+
+def _drop_implied(detections: list[Detection]) -> list[Detection]:
+    """Drop a module line when a member of it says the same thing.
+
+    `from dataclasses import dataclass` is one import that matches two
+    features of the same age. Both are true and printing both is noise,
+    so the more specific one speaks for the pair. The module keeps its
+    line whenever it is the older of the two, since then it is saying
+    something the member does not.
+    """
+    members = {
+        (found.lineno, name.partition(".")[0], found.added)
+        for found in detections
+        for name in found.feature.attributes
+    }
+    return [
+        found
+        for found in detections
+        if not any(
+            (found.lineno, module, found.added) in members
+            for module in found.feature.modules
+        )
+    ]
+
+
+def _first_uses(detections: list[Detection]) -> list[Detection]:
+    seen = set()
+    kept = []
+    for found in detections:
+        if found.feature.id not in seen:
+            seen.add(found.feature.id)
+            kept.append(found)
+    return kept
+
+
+def _feature_data(feature: Feature) -> dict:
+    return {
+        "id": feature.id,
+        "name": feature.name,
+        "added": str(feature.added),
+        "or_earlier": feature.or_earlier,
+        "released": released.isoformat()
+        if (released := feature.added.released)
+        else None,
+        "category": feature.category,
+        "pep": feature.pep,
+        "pep_url": feature.pep_url,
+        "docs_url": feature.docs_url,
+    }
+
+
+def _released(version: Version) -> str:
+    """When that release shipped, for the reader's sense of scale."""
+    released = version.released
+    return released.isoformat() if released else ""
+
+
+def _report(results: list[tuple[str, list[Detection]]]) -> None:
+    rows = [
+        (
+            f"{name}:{found.lineno}",
+            found.feature.name,
+            found.feature.since,
+            _released(found.added),
+        )
+        for name, detections in results
+        for found in detections
+    ]
+    if not rows:
+        print("No dated features detected.")
+        return
+    location_width = max(len(location) for location, *_ in rows)
+    name_width = max(len(name) for _, name, *_ in rows)
+    since_width = max(len(since) for *_, since, _ in rows)
+    for location, name, since, released in rows:
+        line = f"{location:<{location_width}}  {name:<{name_width}}  {since:<{since_width}}"
+        print(f"{line}  {released}".rstrip())
+
+    # Non-empty rows guarantee at least one detection, so there is a floor.
+    floor = max(found.added for _, detections in results for found in detections)
+    setters = sorted(
+        {
+            found.feature.name
+            for _, detections in results
+            for found in detections
+            if found.added == floor
+        }
+    )
+    released = _released(floor)
+    since = f", released {released}" if released else ""
+    print(f"\nMinimum: Python {floor}{since} (set by {', '.join(setters)})")
+
+
+def _search(term: str, as_json: bool) -> int:
+    features = lookup(term)
+    if as_json:
+        print(json.dumps([_feature_data(f) for f in features], indent=2))
+        return 0 if features else 1
+    if not features:
+        print(f"No feature matches {term!r}.", file=sys.stderr)
+        return 1
+    if features == enclosing_module(term.casefold().strip()):
+        print(f"No entry for {term}, but it lives in:")
+    for feature in features:
+        released = _released(feature.added)
+        when = f" (released {released})" if released else ""
+        print(f"{feature.name} - Python {feature.since}{when}")
+        for label, url in (("PEP", feature.pep_url), ("Docs", feature.docs_url)):
+            if url:
+                print(f"  {label}: {url}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+
+    if args.search:
+        return _search(args.search, args.json)
+
+    if not args.paths:
+        _parser().error("provide at least one path, or use --search")
+
+    results = []
+    for path in args.paths:
+        try:
+            source, name = _read(path)
+        except OSError as error:
+            print(f"sincewhen: {error}", file=sys.stderr)
+            return 2
+        try:
+            detections = detect(source, filename=name)
+        except SyntaxError as error:
+            print(f"sincewhen: {name}: {error}", file=sys.stderr)
+            return 2
+        detections = _drop_implied(detections)
+        if args.since:
+            detections = [f for f in detections if f.added >= args.since]
+        if not args.all:
+            detections = _first_uses(detections)
+        results.append((name, detections))
+
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    name: [
+                        {
+                            "line": f.lineno,
+                            "column": f.col_offset,
+                            **_feature_data(f.feature),
+                        }
+                        for f in detections
+                    ]
+                    for name, detections in results
+                },
+                indent=2,
+            )
+        )
+    else:
+        _report(results)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sincewhen/detect.py

@@ -0,0 +1,392 @@
+"""Detect dated features in Python source code."""
+
+import ast
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from functools import cache
+from typing import Any
+
+from .features import Feature, load_features
+from .versions import Version
+
+
+@dataclass(frozen=True)
+class Detection:
+    """One use of one feature, at one place in the source."""
+
+    feature: Feature
+    lineno: int
+    col_offset: int
+
+    @property
+    def added(self) -> Version:
+        return self.feature.added
+
+
+# Containers that PEP 585 made subscriptable in 3.9. Subscripting any
+# of these names is either the new generic syntax or a variable that
+# shadows the builtin, which is why the check needs to know what the
+# module binds.
+PEP_585_GENERICS = frozenset(
+    {"list", "dict", "set", "frozenset", "tuple", "type"},
+)
+
+
+def _has_none_key(node: ast.Dict, _bound: frozenset[str]) -> bool:
+    """A `None` key means dict unpacking, as in `{**a}`."""
+    return any(key is None for key in node.keys)
+
+
+def _has_starred_element(
+    node: ast.List | ast.Tuple | ast.Set, _bound: frozenset[str]
+) -> bool:
+    """Unpacking into a literal, as in `[*a]`.
+
+    Store context is excluded because `a, *b = c` is a different
+    feature that arrived in Python 3.0.
+    """
+    if not isinstance(getattr(node, "ctx", ast.Load()), ast.Load):
+        return False
+    return any(isinstance(element, ast.Starred) for element in node.elts)
+
+
+def _has_starred_target(node: ast.List | ast.Tuple, _bound: frozenset[str]) -> bool:
+    """Extended unpacking, as in `a, *rest = values`."""
+    if isinstance(node.ctx, ast.Load):
+        return False
+    return any(isinstance(element, ast.Starred) for element in node.elts)
+
+
+def _has_async_comprehension(
+    node: ast.ListComp | ast.SetComp | ast.DictComp | ast.GeneratorExp,
+    _bound: frozenset[str],
+) -> bool:
+    return any(generator.is_async for generator in node.generators)
+
+
+def _has_multiple_unpackings(node: ast.Call, _bound: frozenset[str]) -> bool:
+    """More than one unpacking in a call, as in `f(*a, *b)`.
+
+    A single `f(*a)` has always been legal, so only the second one
+    dates the call to 3.5.
+    """
+    starred = sum(isinstance(argument, ast.Starred) for argument in node.args)
+    doubled = sum(keyword.arg is None for keyword in node.keywords)
+    return starred > 1 or doubled > 1
+
+
+def _is_async_generator(node: ast.AsyncFunctionDef, _bound: frozenset[str]) -> bool:
+    """An `async def` that yields, which needed 3.6.
+
+    Nested functions are skipped, since a plain generator defined inside
+    a coroutine is not itself an async generator.
+    """
+    return any(
+        isinstance(inner, ast.Yield)
+        for inner in _own_body(node)
+        if not isinstance(inner, ast.Lambda)
+    )
+
+
+def _own_body(node: ast.AST):
+    """Every node inside `node` that is not inside a nested function."""
+    for child in ast.iter_child_nodes(node):
+        if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
+            continue
+        yield child
+        yield from _own_body(child)
+
+
+def _is_builtin_generic(node: ast.Subscript, bound: frozenset[str]) -> bool:
+    """Subscripting a builtin container, as in `list[int]` (PEP 585)."""
+    return (
+        isinstance(node.value, ast.Name)
+        and node.value.id in PEP_585_GENERICS
+        and node.value.id not in bound
+    )
+
+
+def _is_zero_argument_super(node: ast.Call, bound: frozenset[str]) -> bool:
+    """`super()` with no arguments, which needs the 3.0 compiler help."""
+    return (
+        isinstance(node.func, ast.Name)
+        and node.func.id == "super"
+        and "super" not in bound
+        and not node.args
+        and not node.keywords
+    )
+
+
+def _has_complex_decorator(
+    node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, _bound: frozenset[str]
+) -> bool:
+    """A decorator that the pre-3.9 grammar would have rejected.
+
+    Until PEP 614 a decorator had to be a dotted name, optionally
+    called. Anything else -- a subscript, a comparison, a lambda -- is
+    3.9 or newer.
+    """
+    return any(
+        not _is_dotted_name(
+            decorator.func if isinstance(decorator, ast.Call) else decorator
+        )
+        for decorator in node.decorator_list
+    )
+
+
+def _is_dotted_name(node: ast.expr) -> bool:
+    while isinstance(node, ast.Attribute):
+        node = node.value
+    return isinstance(node, ast.Name)
+
+
+def _has_except_and_finally(node: ast.Try, _bound: frozenset[str]) -> bool:
+    """One statement with both `except` and `finally`, unified in 2.5."""
+    return bool(node.handlers and node.finalbody)
+
+
+def _has_several_context_managers(
+    node: ast.With | ast.AsyncWith, _bound: frozenset[str]
+) -> bool:
+    """`with a, b:` rather than two nested `with` statements."""
+    return len(node.items) > 1
+
+
+def _has_metaclass_keyword(node: ast.ClassDef, _bound: frozenset[str]) -> bool:
+    """`class C(metaclass=M)`, which replaced `__metaclass__` in 3.0."""
+    return any(keyword.arg == "metaclass" for keyword in node.keywords)
+
+
+def _has_named_handler(node: ast.ExceptHandler, _bound: frozenset[str]) -> bool:
+    """`except E as name`, the spelling that replaced `except E, name`."""
+    return node.name is not None
+
+
+def _has_cause(node: ast.Raise, _bound: frozenset[str]) -> bool:
+    """`raise X from Y`, which is exception chaining."""
+    return node.cause is not None
+
+
+def _is_bytes(node: ast.Constant, _bound: frozenset[str]) -> bool:
+    """A `b"..."` literal."""
+    return isinstance(node.value, bytes)
+
+
+def _has_starred_subscript(node: ast.Subscript, _bound: frozenset[str]) -> bool:
+    """Unpacking inside a subscript, as in `tuple[*Ts]` (PEP 646).
+
+    A starred subscript is always parsed into a tuple, even when it is
+    the only thing in the brackets, so there is no bare `Starred` slice
+    to look for.
+    """
+    return isinstance(node.slice, ast.Tuple) and any(
+        isinstance(element, ast.Starred) for element in node.slice.elts
+    )
+
+
+# Checks are dispatched by name from the dataset, so the node type is
+# only known at runtime. Each predicate still annotates the node types
+# it actually accepts; `Any` here is the dynamic-dispatch boundary.
+# Every check also receives the names the module binds, so that a
+# feature keyed on a builtin can tell a real use from a shadowed one.
+CHECKS: dict[str, Callable[[Any, frozenset[str]], bool]] = {
+    "has_none_key": _has_none_key,
+    "has_starred_element": _has_starred_element,
+    "has_starred_target": _has_starred_target,
+    "has_async_comprehension": _has_async_comprehension,
+    "has_multiple_unpackings": _has_multiple_unpackings,
+    "is_async_generator": _is_async_generator,
+    "is_builtin_generic": _is_builtin_generic,
+    "is_zero_argument_super": _is_zero_argument_super,
+    "has_complex_decorator": _has_complex_decorator,
+    "has_except_and_finally": _has_except_and_finally,
+    "has_several_context_managers": _has_several_context_managers,
+    "has_metaclass_keyword": _has_metaclass_keyword,
+    "has_named_handler": _has_named_handler,
+    "has_cause": _has_cause,
+    "is_bytes": _is_bytes,
+    "has_starred_subscript": _has_starred_subscript,
+}
+
+
+class _Index:
+    """Feature lookups keyed by what they match on."""
+
+    def __init__(self, features: Iterable[Feature]):
+        self.by_node: dict[str, list[Feature]] = {}
+        self.by_builtin: dict[str, Feature] = {}
+        self.by_module: dict[str, Feature] = {}
+        self.by_attribute: dict[str, Feature] = {}
+        for feature in features:
+            for name in feature.nodes:
+                self.by_node.setdefault(name, []).append(feature)
+            for name in feature.builtins:
+                self.by_builtin[name] = feature
+            for name in feature.modules:
+                self.by_module[name] = feature
+            for name in feature.attributes:
+                self.by_attribute[name] = feature
+
+
+@cache
+def _index() -> _Index:
+    return _Index(load_features())
+
+
+def _bound_names(tree: ast.AST) -> frozenset[str]:
+    """Names the module binds itself.
+
+    A module that defines its own `sum` is not using the builtin, so
+    shadowed names are skipped entirely rather than reported wrongly.
+    """
+    names = set()
+    for node in ast.walk(tree):
+        match node:
+            case ast.Name(id=name, ctx=ast.Store() | ast.Del()):
+                names.add(name)
+            case (
+                ast.FunctionDef(name=name)
+                | ast.AsyncFunctionDef(name=name)
+                | ast.ClassDef(name=name)
+            ):
+                names.add(name)
+            case ast.arg(arg=name):
+                names.add(name)
+            case ast.alias(name=name, asname=asname):
+                names.add(asname or name.partition(".")[0])
+            case ast.Global(names=found) | ast.Nonlocal(names=found):
+                names.update(found)
+            case ast.ExceptHandler(name=str() as name):
+                names.add(name)
+    return frozenset(names)
+
+
+class _Detector(ast.NodeVisitor):
+    def __init__(self, index: _Index, bound_names: frozenset[str]):
+        self.index = index
+        self.bound_names = bound_names
+        self.detections: list[Detection] = []
+        self.aliases: dict[str, str] = {}
+        self._position = (1, 0)
+
+    def visit(self, node: ast.AST) -> None:
+        # Some matched nodes (`arguments`) carry no position of their
+        # own, so the nearest enclosing position is tracked instead.
+        lineno = getattr(node, "lineno", None)
+        previous = self._position
+        if lineno is not None:
+            self._position = (lineno, getattr(node, "col_offset", 0))
+        self._match_node(node)
+        handler = getattr(self, f"visit_{type(node).__name__}", None)
+        if handler is None:
+            self.generic_visit(node)
+        else:
+            handler(node)
+        self._position = previous
+
+    def _match_node(self, node: ast.AST) -> None:
+        for feature in self.index.by_node.get(type(node).__name__, ()):
+            if feature.requires and not getattr(node, feature.requires, None):
+                continue
+            if feature.check and not CHECKS[feature.check](node, self.bound_names):
+                continue
+            self._record(feature)
+
+    def _record(self, feature: Feature) -> None:
+        lineno, col_offset = self._position
+        self.detections.append(Detection(feature, lineno, col_offset))
+
+    def _record_module(self, dotted: str) -> None:
+        """Report the most specific module the dataset knows about.
+
+        `import importlib.resources` needs both `importlib` and
+        `importlib.resources`, but only the submodule is worth saying:
+        it is the newer of the two, so it sets the floor on its own, and
+        naming the parent as well would be noise.
+        """
+        parts = dotted.split(".")
+        for size in range(len(parts), 0, -1):
+            feature = self.index.by_module.get(".".join(parts[:size]))
+            if feature is not None:
+                self._record(feature)
+                return
+
+    def _record_attribute(self, dotted: str) -> None:
+        feature = self.index.by_attribute.get(dotted)
+        if feature is not None:
+            self._record(feature)
+
+    def visit_Name(self, node: ast.Name) -> None:
+        if isinstance(node.ctx, ast.Load):
+            if node.id not in self.bound_names:
+                feature = self.index.by_builtin.get(node.id)
+                if feature is not None:
+                    self._record(feature)
+            elif node.id in self.aliases:
+                self._record_attribute(self.aliases[node.id])
+        self.generic_visit(node)
+
+    def visit_Attribute(self, node: ast.Attribute) -> None:
+        dotted = self._dotted_name(node)
+        if dotted is not None:
+            self._record_attribute(dotted)
+        self.generic_visit(node)
+
+    def visit_Import(self, node: ast.Import) -> None:
+        for alias in node.names:
+            self._record_module(alias.name)
+            bound = alias.asname or alias.name.partition(".")[0]
+            self.aliases[bound] = alias.name if alias.asname else bound
+        self.generic_visit(node)
+
+    def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
+        if node.level == 0 and node.module:
+            for alias in node.names:
+                dotted = f"{node.module}.{alias.name}"
+                # `_record_module` walks back up the dotted path, so
+                # this covers the package too: `from pathlib import
+                # Path` reports `pathlib`, and `from importlib import
+                # resources` reports the submodule instead.
+                self._record_module(dotted)
+                self._record_attribute(dotted)
+                self.aliases[alias.asname or alias.name] = dotted
+        self.generic_visit(node)
+
+    def _dotted_name(self, node: ast.Attribute) -> str | None:
+        """Resolve `m.isclose` to `math.isclose`, following aliases."""
+        parts = []
+        current: ast.expr = node
+        while isinstance(current, ast.Attribute):
+            parts.append(current.attr)
+            current = current.value
+        if not isinstance(current, ast.Name):
+            return None
+        parts.append(self.aliases.get(current.id, current.id))
+        return ".".join(reversed(parts))
+
+
+def detect(source: str | ast.AST, *, filename: str = "<string>") -> list[Detection]:
+    """Find every dated feature used in `source`, in source order.
+
+    A feature used several times is reported once per use.
+    """
+    tree = source if isinstance(source, ast.AST) else ast.parse(source, filename)
+    detector = _Detector(_index(), _bound_names(tree))
+    detector.visit(tree)
+    return sorted(
+        detector.detections,
+        key=lambda found: (found.lineno, found.col_offset, found.feature.id),
+    )
+
+
+def minimum_version(source: str | ast.AST) -> Version | None:
+    """The oldest Python that can run `source`, as far as we can tell.
+
+    Returns `None` when nothing in the dataset was detected, which
+    means no known feature sets a floor rather than that the code runs
+    anywhere.
+    """
+    detections = detect(source)
+    if not detections:
+        return None
+    return max(found.added for found in detections)