api-parity-py 0.0.2__py3-none-any.whl

api_parity_py/__init__.py

@@ -0,0 +1,5 @@
+"""api-parity-py: Python plugin for api-parity (reference + port)."""
+
+from .parity import Status, parity, parity_impl, parity_ref
+
+__all__ = ["Status", "parity", "parity_impl", "parity_ref"]

api_parity_py/__main__.py

@@ -0,0 +1,148 @@
+"""CLI: ``api-parity-py <kind> [--mode walker|annotation] <target> [-o PATH]``.
+
+Supported combos:
+  reference walker     walk a package's public API (default for reference)
+  reference annotation collect ``@parity_ref`` decorators in a package
+  port      walker     walk a package and synthesize implemented port entries
+  port      annotation collect ``@parity_impl`` / ``@parity`` decorators
+                       (default for port)
+"""
+
+import argparse
+import importlib
+import json
+import sys
+from pathlib import Path
+
+from . import walk
+from .parity import collect_port_entries, collect_reference_entries
+
+EXIT_USAGE = 64
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(prog="api-parity-py")
+    ap.add_argument("kind", choices=("reference", "port"))
+    ap.add_argument(
+        "--mode",
+        choices=("walker", "annotation"),
+        default=None,
+        help="how entries are produced (defaults: reference→walker, port→annotation)",
+    )
+    ap.add_argument(
+        "target",
+        help="package name (or comma-separated names, e.g. "
+        "'pyspark.sql.connect,pyspark.sql.session')",
+    )
+    ap.add_argument(
+        "--version-from",
+        default=None,
+        help="module to read `__version__` from (e.g. `pyspark`)",
+    )
+    ap.add_argument(
+        "-o",
+        "--output",
+        default="-",
+        help="output file path, or `-` for stdout (default)",
+    )
+    args = ap.parse_args()
+
+    mode = args.mode or _default_mode(args.kind)
+    envelope = _build_envelope(
+        kind=args.kind,
+        mode=mode,
+        target=args.target,
+        version_from=args.version_from,
+    )
+    if envelope is None:
+        sys.stderr.write(
+            f"api-parity-py: ({args.kind}, mode={mode}) is not yet implemented\n"
+        )
+        return EXIT_USAGE
+
+    text = json.dumps(envelope, indent=2)
+    if args.output == "-":
+        sys.stdout.write(text + "\n")
+    else:
+        Path(args.output).write_text(text + "\n")
+    return 0
+
+
+def _default_mode(kind: str) -> str:
+    return "annotation" if kind == "port" else "walker"
+
+
+def _build_envelope(
+    *,
+    kind: str,
+    mode: str,
+    target: str,
+    version_from: str | None,
+) -> dict | None:
+    if kind == "reference" and mode == "walker":
+        return walk.walk_package(target, version_from=version_from)
+    if kind == "reference" and mode == "annotation":
+        return _collect_annotations(target, version_from, kind="reference")
+    if kind == "port" and mode == "annotation":
+        return _collect_annotations(target, version_from, kind="port")
+    if kind == "port" and mode == "walker":
+        return _walker_as_port(target, version_from)
+    return None
+
+
+def _import_target_packages(target: str) -> list[str]:
+    """Import every submodule of every requested package so decorators run."""
+    packages = [p.strip() for p in target.split(",") if p.strip()]
+    for pkg in packages:
+        walk._preload_submodules(pkg)
+    return packages
+
+
+def _read_version(version_from: str | None) -> str | None:
+    if not version_from:
+        return None
+    try:
+        mod = importlib.import_module(version_from)
+        return getattr(mod, "__version__", None)
+    except Exception:
+        return None
+
+
+def _collect_annotations(
+    target: str, version_from: str | None, *, kind: str,
+) -> dict:
+    packages = _import_target_packages(target)
+    if kind == "port":
+        entries = collect_port_entries()
+    else:
+        entries = collect_reference_entries()
+    return {
+        "schema_version": 1,
+        "kind": kind,
+        "language": "python",
+        "version": _read_version(version_from),
+        "source": ",".join(packages),
+        "entries": entries,
+    }
+
+
+def _walker_as_port(target: str, version_from: str | None) -> dict:
+    """Treat every walked public API as `status=implemented`. Useful for
+    py-vs-py comparisons where neither side is annotated."""
+    ref = walk.walk_package(target, version_from=version_from)
+    entries = []
+    for e in ref["entries"]:
+        impl = e["path"]
+        entries.append({
+            "path": e["path"],
+            "implementation": impl,
+            "status": "implemented",
+            "since": None,
+            "issue": None,
+            "comment": None,
+        })
+    return {**ref, "kind": "port", "entries": entries}
+
+
+if __name__ == "__main__":
+    sys.exit(main())

api_parity_py/parity.py

@@ -0,0 +1,212 @@
+"""Annotation API for Python plugins.
+
+Three decorators:
+
+- ``@parity_impl(path=..., status=...)`` on a class. Sets a parent path
+  for its methods; if both ``path`` and ``status`` are given, also
+  registers a class-level port entry.
+- ``@parity(path=..., status=...)`` on a method. A leading ``.`` makes
+  the path relative to the enclosing ``@parity_impl``'s path. Free
+  functions (no enclosing class) are supported with absolute paths.
+- ``@parity_ref(path=..., kind=...)`` on a class / method / function.
+  For declaring a reference inventory in code (rare; usually a walker
+  is better).
+
+Decorators write to module-level registries that the CLI dumps in
+``port`` / ``reference`` mode with ``--mode annotation``.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Callable
+
+_PORT_ENTRIES: list[dict] = []
+_REFERENCE_ENTRIES: list[dict] = []
+
+
+class Status(str, Enum):
+    IMPLEMENTED = "implemented"
+    PARTIAL = "partial"
+    UNIMPLEMENTED = "unimplemented"
+
+
+def _normalize_status(status: Any) -> str:
+    if isinstance(status, Status):
+        return status.value
+    if isinstance(status, str) and status in {s.value for s in Status}:
+        return status
+    raise TypeError(
+        f"status must be a Status enum or one of {{implemented, partial, unimplemented}}, "
+        f"got {status!r}"
+    )
+
+
+def _validate_port_args(status: str, comment: str | None) -> None:
+    if status == Status.UNIMPLEMENTED.value and not comment:
+        raise ValueError(
+            "status=Unimplemented requires a comment explaining why"
+        )
+
+
+def parity(
+    path: str,
+    status: Status | str,
+    *,
+    since: str | None = None,
+    issue: int | None = None,
+    comment: str | None = None,
+) -> Callable:
+    """Method-level / free-function port annotation.
+
+    Inside a ``@parity_impl`` class, a leading ``.`` in ``path`` is
+    rewritten as ``parent_path + child`` when the class decorator runs.
+    For free functions the path must be absolute.
+    """
+    status_str = _normalize_status(status)
+    _validate_port_args(status_str, comment)
+    meta = {
+        "path": path,
+        "status": status_str,
+        "since": since,
+        "issue": issue,
+        "comment": comment,
+    }
+
+    def decorate(fn: Callable) -> Callable:
+        if path.startswith("."):
+            # Relative path: stash meta so the enclosing `parity_impl`
+            # sweep can resolve it against the parent path.
+            fn._parity_meta = meta  # type: ignore[attr-defined]
+            return fn
+        # Absolute path: register immediately. Works for free functions
+        # and for class methods whose path is unrelated to their enclosing
+        # type. We don't stash meta — that would cause `parity_impl` to
+        # double-register the same entry.
+        impl = f"{fn.__module__}.{fn.__qualname__}"
+        _PORT_ENTRIES.append({**meta, "implementation": impl})
+        return fn
+
+    return decorate
+
+
+def parity_impl(
+    path: Any = None,
+    status: Status | str | None = None,
+    *,
+    since: str | None = None,
+    issue: int | None = None,
+    comment: str | None = None,
+) -> Callable:
+    """Class-level port annotation.
+
+    Two call forms:
+
+    - ``@parity_impl(path="...", status=...)`` — with args, registers a
+      class-level entry and provides a parent path for relative children.
+    - ``@parity_impl`` — bare, no args. Sweeps the class for relative
+      ``@parity`` children but registers no class-level entry (and a
+      child with a leading-``.`` path is an error in this form).
+
+    Either way, the class's ``__dict__`` is swept and any
+    ``@parity``-decorated method with relative path gets joined with the
+    parent's ``path`` and registered.
+    """
+    # Bare-decorator form: Python passes the class as the first arg.
+    if isinstance(path, type) and status is None:
+        return parity_impl()(path)
+
+    parent_path = path
+    parent_status = _normalize_status(status) if status is not None else None
+    if parent_status is not None:
+        _validate_port_args(parent_status, comment)
+
+    def decorate(cls: type) -> type:
+        # Class-level entry, if both path and status given.
+        if parent_path and parent_status:
+            _PORT_ENTRIES.append({
+                "path": parent_path,
+                "implementation": f"{cls.__module__}.{cls.__qualname__}",
+                "status": parent_status,
+                "since": since,
+                "issue": issue,
+                "comment": comment,
+            })
+
+        # Sweep methods. We look at __dict__ rather than dir() so we
+        # only see things defined on this class (not inherited), and so
+        # properties / classmethods come back as their raw descriptor
+        # (which is what the @parity decorator attached the meta to).
+        for name, value in cls.__dict__.items():
+            inner = value
+            if isinstance(value, (classmethod, staticmethod)):
+                inner = value.__func__
+            elif isinstance(value, property):
+                inner = value.fget
+            meta = getattr(inner, "_parity_meta", None)
+            if not meta:
+                continue
+            child_path = meta["path"]
+            if child_path.startswith("."):
+                if not parent_path:
+                    raise ValueError(
+                        f"@parity({child_path!r}) on {cls.__qualname__}.{name} "
+                        f"requires the enclosing @parity_impl to declare a path"
+                    )
+                full_path = f"{parent_path}{child_path}"
+            else:
+                full_path = child_path
+            _PORT_ENTRIES.append({
+                "path": full_path,
+                "implementation": f"{cls.__module__}.{cls.__qualname__}.{name}",
+                "status": meta["status"],
+                "since": meta["since"],
+                "issue": meta["issue"],
+                "comment": meta["comment"],
+            })
+        return cls
+
+    return decorate
+
+
+def parity_ref(path: str, kind: str) -> Callable:
+    """Code-level reference entry. Apply to a class / method / function."""
+    if kind not in {"class", "method", "property", "function"}:
+        raise ValueError(
+            f"kind must be one of class/method/property/function, got {kind!r}"
+        )
+    entry = {"path": path, "kind": kind}
+
+    def decorate(target: Any) -> Any:
+        _REFERENCE_ENTRIES.append(entry)
+        return target
+
+    return decorate
+
+
+def collect_port_entries() -> list[dict]:
+    """Return all port entries registered so far, sorted+deduped by path."""
+    return _sort_dedup(_PORT_ENTRIES, key=("path",))
+
+
+def collect_reference_entries() -> list[dict]:
+    """Return all reference entries registered so far, sorted+deduped."""
+    return _sort_dedup(_REFERENCE_ENTRIES, key=("path", "kind"))
+
+
+def reset_registries() -> None:
+    """Clear both registries. Intended for tests."""
+    _PORT_ENTRIES.clear()
+    _REFERENCE_ENTRIES.clear()
+
+
+def _sort_dedup(entries: list[dict], key: tuple[str, ...]) -> list[dict]:
+    seen: set[tuple] = set()
+    out: list[dict] = []
+    for e in sorted(entries, key=lambda d: tuple(d.get(k) or "" for k in key)):
+        k = tuple(e.get(field) for field in key)
+        if k in seen:
+            continue
+        seen.add(k)
+        out.append(e)
+    return out

api_parity_py/walk.py

@@ -0,0 +1,242 @@
+"""Walk a Python package and emit a reference-side envelope (see SCHEMA.md).
+
+# Algorithm
+
+1. Preload every submodule of each requested package, so classes defined
+   in lazy-imported modules show up in `sys.modules` and forward-reference
+   strings are resolvable.
+2. Walk the loaded modules. For each module under one of the requested
+   packages, list its public classes — but only the ones *defined* there
+   (i.e. `cls.__module__ == mod_name`), so re-exports don't double-count.
+3. For each class, recurse into nested classes (e.g. `SparkSession.Builder`)
+   and emit one entry per public method/property. Class-level data attrs
+   (`MAX_MESSAGE_LENGTH = 128`) and nested classes are handled separately.
+4. Module-level free functions are emitted with `kind = "function"`.
+
+# Path keying
+
+Each entry's `path` is `cls.__module__ + "." + cls.__qualname__` (for a
+class) or `<class path>.<member>` (for a member). `__qualname__` includes
+nesting, so `SparkSession.Builder` keeps its dotted lexical structure.
+
+This means `pyspark.sql.session.SparkSession` and
+`pyspark.sql.connect.session.SparkSession` are distinct entries — both
+classes are tracked, no qualname collision, no priority/preference flag.
+"""
+
+import importlib
+import inspect
+import pkgutil
+import sys
+from collections import Counter
+
+# `inspect.getmembers(object)` brings in `__init__`, `__doc__`, etc.; we
+# strip those by name to keep the public API list focused on real surface.
+_OBJECT_MEMBERS = frozenset(name for name, _ in inspect.getmembers(object))
+# Subclasses of these are "data shapes", not API surface. Filtering by
+# class hierarchy avoids clutter from `Row` (a tuple subclass), exception
+# types, etc.
+_DATA_BASES = (tuple, list, dict, BaseException)
+
+
+def _preload_submodules(package_name: str) -> None:
+    """Import every submodule of `package_name` into `sys.modules`.
+
+    Without this, classes defined in lazy-loaded submodules would never be
+    seen by the discovery walk, and string-form forward-reference type
+    annotations (`'SparkConnectClient'`) wouldn't resolve.
+
+    Submodules whose import raises (e.g. because an optional dep like
+    pandas isn't installed) are collected and summarized on stderr so an
+    incomplete walk is visibly incomplete instead of silently truncated.
+    """
+    try:
+        pkg = importlib.import_module(package_name)
+    except Exception as e:
+        sys.stderr.write(
+            f"api-parity-py: failed to import top-level package "
+            f"{package_name!r}: {type(e).__name__}: {e}\n"
+        )
+        return
+    pkg_path = getattr(pkg, "__path__", None)
+    if pkg_path is None:
+        return
+
+    failures: list[tuple[str, BaseException]] = []
+    total = 0
+    for _, mod_name, _ in pkgutil.walk_packages(pkg_path, prefix=package_name + "."):
+        total += 1
+        try:
+            importlib.import_module(mod_name)
+        except Exception as e:
+            failures.append((mod_name, e))
+
+    if failures:
+        _report_skipped(package_name, total, failures)
+
+
+def _report_skipped(
+    package_name: str,
+    total: int,
+    failures: list[tuple[str, BaseException]],
+) -> None:
+    """Print a grouped, terse summary of import failures to stderr."""
+    reasons: Counter[str] = Counter()
+    examples: dict[str, str] = {}
+    for mod, err in failures:
+        first_line = str(err).splitlines()[0][:120] if str(err) else ""
+        key = f"{type(err).__name__}: {first_line}" if first_line else type(err).__name__
+        reasons[key] += 1
+        examples.setdefault(key, mod)
+
+    sys.stderr.write(
+        f"api-parity-py: {package_name}: "
+        f"skipped {len(failures)}/{total} submodule(s) — inventory will be incomplete\n"
+    )
+    for reason, count in reasons.most_common():
+        sys.stderr.write(f"  - [{count}x] {reason}\n")
+        sys.stderr.write(f"      e.g. {examples[reason]}\n")
+
+
+def _raw_attr(cls: type, name: str) -> object:
+    """Return the raw descriptor for `name` on `cls`, walking the MRO.
+
+    We can't use `getattr(cls, name)` to detect properties — that triggers
+    descriptor invocation and gives us the resolved value (e.g. a `Builder`
+    instance) instead of the descriptor itself. Looking up via `__dict__`
+    on each MRO class returns the raw `property` / `classmethod` object.
+    """
+    for klass in cls.__mro__:
+        if name in klass.__dict__:
+            return klass.__dict__[name]
+    return None
+
+
+def _member_kind(cls: type, name: str, value: object) -> str | None:
+    """Classify a class member, or `None` to drop it from the inventory.
+
+    `isinstance(raw, property)` catches `property` subclasses like pyspark's
+    `classproperty`. Nested classes return None — they're emitted as their
+    own top-level entries by the recursion in `_collect_class`. Callable
+    values (functions, classmethods after descriptor invocation, etc.)
+    become methods.
+    """
+    raw = _raw_attr(cls, name)
+    if isinstance(raw, property):
+        return "property"
+    if isinstance(value, type):
+        return None
+    if callable(value):
+        return "method"
+    # Plain class attributes (constants, default values) are not API surface.
+    return None
+
+
+def _is_api_class(obj: object) -> bool:
+    return isinstance(obj, type) and not issubclass(obj, _DATA_BASES)
+
+
+def _collect_class(cls: type, entries: list[dict]) -> None:
+    """Record `cls` and its members, then recurse into nested classes."""
+    full = f"{cls.__module__}.{cls.__qualname__}"
+    # Cheap dedup: re-exports may cause us to revisit a class. We bail
+    # before doing redundant member walks.
+    if any(e["path"] == full and e["kind"] == "class" for e in entries):
+        return
+    entries.append({"path": full, "kind": "class"})
+
+    for name, value in inspect.getmembers(cls):
+        if name.startswith("_") or name in _OBJECT_MEMBERS:
+            continue
+        kind = _member_kind(cls, name, value)
+        if kind is None:
+            continue
+        entries.append({"path": f"{full}.{name}", "kind": kind})
+
+    # Nested classes: their `__module__` matches the outer class's, and
+    # their `__qualname__` is prefixed with `<outer>.` — those two checks
+    # together filter out unrelated classes that happen to be exposed as
+    # attributes (e.g. types pulled in for type hints).
+    for name, obj in inspect.getmembers(cls):
+        if name.startswith("_") or not _is_api_class(obj):
+            continue
+        if obj.__module__ != cls.__module__:
+            continue
+        if not obj.__qualname__.startswith(cls.__qualname__ + "."):
+            continue
+        _collect_class(obj, entries)
+
+
+def _discover(packages: list[str]) -> list[dict]:
+    """Walk every loaded module under `packages`, emit entries."""
+    entries: list[dict] = []
+    for mod_name, mod in list(sys.modules.items()):
+        if mod is None:
+            continue
+        if not any(mod_name == p or mod_name.startswith(p + ".") for p in packages):
+            continue
+
+        # Top-level classes, recursing into their nested classes.
+        for name, obj in inspect.getmembers(mod):
+            if name.startswith("_") or not _is_api_class(obj):
+                continue
+            # Skip re-exports: only record a class in its defining module.
+            # Without this, `pyspark.sql.SparkSession` (re-export from
+            # `pyspark.sql.session`) would be listed twice with different
+            # paths.
+            if obj.__module__ != mod_name:
+                continue
+            _collect_class(obj, entries)
+
+        # Module-level free functions (e.g. `pyspark.sql.functions.col`).
+        for name, obj in inspect.getmembers(mod, inspect.isfunction):
+            if name.startswith("_"):
+                continue
+            if obj.__module__ != mod_name:
+                continue
+            entries.append({
+                "path": f"{obj.__module__}.{obj.__qualname__}",
+                "kind": "function",
+            })
+
+    # Sort + dedup by (path, kind). Stability matters because the JSON
+    # output is part of the contract — diffing two versions should produce
+    # minimal noise.
+    seen: set[tuple[str, str]] = set()
+    out: list[dict] = []
+    for e in sorted(entries, key=lambda e: (e["path"], e["kind"])):
+        key = (e["path"], e["kind"])
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(e)
+    return out
+
+
+def walk_package(target: str, *, version_from: str | None = None) -> dict:
+    """Build a reference envelope for one or more comma-separated packages.
+
+    `version_from` names a module to import for its `__version__` (e.g.
+    `"pyspark"`). If unimportable or missing the attribute, version is
+    left null; this is informational metadata, not part of the join key.
+    """
+    packages = [p.strip() for p in target.split(",") if p.strip()]
+    for pkg in packages:
+        _preload_submodules(pkg)
+
+    version = None
+    if version_from:
+        try:
+            mod = importlib.import_module(version_from)
+            version = getattr(mod, "__version__", None)
+        except Exception:
+            pass
+
+    return {
+        "schema_version": 1,
+        "kind": "reference",
+        "language": "python",
+        "version": version,
+        "source": ",".join(packages),
+        "entries": _discover(packages),
+    }

api_parity_py-0.0.2.dist-info/METADATA

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: api-parity-py
+Version: 0.0.2
+Summary: Python plugin for api-parity (reference walker + port annotations).
+Requires-Python: >=3.10

api_parity_py-0.0.2.dist-info/RECORD

@@ -0,0 +1,8 @@
+api_parity_py/__init__.py,sha256=KlBe5r7WDDKsGENk_3f6XpDCA4tFJGzCvshMbfya7DA,192
+api_parity_py/__main__.py,sha256=7tlHU_KDHK8oDjUFPVU9138PpnA-RhZIHCgCIIR598g,4371
+api_parity_py/parity.py,sha256=GAIE4jBDov2bgbn8djPaw5FBKQlpbH-TaQQChUerNJ8,7331
+api_parity_py/walk.py,sha256=eGOLLKA27svr4CmsWWww6wPIc0Jv6QHCWeZTC7j6vls,9299
+api_parity_py-0.0.2.dist-info/METADATA,sha256=ZcbCkFdK2jVYNjWMUIGC9XacDwvibqUggQseYfQZJk0,158
+api_parity_py-0.0.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+api_parity_py-0.0.2.dist-info/entry_points.txt,sha256=ZLa5eWK18jaDfYgBmfwoQNWAVqV0fzMdp6zv6b5j3nA,62
+api_parity_py-0.0.2.dist-info/RECORD,,

api_parity_py-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

api_parity_py-0.0.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+api-parity-py = api_parity_py.__main__:main