simplibs-extractor-python 0.1.0__py3-none-any.whl

simplibs/extractor_python/logic/_helpers/find_docstrings.py

@@ -0,0 +1,46 @@
+import ast
+
+
+def find_docstrings(tree: ast.Module) -> set[ast.stmt]:
+    """Find the docstring statements of a module, its classes and its functions.
+
+    Args:
+        tree: The parsed module.
+
+    Returns:
+        The `Expr` statements that are docstrings.
+    """
+
+    # 1. A docstring is a string constant that is the FIRST statement of a module, class or function
+    found: set[ast.stmt] = set()
+    for node in ast.walk(tree):
+        if not isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        if not node.body:
+            continue
+        first = node.body[0]
+        if (
+            isinstance(first, ast.Expr)
+            and isinstance(first.value, ast.Constant)
+            and isinstance(first.value.value, str)
+        ):
+            found.add(first)
+
+    return found
+
+
+_DESIGN_NOTES = """
+# find_docstrings — What Python Itself Calls a Docstring
+
+## Purpose
+The same rule as `ast.get_docstring`, applied to the whole tree at once.
+
+## Only the First Statement
+A string in the middle of a body is an expression nobody uses, not a
+docstring, and stays. An f-string is not a docstring either (it is built at
+run time), and stays.
+
+## A Statement, Not a Line
+A docstring may share its line (`def f(): '''doc'''`) or be wrapped in
+parentheses; the statement's own span covers both.
+"""

simplibs/extractor_python/logic/_helpers/find_imports.py

@@ -0,0 +1,33 @@
+import ast
+
+
+def find_imports(tree: ast.Module) -> set[ast.stmt]:
+    """Find every import statement of a module, at any depth.
+
+    Args:
+        tree: The parsed module.
+
+    Returns:
+        The `Import` and `ImportFrom` statements: at the top, inside functions and classes,
+        in `try` and `if` blocks.
+    """
+
+    # 1. Both kinds, wherever they stand
+    return {node for node in ast.walk(tree) if isinstance(node, (ast.Import, ast.ImportFrom))}
+
+
+_DESIGN_NOTES = """
+# find_imports — Every Import, Not Only the Top Ones
+
+## Purpose
+`remove_python_imports` takes out what this finds.
+
+## Depth
+`try: import ujson / except ImportError: import json`, `if TYPE_CHECKING:` and
+imports inside a function are all imports. Leaving the nested ones would leave
+a file that still names its dependencies.
+
+## `from __future__`
+It is an `ImportFrom` and goes with the rest. For reading the code, a
+`from __future__ import annotations` carries no information.
+"""

simplibs/extractor_python/logic/_helpers/first_doc_line.py

@@ -0,0 +1,37 @@
+import ast
+
+
+def first_doc_line(node: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef) -> str:
+    """Make a trailing comment out of the first line of a docstring.
+
+    Args:
+        node: A class or a function.
+
+    Returns:
+        `"  # First line"`, or an empty string if the node has no docstring.
+    """
+
+    # 1. The cleaned docstring, or nothing
+    doc = ast.get_docstring(node)
+    if not doc:
+        return ""
+
+    # 2. Its first non-empty line; every kind of line end counts, so a comment can never run on
+    lines = doc.strip().splitlines()
+    first = lines[0].strip() if lines else ""
+
+    return f"  # {first}" if first else ""
+
+
+_DESIGN_NOTES = """
+# first_doc_line — A Docstring as a Comment
+
+## Purpose
+The outline keeps what a docstring says in one line, without a string that
+would have to be quoted and escaped.
+
+## `splitlines`, Not `split("\\n")`
+A docstring can hold a `\\r` or a form feed written as an escape. Cut by
+`\\n` only, such a character would end up in the output and `ast` would take it
+for a line end, turning the rest of the comment into code.
+"""

simplibs/extractor_python/logic/_helpers/format_type_params.py

@@ -0,0 +1,29 @@
+import ast
+
+
+def format_type_params(node: ast.AST) -> str:
+    """Write the type parameters of a class or function (`def f[T](x: T)`, Python 3.12+).
+
+    Args:
+        node: A class or a function definition.
+
+    Returns:
+        `"[T, U: int]"`, or an empty string if there are none (and always on older Pythons).
+    """
+
+    # 1. `type_params` exists since Python 3.12; before that there is nothing to write
+    params = getattr(node, "type_params", None)
+    if not params:
+        return ""
+
+    return "[" + ", ".join(ast.unparse(param) for param in params) + "]"
+
+
+_DESIGN_NOTES = """
+# format_type_params — The Brackets After the Name
+
+## Purpose
+`class Box[T]:` and `def first[T](items: list[T]) -> T:` carry their type
+parameters in a field that older Pythons do not have. Without them the outline
+would name a different signature than the source.
+"""

simplibs/extractor_python/logic/_helpers/guarded_parse.py

@@ -0,0 +1,64 @@
+import tokenize
+from collections.abc import Callable
+from typing import TypeVar
+# Outers
+from ...annotations import OnError
+from .._validations import raise_python_source_error
+# Inners
+from .describe_source_failure import describe_source_failure
+
+
+T = TypeVar("T")
+
+
+def guarded_parse(text: str, on_error: OnError, parser: Callable[[str], T]) -> T | None:
+    """Run a parser over a text and decide what a failure means.
+
+    Args:
+        text: The Python source.
+        on_error: `"raise"` for a card, `"keep"` for `None`.
+        parser: What reads the text (`ast.parse`, a tokenizer loop).
+
+    Returns:
+        What the parser returned; `None` if it could not read the text and `on_error` is `"keep"`.
+
+    Raises:
+        ParamError: The text cannot be read and `on_error` is `"raise"`.
+    """
+
+    # 1. Only `\n` line endings: `ast` and `tokenize` would count lines differently from the offsets
+    failure: tuple[str, int | None] | None = None
+    if "\r" in text:
+        failure = ("it has line endings other than \\n", None)
+
+    # 2. Read it
+    else:
+        try:
+            return parser(text)
+        except (SyntaxError, ValueError, RecursionError, tokenize.TokenError) as exc:
+            failure = describe_source_failure(exc)
+
+    # 3. What a failure means
+    if on_error == "keep":
+        return None
+    raise_python_source_error(text, *failure)
+
+
+_DESIGN_NOTES = """
+# guarded_parse — One Place Where Reading Fails
+
+## Purpose
+The steps of this domain all need a reader (`ast.parse`, `tokenize`) and all
+have the same question when it fails. This is where it is answered, once.
+
+## The Exceptions
+`SyntaxError` (and its `IndentationError`), `ValueError` (a null byte, before
+Python 3.12), `RecursionError` (a source nested deeper than the parser
+goes) and `tokenize.TokenError` (an unterminated string or bracket).
+Anything else is a bug and is not hidden.
+
+## Why `\\r` Is a Failure
+`ast` counts a lone `\\r` as a line end, the offset table does not. A cut by
+such a position would land on a wrong line, so the text is refused. The
+extractor never gives it one: the reader hands over `\\n` only.
+"""

simplibs/extractor_python/logic/_helpers/outline_node.py

@@ -0,0 +1,72 @@
+import ast
+# Inners
+from .collect_definitions import collect_definitions
+from .first_doc_line import first_doc_line
+from .format_type_params import format_type_params
+
+
+def outline_node(
+    node: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef,
+    depth: int,
+    docstrings: bool,
+) -> list[str]:
+    """Write the outline of one class or function, with the definitions inside a class.
+
+    Args:
+        node: The definition.
+        depth: How deeply it is indented.
+        docstrings: Add the first line of the docstring as a comment.
+
+    Returns:
+        The lines of the outline; always valid Python when put together.
+    """
+
+    # 1. The decorators, as written
+    pad = "    " * depth
+    lines = [f"{pad}@{ast.unparse(decorator)}" for decorator in node.decorator_list]
+    note = first_doc_line(node) if docstrings else ""
+    params = format_type_params(node)
+
+    # 2. A function: its signature, no body
+    if not isinstance(node, ast.ClassDef):
+        keyword = "async def" if isinstance(node, ast.AsyncFunctionDef) else "def"
+        returns = f" -> {ast.unparse(node.returns)}" if node.returns else ""
+        lines.append(f"{pad}{keyword} {node.name}{params}({ast.unparse(node.args)}){returns}: ...{note}")
+        return lines
+
+    # 3. A class: its bases and keywords (`metaclass=M`, `**options`)
+    bases = [ast.unparse(base) for base in node.bases]
+    bases += [
+        f"{keyword.arg}={ast.unparse(keyword.value)}" if keyword.arg else f"**{ast.unparse(keyword.value)}"
+        for keyword in node.keywords
+    ]
+    head = f"class {node.name}{params}" + (f"({', '.join(bases)})" if bases else "")
+
+    # 4. The members are listed under it; a class with none is one line
+    members = collect_definitions(node.body)
+    if not members:
+        lines.append(f"{pad}{head}: ...{note}")
+        return lines
+    lines.append(f"{pad}{head}:{note}")
+    for member in members:
+        lines.extend(outline_node(member, depth + 1, docstrings))
+
+    return lines
+
+
+_DESIGN_NOTES = """
+# outline_node — One Definition Without Its Body
+
+## Purpose
+The signature of a class or a function, written by `ast.unparse` from the
+parsed tree, so a default value, an annotation or a keyword is always what the
+source says and never a guess of a regex.
+
+## Always Valid Python
+`def f(...): ...` and `class C: ...` are complete statements; a class with
+members is a header followed by an indented block of members. The docstring
+note is a comment, which cannot break anything.
+
+## The Keywords of a Class
+`class A(B, metaclass=M, **options)`: a keyword without a name is `**options`.
+"""

simplibs/extractor_python/logic/_helpers/parse_python.py

@@ -0,0 +1,32 @@
+import ast
+# Outers
+from ...annotations import OnError
+# Inners
+from .guarded_parse import guarded_parse
+
+
+def parse_python(text: str, on_error: OnError) -> ast.Module | None:
+    """Parse a text as Python source.
+
+    Args:
+        text: The Python source.
+        on_error: `"raise"` for a card, `"keep"` for `None`.
+
+    Returns:
+        The syntax tree; `None` if the text cannot be parsed and `on_error` is `"keep"`.
+
+    Raises:
+        ParamError: The text cannot be parsed and `on_error` is `"raise"`.
+    """
+
+    # 1. `ast.parse` under the guard
+    return guarded_parse(text, on_error, ast.parse)
+
+
+_DESIGN_NOTES = """
+# parse_python — The Tree or Nothing
+
+## Purpose
+The first line of every step that works on the structure of a module.
+`None` means "leave the text alone" and the caller returns it as it was.
+"""

simplibs/extractor_python/logic/_validations/__init__.py

@@ -0,0 +1,14 @@
+from .raise_python_source_error import raise_python_source_error
+
+
+_DESIGN_NOTES = """
+# _validations — Cards of the Python Steps
+
+Private. Plain imports, no `__all__`.
+
+## Internal Components Registry
+
+| Component                    | Type     | Description                                 |
+| :--------------------------- | :------- | :------------------------------------------ |
+| `raise_python_source_error`  | Function | The text cannot be read as Python source.   |
+"""

simplibs/extractor_python/logic/_validations/raise_python_source_error.py

@@ -0,0 +1,55 @@
+from typing import NoReturn
+from simplibs.exception import ParamError
+
+
+def raise_python_source_error(text: str, reason: str, line: int | None = None) -> NoReturn:
+    """Raise ParamError when a text cannot be read as Python source.
+
+    Args:
+        text: The text that could not be read.
+        reason: What is wrong with it, in words.
+        line: The line the problem was found on (1-based), if known.
+    """
+
+    # 1. The line of the problem is the best value to show; without it, the start of the text
+    lines = text.split("\n")
+    if line is not None and 1 <= line <= len(lines):
+        shown = lines[line - 1].strip()
+        where = f" (line {line})"
+    else:
+        shown = text[:80]
+        where = ""
+
+    # 2. Raise structured ParamError wrapping a ValueError
+    raise ParamError(
+        error_name="CODE_EXTRACTOR_PYTHON_SOURCE_INVALID",
+        label="text",
+        value=shown,
+        problem=f"The text is not Python source I can read: {reason}{where}.",
+        expected="Valid Python 3 source with `\\n` line endings.",
+        how_to_fix=(
+            "Fix the file, or leave it out of the extraction (an `exclude` rule of the profile).",
+            "A text with `\\r\\n` or `\\r` endings has to be normalized to `\\n` first.",
+            "Pass on_error='keep' to get such a file back unchanged instead of an error.",
+        ),
+        exception=ValueError,
+    )
+
+
+_DESIGN_NOTES = """
+# raise_python_source_error — A File That Python Cannot Read
+
+## Purpose
+One card for every way a Python step can fail to read its text: a syntax
+error, a bad indentation, an unterminated string, a null character, line
+endings other than `\\n`.
+
+## What the Extractor Does With It
+A step that raises marks THAT file `error` (`transform failed: ...`) and the
+run goes on with the next file; the run is unsuccessful. With
+`on_error="keep"` the step never gets here: the text comes back as it is.
+
+## The Value
+The offending line, not the whole text: a card that prints a whole file is not
+a card anybody reads.
+"""

simplibs/extractor_python/logic/python_comment_spans.py

@@ -0,0 +1,73 @@
+import io
+import tokenize
+from simplibs.text import Span
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import OnError
+# Inners
+from ._helpers import SourceOffsets, guarded_parse
+
+
+def _collect_comment_spans(text: str) -> list[Span]:
+    """Run the tokenizer and turn every comment token into a span."""
+
+    # 1. Every COMMENT token, with the blanks in front of it
+    offsets = SourceOffsets(text)
+    spans: list[Span] = []
+    for token in tokenize.generate_tokens(io.StringIO(text).readline):
+        if token.type != tokenize.COMMENT:
+            continue
+        start = offsets.token_offset(*token.start)
+        end = offsets.token_offset(*token.end)
+        while start > 0 and text[start - 1] in " \t":
+            start -= 1
+        spans.append((start, end))
+
+    return spans
+
+
+@validate_call
+def python_comment_spans(text: str, *, on_error: OnError = "raise") -> list[Span]:
+    """Find the comments of a Python source.
+
+    Args:
+        text: The Python source.
+        on_error: `"raise"` for a card when the text cannot be tokenized, `"keep"` for no spans.
+
+    Returns:
+        A span for each comment, from the blanks in front of it to the end of the comment.
+
+    Raises:
+        ParamError: The text cannot be tokenized and `on_error` is `"raise"`.
+    """
+
+    # 1. Tokenize under the guard
+    spans = guarded_parse(text, on_error, _collect_comment_spans)
+
+    # 2. A text that could not be read has no spans
+    return spans or []
+
+
+_DESIGN_NOTES = """
+# python_comment_spans — Comments Found by the Tokenizer
+
+## Purpose
+Where the comments are, as spans that `remove_spans` can cut.
+
+## Why `tokenize` and Not a Regex
+A `#` inside a string (`"a # b"`, `'''# not a comment'''`) is not a comment, and
+telling the two apart is the tokenizer's job. A regex gets it wrong exactly in
+the files where it matters.
+
+## The Blanks in Front
+`x = 1  # note` is cut to `x = 1`, not to `x = 1  `. A comment on a line of its
+own is taken with its indentation, and the cut removes the empty line.
+
+## Shebang and Coding Line
+`#!/usr/bin/env python` and `# -*- coding: utf-8 -*-` are comments to the
+tokenizer and go with the rest: the output is for reading, not for running.
+
+## What It Needs
+Only that the text can be TOKENIZED, not that it is valid Python 3: a file that
+`ast` refuses may still have its comments found.
+"""

simplibs/extractor_python/logic/python_outline.py

@@ -0,0 +1,63 @@
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import OnError
+# Inners
+from ._helpers import collect_definitions, outline_node, parse_python
+
+
+@validate_call
+def python_outline(text: str, *, docstrings: bool = True, on_error: OnError = "raise") -> str:
+    """Write an overview of a Python source: its classes and functions as signatures without bodies.
+
+    Args:
+        text: The Python source.
+        docstrings: Add the first line of each docstring as a trailing comment.
+        on_error: `"raise"` for a card when the text cannot be parsed, `"keep"` to return it unchanged
+            (the whole file, not an outline).
+
+    Returns:
+        The outline, valid Python; an empty string for a file that defines no class or function.
+
+    Raises:
+        ParamError: The text cannot be parsed and `on_error` is `"raise"`.
+
+    Example:
+        python_outline("@dec\\nclass A(B):\\n    def m(self, a: int = 1) -> str:\\n        return ''\\n")
+        # '@dec\\nclass A(B):\\n    def m(self, a: int=1) -> str: ...\\n'
+    """
+
+    # 1. Parse; a text that could not be read is not touched
+    tree = parse_python(text, on_error)
+    if tree is None:
+        return text
+
+    # 2. The outline of every definition of the module
+    lines: list[str] = []
+    for definition in collect_definitions(tree.body):
+        lines.extend(outline_node(definition, 0, docstrings))
+
+    return "\n".join(lines) + "\n" if lines else ""
+
+
+_DESIGN_NOTES = """
+# python_outline — The Shape of a Module in a Few Lines
+
+## Purpose
+The step of the profile `python/outline`: what a file offers, without how it
+does it. For giving a large project to a reader with a limited window, the
+signatures are most of what is needed to ask the right questions.
+
+## What Is Listed
+Classes and functions of the module, methods and nested classes of a class,
+looking through `if`, `try` and `with` blocks. A function inside a function is
+not listed: it is the function's own detail.
+
+## Written From the Tree
+Signatures come from `ast.unparse`, not from the source text, so spacing and
+line breaks are normalized (`a: int = 1` is written `a: int=1`) and the output
+is always valid Python.
+
+## Docstring as a Comment
+`# First line of the docstring` after the signature. No string means nothing
+to quote, escape or close.
+"""

simplibs/extractor_python/logic/remove_python_comments.py

@@ -0,0 +1,44 @@
+from simplibs.text import remove_spans
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import OnError
+# Inners
+from .python_comment_spans import python_comment_spans
+
+
+@validate_call
+def remove_python_comments(text: str, *, on_error: OnError = "raise") -> str:
+    """Remove all comments from a Python source.
+
+    Args:
+        text: The Python source.
+        on_error: `"raise"` for a card when the text cannot be tokenized, `"keep"` to return it unchanged.
+
+    Returns:
+        The text without comments. A line that held only a comment disappears.
+
+    Raises:
+        ParamError: The text cannot be tokenized and `on_error` is `"raise"`.
+    """
+
+    # 1. Find the comments
+    spans = python_comment_spans(text, on_error=on_error)
+
+    # 2. Cut them; strings that merely contain a `#` were never found
+    return remove_spans(text, spans)
+
+
+_DESIGN_NOTES = """
+# remove_python_comments — The Comments Out, The Code Untouched
+
+## Purpose
+A step of the profiles `python/code-clean` and `python/code-no-imports`.
+
+## Guarantee
+For a text that parses, the syntax tree of the result is the same as the tree
+of the original: nothing but comments (and the blanks and empty lines they
+left) is removed.
+
+## Idempotent
+A second run finds nothing and returns the very same text.
+"""

simplibs/extractor_python/logic/remove_python_decorators.py

@@ -0,0 +1,48 @@
+from simplibs.text import remove_spans
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import OnError
+# Inners
+from ._helpers import SourceOffsets, find_decorator_spans, parse_python
+
+
+@validate_call
+def remove_python_decorators(text: str, *, on_error: OnError = "raise") -> str:
+    """Remove the decorators of every class and function of a Python source.
+
+    Args:
+        text: The Python source.
+        on_error: `"raise"` for a card when the text cannot be parsed, `"keep"` to return it unchanged.
+
+    Returns:
+        The text without decorators; the lines they stood on disappear.
+
+    Raises:
+        ParamError: The text cannot be parsed and `on_error` is `"raise"`.
+    """
+
+    # 1. Parse; a text that could not be read is not touched
+    tree = parse_python(text, on_error)
+    if tree is None:
+        return text
+
+    # 2. Cut each decorator from its `@`
+    return remove_spans(text, find_decorator_spans(text, tree, SourceOffsets(text)))
+
+
+_DESIGN_NOTES = """
+# remove_python_decorators — The Function, Not How It Is Wrapped
+
+## Purpose
+A step of the profile `python/code-no-imports`: `@app.route(...)`,
+`@dataclass`, `@property` describe how a function is registered or wrapped,
+not what it does.
+
+## Guarantee
+For a text that parses, the syntax tree of the result equals the tree of the
+original with every `decorator_list` emptied.
+
+## Semantics Change
+A decorator can change what a function IS (`@property`, `@staticmethod`,
+`@contextmanager`). The output is for reading; it is not meant to be run.
+"""

simplibs/extractor_python/logic/remove_python_docstrings.py

@@ -0,0 +1,50 @@
+from simplibs.validate import validate_call
+# Outers
+from ..annotations import OnError
+# Inners
+from ._helpers import SourceOffsets, cut_statements, find_docstrings, parse_python
+
+
+@validate_call
+def remove_python_docstrings(text: str, *, on_error: OnError = "raise") -> str:
+    """Remove the docstrings of a Python source: of the module, its classes and its functions.
+
+    Args:
+        text: The Python source.
+        on_error: `"raise"` for a card when the text cannot be parsed, `"keep"` to return it unchanged.
+
+    Returns:
+        The text without docstrings. A body left empty gets a `pass`.
+
+    Raises:
+        ParamError: The text cannot be parsed and `on_error` is `"raise"`.
+    """
+
+    # 1. Parse; a text that could not be read is not touched
+    tree = parse_python(text, on_error)
+    if tree is None:
+        return text
+
+    # 2. Cut the docstring statements
+    return cut_statements(text, tree, find_docstrings(tree), SourceOffsets(text))
+
+
+_DESIGN_NOTES = """
+# remove_python_docstrings — Documentation Out, Structure Kept
+
+## Purpose
+A step of the profiles `python/code-clean` and `python/code-no-imports`.
+
+## What Is a Docstring
+The string that is the first statement of a module, a class or a function.
+Other strings, even alone on their line, are left.
+
+## A Body Needs a Statement
+`def f():` with only a docstring is `def f(): pass` afterwards; so is a class.
+A module may be empty.
+
+## Guarantee
+For a text that parses, the syntax tree of the result equals the tree of the
+original with the docstring statements taken out (and a `Pass` where a body
+would be empty).
+"""