simplibs-extractor-python 0.1.0__py3-none-any.whl

simplibs/extractor_python/__init__.py

@@ -0,0 +1,67 @@
+from .annotations import OnError
+from .logic import (
+    python_comment_spans,
+    python_outline,
+    remove_python_comments,
+    remove_python_decorators,
+    remove_python_docstrings,
+    remove_python_imports,
+)
+from .plugin import register_all
+
+
+__all__ = [
+    # steps
+    "python_comment_spans",
+    "python_outline",
+    "remove_python_comments",
+    "remove_python_decorators",
+    "remove_python_docstrings",
+    "remove_python_imports",
+    # add-on of the extractor
+    "register_all",
+    # types
+    "OnError",
+]
+
+
+_DESIGN_NOTES = """
+# simplibs-extractor-python — Package Root
+
+## Purpose
+The add-on that teaches `simplibs-extractor` about Python: file rules for
+Python projects, the steps that clean the source (comments, docstrings,
+imports, decorators), an outline step and five ready profiles. Install it and
+the profiles are there:
+
+    simplibs-extractor python src-clean my_project
+
+## What the Root Exports
+The steps as plain functions (`str -> str`), `register_all` (the target of the
+entry point `simplibs.extractor.plugins`) and the annotation meant for the
+caller's own signatures:
+
+    from simplibs.extractor_python import remove_python_comments, python_outline, OnError
+
+## What the Root Deliberately Does NOT Export
+Four domain folders with their own `__all__`, imported through their domain:
+
+    from simplibs.extractor_python.actions import RemovePythonComments
+    from simplibs.extractor_python.file_rules import PYTHON_BASE, build_python_base
+    from simplibs.extractor_python.presets import PYTHON_CLEAN
+    from simplibs.extractor_python.profiles import PYTHON_SRC_CLEAN
+
+and the internals: `_helpers/` and `_validations/` inside the domains.
+
+## Spans
+The steps find ranges of characters as `simplibs.text.Span`. The type is not
+defined here, it is taken from `simplibs-text`.
+
+## Layering
+`annotations` ← `constants` ← `file_rules` ← `logic` ← `actions` ← `presets` ← `profiles` ← `plugin`.
+Each layer imports only from the layers on its left; the root imports from the ones it exports.
+
+## `__all__`
+Defined here and in the domain folders (`actions`, `file_rules`, `presets`,
+`profiles`) only; the layers inside them use plain imports.
+"""

simplibs/extractor_python/actions/PythonOutline.py

@@ -0,0 +1,23 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import python_outline
+
+
+PythonOutline = create_action(
+    python_outline,
+    main_param="text",
+    class_name="PythonOutline",
+)
+"""Action wrapping `python_outline` — turns the Python text flowing in into signatures without bodies."""
+
+
+_DESIGN_NOTES = """
+# PythonOutline — Action Over `python_outline`
+
+## Purpose
+The overview step as a link of a pipeline: `PythonOutline(docstrings=False)`.
+
+## Shape
+Main input: `text`. Options: `docstrings` (first docstring line as a comment, on by default), `on_error`.
+Text in, text out (a valid Python outline).
+"""

simplibs/extractor_python/actions/RemovePythonComments.py

@@ -0,0 +1,22 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import remove_python_comments
+
+
+RemovePythonComments = create_action(
+    remove_python_comments,
+    main_param="text",
+    class_name="RemovePythonComments",
+)
+"""Action wrapping `remove_python_comments` — takes the comments out of the Python text flowing in."""
+
+
+_DESIGN_NOTES = """
+# RemovePythonComments — Action Over `remove_python_comments`
+
+## Purpose
+`RemovePythonComments() >> RemovePythonDocstrings()`: the comment step as a link of a pipeline, a step of a profile.
+
+## Shape
+Main input: `text`. Option: `on_error` (`"raise"` by default, `"keep"`). Text in, text out.
+"""

simplibs/extractor_python/actions/RemovePythonDecorators.py

@@ -0,0 +1,22 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import remove_python_decorators
+
+
+RemovePythonDecorators = create_action(
+    remove_python_decorators,
+    main_param="text",
+    class_name="RemovePythonDecorators",
+)
+"""Action wrapping `remove_python_decorators` — takes the decorators off the classes and functions of the Python text flowing in."""
+
+
+_DESIGN_NOTES = """
+# RemovePythonDecorators — Action Over `remove_python_decorators`
+
+## Purpose
+The decorator step as a link of a pipeline.
+
+## Shape
+Main input: `text`. Option: `on_error`. Text in, text out.
+"""

simplibs/extractor_python/actions/RemovePythonDocstrings.py

@@ -0,0 +1,22 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import remove_python_docstrings
+
+
+RemovePythonDocstrings = create_action(
+    remove_python_docstrings,
+    main_param="text",
+    class_name="RemovePythonDocstrings",
+)
+"""Action wrapping `remove_python_docstrings` — takes the docstrings out of the Python text flowing in."""
+
+
+_DESIGN_NOTES = """
+# RemovePythonDocstrings — Action Over `remove_python_docstrings`
+
+## Purpose
+The docstring step as a link of a pipeline.
+
+## Shape
+Main input: `text`. Option: `on_error`. Text in, text out; a body left empty gets `pass`.
+"""

simplibs/extractor_python/actions/RemovePythonImports.py

@@ -0,0 +1,22 @@
+from simplibs.actions import create_action
+# Outers
+from ..logic import remove_python_imports
+
+
+RemovePythonImports = create_action(
+    remove_python_imports,
+    main_param="text",
+    class_name="RemovePythonImports",
+)
+"""Action wrapping `remove_python_imports` — takes the import statements out of the Python text flowing in."""
+
+
+_DESIGN_NOTES = """
+# RemovePythonImports — Action Over `remove_python_imports`
+
+## Purpose
+The import step as a link of a pipeline.
+
+## Shape
+Main input: `text`. Option: `on_error`. Text in, text out.
+"""

simplibs/extractor_python/actions/__init__.py

@@ -0,0 +1,45 @@
+from .PythonOutline import PythonOutline
+from .RemovePythonComments import RemovePythonComments
+from .RemovePythonDecorators import RemovePythonDecorators
+from .RemovePythonDocstrings import RemovePythonDocstrings
+from .RemovePythonImports import RemovePythonImports
+
+
+__all__ = [
+    "PythonOutline",
+    "RemovePythonComments",
+    "RemovePythonDecorators",
+    "RemovePythonDocstrings",
+    "RemovePythonImports",
+]
+
+
+_DESIGN_NOTES = """
+# simplibs.extractor_python.actions — The Steps as Pipeline Actions
+
+## Purpose
+Every step of `logic/` as an `Action` of `simplibs-actions`, so it can be
+chained with `>>` and used as the `steps` of an extractor profile.
+
+    from simplibs.extractor_python.actions import RemovePythonComments, RemovePythonImports
+    steps = RemovePythonComments() >> RemovePythonImports()
+
+## Domain Folder
+`actions/` has its own `__all__` and is not re-exported from the package root:
+a function and an action are different kinds of thing and stay visibly
+different at the import line.
+
+## Thin Wrappers
+Each action is `create_action(function, main_param="text", ...)`. The options
+of the function (`on_error`, `docstrings`) are the options of the action.
+
+## Internal Components Registry
+
+| Component                | Type   | Description                                         |
+| :----------------------- | :----- | :-------------------------------------------------- |
+| `PythonOutline`          | Action | Over `python_outline`.                              |
+| `RemovePythonComments`   | Action | Over `remove_python_comments`.                      |
+| `RemovePythonDecorators` | Action | Over `remove_python_decorators`.                    |
+| `RemovePythonDocstrings` | Action | Over `remove_python_docstrings`.                    |
+| `RemovePythonImports`    | Action | Over `remove_python_imports`.                       |
+"""

simplibs/extractor_python/annotations/OnError.py

@@ -0,0 +1,25 @@
+from typing import Literal
+
+
+OnError = Literal["raise", "keep"]
+"""What a Python step does with a source it cannot read: raise a card, or return the text unchanged."""
+
+
+_DESIGN_NOTES = """
+# OnError — What to Do With a Source That Is Not Valid
+
+## Purpose
+The type of the `on_error` option of the Python steps: a file that is not valid
+Python (an old Python 2 file, a template with `{{ }}` in it, a half-written
+module) cannot be read by `ast` or `tokenize`.
+
+## The Two Answers
+* `"raise"` — the step raises a card. The extractor marks that file `error`,
+  names the reason in the report and goes on with the next file. The default:
+  nothing is hidden.
+* `"keep"` — the text comes back unchanged and nothing is said. For your own
+  profile, when "not cleaned" is better than "not there".
+
+## Why a Literal
+A typo (`"Raise"`, `"ignore"`) fails at the signature, not deep inside a step.
+"""

simplibs/extractor_python/annotations/__init__.py

@@ -0,0 +1,18 @@
+from .OnError import OnError
+
+
+_DESIGN_NOTES = """
+# annotations — Shared Types of simplibs-extractor-python
+
+Plain imports, no `__all__` (the package root decides what is public).
+
+## Internal Components Registry
+
+| Component | Type    | Description                                                             |
+| :-------- | :------ | :---------------------------------------------------------------------- |
+| `OnError` | Literal | `"raise"` or `"keep"`: what a Python step does with an invalid source.  |
+
+## Not Here
+`Span` (a half-open range of characters) is a type of `simplibs-text`; the
+steps import it from there.
+"""

simplibs/extractor_python/constants/PYTHON_CONFIG_NAMES.py

@@ -0,0 +1,27 @@
+from typing import Final
+
+
+PYTHON_CONFIG_NAMES: Final[tuple[str, ...]] = (
+    "setup.py",
+    "setup.cfg",
+    "pyproject.toml",
+    "Pipfile",
+    "Pipfile.lock",
+    "requirements.txt",
+    "tox.ini",
+    "pytest.ini",
+)
+"""File names of Python project configuration."""
+
+
+_DESIGN_NOTES = """
+# PYTHON_CONFIG_NAMES — Packaging, Not Logic
+
+## Purpose
+Used by `PYTHON_CONFIG`. These files describe how the project is built and
+tested; they are not what it does.
+
+## Case
+Names are compared exactly. `Pipfile` is written with a capital letter by the
+tool that owns it.
+"""

simplibs/extractor_python/constants/PYTHON_OUTPUT_DIR_NAMES.py

@@ -0,0 +1,19 @@
+from typing import Final
+
+
+PYTHON_OUTPUT_DIR_NAMES: Final[tuple[str, ...]] = ("build", "dist", "out")
+"""Names of folders that hold generated build output of a Python project."""
+
+
+_DESIGN_NOTES = """
+# PYTHON_OUTPUT_DIR_NAMES — Generated, Not Written
+
+## Purpose
+The names behind `OUTPUT_DIRS`. Build output repeats the source in another
+shape; reading it twice only costs tokens.
+
+## Own List
+The library keeps its own list instead of borrowing one: what counts as
+build output differs from language to language, and each add-on decides it
+for its own language.
+"""

simplibs/extractor_python/constants/PYTHON_SUFFIXES.py

@@ -0,0 +1,17 @@
+from typing import Final
+
+
+PYTHON_SUFFIXES: Final[tuple[str, ...]] = (".py",)
+"""The suffixes of Python source files."""
+
+
+_DESIGN_NOTES = """
+# PYTHON_SUFFIXES — What Counts as Python Source
+
+## Purpose
+One place for the suffixes the Python rules and profiles look for.
+
+## Only `.py`
+Stub files (`.pyi`) and Windows scripts (`.pyw`) are not source to read: a
+stub only repeats the signatures of a module that is in the output anyway.
+"""

simplibs/extractor_python/constants/PYTHON_TEST_DIR_NAMES.py

@@ -0,0 +1,18 @@
+from typing import Final
+
+
+PYTHON_TEST_DIR_NAMES: Final[tuple[str, ...]] = ("tests", "test")
+"""Names of folders that hold Python tests."""
+
+
+_DESIGN_NOTES = """
+# PYTHON_TEST_DIR_NAMES — Where Python Tests Live
+
+## Purpose
+Used by `PYTHON_TESTS`: every file in a folder with one of these names is a
+test, whatever the file is called.
+
+## Whole Names Only
+A folder is compared as a whole (`tests`), never as a part of a name, so
+`contest` or `latest` are not tests.
+"""

simplibs/extractor_python/constants/PYTHON_VENV_DIR_NAMES.py

@@ -0,0 +1,18 @@
+from typing import Final
+
+
+PYTHON_VENV_DIR_NAMES: Final[tuple[str, ...]] = ("venv", ".venv", "env", "virtualenv")
+"""Names under which a Python virtual environment usually lives."""
+
+
+_DESIGN_NOTES = """
+# PYTHON_VENV_DIR_NAMES — Where Installed Packages Hide
+
+## Purpose
+The names behind `VENV_DIRS`. A virtual environment holds thousands of files
+that are not the project.
+
+## Known Limit
+A folder called `env` that is something else (a settings package, say) is
+excluded by the strict rules. The base rules do not look at these names.
+"""

simplibs/extractor_python/constants/__init__.py

@@ -0,0 +1,26 @@
+from .PYTHON_CONFIG_NAMES import PYTHON_CONFIG_NAMES
+from .PYTHON_OUTPUT_DIR_NAMES import PYTHON_OUTPUT_DIR_NAMES
+from .PYTHON_SUFFIXES import PYTHON_SUFFIXES
+from .PYTHON_TEST_DIR_NAMES import PYTHON_TEST_DIR_NAMES
+from .PYTHON_VENV_DIR_NAMES import PYTHON_VENV_DIR_NAMES
+
+
+_DESIGN_NOTES = """
+# constants — Fixed Values of simplibs-extractor-python
+
+Plain imports, no `__all__` (the package root decides what is public).
+
+## Internal Components Registry
+
+| Component                 | Type     | Description                                      |
+| :------------------------ | :------- | :----------------------------------------------- |
+| `PYTHON_CONFIG_NAMES`     | Constant | File names of Python project configuration.      |
+| `PYTHON_OUTPUT_DIR_NAMES` | Constant | Folders of generated build output.               |
+| `PYTHON_SUFFIXES`         | Constant | Suffixes of Python files.                        |
+| `PYTHON_TEST_DIR_NAMES`   | Constant | Folders that hold Python tests.                  |
+| `PYTHON_VENV_DIR_NAMES`   | Constant | Folders of Python virtual environments.          |
+
+## Layering & Design
+- Every value is a tuple of plain strings: the rules are built from them in `file_rules`.
+- The names of version-control folders are not here: they come from `IGNORE_VCS` of `simplibs-path`.
+"""

simplibs/extractor_python/file_rules/__init__.py

@@ -0,0 +1,62 @@
+from .atomic import (
+    DOT_NAME,
+    INIT_NAME,
+    IN_DUNDER_DIR,
+    MAIN_NAME,
+    OUTPUT_DIRS,
+    VCS_DIRS,
+    VENV_DIRS,
+)
+from .python import (
+    PYTHON_BASE,
+    PYTHON_CONFIG,
+    PYTHON_STRICT_EXCLUDE,
+    PYTHON_TESTS,
+    build_python_base,
+)
+
+
+__all__ = [
+    # atomic
+    "DOT_NAME",
+    "INIT_NAME",
+    "IN_DUNDER_DIR",
+    "MAIN_NAME",
+    "OUTPUT_DIRS",
+    "VCS_DIRS",
+    "VENV_DIRS",
+    # python
+    "PYTHON_BASE",
+    "PYTHON_CONFIG",
+    "PYTHON_STRICT_EXCLUDE",
+    "PYTHON_TESTS",
+    "build_python_base",
+]
+
+
+_DESIGN_NOTES = """
+# simplibs.extractor_python.file_rules — Which Files Are Worth Reading
+
+## Purpose
+Rules on the path of a file, relative to the source folder. The profiles use
+them as `include` and `exclude`; they compose with `&`, `|` and `~`, so a
+program can build its own selection from them:
+
+    from simplibs.extractor_python.file_rules import PYTHON_BASE, PYTHON_TESTS
+    only_tests = PYTHON_BASE & PYTHON_TESTS
+
+## Domain Folder
+`file_rules/` has its own `__all__` and is not re-exported from the package root.
+
+## Internal Components Registry
+
+| Component | Type    | Description                                                   |
+| :-------- | :------ | :------------------------------------------------------------ |
+| `atomic`  | Package | The shared pieces: dot names, init, main, VCS folders...      |
+| `python`  | Package | Rules for Python projects.                                    |
+
+## Layering & Design
+- `atomic` ← `python`.
+- Every rule is built once, when the package is imported.
+- A rule about the folders above a file is `AnyParentNameIs` of `simplibs-path`; a set of names is `IsIn` of `simplibs-rules`.
+"""

simplibs/extractor_python/file_rules/atomic/DOT_NAME.py

@@ -0,0 +1,19 @@
+from typing import Final
+from simplibs.path.rules import NameIs
+from simplibs.rules import Rule, StartsWith
+
+
+DOT_NAME: Final[Rule] = NameIs(StartsWith("."))
+"""The file name starts with a dot (`.env`, `.hidden.py`)."""
+
+
+_DESIGN_NOTES = """
+# DOT_NAME — Hidden by Convention
+
+## Purpose
+Dot files are settings and tool state, not source. Used by both base rules.
+
+## Name Only
+Only the name of the file is judged. A visible file inside a `.cache` folder
+is not hidden by this rule (the strict exclusions look at folders).
+"""

simplibs/extractor_python/file_rules/atomic/INIT_NAME.py

@@ -0,0 +1,17 @@
+from typing import Final
+from simplibs.path.rules import NameIs
+from simplibs.rules import Equals, Rule
+
+
+INIT_NAME: Final[Rule] = NameIs(Equals("__init__.py"))
+"""The file is called `__init__.py`."""
+
+
+_DESIGN_NOTES = """
+# INIT_NAME — The Face of a Package
+
+## Purpose
+`__init__.py` holds the public names and `__all__` of a package, so it is
+usually the best single file for understanding one. `build_python_base()`
+takes it unless `include_init=False`.
+"""

simplibs/extractor_python/file_rules/atomic/IN_DUNDER_DIR.py

@@ -0,0 +1,16 @@
+from typing import Final
+from simplibs.path.rules import AnyParentNameIs
+from simplibs.rules import Rule, StartsWith
+
+
+IN_DUNDER_DIR: Final[Rule] = AnyParentNameIs(StartsWith("__"))
+"""Some folder above the file has a name that starts with two underscores (`__pycache__`)."""
+
+
+_DESIGN_NOTES = """
+# IN_DUNDER_DIR — Folders Python Makes by Itself
+
+## Purpose
+`__pycache__` and `__pypackages__` hold what the interpreter produced, never
+source. Used by `build_python_base()`.
+"""

simplibs/extractor_python/file_rules/atomic/MAIN_NAME.py

@@ -0,0 +1,16 @@
+from typing import Final
+from simplibs.path.rules import NameIs
+from simplibs.rules import Equals, Rule
+
+
+MAIN_NAME: Final[Rule] = NameIs(Equals("__main__.py"))
+"""The file is called `__main__.py`."""
+
+
+_DESIGN_NOTES = """
+# MAIN_NAME — What Runs With `python -m`
+
+## Purpose
+The entry point of a package. `build_python_base()` takes it unless
+`include_main=False`.
+"""

simplibs/extractor_python/file_rules/atomic/OUTPUT_DIRS.py

@@ -0,0 +1,17 @@
+from typing import Final
+from simplibs.path.rules import AnyParentNameIs
+from simplibs.rules import IsIn, Rule
+# Outers
+from ...constants import PYTHON_OUTPUT_DIR_NAMES
+
+
+OUTPUT_DIRS: Final[Rule] = AnyParentNameIs(IsIn(PYTHON_OUTPUT_DIR_NAMES))
+"""The file lies inside a build-output folder."""
+
+
+_DESIGN_NOTES = """
+# OUTPUT_DIRS — Generated Files
+
+## Purpose
+Part of the strict Python exclusion. See `PYTHON_OUTPUT_DIR_NAMES`.
+"""

simplibs/extractor_python/file_rules/atomic/VCS_DIRS.py

@@ -0,0 +1,26 @@
+from typing import Final
+from simplibs.path import IGNORE_VCS
+from simplibs.path.rules import AnyParentNameIs
+from simplibs.rules import IsIn, Rule
+
+
+VCS_DIRS: Final[Rule] = AnyParentNameIs(IsIn(IGNORE_VCS))
+"""The file lies inside a version-control folder (`.git`, `.svn`, `.hg`, `.bzr`)."""
+
+
+_DESIGN_NOTES = """
+# VCS_DIRS — Not the Project's Own Files
+
+## Purpose
+Part of the strict Python exclusion. The names are `IGNORE_VCS` of
+`simplibs-path`: version control is the same for every language, so the
+library borrows the list instead of keeping its own.
+
+## What Changed From the Old Code
+The old list `VCS_DIR_NAMES` held also `node_modules` and `tmp`. Neither is
+version control: `node_modules` belongs to the JavaScript add-on, and `tmp`
+is dropped.
+
+## Whole Names Only
+`IsIn` compares a folder name as a whole, so `.github` is not `.git`.
+"""

simplibs/extractor_python/file_rules/atomic/VENV_DIRS.py

@@ -0,0 +1,18 @@
+from typing import Final
+from simplibs.path.rules import AnyParentNameIs
+from simplibs.rules import IsIn, Rule
+# Outers
+from ...constants import PYTHON_VENV_DIR_NAMES
+
+
+VENV_DIRS: Final[Rule] = AnyParentNameIs(IsIn(PYTHON_VENV_DIR_NAMES))
+"""The file lies inside a Python virtual environment folder."""
+
+
+_DESIGN_NOTES = """
+# VENV_DIRS — The Interpreter's Own Packages
+
+## Purpose
+Part of the strict Python exclusion. See `PYTHON_VENV_DIR_NAMES` for the names
+and the known limit.
+"""

simplibs/extractor_python/file_rules/atomic/__init__.py

@@ -0,0 +1,30 @@
+from .DOT_NAME import DOT_NAME
+from .INIT_NAME import INIT_NAME
+from .IN_DUNDER_DIR import IN_DUNDER_DIR
+from .MAIN_NAME import MAIN_NAME
+from .OUTPUT_DIRS import OUTPUT_DIRS
+from .VCS_DIRS import VCS_DIRS
+from .VENV_DIRS import VENV_DIRS
+
+
+_DESIGN_NOTES = """
+# atomic — The Shared Pieces of the File Rules
+
+Plain imports, no `__all__` (the package root decides what is public).
+
+## Internal Components Registry
+
+| Component       | Type | Description                                                  |
+| :-------------- | :--- | :----------------------------------------------------------- |
+| `DOT_NAME`      | Rule | The name starts with a dot.                                  |
+| `INIT_NAME`     | Rule | The name is `__init__.py`.                                   |
+| `IN_DUNDER_DIR` | Rule | A folder above the file starts with two underscores.         |
+| `MAIN_NAME`     | Rule | The name is `__main__.py`.                                   |
+| `OUTPUT_DIRS`   | Rule | A folder above the file is a build-output folder.            |
+| `VCS_DIRS`      | Rule | A folder above the file is a version-control folder.         |
+| `VENV_DIRS`     | Rule | A folder above the file is a virtual environment.            |
+
+## Layering & Design
+- Every piece is one rule of one idea; the `python` folder only combines them.
+- The rules about folders use `AnyParentNameIs` of `simplibs-path` and `IsIn` of `simplibs-rules`.
+"""

simplibs/extractor_python/file_rules/python/PYTHON_BASE.py

@@ -0,0 +1,17 @@
+from typing import Final
+from simplibs.rules import Rule
+# Inners
+from .build_python_base import build_python_base
+
+
+PYTHON_BASE: Final[Rule] = build_python_base()
+"""The default rule for Python files worth reading: `__init__.py` and `__main__.py` included."""
+
+
+_DESIGN_NOTES = """
+# PYTHON_BASE — The Default Choice of Python Files
+
+## Purpose
+`build_python_base()` with its defaults, built once. Every Python profile of
+this library uses it as `include`.
+"""