simplibs-extractor 0.1.0__py3-none-any.whl

simplibs/extractor/__init__.py

@@ -0,0 +1,121 @@
+# Running an extraction
+from .logic import extract, extract_many
+from .scripts import extract_here, generate_local_script
+
+# Profiles and formats
+from .registry import (
+    expand_profile_keys,
+    get_profile,
+    get_renderer,
+    list_formats,
+    list_profiles,
+    list_systems,
+    register_profile,
+    register_renderer,
+)
+from .plugins import load_plugins
+
+# Data objects
+from .definitions import Entry, ExtractionConfig, ExtractionResult, FileReport, Profile
+from .logic import RenderContext
+
+# Reporting
+from .logic import estimate_tokens, format_summary
+
+# Annotations and constants
+from .annotations import FileStatus, OutputFormat, OverwritePolicy, StepsSpec
+from .constants import DEFAULT_PROFILE_KEY, GENERATED_MARKER
+
+
+__all__ = [
+    # Running an extraction
+    "extract",
+    "extract_many",
+    "extract_here",
+    "generate_local_script",
+    # Profiles and formats
+    "expand_profile_keys",
+    "get_profile",
+    "get_renderer",
+    "list_formats",
+    "list_profiles",
+    "list_systems",
+    "register_profile",
+    "register_renderer",
+    "load_plugins",
+    # Data objects
+    "Entry",
+    "ExtractionConfig",
+    "ExtractionResult",
+    "FileReport",
+    "Profile",
+    "RenderContext",
+    # Reporting
+    "estimate_tokens",
+    "format_summary",
+    # Annotations and constants
+    "FileStatus",
+    "OutputFormat",
+    "OverwritePolicy",
+    "StepsSpec",
+    "DEFAULT_PROFILE_KEY",
+    "GENERATED_MARKER",
+]
+
+
+_DESIGN_NOTES = """
+# simplibs.extractor — Universal File Extractor
+
+Gathers the text files of a folder into one document: select, read,
+transform, render, write, report. The core knows no language; add-ons bring
+profiles (`simplibs-code-extractor` is the first).
+
+## Layers (each one looks only to the left)
+
+`annotations` <- `constants` <- `exceptions` <- `definitions` <- `registry`
+<- `logic` <- `plugins` <- `scripts` <- `cli`
+
+## The Public Surface
+
+Everything a program needs is in this module. The command line is the one
+exception: `from simplibs.extractor.cli import run_cli`. It is not imported
+here, so a program that only calls `extract()` does not load the prompt and
+logger libraries it would never use.
+
+## Internal Components Registry
+
+| Component                | Type      | Description                                                    |
+| :----------------------- | :-------- | :------------------------------------------------------------- |
+| `extract`                | Function  | One extraction through all six phases.                         |
+| `extract_many`           | Function  | Several extractions; a failed one does not stop the rest.      |
+| `extract_here`           | Function  | Extracts the folder of a script into a file next to it.        |
+| `generate_local_script`  | Function  | Writes a ready-to-run local extraction script.                 |
+| `register_profile`       | Function  | Adds a profile to the registry.                                |
+| `get_profile`            | Function  | Returns a profile by its `system/name` key.                    |
+| `list_profiles`          | Function  | Lists the registered profiles.                                 |
+| `list_systems`           | Function  | Lists the systems that have a profile.                         |
+| `expand_profile_keys`    | Function  | Turns typed names into full profile keys.                      |
+| `register_renderer`      | Function  | Adds an output format.                                         |
+| `get_renderer`           | Function  | Returns the renderer of a format.                              |
+| `list_formats`           | Function  | Lists the registered output formats.                           |
+| `load_plugins`           | Function  | Lets every installed add-on register its profiles.             |
+| `Profile`                | Dataclass | A named set of selection rules and transformation steps.       |
+| `ExtractionConfig`       | Dataclass | Everything one run needs to know.                              |
+| `ExtractionResult`       | Dataclass | The outcome of one run; the source of every report.            |
+| `FileReport`             | Dataclass | What happened to one file.                                     |
+| `Entry`                  | Dataclass | One file ready to be rendered.                                 |
+| `RenderContext`          | Dataclass | What a renderer knows besides the entries.                     |
+| `format_summary`         | Function  | Readable text from an `ExtractionResult`.                      |
+| `estimate_tokens`        | Function  | A rough token count of a text.                                 |
+| `FileStatus`             | Type      | `ok`, `skipped` or `error`.                                    |
+| `OutputFormat`           | Type      | The name of an output format.                                  |
+| `OverwritePolicy`        | Type      | `overwrite`, `unique`, `backup` or `error`.                    |
+| `StepsSpec`              | Type      | One action, a suffix -> action mapping, or nothing.            |
+| `DEFAULT_PROFILE_KEY`    | Constant  | The built-in profile `generic/all`.                            |
+| `GENERATED_MARKER`       | Constant  | The text that marks a file as an output of this library.       |
+
+## Importing Has Two Side Effects
+The built-in profile `generic/all` and the formats `markdown` and `plain` are
+registered. Nothing else happens until a function is called; add-ons are only
+looked up by `load_plugins()` (the command line calls it).
+"""

simplibs/extractor/__main__.py

@@ -0,0 +1,12 @@
+from .cli import run_cli
+
+
+_DESIGN_NOTES = """
+# __main__ — `python -m simplibs.extractor`
+
+The same command as the console script `simplibs-extractor`, for
+environments where the script's folder is not on the path.
+"""
+
+if __name__ == "__main__":
+    raise SystemExit(run_cli())

simplibs/extractor/annotations/FileStatus.py

@@ -0,0 +1,16 @@
+from typing import Literal
+
+
+FileStatus = Literal["ok", "skipped", "error"]
+"""The outcome of one file: included in the output, left out on purpose, or failed."""
+
+
+_DESIGN_NOTES = """
+# FileStatus — Three Outcomes, Nothing Else
+
+## Purpose
+`skipped` is not a failure (binary file, too large, generated by this
+library) and `error` is not a stop (the run continues). Keeping the two apart
+is what lets the report and the exit code tell "nothing wrong" from "something
+needs a look".
+"""

simplibs/extractor/annotations/OutputFormat.py

@@ -0,0 +1,25 @@
+from typing import Annotated
+# Inners
+from ._helpers import is_output_format
+
+
+OutputFormat = Annotated[str, is_output_format]
+"""The name of an output format (`"markdown"`, `"plain"`, or a registered one).
+
+A lowercase name that starts with a letter. Whether a renderer with that
+name exists is checked when the configuration is resolved.
+"""
+
+
+_DESIGN_NOTES = """
+# OutputFormat — a Name, Not a Closed List
+
+## Purpose
+Renderers are registered by name, so the set of formats is open. A `Literal`
+would freeze it at the two built-in formats and make every added renderer a
+typing error.
+
+## Why `Annotated`
+An editor sees a plain `str`; `@validate_call` reads the predicate from the
+metadata and rejects `"Markdown"`, `""` or `"md file"` at the signature.
+"""

simplibs/extractor/annotations/OverwritePolicy.py

@@ -0,0 +1,24 @@
+from typing import Literal
+
+
+OverwritePolicy = Literal["overwrite", "unique", "backup", "error"]
+"""What happens when the output file already exists.
+
+`"overwrite"` replaces it atomically, `"unique"` writes `name (1).ext` next to
+it, `"backup"` keeps the old file as a backup and then replaces it, `"error"`
+refuses to write.
+"""
+
+
+_DESIGN_NOTES = """
+# OverwritePolicy — Four Answers to "It Already Exists"
+
+## Purpose
+The old code knew only `"error"`. The ecosystem already has the building
+blocks for the other three (`SafeFileWriter`, `make_unique_path`,
+`backup_file`), so the policy is a closed, readable choice.
+
+## Default
+`"overwrite"` — an extractor is re-run constantly and its output is
+generated, never hand-edited. The write itself is atomic.
+"""

simplibs/extractor/annotations/StepsSpec.py

@@ -0,0 +1,26 @@
+from collections.abc import Mapping
+from typing import TypeAlias
+from simplibs.actions import Action
+
+
+StepsSpec: TypeAlias = Action | Mapping[str, Action] | None
+"""How a profile transforms file text.
+
+`None` keeps the text as it is. One `Action` is applied to every file. A
+mapping `suffix -> Action` applies the action of the file's own suffix
+(`".py"`); the key `"*"` is the fallback for every other file.
+"""
+
+
+_DESIGN_NOTES = """
+# StepsSpec — One Pipeline or One Pipeline per Language
+
+## Purpose
+A step receives only the text, never the path, so it cannot know which
+language it is cleaning. A profile that cleans Python and JavaScript
+differently therefore needs a mapping from suffix to action.
+
+## Why Not Always a Mapping
+Most profiles apply one action to everything; forcing `{"*": action}` on them
+would be noise.
+"""

simplibs/extractor/annotations/__init__.py

@@ -0,0 +1,26 @@
+from .FileStatus import FileStatus
+from .OutputFormat import OutputFormat
+from .OverwritePolicy import OverwritePolicy
+from .StepsSpec import StepsSpec
+
+
+_DESIGN_NOTES = """
+# annotations — Shared Types of simplibs-extractor
+
+Plain imports, no `__all__` (the package root decides what is public).
+`PathLike` and `ByteCount` are not redefined here — they come from
+`simplibs.path` and `simplibs.file`.
+
+## Internal Components Registry
+
+| Component         | Type            | Description                                             |
+| :---------------- | :-------------- | :------------------------------------------------------ |
+| `FileStatus`      | Literal Type    | Outcome of one file: `ok`, `skipped`, `error`.          |
+| `OutputFormat`    | Annotated Type  | The name of an output format.                           |
+| `OverwritePolicy` | Literal Type    | What to do when the output exists.                      |
+| `StepsSpec`       | Union Type      | One action, a suffix -> action mapping, or nothing.     |
+
+## Changed in Batch 2
+`is_output_format` is no longer re-exported: `Profile` is now checked by
+`@validate_dataclass`, which reads the `OutputFormat` annotation itself.
+"""

simplibs/extractor/annotations/_helpers/__init__.py

@@ -0,0 +1,15 @@
+from .is_output_format import is_output_format
+
+
+_DESIGN_NOTES = """
+# annotations/_helpers — Private Helpers of the Annotations
+
+Plain imports only. Used by the annotations of this folder, not exported
+from the package root.
+
+## Internal Components Registry
+
+| Component          | Type     | Description                                              |
+| :----------------- | :------- | :------------------------------------------------------- |
+| `is_output_format` | Function | Validates whether a value has the shape of a format name. |
+"""

simplibs/extractor/annotations/_helpers/is_output_format.py

@@ -0,0 +1,36 @@
+import re
+
+
+_FORMAT_NAME = re.compile(r"[a-z][a-z0-9_-]*")
+
+
+def is_output_format(name: str) -> bool:
+    """Check whether a value has the shape of an output format name.
+
+    Args:
+        name: The candidate (`"markdown"`, `"plain"`, `"json-manifest"`).
+
+    Returns:
+        True for a lowercase name made of letters, digits, `_` and `-` that
+        starts with a letter; False for anything else, including non-strings.
+    """
+
+    # 1. Only a string can name a format
+    if not isinstance(name, str):
+        return False
+
+    # 2. Lowercase word characters only — no spaces, no path separators
+    return _FORMAT_NAME.fullmatch(name) is not None
+
+
+_DESIGN_NOTES = """
+# is_output_format — "Does This Look Like a Format Name?"
+
+## Purpose
+The single rule behind the `OutputFormat` annotation.
+
+## Shape, Not Existence
+Whether a renderer with that name is registered is known only to the
+registry, a layer further to the right. This helper checks the shape; the
+registry check happens when the configuration is resolved.
+"""

simplibs/extractor/cli/__init__.py

@@ -0,0 +1,32 @@
+from .build_configs import build_configs
+from .build_parser import build_parser
+from .collect_dry_run import collect_dry_run
+from .collect_profile_keys import collect_profile_keys
+from .collect_source import collect_source
+from .handle_list_options import handle_list_options
+from .normalize_positionals import normalize_positionals
+from .run_cli import run_cli
+from .run_init import run_init
+
+
+_DESIGN_NOTES = """
+# cli — The Command Line of simplibs-extractor
+
+Plain imports, no `__all__`. `run_cli` is the console-script entry point
+(`simplibs-extractor = simplibs.extractor.cli:run_cli`); it returns the exit
+code, which a console script passes to `sys.exit`.
+
+## Internal Components Registry
+
+| Component               | Type     | Description                                              |
+| :---------------------- | :------- | :------------------------------------------------------- |
+| `build_configs`         | Function | One config per profile key.                              |
+| `build_parser`          | Function | The argument parser.                                     |
+| `collect_dry_run`       | Function | The dry-run flag, asked for only inside the wizard.      |
+| `collect_profile_keys`  | Function | Profile keys from the command line or from questions.    |
+| `collect_source`        | Function | The source folder from the command line or a question.   |
+| `handle_list_options`   | Function | Answers `--list-systems` and `--list-profiles`.          |
+| `normalize_positionals` | Function | Sorts the three positional words into their roles.       |
+| `run_cli`               | Function | The command; returns the exit code.                      |
+| `run_init`              | Function | The `init` subcommand.                                   |
+"""

simplibs/extractor/cli/_helpers/__init__.py

@@ -0,0 +1,22 @@
+from .describe_result import describe_result
+from .is_profile_reference import is_profile_reference
+from .print_error import print_error
+from .profile_options import profile_options
+from .system_options import system_options
+
+
+_DESIGN_NOTES = """
+# cli/_helpers — Private Helpers of the Command Line
+
+Plain imports only.
+
+## Internal Components Registry
+
+| Component              | Type     | Description                                              |
+| :--------------------- | :------- | :------------------------------------------------------- |
+| `describe_result`      | Function | One line saying why an extraction failed.                |
+| `is_profile_reference` | Function | Whether a word is a registered system or profile keys.   |
+| `print_error`          | Function | Prints an error card to standard error.                  |
+| `profile_options`      | Function | `{name: description}` of one system's profiles.          |
+| `system_options`       | Function | `{system: "N profile(s)"}` of every registered system.   |
+"""

simplibs/extractor/cli/_helpers/describe_result.py

@@ -0,0 +1,34 @@
+# Outers
+from ...definitions import ExtractionResult
+
+
+def describe_result(result: ExtractionResult) -> str:
+    """Describe a failed extraction in one line for the batch report.
+
+    Args:
+        result: A result that did not succeed.
+
+    Returns:
+        `"<profile key>: <why>"`: the fatal reason, the number of failed
+        files, or a note that nothing was selected.
+    """
+
+    # 1. The whole run stopped
+    if result.fatal:
+        return f"{result.profile_key}: {result.fatal}"
+
+    # 2. Some files failed
+    if result.error_files:
+        return f"{result.profile_key}: {len(result.error_files)} file(s) failed"
+
+    # 3. Nothing failed, so nothing was selected
+    return f"{result.profile_key}: no file was selected"
+
+
+_DESIGN_NOTES = """
+# describe_result — Why a Run Counts as Failed
+
+## Purpose
+`ExtractionResult.success` is False for three different reasons. The report
+at the end of a batch should say which.
+"""

simplibs/extractor/cli/_helpers/is_profile_reference.py

@@ -0,0 +1,34 @@
+# Outers
+from ...registry import list_profiles, list_systems
+
+
+def is_profile_reference(value: str) -> bool:
+    """Check whether a command-line word names a registered system or registered profile keys.
+
+    Args:
+        value: The word (`"python"`, `"generic/all"`, `"python/a,python/b"`, `"./src"`).
+
+    Returns:
+        True if it is a system, or a comma list made only of registered full keys.
+    """
+
+    # 1. A system
+    if value in list_systems():
+        return True
+
+    # 2. Full keys, one or several; every part must be registered
+    keys = {profile.key for profile in list_profiles()}
+    return all(part.strip() in keys for part in value.split(","))
+
+
+_DESIGN_NOTES = """
+# is_profile_reference — Profile or Folder?
+
+## Purpose
+`simplibs-extractor src` could mean "the system `src`" or "the folder
+`src`". A word that is a registered system or key is a profile reference;
+anything else may be a folder.
+
+## Why the Registry Decides
+Not the shape of the word: `generic/all` and `./src` both contain a slash.
+"""

simplibs/extractor/cli/_helpers/print_error.py

@@ -0,0 +1,25 @@
+import sys
+
+
+def print_error(error: BaseException) -> None:
+    """Print an error card to standard error.
+
+    Args:
+        error: The exception; a `simplibs` card prints as its framed block.
+    """
+
+    # 1. Diagnostics go to standard error, so they never mix with a piped result
+    print(error, file=sys.stderr)
+
+
+_DESIGN_NOTES = """
+# print_error — One Way to Show a Refused Command
+
+## Purpose
+`run_cli` and `run_init` end a refused command the same way. The card is
+printed as it is: it already says what, why and how to fix it.
+
+## Not the Logger
+A logger prefixes the first line of a multi-line card with time and level,
+which breaks its frame.
+"""

simplibs/extractor/cli/_helpers/profile_options.py

@@ -0,0 +1,25 @@
+# Outers
+from ...registry import list_profiles
+
+
+def profile_options(system: str) -> dict[str, str]:
+    """Describe the profiles of one system for a list or a choice.
+
+    Args:
+        system: The system (`"python"`).
+
+    Returns:
+        `{profile name: description}`, sorted by name; empty for an unknown system.
+    """
+
+    # 1. The names are the system's own (`code-clean`), without the `system/` part
+    return {profile.name: profile.description for profile in list_profiles(system)}
+
+
+_DESIGN_NOTES = """
+# profile_options — Names and Descriptions of One System
+
+## Purpose
+Feeds `--list-profiles` and the question "Which profiles?". An empty result
+is how the caller learns that a system does not exist.
+"""

simplibs/extractor/cli/_helpers/system_options.py

@@ -0,0 +1,22 @@
+# Outers
+from ...registry import list_profiles, list_systems
+
+
+def system_options() -> dict[str, str]:
+    """Describe the registered systems for a list or a choice.
+
+    Returns:
+        `{system: "N profile(s)"}`, sorted by system.
+    """
+
+    # 1. One entry per system, with the number of its profiles
+    return {system: f"{len(list_profiles(system))} profile(s)" for system in list_systems()}
+
+
+_DESIGN_NOTES = """
+# system_options — What `--list-systems` and the First Question Show
+
+## Purpose
+One function for both, so a list and a question never disagree about which
+systems exist.
+"""

simplibs/extractor/cli/_validations/__init__.py

@@ -0,0 +1,22 @@
+from .raise_duplicate_output_error import raise_duplicate_output_error
+from .raise_list_unknown_system_error import raise_list_unknown_system_error
+from .raise_output_is_file_error import raise_output_is_file_error
+from .raise_too_many_positionals_error import raise_too_many_positionals_error
+from .raise_unknown_system_error import raise_unknown_system_error
+
+
+_DESIGN_NOTES = """
+# cli/_validations — Private Diagnostics of the Command Line
+
+Plain imports only. Every function raises one `ParamError` and returns nothing.
+
+## Internal Components Registry
+
+| Component                         | Type     | Description                                                   |
+| :-------------------------------- | :------- | :------------------------------------------------------------ |
+| `raise_duplicate_output_error`    | Function | Two profiles would write the same output file.                |
+| `raise_list_unknown_system_error` | Function | `--list-profiles` names a system without profiles.            |
+| `raise_output_is_file_error`      | Function | `-o` points to a file, not a folder.                          |
+| `raise_too_many_positionals_error`| Function | Full profile keys were followed by two more words.            |
+| `raise_unknown_system_error`      | Function | The system named (or chosen) has no registered profile.       |
+"""

simplibs/extractor/cli/_validations/raise_duplicate_output_error.py

@@ -0,0 +1,32 @@
+from collections.abc import Sequence
+from typing import NoReturn
+from simplibs.exception import ParamError
+
+
+def raise_duplicate_output_error(keys: Sequence[str]) -> NoReturn:
+    """Raise ParamError when two or more profiles would write the same output file.
+
+    Args:
+        keys: The profile keys of the run.
+    """
+
+    # 1. Raise structured ParamError wrapping a ValueError
+    raise ParamError(
+        error_name="EXTRACTOR_CONFIG_INVALID",
+        label="profiles",
+        value=list(keys),
+        problem="Two or more of these profiles write the same output file.",
+        expected="Profiles with different output names, or one profile per run.",
+        how_to_fix="Run the profiles separately, or give them different output_name values.",
+        exception=ValueError,
+    )
+
+
+_DESIGN_NOTES = """
+# raise_duplicate_output_error — Two Profiles, One File
+
+## Purpose
+The profiles of one run would overwrite each other. Found before anything
+runs, it costs nothing; found by `extract_many` it would cost the files the
+earlier profiles had already written.
+"""

simplibs/extractor/cli/_validations/raise_list_unknown_system_error.py

@@ -0,0 +1,34 @@
+from typing import NoReturn
+from simplibs.exception import ParamError
+# Outers
+from .._helpers import system_options
+
+
+def raise_list_unknown_system_error(system: str) -> NoReturn:
+    """Raise ParamError when `--list-profiles` names a system without profiles.
+
+    Args:
+        system: The system that was asked for.
+    """
+
+    # 1. Raise structured ParamError wrapping a ValueError
+    raise ParamError(
+        error_name="EXTRACTOR_CONFIG_INVALID",
+        label="option '--list-profiles'",
+        value=system,
+        problem=f"No profile is registered for the system {system!r}.",
+        expected=f"One of the registered systems: {', '.join(system_options()) or '(none)'}.",
+        how_to_fix="Run --list-systems to see what is installed.",
+        exception=ValueError,
+    )
+
+
+_DESIGN_NOTES = """
+# raise_list_unknown_system_error — A Typo Is Not an Empty List
+
+## Purpose
+An empty list would look like "no profiles exist" and hide the typo.
+
+## Close Relative of `raise_unknown_system_error`
+Same problem, other option: the label and the advice differ.
+"""

simplibs/extractor/cli/_validations/raise_output_is_file_error.py

@@ -0,0 +1,30 @@
+from pathlib import Path
+from typing import NoReturn
+from simplibs.exception import ParamError
+
+
+def raise_output_is_file_error(out_dir: Path) -> NoReturn:
+    """Raise ParamError when the output folder given with `-o` is a file.
+
+    Args:
+        out_dir: The path that was meant to be a folder.
+    """
+
+    # 1. Raise structured ParamError wrapping a ValueError
+    raise ParamError(
+        error_name="EXTRACTOR_CONFIG_INVALID",
+        label="option '-o'",
+        value=str(out_dir),
+        problem=f"'{out_dir}' is a file, but -o takes a folder.",
+        expected="A folder (existing or to be created).",
+        how_to_fix="Pass a folder; the file name comes from the profile.",
+        exception=ValueError,
+    )
+
+
+_DESIGN_NOTES = """
+# raise_output_is_file_error — `-o` Takes a Folder
+
+## Purpose
+The file name always comes from the profile, so `-o` can only be a folder.
+"""