FTL-Extract 0.9.0__py3-none-win_amd64.whl

ftl_extract/__init__.py

@@ -0,0 +1,4 @@
+from ftl_extract.__version__ import __version__
+from ftl_extract.code_extractor import extract_fluent_keys
+
+__all__ = ("__version__", "extract_fluent_keys")

ftl_extract/__version__.py

@@ -0,0 +1,3 @@
+import importlib.metadata
+
+__version__ = importlib.metadata.version(__package__)

ftl_extract/cli.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+from pathlib import Path
+from time import perf_counter_ns
+from typing import Literal
+
+import click
+
+from ftl_extract.const import (
+    COMMENT_KEYS_MODE,
+    DEFAULT_EXCLUDE_DIRS,
+    DEFAULT_FTL_FILE,
+    DEFAULT_I18N_KEYS,
+    DEFAULT_IGNORE_ATTRIBUTES,
+    DEFAULT_IGNORE_KWARGS,
+)
+from ftl_extract.ftl_extractor import extract
+from ftl_extract.stub.generator import generate_stubs
+
+
+@click.group("ftl")
+@click.version_option()
+def ftl() -> None: ...
+
+
+@ftl.command("extract")
+@click.argument("code_path", type=click.Path(exists=True, path_type=Path))
+@click.argument("output_path", type=click.Path(path_type=Path))
+@click.option(
+    "--language",
+    "-l",
+    multiple=True,
+    default=("en",),
+    show_default=True,
+    help="Language of translation.",
+)
+@click.option(
+    "--i18n-keys",
+    "-k",
+    default=DEFAULT_I18N_KEYS,
+    multiple=True,
+    show_default=True,
+    help="Names of function that is used to get translation.",
+)
+@click.option(
+    "--i18n-keys-append",
+    "-K",
+    default=(),
+    multiple=True,
+    help="Append names of function that is used to get translation.",
+)
+@click.option(
+    "--i18n-keys-prefix",
+    "-p",
+    default=(),
+    multiple=True,
+    help="Prefix names of function that is used to get translation. `self.i18n.*()`",
+)
+@click.option(
+    "--exclude-dirs",
+    "-e",
+    multiple=True,
+    default=DEFAULT_EXCLUDE_DIRS,
+    show_default=True,
+    help="Exclude directories.",
+)
+@click.option(
+    "--exclude-dirs-append",
+    "-E",
+    default=(),
+    multiple=True,
+    help="Append directories to exclude.",
+)
+@click.option(
+    "--ignore-attributes",
+    "-i",
+    default=DEFAULT_IGNORE_ATTRIBUTES,
+    multiple=True,
+    show_default=True,
+    help="Ignore attributes, like `i18n.set_locale`.",
+)
+@click.option(
+    "--append-ignore-attributes",
+    "-I",
+    multiple=True,
+    help="Append attributes to ignore.",
+)
+@click.option(
+    "--ignore-kwargs",
+    default=DEFAULT_IGNORE_KWARGS,
+    multiple=True,
+    show_default=True,
+    help="Ignore kwargs, like `when` from `aiogram_dialog.I18nFormat(..., when=...)`.",
+)
+@click.option(
+    "--comment-junks",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="Comments Junk elements.",
+)
+@click.option(
+    "--default-ftl-file",
+    default=DEFAULT_FTL_FILE,
+    show_default=True,
+    type=click.Path(path_type=Path),
+)
+@click.option(
+    "--comment-keys-mode",
+    default=COMMENT_KEYS_MODE[0],
+    show_default=True,
+    help="Comment keys mode.",
+    type=click.Choice(COMMENT_KEYS_MODE, case_sensitive=False),
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="Do not write to output files.",
+)
+@click.option(
+    "--verbose",
+    "-v",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="Verbose output.",
+)
+def cli_extract(
+    code_path: Path,
+    output_path: Path,
+    language: tuple[str, ...],
+    i18n_keys: tuple[str, ...],
+    i18n_keys_append: tuple[str, ...],
+    i18n_keys_prefix: tuple[str, ...],
+    exclude_dirs: tuple[str, ...],
+    exclude_dirs_append: tuple[str, ...],
+    ignore_attributes: tuple[str, ...],
+    append_ignore_attributes: tuple[str, ...],
+    ignore_kwargs: tuple[str, ...],
+    comment_junks: bool,
+    default_ftl_file: Path,
+    comment_keys_mode: Literal["comment", "warn"],
+    dry_run: bool,
+    verbose: bool,
+) -> None:
+    click.echo(f"Extracting from {code_path}")
+    start_time = perf_counter_ns()
+
+    statistics = extract(
+        code_path=code_path,
+        output_path=output_path,
+        language=language,
+        i18n_keys=i18n_keys,
+        i18n_keys_append=i18n_keys_append,
+        i18n_keys_prefix=i18n_keys_prefix,
+        exclude_dirs=exclude_dirs,
+        exclude_dirs_append=exclude_dirs_append,
+        ignore_attributes=ignore_attributes,
+        append_ignore_attributes=append_ignore_attributes,
+        ignore_kwargs=ignore_kwargs,
+        comment_junks=comment_junks,
+        default_ftl_file=default_ftl_file,
+        comment_keys_mode=comment_keys_mode,
+        dry_run=dry_run,
+    )
+
+    if verbose:
+        click.echo("Extraction statistics:")
+        click.echo(f"  - Py files count: {statistics.py_files_count}")
+        click.echo(f"  - FTL files count: {statistics.ftl_files_count}")
+        click.echo(f"  - FTL keys in code: {statistics.ftl_in_code_keys_count}")
+        click.echo(f"  - FTL keys stored: {statistics.ftl_stored_keys_count}")
+        click.echo(f"  - FTL keys updated: {statistics.ftl_keys_updated}")
+        click.echo(f"  - FTL keys added: {statistics.ftl_keys_added}")
+        click.echo(f"  - FTL keys commented: {statistics.ftl_keys_commented}")
+
+    click.echo(f"[Python] Done in {(perf_counter_ns() - start_time) * 1e-9:.3f}s.")
+
+
+@ftl.command("stub")
+@click.argument("locale_path", type=click.Path(exists=True, path_type=Path))
+@click.argument("output_path", type=click.Path(path_type=Path))
+@click.option(
+    "--export-tree",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="Export tree structure of FTL messages.",
+)
+def cli_stub(locale_path: Path, output_path: Path, export_tree: bool) -> None:
+    click.echo(f"Generating stubs from {locale_path}")
+    start_time = perf_counter_ns()
+
+    generate_stubs(locale_path, output_path, export_tree)
+
+    click.echo(f"[Python] Done in {(perf_counter_ns() - start_time) * 1e-9:.3f}s.")

ftl_extract/code_extractor.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+from typing import TYPE_CHECKING, cast
+
+from fluent.syntax import ast as fluent_ast
+
+from ftl_extract.exceptions import (
+    FTLExtractorDifferentPathsError,
+    FTLExtractorDifferentTranslationError,
+)
+from ftl_extract.matcher import FluentKey, I18nMatcher
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+
+    from ftl_extract.matcher import FluentKey
+    from ftl_extract.utils import ExtractionStatistics
+
+
+def find_py_files(*, search_path: Path, exclude_dirs: frozenset[Path]) -> Iterator[Path]:
+    """
+    First step: find all .py files in given path.
+
+    :param search_path: Path to directory with .py files.
+    :type search_path: Path
+    :param exclude_dirs: Exclude directories from search.
+    :type exclude_dirs: frozenset[Path]
+    :return: Iterator with Path to .py files.
+    :rtype: Iterator[Path]
+    """
+    if search_path.is_dir():
+        for path in search_path.rglob("[!{.}]*.py"):
+            if path.is_file() and not any(
+                path.is_relative_to(exclude_dir) for exclude_dir in exclude_dirs
+            ):
+                yield path
+    else:
+        # If search_path is not a directory, check if it is a file
+        if search_path.is_file() and search_path.suffix == ".py":
+            yield search_path
+        else:
+            # If search_path is not a directory or file, return an empty iterator
+            yield from iter([])
+
+
+def parse_file(
+    *,
+    path: Path,
+    i18n_keys: Iterable[str],
+    i18n_keys_prefix: Iterable[str],
+    ignore_attributes: Iterable[str],
+    ignore_kwargs: Iterable[str],
+    default_ftl_file: Path,
+) -> dict[str, FluentKey]:
+    """
+    Second step: parse given .py file and find all i18n calls.
+
+    :param path: Path to .py file.
+    :type path: Path
+    :param i18n_keys: Names of function that is used to get translation.
+    :type i18n_keys: Iterable[str]
+    :param i18n_keys_prefix: Prefix names of function that is used to get translation.
+    :type i18n_keys_prefix: Iterable[str]
+    :param ignore_attributes: Ignore attributes, like `i18n.set_locale`.
+    :type ignore_attributes: Iterable[str]
+    :param ignore_kwargs: Ignore kwargs, like `when` from
+    `aiogram_dialog.I18nFormat(..., when=...)`.
+    :type ignore_kwargs: Iterable[str]
+    :param default_ftl_file: Default name of FTL file.
+    :type default_ftl_file: Path
+    :return: Dict with `key` and `FluentKey`.
+    :rtype: dict[str, FluentKey]
+    """
+    node = ast.parse(path.read_bytes())
+    matcher = I18nMatcher(
+        code_path=path,
+        default_ftl_file=default_ftl_file,
+        i18n_keys=i18n_keys,
+        i18n_keys_prefix=i18n_keys_prefix,
+        ignore_attributes=ignore_attributes,
+        ignore_kwargs=ignore_kwargs,
+    )
+    matcher.visit(node)
+    return matcher.fluent_keys
+
+
+def post_process_fluent_keys(*, fluent_keys: dict[str, FluentKey], default_ftl_file: Path) -> None:
+    """
+    Third step: post-process parsed `FluentKey`.
+
+    :param fluent_keys: Dict with `key` and `FluentKey` that will be post-processed.
+    :type fluent_keys: dict[str, FluentKey]
+    :param default_ftl_file: Default name of FTL file.
+    :type default_ftl_file: Path
+    """
+    for fluent_key in fluent_keys.values():
+        if not isinstance(fluent_key.path, Path):
+            fluent_key.path = Path(fluent_key.path)
+
+        if not fluent_key.path.suffix:  # if path looks like directory (no suffix)
+            fluent_key.path /= default_ftl_file
+
+
+def find_conflicts(
+    *,
+    current_fluent_keys: dict[str, FluentKey],
+    new_fluent_keys: dict[str, FluentKey],
+) -> None:
+    """
+    Fourth step: find conflicts between current and new `FluentKey`s.
+
+    If conflict is found, raise `ValueError`.
+
+    Conflict is when `key` is the same, but `path` or `kwargs` are different.
+    """
+    # Find common keys
+    conflict_keys = set(current_fluent_keys.keys()) & set(new_fluent_keys.keys())
+
+    if not conflict_keys:
+        return
+
+    for key in conflict_keys:
+        if current_fluent_keys[key].path != new_fluent_keys[key].path:
+            raise FTLExtractorDifferentPathsError(
+                key,
+                current_fluent_keys[key].path,
+                new_fluent_keys[key].path,
+            )
+
+        if not current_fluent_keys[key].translation.equals(new_fluent_keys[key].translation):
+            raise FTLExtractorDifferentTranslationError(
+                key,
+                cast(fluent_ast.Message, current_fluent_keys[key].translation),
+                cast(fluent_ast.Message, new_fluent_keys[key].translation),
+            )
+
+
+def extract_fluent_keys(
+    *,
+    path: Path,
+    i18n_keys: Iterable[str],
+    i18n_keys_prefix: Iterable[str],
+    exclude_dirs: frozenset[Path],
+    ignore_attributes: Iterable[str],
+    ignore_kwargs: Iterable[str],
+    default_ftl_file: Path,
+    statistics: ExtractionStatistics | None = None,
+) -> dict[str, FluentKey]:
+    """
+    Extract all `FluentKey`s from given path.
+
+    :param path: Path to [.py file] / [directory with .py files].
+    :type path: Path
+    :param i18n_keys: Names of function that is used to get translation.
+    :type i18n_keys: Iterable[str]
+    :param i18n_keys_prefix: Prefix names of function that is used to get translation.
+    :type i18n_keys_prefix: Iterable[str]
+    :param exclude_dirs: Exclude directories from search.
+    :type exclude_dirs: frozenset[Path]
+    :param ignore_attributes: Ignore attributes, like `i18n.set_locale`.
+    :type ignore_attributes: Iterable[str]
+    :param ignore_kwargs: Ignore kwargs, like `when` from
+    `aiogram_dialog.I18nFormat(..., when=...)`.
+    :type ignore_kwargs: Iterable[str]
+    :param default_ftl_file: Default name of FTL file.
+    :type default_ftl_file: Path
+    :param statistics: Statistics of extraction.
+    :type statistics: ExtractionStatistics
+    :return: Dict with `key` and `FluentKey`.
+    :rtype: dict[str, FluentKey]
+
+    """
+    fluent_keys: dict[str, FluentKey] = {}
+
+    for file in find_py_files(search_path=path, exclude_dirs=exclude_dirs):
+        keys = parse_file(
+            path=file,
+            i18n_keys=i18n_keys,
+            i18n_keys_prefix=i18n_keys_prefix,
+            ignore_attributes=ignore_attributes,
+            ignore_kwargs=ignore_kwargs,
+            default_ftl_file=default_ftl_file,
+        )
+        post_process_fluent_keys(fluent_keys=keys, default_ftl_file=default_ftl_file)
+        find_conflicts(current_fluent_keys=fluent_keys, new_fluent_keys=keys)
+        fluent_keys.update(keys)
+
+        if statistics and keys:
+            statistics.py_files_count += 1
+
+    return fluent_keys
+
+
+def sort_fluent_keys_by_path(*, fluent_keys: dict[str, FluentKey]) -> dict[Path, list[FluentKey]]:
+    """
+    Sort `FluentKey`s by their paths.
+
+    :param fluent_keys: Dict with `key` and `FluentKey`.
+    :type fluent_keys: dict[str, FluentKey]
+    :return: Dict with `Path` and list of `FluentKey`.
+    :rtype: dict[Path, list[FluentKey]]
+    """
+    sorted_fluent_keys: dict[Path, list[FluentKey]] = {}
+    for fluent_key in fluent_keys.values():
+        sorted_fluent_keys.setdefault(fluent_key.path, []).append(fluent_key)
+
+    return sorted_fluent_keys

ftl_extract/const.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Final
+
+I18N_LITERAL: Final[str] = "i18n"
+GET_LITERAL: Final[str] = "get"
+PATH_LITERAL: Final[str] = "_path"
+DEFAULT_I18N_KEYS: Final[tuple[str, ...]] = (I18N_LITERAL, "L", "LazyProxy", "LazyFilter")
+DEFAULT_IGNORE_ATTRIBUTES: Final[tuple[str, ...]] = (
+    "set_locale",
+    "use_locale",
+    "use_context",
+    "set_context",
+)
+DEFAULT_IGNORE_KWARGS: Final[tuple[str, ...]] = ()
+DEFAULT_FTL_FILE: Final[Path] = Path("_default.ftl")
+FTL_DEBUG_VAR_NAME: Final[str] = "FTL_DEBUG"
+COMMENT_KEYS_MODE: Final[tuple[str, ...]] = ("comment", "warn")
+DEFAULT_EXCLUDE_DIRS: Final[tuple[str, ...]] = (
+    ".venv",
+    "venv",
+    ".git",
+    "__pycache__",
+    ".pytest_cache",
+)

ftl_extract/exceptions.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from os import environ
+from pprint import pformat
+from typing import TYPE_CHECKING
+
+from ftl_extract.const import FTL_DEBUG_VAR_NAME
+from ftl_extract.utils import to_json_no_span
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from fluent.syntax import ast
+
+
+class FTLExtractorError(Exception):
+    pass
+
+
+class FTLExtractorDifferentPathsError(FTLExtractorError):
+    def __init__(self, key: str, current_path: Path, new_path: Path) -> None:
+        self.current_path = current_path
+        self.new_path = new_path
+        super().__init__(
+            f"Key {key!r} already exists with different path: "
+            f"{self.current_path} != {self.new_path}",
+        )
+
+
+class FTLExtractorDifferentTranslationError(FTLExtractorError):
+    def __init__(
+        self,
+        key: str,
+        current_translation: ast.Message,
+        new_translation: ast.Message,
+    ) -> None:
+        self.current_translation = current_translation
+        self.new_translation = new_translation
+
+        if bool(environ.get(FTL_DEBUG_VAR_NAME, "")) is True:
+            super().__init__(
+                f"Translation {key!r} already exists with different elements:\n"
+                f"current_translation: "
+                f"{pformat(self.current_translation.to_json(fn=to_json_no_span))}\n!= "
+                f"new_translation: {pformat(self.new_translation.to_json(fn=to_json_no_span))}",
+            )
+        else:
+            super().__init__(
+                f"Translation {key!r} already exists with different elements: "
+                f"{self.current_translation} != {self.new_translation}",
+            )
+
+
+class FTLExtractorCantFindReferenceError(FTLExtractorError):
+    def __init__(self, key: str, key_path: Path, reference_key: str) -> None:
+        self.key = key
+        self.key_path = key_path
+        self.reference_key = reference_key
+        super().__init__(f"Can't find reference {reference_key!r} for key {key!r} at {key_path}")
+
+
+class FTLExtractorCantFindTermError(FTLExtractorError):
+    def __init__(self, key: str, locale: str, key_path: Path, term_key: str) -> None:
+        self.key = key
+        self.locale = locale
+        self.key_path = key_path
+        self.term_key = term_key
+        super().__init__(
+            f"Can't find term {term_key!r} for key {key!r} with locale {locale!r} at: {key_path}",
+        )