lingva 5.0.7__py3-none-any.whl

lingva/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+__version__ = version("lingva")

lingva/extract.py

@@ -0,0 +1,492 @@
+import os
+import re
+import sys
+import tempfile
+from collections import OrderedDict
+from configparser import ConfigParser as SafeConfigParser
+from datetime import datetime
+from operator import attrgetter
+
+import click
+import polib
+
+from lingva import __version__
+from lingva.extractors import EXTENSIONS, EXTRACTORS, get_extractor, register_extractors
+from lingva.extractors.babel import register_babel_plugins
+
+
+def po_timestamp():
+    now = datetime.now().astimezone()
+    return f"{now:%Y-%m-%d %H:%M%z}"
+
+
+def _same_text(a, b):
+    a = re.sub(r"\s+", " ", a)
+    b = re.sub(r"\s+", " ", b)
+    return a == b
+
+
+class POEntry(polib.POEntry):
+    def __init__(self, *a, **kw):
+        polib.POEntry.__init__(self, *a, **kw)
+        self._comments = []
+        self._tcomments = []
+
+    @property
+    def comment(self):
+        return "\n".join(self._comments)
+
+    @comment.setter
+    def comment(self, value):
+        pass
+
+    @property
+    def tcomment(self):
+        return "\n".join(self._tcomments)
+
+    @tcomment.setter
+    def tcomment(self, value):
+        pass
+
+    def __eq__(self, other):
+        r = super().__eq__(other)
+        if not r:
+            return False
+        return _same_text(other.comment, self.comment) and _same_text(
+            other.tcomment, self.tcomment
+        )
+
+    def update(self, message, add_occurrences=True):
+        if add_occurrences:
+            self.occurrences.append((message.location[0], str(message.location[1])))
+        self.flags.extend(f for f in message.flags if f not in self.flags)
+        if message.comment not in self._comments:
+            self._comments.append(message.comment)
+        if message.tcomment not in self._tcomments:
+            self._tcomments.append(message.tcomment)
+
+
+class POFile(polib.POFile):
+    copyright_holder = None
+    package_name = None
+
+    def metadata_as_entry(self):
+        entry = polib.POFile.metadata_as_entry(self)
+        year = datetime.now().year
+        header = ["SOME DESCRIPTIVE TITLE"]
+        if self.copyright_holder:
+            header.append(f"Copyright (C) {year} {self.copyright_holder}")
+        header.append(
+            f"This file is distributed under the same license as the {self.package_name} package."
+        )
+        header.append(f"FIRST AUTHOR <EMAIL@ADDRESS>, {year}.")
+        entry.tcomment = "\n".join(header)
+        return entry
+
+
+def no_duplicates(iterator):
+    seen = set()
+    for item in iterator:
+        if item in seen:
+            continue
+        seen.add(item)
+        yield item
+
+
+def list_files(files_from, sources):
+    if files_from:
+        for filename in files_from:
+            if filename.startswith("#") or not filename.strip():
+                continue
+            yield filename.rstrip()
+    for file in sources:
+        if os.path.isfile(file):
+            yield file
+        elif os.path.isdir(file):
+            for dirpath, dirnames, filenames in os.walk(file):
+                for file in filenames:
+                    if get_extractor(file) is not None:
+                        yield os.path.join(dirpath, file)
+        else:
+            click.echo(f"Invalid file type for {file}", err=True)
+            sys.exit(1)
+
+
+def find_file(filename, search_path=None):
+    """Return the filename for a given file, checking search paths."""
+    for path in (os.path.curdir, *(search_path or ())):
+        filename = os.path.join(path, filename)
+        if os.path.isfile(filename):
+            return filename
+    return None
+
+
+def strip_linenumbers(entry):
+    seen = set()
+    occurrences = []
+    for location, line in entry.occurrences:
+        if location in seen:
+            continue
+        occurrences.append((location, ""))
+        seen.add(location)
+    entry.occurrences = occurrences
+
+
+def create_catalog(width, copyright_holder, package_name, package_version, msgid_bugs_address):
+    catalog = POFile(wrapwidth=width)
+    catalog.copyright_holder = copyright_holder
+    catalog.package_name = package_name
+    catalog.metadata_is_fuzzy = True
+    catalog.metadata = OrderedDict()
+    catalog.metadata["Project-Id-Version"] = " ".join(
+        filter(None, [package_name, package_version])
+    )
+    if msgid_bugs_address:
+        catalog.metadata["Report-Msgid-Bugs-To"] = msgid_bugs_address
+    po_time = po_timestamp()
+    catalog.metadata["POT-Creation-Date"] = po_time
+    catalog.metadata["PO-Revision-Date"] = po_time
+    catalog.metadata["Last-Translator"] = "FULL NAME <EMAIL@ADDRESS>"
+    catalog.metadata["Language-Team"] = "LANGUAGE <LL@li.org>"
+    catalog.metadata["Language"] = "LANGUAGE"
+    catalog.metadata["MIME-Version"] = "1.0"
+    catalog.metadata["Content-Type"] = "text/plain; charset=UTF-8"
+    catalog.metadata["Content-Transfer-Encoding"] = "8bit"
+    catalog.metadata["Generated-By"] = f"Lingva {__version__}"
+    return catalog
+
+
+def _register_extension(extension, extractor):
+    if extractor not in EXTRACTORS:
+        click.echo(
+            f"Unknown extractor {extractor}. Check --list-extractors for available options",
+            err=True,
+        )
+        sys.exit(1)
+    EXTENSIONS[extension] = extractor
+
+
+def read_config(cfg_file):
+    config = SafeConfigParser()
+    config.read_file(cfg_file)
+    for section in config.sections():
+        if section == "extensions":
+            for extension, extractor in config.items(section):
+                _register_extension(extension, extractor)
+        elif section.startswith("extractor:"):
+            extractor = section[10:]
+            if extractor not in EXTRACTORS:
+                click.echo(
+                    f"Unknown extractor {extractor}. "
+                    "Check --list-extractors for available options",
+                    err=True,
+                )
+                sys.exit(1)
+            extractor_config = dict(config.items(section))
+            EXTRACTORS[extractor].update_config(**extractor_config)
+        elif section.startswith("extension"):
+            click.echo(
+                f'Use of {section} section is obsolete. Please use the "extensions" section.',
+                err=True,
+            )
+            extension = section[10:]
+            plugin = config.get(section, "plugin")
+            if not plugin:
+                click.echo(f"No plugin defined for extension {extension}", err=True)
+            _register_extension(extension, plugin)
+
+
+def _summarise(catalog):
+    summary = {}
+    for entry in catalog:
+        if entry.obsolete:
+            continue
+        summary[(entry.msgid, entry.msgctxt)] = entry
+    return summary
+
+
+def identical(a, b):
+    """Check if two catalogs are identical, ignoring metadata."""
+    a = _summarise(a)
+    b = _summarise(b)
+    return a == b
+
+
+def _location_sort_key(msg):
+    locations = [(fn, int(line)) for (fn, line) in msg.occurrences]
+    locations.sort()  # Sort so first occurence is always used.
+    return locations
+
+
+class ExtractorOptions:
+    def __init__(self, comment_tag, domain, keywords):
+        self.comment_tag = comment_tag
+        self.domain = domain
+        self.keywords = keywords
+
+
+def extract(
+    cfg_file=None,
+    files_from=None,
+    directory=None,
+    sources=None,
+    list_extractors=None,
+    quiet=False,
+    output="messages.pot",
+    location=True,
+    linenumbers=True,
+    width=79,
+    sort_order=None,
+    allow_empty=False,
+    domain=None,
+    keywords=None,
+    comment_tag=None,
+    copyright_holder=None,
+    package_name="PACKAGE",
+    package_version="1.0",
+    msgid_bugs_address=None,
+):
+    """Extract translatable strings."""
+    register_extractors()
+    register_babel_plugins()
+
+    if comment_tag is None:
+        comment_tag = True
+    if list_extractors:
+        for extractor in sorted(EXTRACTORS):
+            click.echo(f"{extractor:<17} {EXTRACTORS[extractor].__doc__ or ''}")
+        return
+
+    if cfg_file:
+        read_config(cfg_file)
+    else:
+        user_home = os.path.expanduser("~")
+        global_config = os.path.join(user_home, ".config", "lingva")
+        if os.path.exists(global_config):
+            read_config(open(global_config))
+
+    catalog = create_catalog(
+        width, copyright_holder, package_name, package_version, msgid_bugs_address
+    )
+
+    scanned = 0
+    if directory and not isinstance(directory, list):
+        directory = list(directory)
+    for filename in no_duplicates(list_files(files_from, sources)):
+        real_filename = find_file(filename, directory)
+        if real_filename is None:
+            click.echo(f"Can not find file {filename}", err=True)
+            sys.exit(1)
+        extractor = get_extractor(real_filename)
+        if extractor is None:
+            click.echo(f"No extractor available for file {filename}", err=True)
+            sys.exit(1)
+
+        extractor_options = ExtractorOptions(
+            comment_tag=comment_tag,
+            domain=domain,
+            keywords=keywords,
+        )
+        for message in extractor(real_filename, extractor_options):
+            entry = catalog.find(message.msgid, msgctxt=message.msgctxt)
+            if entry is None:
+                entry = POEntry(msgctxt=message.msgctxt, msgid=message.msgid)
+                if message.msgid_plural:
+                    entry.msgid_plural = message.msgid_plural
+                    entry.msgstr_plural[0] = ""
+                    entry.msgstr_plural[1] = ""
+                catalog.append(entry)
+            entry.update(message, add_occurrences=location)
+        scanned += 1
+    if not scanned:
+        click.echo("No files scanned, aborting", err=True)
+        sys.exit(1)
+    if not catalog and not allow_empty:
+        click.echo("No translatable strings found, aborting", err=True)
+        sys.exit(2)
+
+    if sort_order == "msgid":
+        catalog.sort(key=attrgetter("msgid"))
+    elif sort_order == "location":
+        catalog.sort(key=_location_sort_key)
+
+    if not linenumbers:
+        for entry in catalog:
+            strip_linenumbers(entry)
+
+    if os.path.exists(output):
+        old_catalog: POFile | None = None
+        try:
+            old_catalog = polib.pofile(output)
+        except (OSError, UnicodeDecodeError):
+            pass
+        if old_catalog is not None and identical(catalog, old_catalog):
+            if not quiet:
+                click.echo(f"No changes found - not replacing {output}")
+            return
+        os.unlink(output)
+    fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(output), text=True)
+    with open(fd, "w", encoding=catalog.encoding) as f:
+        f.write(catalog.__unicode__())
+    os.rename(tmpfile, output)
+
+
+@click.command()
+@click.option(
+    "-c",
+    "--config",
+    "cfg_file",
+    metavar="CONFIG",
+    help="Read configuration from CONFIG file",
+    type=click.File(),
+)
+# Input options
+@click.option(
+    "-f",
+    "--files-from",
+    metavar="FILE",
+    type=click.File(),
+    help="Get list of files to process from FILE",
+)
+@click.option(
+    "-D",
+    "--directory",
+    metavar="DIRECTORY",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+    multiple=True,
+    help="Add DIRECTORY to list of paths to check for input files",
+)
+@click.argument("sources", nargs=-1, type=click.Path(exists=True))
+@click.option("--list-extractors", is_flag=True, help="List all known extraction plugins")
+@click.option(
+    "-q",
+    "--quiet",
+    "quiet",
+    default=False,
+    is_flag=True,
+    help="Show error messages only",
+)
+# Output options
+@click.option(
+    "-o",
+    "--output",
+    metavar="FILE",
+    type=click.Path(exists=False, dir_okay=False, writable=True),
+    default="messages.pot",
+    help="Filename for generated POT file",
+)
+@click.option(
+    "--add-location/--no-location",
+    "location",
+    default=True,
+    help="Include location information",
+)
+@click.option(
+    "--linenumbers/--no-linenumbers",
+    default=True,
+    help="Include line numbers in location information",
+)
+@click.option("-w", "--width", metavar="NUMBER", default=79, help="Output width")
+@click.option(
+    "-s",
+    "--sort-output",
+    "sort_order",  # babel compatibility
+    flag_value="msgid",
+    help="Order messages by their msgid",
+)
+@click.option(
+    "-F",
+    "--sort-by-file",
+    "sort_order",
+    flag_value="location",
+    help="Order messages by file location",
+)
+@click.option(
+    "--allow-empty/--no-allow-empty",
+    "allow_empty",
+    default=False,
+    help="Allow output file with no msg entries",
+)
+# Extraction configuration
+@click.option("-d", "--domain", help="Domain to extract")
+@click.option(
+    "-k",
+    "--keyword",
+    "keywords",
+    metavar="WORD",
+    multiple=True,
+    help="Look for WORD as additional keyword",
+)
+@click.option(
+    "-C",
+    "--add-comments",
+    "comment_tag",
+    metavar="TAG",
+    help="Add comments prefixed by TAG to messages, or all if no tag is given",
+)
+# POT metadata
+@click.option(
+    "--copyright-holder",
+    metavar="STRING",
+    help="Specifies the copyright holder for the texts",
+)
+@click.option(
+    "--package-name",
+    metavar="NAME",
+    default="PACKAGE",
+    help="Package name to use in the generated POT file",
+)
+@click.option(
+    "--package-version",
+    metavar="Version",
+    default="1.0",
+    help="Package version to use in the generated POT file",
+)
+@click.option("--msgid-bugs-address", metavar="EMAIL", help="Email address bugs should be send to")
+def main(
+    cfg_file,
+    files_from,
+    directory,
+    sources,
+    list_extractors,
+    quiet,
+    output,
+    location,
+    linenumbers,
+    width,
+    sort_order,
+    allow_empty,
+    domain,
+    keywords,
+    comment_tag,
+    copyright_holder,
+    package_name,
+    package_version,
+    msgid_bugs_address,
+):
+    """Main entrypoint."""
+    extract(
+        cfg_file,
+        files_from,
+        directory,
+        sources,
+        list_extractors,
+        quiet,
+        output,
+        location,
+        linenumbers,
+        width,
+        sort_order,
+        allow_empty,
+        domain,
+        keywords,
+        comment_tag,
+        copyright_holder,
+        package_name,
+        package_version,
+        msgid_bugs_address,
+    )
+
+
+if __name__ == "__main__":
+    main()

lingva/extractors/__init__.py

@@ -0,0 +1,178 @@
+import abc
+import collections
+import os
+import re
+import sys
+from importlib.metadata import entry_points
+
+from .compat import add_metaclass
+
+Message = collections.namedtuple(
+    "Message", "msgctxt msgid msgid_plural flags comment tcomment location"
+)
+
+EXTRACTORS = {}
+EXTENSIONS = {}
+
+
+def get_extractor(filename):
+    ext = os.path.splitext(filename)[1]
+    try:
+        return EXTRACTORS[EXTENSIONS[ext]]
+    except KeyError:
+        return None
+
+
+# Based on http://www.cplusplus.com/reference/cstdio/printf/
+# Note that we skip the space-flag in this list, since this creates too
+# many false positives.
+_C_FORMAT = re.compile(
+    r"""
+        %
+        [+#0-]?               # flags
+        (\d+|\*)?             # width
+        (\.(\d+|\*))?         # precision
+        (hh?|ll?|j|z|t|L)?    # length
+        [diuoxXfFeEgGaAcspn%] # specifier
+        """,
+    re.VERBOSE,
+)
+
+
+def check_c_format(buf, flags):
+    if "no-c-format" in flags or "c-format" in flags:
+        return
+    formats = list(re.finditer("%(?!%)", buf))
+    if formats and all(_C_FORMAT.match(buf[m.start() :]) is not None for m in formats):
+        flags.append("c-format")
+
+
+# Based on http://docs.python.org/2/library/string.html#format-string-syntax
+_PYTHON_FORMAT = re.compile(
+    r"""
+        \{
+            (([_A-Za-z](\w*)(\.[_a-z]\w*|\[\d+\])?)|\w+)?  # fieldname
+            (![rs])?  # conversion
+            (:\.?[<>=^]?[+ -]?\w*,?(\.\w+)?[bcdeEfFgGnosxX%]?)?  # format_spec
+        \}
+        """,
+    re.VERBOSE,
+)
+
+
+def check_python_format(buf, flags):
+    if "no-python-format" in flags or "python-format" in flags:
+        return
+    if _PYTHON_FORMAT.search(buf) is not None:
+        flags.append("python-format")
+
+
+def check_comment_flags(comment):
+    flags = re.match("\\[\\s*(.*?)\\s*\\]\\s*(.*)", comment)
+    if flags is not None:
+        return (re.split("\\s*,\\s*", flags.group(1)), flags.group(2))
+    else:
+        return [], comment
+
+
+class Keyword:
+    msgctxt_param = None
+    domain_param = None
+    comment = ""
+    required_arguments = None
+
+    _comment_arg = re.compile(r'^"(.*)"$')
+
+    def __init__(
+        self,
+        function,
+        msgid_param=1,
+        msgid_plural_param=None,
+        domain_param=None,
+        msgctxt_param=None,
+    ):
+        self.function = function
+        self.msgid_param = msgid_param
+        self.msgid_plural_param = msgid_plural_param
+        self.domain_param = domain_param
+        self.msgctxt_param = msgctxt_param
+
+    @classmethod
+    def from_spec(cls, spec):
+        if ":" not in spec:
+            return cls(spec)
+        try:
+            function, args = spec.split(":", 1)
+            kw = cls(function)
+            seen_msgid_param = False
+            while args:
+                if cls._comment_arg.match(args) is not None:
+                    kw.comment = args[1:-1]
+                    break
+                param, args = args.split(",", 1) if "," in args else (args, "")
+                if param.endswith("c"):
+                    kw.msgctxt_param = int(param[:-1])
+                elif param.endswith("d"):
+                    kw.domain_param = int(param[:-1])
+                elif param.endswith("t"):
+                    kw.required_arguments = int(param[:-1])
+                elif not seen_msgid_param:
+                    kw.msgid_param = int(param)
+                    seen_msgid_param = True
+                else:
+                    kw.msgid_plural_param = int(param)
+        except SyntaxError:
+            raise ValueError(f"Invalid keyword spec: {spec}")
+        return kw
+
+
+def update_keywords(keywords, specs):
+    for spec in specs:
+        if not spec:
+            keywords.clear()
+        try:
+            kw = Keyword.from_spec(spec)
+        except ValueError as e:
+            print(e, file=sys.stderr)
+            sys.exit(1)
+        keywords[kw.function] = kw
+
+
+@add_metaclass(abc.ABCMeta)
+class Extractor:
+    default_config = {}
+
+    def __init__(self, config=None):
+        self.config = self.default_config.copy()
+        if config:
+            self.config.update(config)
+
+    def update_config(self, **kw):
+        self.config.update(kw)
+
+    @abc.abstractproperty
+    def extensions(self):
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def __call__(self, filename, options, fileobj=None, lineno=0):
+        raise NotImplementedError()
+
+
+def register_extractors():
+    try:
+        extractor_entry_points = entry_points(group="lingva.extractors")
+    except TypeError:  # <= Python 3.9
+        extractor_entry_points = entry_points()["lingva.extractors"]
+
+    for entry_point in extractor_entry_points:
+        try:
+            extractor = entry_point.load()
+        except ModuleNotFoundError:
+            extractor = None
+        if extractor:
+            if not issubclass(extractor, Extractor):
+                raise ValueError("Registered extractor must derive from ``Extractor``")
+            EXTRACTORS[entry_point.name] = extractor()
+            for extension in extractor.extensions:
+                EXTENSIONS[extension] = entry_point.name