hatch-openzim 0.1.0__py3-none-any.whl

hatch_openzim/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

hatch_openzim/build_hook.py

@@ -0,0 +1,19 @@
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+
+from hatch_openzim.files_install import process as process_files_install
+
+
+class OpenzimBuildHook(BuildHookInterface):
+    """Hatch build hook to perform custom openzim actions
+
+    This hook performs:
+    - files installation
+    """
+
+    PLUGIN_NAME = "openzim-build"
+
+    def initialize(self, version, build_data):  # noqa: ARG002
+        if "toml-config" in self.config:
+            process_files_install(openzim_toml_location=self.config["toml-config"])
+        else:
+            process_files_install()

hatch_openzim/files_install.py

@@ -0,0 +1,236 @@
+import shutil
+import subprocess
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Any, Dict, List
+from urllib.request import urlopen
+
+try:
+    import tomllib
+except ImportError:  # pragma: no cover
+    import toml as tomllib
+
+from hatch_openzim.shared import logger
+
+DEFAULT_OPENZIM_TOML_LOCATION = "openzim.toml"
+
+
+def process(openzim_toml_location: str = DEFAULT_OPENZIM_TOML_LOCATION):
+    """performs openZIM operations on files, i.e. download and extract ZIPs
+
+    Configuration is read from openzim.toml which must be in root of the hatch project,
+    next to pyproject.toml
+
+    openzim_toml_location: location of the openzim.toml file with instructions about
+    files to install locally
+    """
+    config_path = Path(openzim_toml_location)
+    if not config_path.exists():
+        if openzim_toml_location != DEFAULT_OPENZIM_TOML_LOCATION:
+            raise Exception(f"File is missing at {openzim_toml_location}")
+        else:
+            return
+    config = tomllib.loads(config_path.read_text())
+    files_config = config.get("files", None)
+    if not files_config:
+        return
+
+    for section_name, section_data in files_config.items():
+        _process_section(section_name=section_name, section_data=section_data)
+
+
+def _process_section(section_name: str, section_data: Dict[str, Any]):
+    """processes all actions required for one section (i.e. one target folder)"""
+
+    logger.info(f"Processing {section_name} section")
+    section_config = section_data.get("config", None)
+    if not section_config:
+        raise Exception("config table is mandatory")
+    base_target_dir = section_config.get("target_dir", None)
+    if not base_target_dir:
+        raise Exception("target_dir is mandatory in config table")
+    base_target_dir = Path(base_target_dir)
+    section_actions = section_data.get("actions", None)
+    if not section_actions:
+        logger.info("  No actions to process")
+        return
+    base_target_dir = Path(section_config["target_dir"])
+    base_target_dir.mkdir(parents=True, exist_ok=True)
+    logger.info(f"  Installing files in {base_target_dir}")
+    for action_name, action_config in section_actions.items():
+        _process_one_action(base_target_dir, action_name, action_config)
+
+    execute_after = section_config.get("execute_after", None)
+    if execute_after:
+        _process_execute_after(base_target_dir=base_target_dir, actions=execute_after)
+
+    logger.info(" All done")
+
+
+def _process_one_action(
+    base_target_dir: Path, action_name: str, action_data: Dict[str, Any]
+):
+    """processes one action (basically trigger the right kind of action)"""
+    logger.info(f"  Processing {action_name} action")
+    source = action_data.get("source", None)
+    if not source:
+        raise Exception("source is not configured")
+    action = action_data.get("action", None)
+    if not action:
+        raise Exception("action is not configured")
+
+    if action == "get_file":
+        _process_get_file_action(
+            base_target_dir=base_target_dir, source=source, action_data=action_data
+        )
+
+    elif action == "extract_all":
+        _process_extract_all_action(
+            base_target_dir=base_target_dir, source=source, action_data=action_data
+        )
+
+    elif action == "extract_items":
+        _process_extract_items_action(
+            base_target_dir=base_target_dir, source=source, action_data=action_data
+        )
+
+    else:
+        raise Exception(f"Unsupported action '{action}'")
+
+    execute_after = action_data.get("execute_after", None)
+    if execute_after:
+        _process_execute_after(base_target_dir=base_target_dir, actions=execute_after)
+
+    logger.info("    Done")
+
+
+def _process_execute_after(base_target_dir: Path, actions: List[str]):
+    """execute actions after file(s) installation"""
+
+    for action in actions:
+        logger.info(f"    Executing '{action}'")
+        process = subprocess.run(
+            action,
+            shell=True,  # noqa: S602 # nosec: B602
+            cwd=base_target_dir,
+            text=True,
+            check=True,
+            capture_output=True,
+        )
+        if process.stdout:
+            logger.info(f"      stdout:\n{process.stdout}")
+
+
+def _process_get_file_action(
+    base_target_dir: Path, source: str, action_data: Dict[str, Any]
+):
+    """downloads one file to a given location"""
+    target_file = action_data.get("target_file", None)
+    if not target_file:
+        raise Exception("target_file is mandatory when action='get_file'")
+    target_dir = action_data.get("target_dir", None)
+    local_dir = base_target_dir
+    if target_dir:
+        local_dir = base_target_dir / str(target_dir)
+        local_dir.mkdir(parents=True, exist_ok=True)
+    local_file = local_dir / str(target_file)
+    if local_file.exists():
+        logger.info("    Skipping, local_file is already present")
+        return
+    _download_file(source, local_file)
+
+
+def _process_extract_all_action(
+    base_target_dir: Path, source: str, action_data: Dict[str, Any]
+):
+    """extacts all Zip content to a given location
+
+    Supports the remove attribute which allows to delete some Zip content
+    """
+    target_dir = action_data.get("target_dir", None)
+    if not target_dir:
+        raise Exception("target_dir is mandatory when action='extract_all'")
+    target_dir = base_target_dir / str(target_dir)
+    if target_dir.exists():
+        logger.info("    Skipping, target_dir is already present")
+        return
+    if not target_dir.parent.exists():
+        target_dir.parent.mkdir(parents=True, exist_ok=True)
+    _extract_zip_from_url(url=source, extract_to=target_dir)
+    if "remove" in action_data:
+        _remove_items(globs=action_data["remove"], directory=target_dir)
+
+
+def _process_extract_items_action(
+    base_target_dir: Path, source: str, action_data: Dict[str, Any]
+):
+    """extacts some items of a content to some given locations
+
+    Supports the remove attribute which allows to delete some items content after
+    extraction
+    """
+    zip_paths = action_data.get("zip_paths", None)
+    if not zip_paths:
+        raise Exception("zip_paths is mandatory when action='extract_items'")
+    target_paths = action_data.get("target_paths", None)
+    if not target_paths:
+        raise Exception("target_paths is mandatory when action='extract_items'")
+    if len(zip_paths) != len(target_paths):
+        raise Exception(
+            f"zip_paths and target_paths must have same length ({len(zip_paths)} !="
+            f" {len(target_paths)})"
+        )
+
+    # do not re-install if asset has already been installed
+    if any(
+        (base_target_dir / str(target_path)).exists() for target_path in target_paths
+    ):
+        logger.info("    Skipping, at least one target path is already present")
+        return
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        _extract_zip_from_url(url=source, extract_to=tempdir)
+        for index, zip_path in enumerate(zip_paths):
+            item_src = Path(tempdir) / str(zip_path)
+            item_dst = base_target_dir / str(target_paths[index])
+            shutil.move(src=str(item_src), dst=item_dst)
+
+    if "remove" in action_data:
+        _remove_items(globs=action_data["remove"], directory=base_target_dir)
+
+
+def _remove_items(directory: Path, globs: List[str]):
+    """removes all files in directory matching one of the provided glob patterns"""
+    for pattern in globs:
+        matches = directory.glob(pattern)
+        for match in matches:
+            if match.is_file():
+                match.unlink()
+            elif match.is_dir():  # pragma: no branch
+                shutil.rmtree(match)
+
+
+def _download_file(url, download_to):
+    """downloads a file to a given location"""
+    if not url.startswith(("http:", "https:")):
+        raise ValueError("URL must start with 'http:' or 'https:'")
+    with urlopen(url) as response, open(download_to, "wb") as file:  # noqa: S310
+        file.write(response.read())
+
+
+def _extract_zip_from_url(url, extract_to):
+    """downloads ZIP from URL and extract in given directory
+
+    Nota: the ZIP is temporarily saved on disk (there is no convenient function
+    to stream the web resource to the ZIP extractor in Python standard library)
+    """
+
+    if not url.startswith(("http:", "https:")):
+        raise ValueError("URL must start with 'http:' or 'https:'")
+    with urlopen(url) as response:  # noqa: S310
+        with tempfile.NamedTemporaryFile(delete=False) as temp_zip:
+            shutil.copyfileobj(response, temp_zip)
+        with zipfile.ZipFile(temp_zip.name, "r") as zip_file:
+            zip_file.extractall(extract_to)
+        Path(temp_zip.name).unlink()

hatch_openzim/hooks.py

@@ -0,0 +1,16 @@
+from typing import Type
+
+from hatchling.plugin import hookimpl
+
+from hatch_openzim.build_hook import OpenzimBuildHook
+from hatch_openzim.metadata_hook import OpenzimMetadataHook
+
+
+@hookimpl
+def hatch_register_build_hook() -> Type[OpenzimBuildHook]:
+    return OpenzimBuildHook
+
+
+@hookimpl
+def hatch_register_metadata_hook() -> Type[OpenzimMetadataHook]:
+    return OpenzimMetadataHook

hatch_openzim/metadata.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+
+from hatch_openzim.utils import get_github_info, get_python_versions
+
+
+def update(root: str, config: dict, metadata: dict):
+    """Update the project table's metadata."""
+
+    # Check for absence of metadata we will set + presence in the dynamic property
+    for metadata_key in ["urls", "authors", "keywords", "license", "classifiers"]:
+        if config.get(f"preserve-{metadata_key}", False):
+            # Do not check if we intend to preserve the value set manually
+            continue
+
+        if metadata_key in metadata:
+            raise ValueError(
+                f"'{metadata_key}' must not be listed in the 'project' table when using"
+                " openzim metadata hook."
+            )
+        if metadata_key not in metadata.get("dynamic", []):
+            raise ValueError(
+                f"'{metadata_key}' must be listed in 'project.dynamic' when using "
+                "openzim metadata hook."
+            )
+
+    github_info = get_github_info(git_config_path=Path(root) / ".git/config")
+
+    organization = config.get("organization", github_info.organization)
+
+    if not config.get("preserve-urls", False):
+        metadata["urls"] = {
+            "Donate": "https://www.kiwix.org/en/support-us/",
+            "Homepage": github_info.homepage,
+        }
+
+    if not config.get("preserve-authors", False):
+        if str(organization).lower() in ("kiwix", "offspot"):
+            authors = [{"name": "Kiwix", "email": "dev@kiwix.org"}]
+        else:
+            authors = [{"name": "openZIM", "email": "dev@openzim.org"}]
+        authors.extend(config.get("additional-authors", []))
+        metadata["authors"] = authors
+
+    if not config.get("preserve-keywords", False):
+        if str(organization).lower() in ("kiwix", "offspot"):
+            keywords = ["kiwix"]
+        else:
+            keywords = ["openzim"]
+        if config.get("kind", "") == "scraper":
+            keywords.extend(["zim", "offline"])
+        keywords.extend(config.get("additional-keywords", []))
+        metadata["keywords"] = keywords
+
+    if not config.get("preserve-license", False):
+        metadata["license"] = {"text": "GPL-3.0-or-later"}
+
+    if not config.get("preserve-classifiers", False):
+        classifiers = [
+            "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)"
+        ]
+        for python_version in get_python_versions(metadata["requires-python"]):
+            classifiers.append(f"Programming Language :: Python :: {python_version}")
+        metadata["classifiers"] = classifiers

hatch_openzim/metadata_hook.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from hatchling.metadata.plugin.interface import MetadataHookInterface
+
+from hatch_openzim.metadata import update
+
+
+class OpenzimMetadataHook(MetadataHookInterface):
+    """Hatch metadata hook to populate 'project' data
+
+    This hook populates:
+    - project urls
+    """
+
+    PLUGIN_NAME = "openzim-metadata"
+
+    def update(self, metadata: dict):
+        """Update the project table's metadata."""
+        update(root=self.root, config=self.config, metadata=metadata)

hatch_openzim/shared.py

@@ -0,0 +1,16 @@
+import logging
+import os
+import sys
+
+DEFAULT_FORMAT = "%(name)s:%(levelname)s:%(message)s"
+
+# create logger
+logger = logging.getLogger("hatch_openzim")
+logger.setLevel(logging.DEBUG)
+
+# setup console logging
+console_handler = logging.StreamHandler(sys.stdout)
+console_handler.setFormatter(logging.Formatter(DEFAULT_FORMAT))
+log_level = logging.getLevelName(os.getenv("HATCH_OPENZIM_LOG_LEVEL", "INFO"))
+console_handler.setLevel(log_level)
+logger.addHandler(console_handler)

hatch_openzim/utils.py

@@ -0,0 +1,80 @@
+import configparser
+import re
+from collections import namedtuple
+from pathlib import Path
+from typing import List
+
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
+
+from hatch_openzim.shared import logger
+
+REMOTE_REGEXP = re.compile(
+    r"""^(?:git@|https:\/\/)github.com[:\/](?P<organization>.*?)\/"""
+    r"""(?P<repository>.*?)(?:.git)?$"""
+)
+
+GithubInfo = namedtuple("GithubInfo", ["homepage", "organization", "repository"])
+
+DEFAULT_GITHUB_INFO = GithubInfo(
+    homepage="https://www.kiwix.org", organization=None, repository=None
+)
+
+
+def get_github_info(git_config_path: Path, remote: str = "origin") -> GithubInfo:
+    if not git_config_path.exists() or not git_config_path.is_file():
+        return DEFAULT_GITHUB_INFO
+
+    try:
+        config = configparser.ConfigParser()
+        config.read(git_config_path)
+        git_remote_url = config[f'remote "{remote}"']["url"]
+        match = REMOTE_REGEXP.match(git_remote_url)
+        if not match:
+            raise Exception(f"Unexpected remote url: {git_remote_url}")
+        return GithubInfo(
+            homepage=f"https://github.com/{match.group('organization')}/"
+            f"{match.group('repository')}",
+            organization=match.group("organization"),
+            repository=match.group("repository"),
+        )
+    except Exception as exc:
+        logger.error("Failed to read Github URL", exc_info=exc)
+        return DEFAULT_GITHUB_INFO
+
+
+def get_python_versions(requires_python: str) -> List[str]:
+    """
+    Returns the list of major and major.minor versions compatible with the specifier
+
+    E.g. if requires_python is ">=3.10,<3.12", the result is "3", "3.10", "3.11"
+
+    Nota: this does not work for requirements overlapping 3.x and 4.x, or later, because
+     latest 3.x version is not yet known
+    """
+    specifier_set = SpecifierSet(requires_python)
+
+    last_py1_minor = 6
+    last_py2_minor = 7
+
+    major_versions = []
+    minor_versions = []
+    for major in range(1, 10):  # this will work up to Python 10 ...
+        major_added = False
+        last_minor = 100  # this supposes we will never have Python x.100
+        if major == 1:
+            last_minor = last_py1_minor
+        elif major == 2:  # noqa: PLR2004
+            last_minor = last_py2_minor
+        for minor in range(last_minor + 1):
+            if specifier_set.contains(Version(f"{major}.{minor}")):
+                if not major_added:
+                    if len(major_versions) > 0 and major >= 4:  # noqa: PLR2004
+                        raise Exception(
+                            "Multiple major versions is not supported for 3 and up"
+                        )
+                    major_added = True
+                    major_versions.append(f"{major}")
+                minor_versions.append(f"{major}.{minor}")
+
+    return major_versions + minor_versions