ghga-transpiler 1.2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

scripts/update_config_docs.py

@@ -1,158 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
-# for the German Human Genome-Phenome Archive (GHGA)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Generates a JSON schema from the service's Config class as well as a corresponding
-example config yaml (or check whether these files are up to date).
-"""
-
-import importlib
-import json
-import subprocess
-import sys
-from difflib import unified_diff
-from pathlib import Path
-from typing import Any, Type
-
-import yaml
-from pydantic import BaseSettings
-from script_utils.cli import echo_failure, echo_success, run
-
-HERE = Path(__file__).parent.resolve()
-REPO_ROOT_DIR = HERE.parent
-DEV_CONFIG_YAML = REPO_ROOT_DIR / ".devcontainer" / ".dev_config.yaml"
-GET_PACKAGE_NAME_SCRIPT = HERE / "get_package_name.py"
-EXAMPLE_CONFIG_YAML = REPO_ROOT_DIR / "example_config.yaml"
-CONFIG_SCHEMA_JSON = REPO_ROOT_DIR / "config_schema.json"
-
-
-class ValidationError(RuntimeError):
-    """Raised when validation of config documentation fails."""
-
-
-def get_config_class() -> Type[BaseSettings]:
-    """
-    Dynamically imports and returns the Config class from the current service.
-    This makes the script service repo agnostic.
-    """
-    # get the name of the microservice package
-    with subprocess.Popen(
-        args=[GET_PACKAGE_NAME_SCRIPT],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.DEVNULL,
-    ) as process:
-        assert (
-            process.wait() == 0 and process.stdout is not None
-        ), "Failed to get package name."
-        package_name = process.stdout.read().decode("utf-8").strip("\n")
-
-    # import the Config class from the microservice package:
-    config_module: Any = importlib.import_module(f"{package_name}.config")
-    config_class = config_module.Config
-
-    return config_class
-
-
-def get_dev_config():
-    """Get dev config object."""
-    config_class = get_config_class()
-    return config_class(config_yaml=DEV_CONFIG_YAML)
-
-
-def get_schema() -> str:
-    """Returns a JSON schema generated from a Config class."""
-
-    config = get_dev_config()
-    return config.schema_json(indent=2)
-
-
-def get_example() -> str:
-    """Returns an example config YAML."""
-
-    config = get_dev_config()
-    normalized_config_dict = json.loads(config.json())
-    return yaml.dump(normalized_config_dict)  # pyright: ignore
-
-
-def update_docs():
-    """Update the example config and config schema files documenting the config
-    options."""
-
-    example = get_example()
-    with open(EXAMPLE_CONFIG_YAML, "w", encoding="utf-8") as example_file:
-        example_file.write(example)
-
-    schema = get_schema()
-    with open(CONFIG_SCHEMA_JSON, "w", encoding="utf-8") as schema_file:
-        schema_file.write(schema)
-
-
-def print_diff(expected: str, observed: str):
-    """Print differences between expected and observed files."""
-    echo_failure("Differences in Config YAML:")
-    for line in unified_diff(
-        expected.splitlines(keepends=True),
-        observed.splitlines(keepends=True),
-        fromfile="expected",
-        tofile="observed",
-    ):
-        print("   ", line.rstrip())
-
-
-def check_docs():
-    """Check whether the example config and config schema files documenting the config
-    options are up to date.
-
-    Raises:
-        ValidationError: if not up to date.
-    """
-
-    example_expected = get_example()
-    with open(EXAMPLE_CONFIG_YAML, "r", encoding="utf-8") as example_file:
-        example_observed = example_file.read()
-    if example_expected != example_observed:
-        print_diff(example_expected, example_observed)
-        raise ValidationError(
-            f"Example config YAML at '{EXAMPLE_CONFIG_YAML}' is not up to date."
-        )
-
-    schema_expected = get_schema()
-    with open(CONFIG_SCHEMA_JSON, "r", encoding="utf-8") as schema_file:
-        schema_observed = schema_file.read()
-    if schema_expected != schema_observed:
-        raise ValidationError(
-            f"Config schema JSON at '{CONFIG_SCHEMA_JSON}' is not up to date."
-        )
-
-
-def main(check: bool = False):
-    """Update or check the config documentation files."""
-
-    if check:
-        try:
-            check_docs()
-        except ValidationError as error:
-            echo_failure(f"Validation failed: {error}")
-            sys.exit(1)
-        echo_success("Config docs are up to date.")
-        return
-
-    update_docs()
-    echo_success("Successfully updated the config docs.")
-
-
-if __name__ == "__main__":
-    run(main)

scripts/update_readme.py

@@ -1,221 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
-# for the German Human Genome-Phenome Archive (GHGA)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Generate documentation for this package using different sources."""
-
-import json
-import subprocess  # nosec
-import sys
-from pathlib import Path
-from string import Template
-
-import jsonschema2md
-from pydantic import BaseModel, Field
-from script_utils.cli import echo_failure, echo_success, run
-from setuptools.config.setupcfg import read_configuration
-from stringcase import spinalcase, titlecase
-
-ROOT_DIR = Path(__file__).parent.parent.resolve()
-SETUP_CFG_PATH = ROOT_DIR / "setup.cfg"
-DESCRIPTION_PATH = ROOT_DIR / ".description.md"
-DESIGN_PATH = ROOT_DIR / ".design.md"
-README_TEMPLATE_PATH = ROOT_DIR / ".readme_template.md"
-CONFIG_SCHEMA_PATH = ROOT_DIR / "config_schema.json"
-OPENAPI_YAML_REL_PATH = "./openapi.yaml"
-README_PATH = ROOT_DIR / "README.md"
-
-
-class PackageHeader(BaseModel):
-    """A basic summary of a package."""
-
-    shortname: str = Field(
-        ...,
-        description=(
-            "The abbreviation of the package name. Is identical to the package name."
-        ),
-    )
-    version: str = Field(..., description="The version of the package.")
-    summary: str = Field(
-        ..., description="A short 1 or 2 sentence summary of the package."
-    )
-
-
-class PackageName(BaseModel):
-    """The name of a package and it's different representations."""
-
-    name: str = Field(..., description="The full name of the package in spinal case.")
-    title: str = Field(..., description="The name of the package formatted as title.")
-
-
-class PackageDetails(PackageHeader, PackageName):
-    """A container for details on a package used to build documentation."""
-
-    description: str = Field(
-        ..., description="A markdown-formatted description of the package."
-    )
-    design_description: str = Field(
-        ...,
-        description=(
-            "A markdown-formatted description of overall architecture and design of"
-            + " the package."
-        ),
-    )
-    config_description: str = Field(
-        ...,
-        description=(
-            "A markdown-formatted list of all configuration parameters of this package."
-        ),
-    )
-    openapi_doc: str = Field(
-        ...,
-        description=(
-            "A markdown-formatted description rendering or linking to an OpenAPI"
-            " specification of the package."
-        ),
-    )
-
-
-def read_package_header() -> PackageHeader:
-    """Read basic information about the package from the setup.cfg."""
-
-    setup_config = read_configuration(SETUP_CFG_PATH)
-    setup_metadata = setup_config["metadata"]
-    return PackageHeader(
-        shortname=setup_metadata["name"],
-        version=setup_metadata["version"],
-        summary=setup_metadata["description"],
-    )
-
-
-def read_package_name() -> PackageName:
-    """Infer the package name from the name of the git origin."""
-
-    with subprocess.Popen(
-        args="basename -s .git `git config --get remote.origin.url`",
-        cwd=ROOT_DIR,
-        stdout=subprocess.PIPE,
-        shell=True,
-    ) as process:
-        stdout, _ = process.communicate()
-
-    if not stdout:
-        raise RuntimeError("The name of the git origin could not be resolved.")
-    git_origin_name = stdout.decode("utf-8").strip()
-
-    return PackageName(
-        name=spinalcase(git_origin_name), title=titlecase(git_origin_name)
-    )
-
-
-def read_package_description() -> str:
-    """Read the package description."""
-
-    return DESCRIPTION_PATH.read_text()
-
-
-def read_design_description() -> str:
-    """Read the design description."""
-
-    return DESIGN_PATH.read_text()
-
-
-def generate_config_docs() -> str:
-    """Generate markdown-formatted documentation for the configration parameters
-    listed in the config schema."""
-
-    parser = jsonschema2md.Parser(
-        examples_as_yaml=False,
-        show_examples="all",
-    )
-
-    if not CONFIG_SCHEMA_PATH.exists():
-        return ""
-
-    with open(CONFIG_SCHEMA_PATH, "r", encoding="utf-8") as json_file:
-        config_schema = json.load(json_file)
-
-    md_lines = parser.parse_schema(config_schema)
-
-    # ignore everything before the properites header:
-    properties_index = md_lines.index("## Properties\n\n")
-    md_lines = md_lines[properties_index + 1 :]
-
-    return "\n".join(md_lines)
-
-
-def generate_openapi_docs() -> str:
-    """Generate markdown-formatted documentation linking to or rendering an OpenAPI
-    specification of the package. If no OpenAPI specification is present, return an
-    empty string."""
-
-    open_api_yaml_path = ROOT_DIR / OPENAPI_YAML_REL_PATH
-
-    if not open_api_yaml_path.exists():
-        return ""
-
-    return (
-        "## HTTP API\n"
-        + "An OpenAPI specification for this service can be found"
-        + f" [here]({OPENAPI_YAML_REL_PATH})."
-    )
-
-
-def get_package_details() -> PackageDetails:
-    """Get details required to build documentation for the package."""
-
-    header = read_package_header()
-    name = read_package_name()
-    description = read_package_description()
-    config_description = generate_config_docs()
-    return PackageDetails(
-        **header.dict(),
-        **name.dict(),
-        description=description,
-        config_description=config_description,
-        design_description=read_design_description(),
-        openapi_doc=generate_openapi_docs(),
-    )
-
-
-def generate_single_readme(*, details: PackageDetails) -> str:
-    """Generate a single markdown-formatted readme file for the package based on the
-    provided details."""
-
-    template_content = README_TEMPLATE_PATH.read_text()
-    template = Template(template_content)
-    return template.substitute(details.dict())
-
-
-def main(check: bool = False) -> None:
-    """Update the readme markdown."""
-
-    details = get_package_details()
-    readme_content = generate_single_readme(details=details)
-
-    if check:
-        if README_PATH.read_text() != readme_content:
-            echo_failure("README.md is not up to date.")
-            sys.exit(1)
-        echo_success("README.md is up to date.")
-        return
-
-    README_PATH.write_text(readme_content)
-    echo_success("Successfully updated README.md.")
-
-
-if __name__ == "__main__":
-    run(main)

scripts/update_template_files.py

@@ -1,250 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
-# for the German Human Genome-Phenome Archive (GHGA)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""This script evaluates the entries in .static_files, .mandatory_files and
-.deprecated_files and compares them with the microservice template repository,
-or verifies their existence or non-existence depending on the list they are in.
-"""
-
-import difflib
-import os
-import shutil
-import stat
-import sys
-import urllib.error
-import urllib.parse
-import urllib.request
-from pathlib import Path
-
-try:
-    from script_utils.cli import echo_failure, echo_success, run
-except ImportError:
-    echo_failure = echo_success = print
-
-    def run(main_fn):
-        """Run main function without cli tools (typer)."""
-        main_fn(check="--check" in sys.argv[1:])
-
-
-REPO_ROOT_DIR = Path(__file__).parent.parent.resolve()
-
-DEPRECATED_FILES = ".deprecated_files"
-MANDATORY_FILES = ".mandatory_files"
-STATIC_FILES = ".static_files"
-IGNORE_SUFFIX = "_ignore"
-RAW_TEMPLATE_URL = (
-    "https://raw.githubusercontent.com/ghga-de/microservice-repository-template/main/"
-)
-
-
-class ValidationError(RuntimeError):
-    """Raised when files need to be updated."""
-
-
-def get_file_list(list_name: str) -> list[str]:
-    """Return a list of all file names specified in a given list file."""
-    list_path = REPO_ROOT_DIR / list_name
-    with open(list_path, "r", encoding="utf8") as list_file:
-        file_list = [
-            clean_line
-            for clean_line in (
-                line.rstrip() for line in list_file if not line.startswith("#")
-            )
-            if clean_line
-        ]
-    if not list_name.endswith(IGNORE_SUFFIX):
-        ignore_list_name = list_name + IGNORE_SUFFIX
-        try:
-            file_set_ignore = set(get_file_list(ignore_list_name))
-        except FileNotFoundError:
-            print(f"  - {ignore_list_name} is missing, no exceptions from the template")
-        else:
-            file_list = [line for line in file_list if line not in file_set_ignore]
-    return file_list
-
-
-def get_template_file_content(relative_file_path: str):
-    """Get the content of the template file corresponding to the given path."""
-    remote_file_url = urllib.parse.urljoin(RAW_TEMPLATE_URL, relative_file_path)
-    remote_file_request = urllib.request.Request(remote_file_url)
-    try:
-        with urllib.request.urlopen(remote_file_request) as remote_file_response:
-            return remote_file_response.read().decode(
-                remote_file_response.headers.get_content_charset("utf-8")
-            )
-    except urllib.error.HTTPError as remote_file_error:
-        print(
-            f"  - WARNING: request to remote file {remote_file_url} returned"
-            f" status code {remote_file_error.code}"
-        )
-        return None
-
-
-def diff_content(local_file_path, local_file_content, template_file_content) -> bool:
-    """Show diff between given local and remote template file content."""
-    if local_file_content != template_file_content:
-        print(f"  - {local_file_path}: differs from template")
-        for line in difflib.unified_diff(
-            template_file_content.splitlines(keepends=True),
-            local_file_content.splitlines(keepends=True),
-            fromfile="template",
-            tofile="local",
-        ):
-            print("   ", line.rstrip())
-        return True
-    return False
-
-
-def check_file(relative_file_path: str, diff: bool = False) -> bool:
-    """Compare file at the given path with the given content.
-
-    Returns True if there are differences.
-    """
-    local_file_path = REPO_ROOT_DIR / Path(relative_file_path)
-
-    if not local_file_path.exists():
-        print(f"  - {local_file_path} does not exist")
-        return True
-
-    if diff:
-        template_file_content = get_template_file_content(relative_file_path)
-
-        if template_file_content is None:
-            print(f"  - {local_file_path}: cannot check, remote is missing")
-            return True
-
-        with open(local_file_path, "r", encoding="utf8") as file:
-            return diff_content(local_file_path, file.read(), template_file_content)
-
-    return False
-
-
-def update_file(relative_file_path: str, diff: bool = False) -> bool:
-    """Update file at the given relative path.
-
-    Returns True if there are updates.
-    """
-
-    local_file_path = REPO_ROOT_DIR / Path(relative_file_path)
-    local_parent_dir = local_file_path.parent
-
-    if not local_parent_dir.exists():
-        local_parent_dir.mkdir(parents=True)
-
-    if diff or not local_file_path.exists():
-        template_file_content = get_template_file_content(relative_file_path)
-
-        if template_file_content is None:
-            print(f"  - {local_file_path}: cannot update, remote is missing")
-            return True
-
-        if diff and local_file_path.exists():
-            with open(local_file_path, "r", encoding="utf8") as file:
-                if file.read() == template_file_content:
-                    return False
-
-        executable = template_file_content.startswith("#!")
-        executable_flags = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
-        with open(local_file_path, "w", encoding="utf8") as file:
-            file.write(template_file_content)
-            mode = os.fstat(file.fileno()).st_mode
-            if executable:
-                mode |= executable_flags
-            else:
-                mode &= ~executable_flags
-            os.fchmod(file.fileno(), stat.S_IMODE(mode))
-
-        print(f"  - {local_file_path}: updated")
-        return True
-
-    return False
-
-
-def update_files(files: list[str], check: bool = False, diff: bool = False) -> bool:
-    """Update or check all the files in the given list.
-
-    Returns True if there are updates.
-    """
-    updates = False
-    update_or_check_file = check_file if check else update_file
-    for relative_file_path in files:
-        if update_or_check_file(relative_file_path, diff=diff):
-            updates = True
-    return updates
-
-
-def remove_files(files: list[str], check: bool = False) -> bool:
-    """Remove or check all the files in the given list.
-
-    Returns True if there are updates.
-    """
-    updates = False
-    for relative_file_path in files:
-        local_file_path = REPO_ROOT_DIR / Path(relative_file_path)
-
-        if local_file_path.exists():
-            if check:
-                print(f"  - {local_file_path}: deprecated, but exists")
-            else:
-                if local_file_path.is_dir():
-                    shutil.rmtree(local_file_path)
-                else:
-                    local_file_path.unlink()
-                print(f"  - {local_file_path}: removed, since it is deprecated")
-            updates = True
-    return updates
-
-
-def main(check: bool = False):
-    """Update the static files in the service template."""
-    updated = False
-    if not check:
-        update_files([STATIC_FILES], diff=True, check=False)
-
-    print("Static files...")
-    files_to_update = get_file_list(STATIC_FILES)
-    if check:
-        files_to_update.append(STATIC_FILES)
-    files_to_update.extend((MANDATORY_FILES, DEPRECATED_FILES))
-    if update_files(files_to_update, diff=True, check=check):
-        updated = True
-
-    print("Mandatory files...")
-    files_to_guarantee = get_file_list(MANDATORY_FILES)
-    if update_files(files_to_guarantee, check=check):
-        updated = True
-
-    print("Deprecated files...")
-    files_to_remove = get_file_list(DEPRECATED_FILES)
-    if remove_files(files_to_remove, check=check):
-        updated = True
-
-    if check:
-        if updated:
-            echo_failure("Validating files from template failed.")
-            sys.exit(1)
-        echo_success("Successfully validated files from template.")
-    else:
-        echo_success(
-            "Successfully updated files from template."
-            if updated
-            else "No updates from the template were necessary."
-        )
-
-
-if __name__ == "__main__":
-    run(main)

tests/fixtures/__init__.py

@@ -1,16 +0,0 @@
-# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
-# for the German Human Genome-Phenome Archive (GHGA)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Fixtures that are used in both integration and unit tests"""

tests/fixtures/configs/0.0.0.yaml

@@ -1,16 +0,0 @@
-ghga_metadata_version: 0.0.0
-default_settings:
-  header_row: 1
-  start_row: 2
-  start_column: 1
-  end_column: 3
-worksheets:
-  - sheet_name: books
-    settings:
-      name: books
-  - sheet_name: publisher
-    settings:
-      name: publisher
-      transformations:
-        publisher_names: !!python/object/apply:ghga_transpiler.transformations.to_list []
-        attributes: !!python/object/apply:ghga_transpiler.transformations.to_attributes []

tests/fixtures/configs/__init__.py

@@ -1,15 +0,0 @@
-# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
-# for the German Human Genome-Phenome Archive (GHGA)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#

tests/fixtures/test_data_objects/__init__.py

@@ -1,16 +0,0 @@
-# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
-# for the German Human Genome-Phenome Archive (GHGA)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Data that are used in unit tests"""