ichec-platform-core 0.0.1__py3-none-any.whl

ichec_platform_core/filesystem/__init__.py

@@ -0,0 +1 @@
+from .filesystem import *  # NOQA

ichec_platform_core/filesystem/cli.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+import logging
+
+from ichec_platform_core.cli_utils import launch_common
+
+from .filesystem import read_file, replace_in_files
+
+
+logger = logging.getLogger(__name__)
+
+
+def cli_replace_in_files(args):
+
+    launch_common(args)
+
+    search_term = read_file(args.search.resolve()).strip()
+    replace_term = read_file(args.insert.resolve()).strip()
+
+    target_dir = args.target.resolve()
+    logger.info(
+        "Replacing '%s' with '%s' in files in %s", search_term, replace_term, target_dir
+    )
+    count = replace_in_files(target_dir, search_term, replace_term, args.extension)
+    logger.info("Replaced in %d files", count)
+
+
+def register(subparsers):
+
+    parser = subparsers.add_parser("filesystem")
+    subparsers = parser.add_subparsers(required=True)
+
+    replace_files_parser = subparsers.add_parser("replace_in_files")
+
+    replace_files_parser.add_argument(
+        "--target",
+        type=Path,
+        help="Path to the directory containing the files",
+    )
+    replace_files_parser.add_argument(
+        "--search", type=Path, help="File containing the term to be replaced"
+    )
+    replace_files_parser.add_argument(
+        "--insert", type=Path, help="File containing the term to be inserted"
+    )
+    replace_files_parser.add_argument(
+        "--extension",
+        type=str,
+        default="",
+        help="Only process files with this extension, if set",
+    )
+
+    replace_files_parser.set_defaults(func=cli_replace_in_files)

ichec_platform_core/filesystem/filesystem.py

@@ -0,0 +1,268 @@
+"""
+This module has wrappers over Python built-ins for filsystem operations
+"""
+
+import shutil
+from pathlib import Path
+import os
+
+from ichec_platform_core.runtime import ctx
+
+
+def copy(src: Path, dst: Path) -> None:
+    """
+    Copy the file or directory from `src` to `dst`
+    :param src: The location to copy from
+    :param dst: The location to copy to
+    """
+
+    if ctx.can_modify():
+        shutil.copy(src, dst)
+    else:
+        ctx.add_cmd(f"copy {src} {dst}")
+
+
+def make_archive(archive: Path, fmt: str, src: Path, rename: bool = False) -> None:
+    """
+    Create an archive with name `archive_name` and chosen format, e.g. zip
+    from the content in `src`
+    :param Path archive_name: The path to the created archive
+    :param str fmt: The desired archive format, e.g. zip
+    :param Path src: The src of the content to be archived
+    :param bool rename: If true rename source directory to have the archive name
+    """
+
+    if ctx.can_modify():
+        if rename:
+            shutil.copytree(src, archive)
+            src = archive
+        shutil.make_archive(str(archive), fmt, src)
+        if rename:
+            shutil.rmtree(archive)
+    else:
+        ctx.add_cmd(f"make_archive {archive} {fmt} {src}")
+
+
+def unpack_archive(src: Path, dst: Path) -> None:
+    """
+    Extract the archive at `src` to `dst`
+    """
+
+    if ctx.can_modify():
+        shutil.unpack_archive(src, dst)
+    else:
+        ctx.add_cmd(f"unpack_archive {src} {dst}")
+
+
+def copy_file(src: Path, dst: Path):
+    """
+    Copy a file in src to the target directory in dst
+
+    Make the dst directory if needed
+    """
+    os.makedirs(dst, exist_ok=True)
+    shutil.copy(src, dst / src.name)
+
+
+def copy_files(src: Path, dst: Path):
+    """
+    Copy all files from the `src` directory to the `dst` directory
+
+    Create the dst directory first if needed.
+
+    :param Path src: The directory to copy from
+    :param Path dst: The directory to copy to
+    """
+    os.makedirs(dst, exist_ok=True)
+    for direntry in src.iterdir():
+        if direntry.is_file():
+            shutil.copy(direntry, dst)
+
+
+def copy_files_relative(paths: list[Path], src: Path, dst: Path):
+    """
+    For each relative path in `paths` copy the file at `src/path` to
+    `dst/path`.
+
+    Tries to create the dst directory first.
+
+    :param list[Path] paths: The list of paths to copy
+    :param Path src: The source path to copy relative to
+    :param Path dst: The destination path to copy into.
+    """
+
+    for path in paths:
+        os.makedirs(dst / path.parent, exist_ok=True)
+        shutil.copy(src / path, dst / path)
+
+
+def clear_dir(src: Path):
+    """
+    If there is a directory at `src` delete it and make a new one
+    """
+
+    if src.exists():
+        shutil.rmtree(src)
+    os.makedirs(src)
+
+
+def read_file(src: Path):
+    """
+    Read a file at the provided path
+    """
+    with open(src, "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def read_file_lines(src: Path):
+    """
+    Read a file as lines
+    """
+
+    with open(src, "r", encoding="utf-8") as f:
+        return f.readlines()
+
+
+def write_file_lines(
+    src: Path, lines: list[str], make_parents: bool = True, make_exe: bool = False
+):
+    """
+    Write a file with lines
+    """
+
+    if make_parents:
+        os.makedirs(src.parent, exist_ok=True)
+
+    with open(src, "w", encoding="utf-8") as f:
+        for line in lines:
+            f.write(line)
+
+    if make_exe:
+        os.chmod(src, 0o0755)
+
+
+def write_file(src: Path, content: str, make_parents: bool = True):
+    """
+    Write a file
+    """
+    if make_parents:
+        os.makedirs(src.parent, exist_ok=True)
+
+    with open(src, "w", encoding="utf-8") as f:
+        f.write(content)
+
+
+def replace_in_file(src: Path, search_term: str, replace_term: str) -> bool:
+    """
+    Replace the search term in a file
+    """
+
+    content = read_file_lines(src)
+    has_update = False
+    for idx, line in enumerate(content):
+        replaced_line = line.replace(search_term, replace_term)
+        if replaced_line != line:
+            content[idx] = replaced_line
+            has_update = True
+    if has_update:
+        write_file_lines(src, content)
+        return True
+    return False
+
+
+def replace_in_files(
+    src: Path, search_term: str, replace_term: str, extension: str
+) -> int:
+    """
+    Replace a search term in multiple files
+    """
+
+    count = 0
+    for direntry in src.iterdir():
+        if direntry.is_file():
+            if not extension or direntry.suffix == f".{extension}":
+                replaced = replace_in_file(direntry, search_term, replace_term)
+                if replaced:
+                    count += 1
+    return count
+
+
+def file_contains_string(src: Path, search_term: str) -> bool:
+    """
+    True if a file contains the search term
+    """
+    content = read_file(src)
+    return search_term in content
+
+
+def get_dirs(path: Path, prefix: str = "") -> list[Path]:
+    """
+    Return all directories in a path, optionally filter directory
+    name on a prefix
+    """
+    if prefix:
+        return [
+            entry
+            for entry in path.iterdir()
+            if entry.is_dir() and entry.name.startswith(prefix)
+        ]
+    return [entry for entry in path.iterdir() if entry.is_dir()]
+
+
+def is_file_with_ext(f, extension: str) -> bool:
+    """
+    True if the path item is a file with given extension
+    """
+    return f.is_file() and str(f).endswith(f".{extension}")
+
+
+def get_files(path: Path, extension: str) -> list:
+    """
+    Return all files in this dir with the provided extension
+    """
+    return [f for f in path.glob("*") if is_file_with_ext(f, extension)]
+
+
+def is_file_with_extensions(
+    f, extensions: tuple[str], excludes: list[str] | None = None
+):
+    """
+    True if the path item is a file and has one of the provided extensions
+    """
+
+    if not f.is_file():
+        return False
+    for ext in extensions:
+        check_str = str(f).lower()
+        if check_str.endswith(f".{ext.lower()}"):
+            if excludes:
+                for exclude in excludes:
+                    if check_str.endswith(f"{exclude.lower()}.{ext.lower()}"):
+                        return False
+            return True
+    return False
+
+
+def get_files_recursive(
+    path: Path, extensions: tuple[str], excludes: list[str] | None = None
+):
+    """
+    Get all provided files recursively, filter on extensions and excludes
+    """
+    return [
+        f for f in path.rglob("*") if is_file_with_extensions(f, extensions, excludes)
+    ]
+
+
+def get_json_files(path: Path):
+    """
+    Get all json files in this dir
+    """
+    return get_files(path, extension="json")
+
+
+def get_csv_files(path: Path):
+    """
+    Get all csv files in this dir
+    """
+    return get_files(path, extension="csv")

ichec_platform_core/logging_utils.py

@@ -0,0 +1,26 @@
+import logging
+
+
+def setup_default_logger(log_filename: str | None = None):
+    fmt = "%(asctime)s%(msecs)03d | %(filename)s:%(lineno)s:%(funcName)s | %(message)s"
+    logging.basicConfig(
+        filename=log_filename,
+        filemode="w",
+        format=fmt,
+        datefmt="%Y%m%dT%H:%M:%S:",
+        level=logging.INFO,
+    )
+
+
+class LogLine:
+    """
+    Helper class to represent a log line
+    """
+
+    def __init__(self, timestamp: float, thread_id: int, message: str):
+        self.timestamp = timestamp
+        self.message = message
+        self.thread_id = thread_id
+
+    def __str__(self):
+        return f"{self.timestamp} | {self.thread_id} | {self.message}"

ichec_platform_core/main_cli.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+"""
+This is the main entrypoint for the ichec_platform_core utility
+"""
+
+import argparse
+import logging
+
+from ichec_platform_core.cli_utils import launch_common
+import ichec_platform_core.network.cli
+import ichec_platform_core.filesystem.cli
+import ichec_platform_core.version_control.cli
+
+
+logger = logging.getLogger(__name__)
+
+
+def main_cli():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dry_run",
+        type=int,
+        default=0,
+        help="Dry run script - 0 can modify, 1 can read, 2 no modify - no read",
+    )
+    subparsers = parser.add_subparsers(required=True)
+
+    ichec_platform_core.network.cli.register(subparsers)
+    ichec_platform_core.filesystem.cli.register(subparsers)
+    ichec_platform_core.version_control.cli.register(subparsers)
+
+    args = parser.parse_args()
+    launch_common(args)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main_cli()

ichec_platform_core/network/__init__.py

@@ -0,0 +1 @@
+from .http_client import *  # NOQA

ichec_platform_core/network/cli.py

@@ -0,0 +1,44 @@
+import os
+from pathlib import Path
+import logging
+
+from ichec_platform_core.cli_utils import launch_common
+
+from .http_client import HttpClient
+
+logger = logging.getLogger(__name__)
+
+
+def download(args):
+
+    launch_common(args)
+    logger.info("Attempting to download from %s", args.url)
+
+    headers = {args.token_header: args.token}
+
+    http_client = HttpClient()
+    http_client.download_file(args.url, args.download_dir, headers)
+
+
+def register(subparsers):
+
+    parser = subparsers.add_parser("network")
+
+    subparsers = parser.add_subparsers(required=True)
+
+    download_parser = subparsers.add_parser("download")
+
+    download_parser.add_argument("url", type=str, help="Url to download")
+    download_parser.add_argument(
+        "--token", type=str, help="Optional auth token", default=""
+    )
+    download_parser.add_argument(
+        "--token_header", type=str, help="Optional auth token header key", default=""
+    )
+    download_parser.add_argument(
+        "--download_dir",
+        type=Path,
+        help="Directory to download to",
+        default=Path(os.getcwd()),
+    )
+    download_parser.set_defaults(func=download)

ichec_platform_core/network/http_client.py

@@ -0,0 +1,125 @@
+"""
+A basic low-dependency http client
+"""
+
+import urllib.request
+import urllib.parse
+from pathlib import Path
+import os
+import json
+
+from ichec_platform_core.runtime import ctx
+
+
+class HttpClient:
+    """
+    A basic http client with low dependencies (urllib)
+    """
+
+    def __init__(self):
+        self.default_encode_charset = "utf-8"
+        self.default_decode_charset = "utf-8"
+
+    def post_json(self, url: str, payload, headers: dict[str, str] | None):
+        if not headers:
+            headers = {}
+
+        headers["content-type"] = "application/json"
+        return self.make_post_request(url, headers, json.dumps(payload))
+
+    def make_get_request(self, url: str, headers: dict[str, str] | None = None):
+        if not headers:
+            headers = {}
+        return self._make_request(url, "GET", headers)
+
+    def make_put_request(
+        self, url: str, headers: dict[str, str] | None = None, payload=None
+    ):
+        if not headers:
+            headers = {}
+        return self._make_request(url, "PUT", headers, payload)
+
+    def make_post_request(
+        self, url: str, headers: dict[str, str] | None = None, payload=None
+    ):
+        if not headers:
+            headers = {}
+        return self._make_request(url, "POST", headers, payload)
+
+    def download_file(
+        self, url: str, path: Path, headers: dict[str, str] | None = None
+    ):
+        if not headers:
+            headers = {}
+
+        if not ctx.can_read():
+            ctx.add_cmd(f"download file {url} {path} {headers}")
+
+        with open(path, "wb") as f:
+            req = urllib.request.Request(url, headers=headers)
+            with urllib.request.urlopen(req) as response:
+                self._check_http_status(response)
+                f.write(response.read())
+
+    def upload_file(self, url: str, path: Path, headers: dict[str, str] | None = None):
+
+        if not headers:
+            headers = {}
+
+        if not ctx.can_modify():
+            ctx.add_cmd(f"upload file {url} {path} {headers}")
+            return ""
+
+        if not path.exists():
+            raise RuntimeError(f"Attempted to upload file without valid path: {path}")
+
+        headers["Content-Length"] = str(os.stat(path).st_size)
+        headers["Content-Type"] = "application/octet-stream"
+
+        with open(path, "rb") as f:
+            req = urllib.request.Request(url, headers=headers, data=f, method="PUT")
+            with urllib.request.urlopen(req) as response:
+                self._check_http_status(response)
+                response_str = response.read().decode(self._get_charset(response))
+        return response_str
+
+    def _make_request(
+        self, url: str, method: str, headers: dict[str, str] | None = None, payload=None
+    ):
+
+        if not headers:
+            headers = {}
+
+        if method == "GET":
+            if not ctx.can_read():
+                ctx.add_cmd(f"make_request GET {url} {headers}")
+                return ""
+        elif method in ["PUT", "POST"]:
+            if not ctx.can_modify():
+                ctx.add_cmd(f"make_request {method} {url} {headers} {payload}")
+                return ""
+
+        req = urllib.request.Request(
+            url, headers=headers, data=self._endcode_payload(payload), method=method
+        )
+        with urllib.request.urlopen(req) as response:
+            self._check_http_status(response)
+            response_str = response.read().decode(self._get_charset(response))
+        return response_str
+
+    def _check_http_status(self, response):
+        if response.status >= 300:
+            body = response.read().decode(self._get_charset(response))
+            msg = f"code {response.status}, reason {response.reason}, and body {body}"
+            raise RuntimeError(f"Http Request failed with: {msg}")
+
+    def _endcode_payload(self, payload):
+        if not payload:
+            return payload
+        return payload.encode("utf-8")
+
+    def _get_charset(self, response):
+        charset = response.headers.get_content_charset()
+        if not charset:
+            charset = "utf-8"
+        return charset

ichec_platform_core/project/__init__.py

@@ -0,0 +1,2 @@
+from .milestone import *  # NOQA
+from .version import Version  # NOQA

ichec_platform_core/project/milestone.py

@@ -0,0 +1,34 @@
+"""
+This module has milestone related project elements
+"""
+
+from datetime import datetime
+
+from pydantic import BaseModel
+
+from .person import Person
+
+
+class Issue(BaseModel, frozen=True):
+    """
+    An issue is a user story or defect to work on
+    """
+
+    title: str = ""
+    description: str = ""
+    start_date: datetime | None = None
+    due_date: datetime | None = None
+    assignee: Person | None = None
+
+
+class Milestone(BaseModel, frozen=True):
+    """
+    A milestone is a collection of work items with start and
+    planned end dates
+    """
+
+    title: str = ""
+    description: str = ""
+    start_date: datetime | None = None
+    due_date: datetime | None = None
+    issues: list[Issue] = []

ichec_platform_core/project/person.py

@@ -0,0 +1,17 @@
+"""
+Class representing a person, eg a user on a platform
+"""
+
+from pydantic import BaseModel
+
+
+class Person(BaseModel, frozen=True):
+    """
+    A person class - used for referencing project
+    contributors.
+    """
+
+    name: str = ""
+    email: str = ""
+    orcid: str = ""
+    username: str

ichec_platform_core/project/version.py

@@ -0,0 +1,68 @@
+"""
+This module has functionality for representing and
+manipulating project version numbers.
+"""
+
+from datetime import date
+
+from pydantic import BaseModel
+
+
+class Version(BaseModel, frozen=True):
+    """
+    Representation of a package version number.
+
+    This resembles a semver scheme but doesn't necessarily
+    follow one.
+
+    Args:
+        content (str): Version in string form x.y.zzzz
+        scheme (str): The versioning scheme to use, 'semver' or 'date'
+
+    Attributes:
+        major (int): Major version number
+        minor (int): Minor version number
+        patch (int): Path version number
+        scheme (str): Versioning scheme to use
+    """
+
+    major: int
+    minor: int
+    patch: int
+    scheme: str = "semver"
+
+    def as_string(self) -> str:
+        return f"{self.major}.{self.minor}.{self.patch}"
+
+
+def increment(version: Version, field="patch", today=date.today()) -> Version:
+    major = version.major
+    minor = version.minor
+    patch = version.patch
+
+    if version.scheme == "semver":
+        if field == "patch":
+            patch += 1
+        elif field == "minor":
+            patch = 0
+            minor += 1
+        elif field == "major":
+            major += 1
+            minor = 0
+            patch = 0
+    elif version.scheme == "date":
+        major = today.year - 2000
+        minor = today.month
+        if minor != version.minor:
+            patch = 1
+        else:
+            patch = version.patch + 1
+    return version.model_copy(update={"major": major, "minor": minor, "patch": patch})
+
+
+def parse(content: str, scheme: str = "semver") -> Version:
+    """
+    Parse the version from the input content, assumes 'major.minor.patch' format.
+    """
+    major, minor, patch = content.split(".")
+    return Version(major=int(major), minor=int(minor), patch=int(patch), scheme=scheme)