mango-lib 0.1.0__tar.gz → 0.2.0__tar.gz

mango_lib-0.2.0/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.3
+Name: mango-lib
+Version: 0.2.0
+Summary: Library of utility functions used across ManGO Portal, ManGO Flow and ManGO Ingest amongst others
+Author: Paul Borgermans, Mariana Montes
+Author-email: Paul Borgermans <paul.borgermans@kuleuven.be>, Mariana Montes <mariana.montes@kuleuven.be>
+Requires-Dist: click>=8.1
+Requires-Dist: humanize>=4.15.0
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: mango-mdconverter>=0.0.10
+Requires-Dist: nh3>=0.3.5
+Requires-Dist: python-irodsclient>=3.2.0
+Requires-Dist: rich>=15.0.0
+Requires-Dist: mango-mdconverter>=0.0.10 ; extra == 'mango-mdconverter'
+Requires-Python: >=3.12
+Provides-Extra: mango-mdconverter
+Description-Content-Type: text/markdown
+
+# ManGO Lib
+
+This package provides common or useful tools that are in part extracted from the ManGO family of applications and frameworks developed and used by KULeuven ICTS-FOZ-RDM team.
+
+> [!WARNING]
+> ManGO Lib is currently under heavy development as it is the cornerstone for the ongoing ManGO platform refactoring
+> Therefore there is no PyPi installable package (yet)
+
+## Documentation
+
+The documentation is curated in a dedicated repository covering the entire ManGO family and is located at:
+
+...
+
+## Installation
+
+### With `uv`
+
+```bash
+uv add mango-lib
+```
+
+### With `pip`
+
+```bash
+pip install mango-lib
+```
+
+## Running the unit tests
+
+Requirements: `uv` and `pytest`
+
+- Checkout the repository locally
+- From the top level run:
+
+```bash
+uv run pytest -v -s
+```
+
+
+
+

mango_lib-0.2.0/README.md

@@ -0,0 +1,42 @@
+# ManGO Lib
+
+This package provides common or useful tools that are in part extracted from the ManGO family of applications and frameworks developed and used by KULeuven ICTS-FOZ-RDM team.
+
+> [!WARNING]
+> ManGO Lib is currently under heavy development as it is the cornerstone for the ongoing ManGO platform refactoring
+> Therefore there is no PyPi installable package (yet)
+
+## Documentation
+
+The documentation is curated in a dedicated repository covering the entire ManGO family and is located at:
+
+...
+
+## Installation
+
+### With `uv`
+
+```bash
+uv add mango-lib
+```
+
+### With `pip`
+
+```bash
+pip install mango-lib
+```
+
+## Running the unit tests
+
+Requirements: `uv` and `pytest`
+
+- Checkout the repository locally
+- From the top level run:
+
+```bash
+uv run pytest -v -s
+```
+
+
+
+

mango_lib-0.2.0/pyproject.toml

@@ -0,0 +1,42 @@
+[project]
+name = "mango-lib"
+version = "0.2.0"
+description = "Library of utility functions used across ManGO Portal, ManGO Flow and ManGO Ingest amongst others"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "click>=8.1",
+    "humanize>=4.15.0",
+    "jinja2>=3.1.6",
+    "mango-mdconverter>=0.0.10",
+    "nh3>=0.3.5",
+    "python-irodsclient>=3.2.0",
+    "rich>=15.0.0",
+]
+
+[[project.authors]]
+name = "Paul Borgermans"
+email = "paul.borgermans@kuleuven.be"
+
+[[project.authors]]
+name = "Mariana Montes"
+email = "mariana.montes@kuleuven.be"
+
+[project.scripts]
+mango = "mango_lib.cli:mango"
+
+[project.optional-dependencies]
+mango-mdconverter = ["mango-mdconverter>=0.0.10"]
+
+[build-system]
+requires = ["uv_build>=0.11,<0.12.0"]
+build-backend = "uv_build"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.3,<9.1.0",
+    "pytest-cases>=3.10.1",
+]

mango_lib-0.2.0/pyproject.toml.orig

@@ -0,0 +1,42 @@
+
+[project]
+name = "mango-lib"
+version = "0.2.0"
+description = "Library of utility functions used across ManGO Portal, ManGO Flow and ManGO Ingest amongst others"
+readme = "README.md"
+authors = [
+    { name = "Paul Borgermans", email = "paul.borgermans@kuleuven.be" },
+    { name = "Mariana Montes", email = "mariana.montes@kuleuven.be"}
+]
+requires-python = ">=3.12"
+dependencies = [
+    "click>=8.1",
+    "humanize>=4.15.0",
+    "jinja2>=3.1.6",
+    "mango-mdconverter>=0.0.10",
+    "nh3>=0.3.5",
+    "python-irodsclient>=3.2.0",
+    "rich>=15.0.0",
+]
+
+[project.scripts]
+mango = "mango_lib.cli:mango"
+
+[project.optional-dependencies]
+mango-mdconverter = [
+    "mango-mdconverter>=0.0.10",
+]
+
+
+[build-system]
+requires = ["uv_build>=0.11,<0.12.0"]
+build-backend = "uv_build"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.3,<9.1.0",
+    "pytest-cases>=3.10.1",
+]

mango_lib-0.2.0/src/mango_lib/__init__.py

@@ -0,0 +1,4 @@
+from mango_lib.cli import mango
+from mango_lib.irods.auth import get_zone_operator_session
+
+__all__ = ["get_zone_operator_session", "mango"]

mango_lib-0.2.0/src/mango_lib/archive/mango_package.py

@@ -0,0 +1,344 @@
+"""Attempt at isolating the packaging function in a module with everything it needs
+to run from local/iRODS to iRODS, with metadata if the source is iRODS,
+WITHOUT celery or steps or any of that.
+This should be added to the orchestrator before providing the orchestrator to the function.
+The only compulsory arguments of the packaging function are
+- an iterator of files
+- a buffered writer to tar things into
+- a base path to compute the relative path
+"""
+
+import functools
+import hashlib
+import io
+import json
+import pathlib
+import tarfile
+from collections.abc import Callable, Generator, Iterable
+
+from irods.collection import iRODSCollection
+from irods.data_object import iRODSDataObject
+from irods.session import iRODSSession
+from mango_mdconverter.md2dict import convert_metadata_to_dict
+
+from mango_lib.archive import mango_tar
+
+MANIFEST_PREFIX = "data"
+MANIFEST_NAME = "manifest-sha256.txt"
+LOCAL_FOLDER = pathlib.Path(".")
+
+MANIFEST_PATH = LOCAL_FOLDER / MANIFEST_NAME
+
+
+def tar_prefix(dataset):
+    return f"{dataset}/bag/data"
+
+
+def add_irods_metadata_to_tar(
+    orchestrator: mango_tar.TarOrchestrator,
+    item: mango_tar.TarInputItem,
+    alt_tar: io.BufferedWriter | None = None,
+):
+    if not isinstance(item, mango_tar.iRODSInputItem):
+        return
+    metadata = convert_metadata_to_dict(item.item.metadata.items())
+
+    if len(metadata) == 0:
+        # skip further processing
+        print(f"NO metadata for {item.name}")
+
+        return
+
+    bytes_buff = json.dumps(metadata).encode()
+    metadata_base_path = (
+        item.item.path
+        if isinstance(item.item, iRODSDataObject)
+        else str(pathlib.Path(item.item.path) / item.item.name)
+    )
+    tar_input_item = mango_tar.BytesInputItem(
+        bytes_buff,
+        needs_checksum=True,
+        path=f"{metadata_base_path}.metadata.json",
+        prefix=item.prefix,
+        alt_prefix=item.alt_prefix,
+        rel_path=item.rel_path,
+    )
+    mango_tar.record_checksum(orchestrator, tar_input_item)
+    mango_tar.add_bytes_item_to_tar(orchestrator.dest_tar, tar_input_item)
+
+    if alt_tar:
+        mango_tar.add_bytes_item_to_tar(alt_tar, tar_input_item)
+
+
+def bag_tar(
+    orchestrator: mango_tar.TarOrchestrator,
+    dataset_name: str,
+    manifest_path: pathlib.Path = MANIFEST_PATH,
+    rel_path: str = str(LOCAL_FOLDER),
+):
+    manifest_tar_input = mango_tar.FileInputItem(
+        manifest_path, rel_path=rel_path, prefix=f"{dataset_name}/bag"
+    )
+    with manifest_path.open("rb") as manifest_fp:
+        mango_tar.stream_fp_to_tar(
+            manifest_fp,
+            input_item=manifest_tar_input,
+            tar_dest=orchestrator.dest_tar,
+            read_buffer_size=1048576,
+            orchestrator=orchestrator,
+        )
+    orchestrator.at_current_file_end(
+        item=manifest_tar_input, dest_tar=orchestrator.dest_tar, callbacks=None
+    )
+    bagit_contents = "BagIt version 0.97\nTag-File-Character-Encoding: UTF-8\n".encode()
+    bagit_item = mango_tar.BytesInputItem(
+        bagit_contents, needs_checksum=False, path=f"{dataset_name}/bag/bagit.txt"
+    )
+    mango_tar.add_bytes_item_to_tar(orchestrator.dest_tar, bagit_item)
+    orchestrator.at_current_file_end(
+        item=bagit_item, dest_tar=orchestrator.dest_tar, callbacks=None
+    )
+
+
+def packaging_orchestrator(
+    orchestrator: mango_tar.TarOrchestrator = None,
+    manifest_path: pathlib.Path = MANIFEST_PATH,
+    dataset_name: str = "",
+    metadata_tar: io.BufferedWriter | None = None,
+):
+
+    if not isinstance(orchestrator, mango_tar.TarOrchestrator):
+        orchestrator = mango_tar.TarOrchestrator()  # initializing!
+
+    # CHECKSUMS FOR MANIFEST
+    orchestrator.add_callback("file_end", "checksum-save", mango_tar.record_checksum)
+
+    flush_checksums = functools.partial(
+        mango_tar.flush_checksums,
+        manifest_path=manifest_path,
+        format="bagit",
+        name_field="alt_name",
+    )
+    orchestrator.add_callback("delay_point", "update-manifest", flush_checksums)
+
+    orchestrator.add_callback(
+        "end",
+        "finish-manifest",
+        flush_checksums,
+    )
+
+    orchestrator.add_callback(
+        "end",
+        "make-bagit",
+        functools.partial(
+            bag_tar,
+            dataset_name=dataset_name,
+            manifest_path=manifest_path,
+            rel_path=str(manifest_path.parent),
+        ),
+    )
+
+    # add metadata
+    add_metadata_partial = functools.partial(
+        add_irods_metadata_to_tar,
+        alt_tar=metadata_tar,
+    )
+    orchestrator.add_callback("file_end", "add_metadata", add_metadata_partial)
+    orchestrator.add_callback(
+        "collection_item", "add_collection_metadata", add_metadata_partial
+    )
+    orchestrator.add_callback("abort", "flush-checksums", flush_checksums)
+    orchestrator.add_callback("exception", "flush-checksums", flush_checksums)
+
+    return orchestrator
+
+
+def parse_file_iterator(
+    file_iterator: Iterable, rel_path: str, dataset_name: str
+) -> Generator[mango_tar.TarInputItem, None, None]:
+    for file in file_iterator:
+        if isinstance(file, mango_tar.TarInputItem):
+            yield file
+        elif isinstance(file, iRODSDataObject):
+            class_name = mango_tar.iRODSInputItem
+        elif isinstance(file, pathlib.Path):
+            class_name = mango_tar.FileInputItem
+        else:
+            raise TypeError(f"File type not supported for: {file}")
+        yield class_name(
+            file,
+            needs_checksum=True,
+            rel_path=rel_path,
+            prefix=tar_prefix(dataset_name),
+            alt_prefix=MANIFEST_PREFIX,
+        )
+
+
+def parse_folder_iterator(
+    folder_iterator: Iterable, rel_path: str, dataset_name: str
+) -> Generator[mango_tar.iRODSInputItem, None, None]:
+    # if isinstance(folder_iterator, iRODSTarInputJSONLReader):
+    #     return folder_iterator.get_next_object()
+    for folder in folder_iterator:
+        if not isinstance(folder, iRODSCollection):
+            raise TypeError("Item must be an iRODS Collection")
+        yield mango_tar.iRODSInputItem(
+            folder,
+            rel_path=rel_path,
+            prefix=tar_prefix(dataset_name),
+            alt_prefix=MANIFEST_PREFIX,
+        )
+
+
+def package_dataset(
+    file_iterator: Iterable,
+    dest_tar_object: iRODSDataObject | pathlib.Path,
+    base_path: str,  # to compute relative paths
+    dataset_name: str = "dummy_dataset",
+    folder_iterator: Iterable | None = None,
+    orchestrator: mango_tar.TarOrchestrator | None = None,
+    local_folder: pathlib.Path = LOCAL_FOLDER,
+    manifest_name: str = MANIFEST_NAME,
+    add_metadata_tar: bool = False,
+    last_good_write: dict | None = None,
+    # rocrate_source=None,
+    return_orchestrator: bool = False,
+) -> tuple:
+    manifest_path = local_folder / manifest_name
+    with manifest_path.open("w"):
+        pass
+
+    files = parse_file_iterator(file_iterator, base_path, dataset_name)
+    collections = parse_folder_iterator(folder_iterator, base_path, dataset_name)
+
+    # metadata tar
+    if add_metadata_tar:
+        alt_metadata_tar_path = local_folder / f"{dataset_name}_metadata.tar"
+        alt_metadata_tar = alt_metadata_tar_path.open("wb")
+    else:
+        alt_metadata_tar = None
+
+    # set up orchestrator, from scratch if none is provided
+    orchestrator = packaging_orchestrator(
+        orchestrator=orchestrator,
+        manifest_path=manifest_path,
+        dataset_name=dataset_name,
+        metadata_tar=alt_metadata_tar,
+    )
+
+    restart_output = mango_tar.restartable_tar(
+        object_iterator=files,
+        collection_iterator=collections,
+        dest_tar_object=dest_tar_object,
+        orchestrator=orchestrator,
+        last_good_write=last_good_write,
+        local_folder=local_folder,
+    )
+    if alt_metadata_tar is not None:
+        alt_metadata_tar.close()
+    return (
+        (restart_output, orchestrator)
+        if return_orchestrator
+        else (restart_output, None)
+    )
+
+
+def compute_sha256(
+    fp,
+    chunk_size: int = 16 * 1024 * 1024,
+    stream_callback: Callable | None = None,
+):
+    """Compute sha256 checksum with a filepointer as input"""
+    hash_sha256 = hashlib.sha256()
+    for chunk in iter(lambda: fp.read(chunk_size), b""):
+        hash_sha256.update(chunk)
+        if stream_callback:
+            stream_callback(len(chunk))
+    return hash_sha256.hexdigest()
+
+
+def compute_checksums_from_irods_tar(
+    irods_session: iRODSSession,
+    path_to_tar: str,
+    output_manifest_path: str,
+    stream_callback: Callable | None = None,
+    file_callback: Callable | None = None,
+):
+    """Finish Me"""
+
+    # @todo implement and do something with the callbacks
+    # or implement something like the tar orchestrator?
+    with irods_session.data_objects.open(path_to_tar, "r") as irods_tar_fp:
+        with tarfile.open(fileobj=irods_tar_fp, mode="r") as tar_fp:
+            with pathlib.Path(output_manifest_path).open("w") as output_manifest_fp:
+                while tar_info := tar_fp.next():
+                    if tar_info.isfile():
+                        with tar_fp.extractfile(tar_info) as fp:
+                            computed_hash = compute_sha256(
+                                fp, stream_callback=stream_callback
+                            )
+                            output_manifest_fp.write(
+                                f"{computed_hash} {tar_info.name}\n"
+                            )
+                            if file_callback:
+                                file_callback(
+                                    tar_info.name, tar_info.size, computed_hash
+                                )
+
+
+def compare_checksums_from_files(
+    from_input: str, from_tar: str, input_prefix="", tar_prefix=""
+):
+    """compare checksums from input lists of which the elements are not necessarily in the
+    same order or even the same list size
+
+    """
+    # Using more_itertools.filter_map
+    cs_input_d = {}  # {path: checksum}
+    cs_tar_d = {}
+    with pathlib.Path(from_input).open("r") as input_mf_fp:
+        for line in input_mf_fp:
+            item = line.rstrip().split(" ", 1)  # path may also contain whitespaces
+            if len(item) == 2:
+                cs_input_d[input_prefix + item[1]] = item[0]
+    with pathlib.Path(from_tar).open("r") as tar_mf_fp:
+        for line in tar_mf_fp:
+            item = line.rstrip().split(" ", 1)
+            if len(item) == 2:
+                cs_tar_d[tar_prefix + item[1]] = item[0]
+
+    not_found = {}
+    mis_match = {}
+
+    for path, checksum in cs_input_d.items():
+        tar_checksum = cs_tar_d.get(path, None)
+        if tar_checksum:
+            if checksum != tar_checksum:
+                mis_match[path] = f"{checksum} {tar_checksum}"
+        else:
+            not_found[path] = checksum
+
+    if not_found or mis_match:
+        # write the problematic paths to a file for later inspection
+        message = ""
+        if not_found:
+            with (pathlib.Path(from_input).parent / "checksums_not_found.txt").open(
+                "w"
+            ) as nf_fp:
+                for item in not_found:
+                    nf_fp.write(f"{not_found[item]} {item}\n")
+            message += f"{len(not_found)} items were not found. "
+        if mis_match:
+            with (pathlib.Path(from_input).parent / "checksums_mis_match.txt").open(
+                "w"
+            ) as missing_fp:
+                for item in mis_match:
+                    # The format is first the checksum as read + SPACE + checksum from tar + PATH
+                    missing_fp.write(f"{mis_match[item]} {item}\n")
+            message += (
+                f"{len(mis_match)} items were found but the checksum does not match."
+            )
+
+        raise ValueError(f"The match is not perfect: {message}")
+
+    return True