amapy-db 1.0.0__tar.gz

amapy-db-1.0.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: amapy-db
+Version: 1.0.0
+Summary: The Database plugin for asset-manager project. It provides functionalities to manage and store metadata related to digital assets, such as storing MD5 hashes of asset files and managing these records efficiently.
+License: Copyright (c) 2024 Roche Diagnostics Computation Science & Informatics
+Requires-Python: <3.11,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: amapy-pluggy==1.*
+Requires-Dist: amapy-utils==1.*
+
+asset-db

amapy-db-1.0.0/README.md

@@ -0,0 +1 @@
+asset-db

amapy-db-1.0.0/amapy_db.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: amapy-db
+Version: 1.0.0
+Summary: The Database plugin for asset-manager project. It provides functionalities to manage and store metadata related to digital assets, such as storing MD5 hashes of asset files and managing these records efficiently.
+License: Copyright (c) 2024 Roche Diagnostics Computation Science & Informatics
+Requires-Python: <3.11,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: amapy-pluggy==1.*
+Requires-Dist: amapy-utils==1.*
+
+asset-db

amapy-db-1.0.0/amapy_db.egg-info/SOURCES.txt

@@ -0,0 +1,22 @@
+README.md
+pyproject.toml
+amapy_db.egg-info/PKG-INFO
+amapy_db.egg-info/SOURCES.txt
+amapy_db.egg-info/dependency_links.txt
+amapy_db.egg-info/requires.txt
+amapy_db.egg-info/top_level.txt
+asset_db/__init__.py
+asset_db/assets_db.py
+asset_db/db.py
+asset_db/file_db.py
+asset_db/manifest_db.py
+asset_db/repo_db.py
+asset_db/states_db.py
+asset_db/store_db.py
+asset_db/store_file_db.py
+asset_db/tests/__init__.py
+asset_db/tests/test_assets_db.py
+asset_db/tests/test_db.py
+asset_db/tests/test_file_db.py
+asset_db/tests/test_manifest_db.py
+asset_db/tests/test_states_db.py

amapy-db-1.0.0/amapy_db.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

amapy-db-1.0.0/amapy_db.egg-info/requires.txt

@@ -0,0 +1,2 @@
+amapy-pluggy==1.*
+amapy-utils==1.*

amapy-db-1.0.0/amapy_db.egg-info/top_level.txt

@@ -0,0 +1 @@
+asset_db

amapy-db-1.0.0/asset_db/__init__.py

@@ -0,0 +1,7 @@
+from .manifest_db import ManifestDB
+from .file_db import FileDB
+from .states_db import StatesDB
+from .assets_db import AssetsDB
+from .repo_db import RepoDB
+from .store_db import StoreDB
+from .store_file_db import StoreFileDB

amapy-db-1.0.0/asset_db/assets_db.py

@@ -0,0 +1,44 @@
+import logging
+
+from asset_db.db import Database
+
+logger = logging.getLogger(__name__)
+
+
+class AssetsDB(Database):
+    """
+    Stores the md5 hashes of asset*.yamls. We use this to check which
+    files need to be re-downloaded during fetch operations
+    """
+    type = "assets-db"
+
+    @property
+    def file_hashes(self):
+        try:
+            return self._file_hashes
+        except AttributeError:
+            self._file_hashes = self.data()
+            return self._file_hashes
+
+    def add_file_hash(self, path, hash):
+        """Adds to existing list of files"""
+        updated = self.update(**{path: hash})
+        setattr(self, "_file_hashes", updated)
+
+    def remove_files(self, *paths):
+        """Deletes specific assets from the list"""
+        data = self.file_hashes
+        for path in paths:
+            deleted = data.pop(path, None)
+            if deleted:
+                pass
+                # logger.info("file {} removed.".format(id))
+            else:
+                logger.info("file not added yet, ignoring remove for:{}".format(id))
+        setattr(self, "_file_hashes", data)
+        self._write_to_file(data)
+
+    def clear(self):
+        """Clears all assets"""
+        setattr(self, "_file_hashes", {})
+        self._write_to_file(data={})

amapy-db-1.0.0/asset_db/db.py

@@ -0,0 +1,107 @@
+import logging
+import os
+from typing import List, AnyStr, Union
+
+from asset_utils.utils import utils
+from asset_utils.utils.file_utils import FileUtils
+
+logger = logging.getLogger(__name__)
+
+DB_FILE = "json"  # yaml
+
+
+class Database:
+    """Yaml wrapper to present a db like interface for writing to yaml files.
+    Singleton, so make sure all DBs inherit directly from this class only
+    Subclasses also behave as singletons
+    """
+    path = None
+    backup_path = None  # in case we want to cache the database
+    _persisted_data = {}
+    _instances = {}
+
+    def __new__(cls, *args, **kwargs):
+        path = kwargs.get('path') or args[0]  # there can be only one instance per path
+        if not path:
+            raise Exception("required param: path is null")
+
+        if cls._instances.get(path, None) is None:
+            cls._instances[path] = super(Database, cls).__new__(cls)
+
+        if cls._instances[path].__class__.__name__ != cls.__name__:
+            raise Exception("Instance already exists with same path but different class type")
+        return cls._instances[path]
+
+    def __init__(self, path, backup_path: str = None):
+        if path:
+            FileUtils.create_file_if_not_exists(path)
+        self.path = path
+        self.backup_path = backup_path
+        # create backup through hardlink
+        if self.backup_path:
+            if not os.path.exists(self.backup_path):
+                FileUtils.create_file_if_not_exists(self.backup_path)
+                if not os.path.samefile(self.path, self.backup_path):
+                    try:
+                        os.unlink(self.backup_path)  # delete existing
+                        os.link(src=self.path, dst=self.backup_path)
+                    except OSError as e:
+                        logger.warning(f"Backup not enabled because of OSError: {str(e)}")
+
+    def copy_to(self, db):
+        db.update(**self.data())
+
+    def update(self, **kwargs):
+        data = self.data() or {}
+        data = utils.update_dict(data, kwargs)
+        self._write_to_file(data=data)
+        return data
+
+    def _write_to_file(self, data):
+        """Stores the dictionary in a json file, overwrites existing data"""
+        self._persisted_data = data
+        self.__class__.write(data=self._persisted_data, path=self.path)
+
+    @classmethod
+    def write(cls, data, path):
+        if DB_FILE == "yaml":
+            FileUtils.write_yaml(abs_path=path, data=data)
+        elif DB_FILE == "json":
+            FileUtils.write_json(abs_path=path, data=data, sort_keys=False)
+        else:
+            raise Exception("Unsupported DB format")
+
+    def _read_from_file(self):
+        """returns data from yaml file"""
+        self._persisted_data = self._persisted_data or self.__class__.read(self.path)
+        return self._persisted_data
+
+    @classmethod
+    def read(cls, path):
+        if not path or not os.path.exists(path):
+            # nothing stored yet
+            return {}
+        if DB_FILE == "json":
+            return FileUtils.read_json(abs_path=path) or {}
+        elif DB_FILE == "yaml":
+            return FileUtils.read_yaml(abs_path=path) or {}
+        else:
+            raise Exception("Unsupported DB format")
+
+    def retrieve(self, key: Union[AnyStr, List]):
+        """Retrieves specific values for specific keys from the json file.
+           If keys are not found, the default value of None is returned.
+        """
+        data = self.data()
+        if isinstance(key, str):
+            return data.get(key, None)
+        elif isinstance(key, list):
+            result = {}
+            for each_key in key:
+                result[each_key] = data.get(each_key, None)
+            return result
+        else:
+            return None
+
+    def data(self):
+        return self._read_from_file()

amapy-db-1.0.0/asset_db/file_db.py

@@ -0,0 +1,49 @@
+import logging
+import os
+
+from asset_db.db import Database
+
+logger = logging.getLogger(__name__)
+
+
+class FileDB(Database):
+    """Stats db that holds file information to manage
+    Important: Stats db is a global db across the entire repo for all assets
+    whereas other dbs are asset specific
+    """
+    type = "file-stats"
+
+    def add_stats(self, **kwargs):
+        """adds to existing list of stats"""
+        self.update(**{"stats": kwargs})
+
+    def get_stats(self) -> dict:
+        """Returns assets and their md5 hashes"""
+        return self.retrieve("stats") or {}
+
+    def remove_stats(self, ids: list):
+        """Deletes specific assets from the list
+
+        Parameters
+        ----------
+        ids : list
+            url ids of asset objects
+        """
+        data = self.data()
+        stats = data.get("stats", {})
+        for id in ids:
+            deleted = stats.pop(id, None)
+            if deleted:
+                pass
+                # logger.info("object {} removed.".format(id))
+            else:
+                logger.warning("object not added yet, ignoring remove for:{}".format(id))
+
+        data["stats"] = stats
+        self._write_to_file(data=data)
+
+    def data(self):
+        if not os.path.exists(self.path):
+            # nothing stored yet
+            return {"stats": {}}
+        return self._read_from_file()

amapy-db-1.0.0/asset_db/manifest_db.py

@@ -0,0 +1,118 @@
+import logging
+
+from asset_db.db import Database
+from asset_utils.utils import utils
+
+logger = logging.getLogger(__name__)
+
+
+class ManifestDB(Database):
+    """
+    Class for managing asset manifest databases.
+    """
+    type = "asset-manifest"
+
+    def add_objects(self, **kwargs) -> None:
+        """Adds a new object to the manifest database.
+
+        Parameters
+        ----------
+        **kwargs : dict
+            Key-value pairs to be added as objects to the database.
+        """
+        data = {"objects": kwargs}
+        self.update(**data)
+
+    def get_objects(self) -> dict:
+        """Retrieves all objects from the manifest database.
+
+        Returns
+        -------
+        dict
+            A dictionary of all objects in the manifest database.
+        """
+        return self.retrieve("objects")
+
+    def get(self, key):
+        """Retrieves a specific value from the manifest database based on the provided key.
+
+        Parameters
+        ----------
+        key : str
+            The key for the value to be retrieved.
+
+        Returns
+        -------
+        The value associated with the key, if found; otherwise, None.
+        """
+        return self.data().get(key)
+
+    def remove_objects(self, ids: list) -> None:
+        """Removes specified objects from the manifest database.
+
+        Parameters
+        ----------
+        ids : list
+            A list of ids representing the objects to be removed.
+
+        Returns
+        -------
+        None
+        """
+        data = self.data()
+        objects = data.get("objects")
+        for _id in ids:
+            deleted = objects.pop(_id, None)
+            if deleted:
+                pass
+            else:
+                logger.info("asset not added yet, ignoring remove for:{}".format(_id))
+        data["objects"] = objects
+        self._write_to_file(data)
+
+    def clear_objects(self) -> None:
+        """Clears all objects from the manifest database.
+
+        Returns
+        -------
+        None
+        """
+        data = self.data()
+        data["objects"] = {}
+        self._write_to_file(data=data)
+
+    def get_version(self) -> str:
+        """Retrieves the version of the asset from the manifest database.
+
+        Returns
+        -------
+        str
+            The version of the asset.
+        """
+        return self.retrieve("version")
+
+    def set_version(self, version) -> None:
+        """Sets the version of the asset in the manifest database.
+
+        Parameters
+        ----------
+        version : str
+            The version to be set for the asset.
+
+        Returns
+        -------
+        None
+        """
+        data = {
+            "version": version,
+            "user": utils.get_user_id(),
+            "time": utils.get_time_stamp()
+        }
+        self.update(**data)
+
+    def get_parent_version(self):
+        return self.retrieve("parent_version")
+
+    def set_parent_version(self, version):
+        data = {"parent_version": version}
+        self.update(**data)

amapy-db-1.0.0/asset_db/repo_db.py

@@ -0,0 +1,42 @@
+import logging
+
+from asset_db.db import Database
+
+logger = logging.getLogger(__name__)
+
+
+class RepoDB(Database):
+    """
+    Stores Assets Meta information
+    """
+    type = "repo-db"
+
+    def add(self, **kwargs):
+        """Adds to existing list of objects"""
+        self.update(**kwargs)
+
+    def get(self, key) -> dict:
+        """Returns objects from yaml file"""
+        return self.retrieve(key=key)
+
+    def remove(self, keys: list):
+        """Deletes specific assets from the list
+
+        Parameters
+        ----------
+        keys : list
+            asset ids
+        """
+        data = self.data()
+        for id in keys:
+            deleted = data.pop(id, None)
+            if deleted:
+                pass
+                # logger.info("asset {} removed.".format(id))
+            else:
+                logger.info("asset not added yet, ignoring remove for:{}".format(id))
+        self._write_to_file(data)
+
+    def clear(self):
+        """Clears all assets"""
+        self._write_to_file(data={})

amapy-db-1.0.0/asset_db/states_db.py

@@ -0,0 +1,116 @@
+import logging
+import os
+
+from asset_db.db import Database
+
+logger = logging.getLogger(__name__)
+
+
+class StatesDB(Database):
+    """Stats db that holds file information to manage
+    Important: States db is asset-specific
+    """
+
+    type = "asset-states"
+
+    def set_asset_state(self, state):
+        self.update(**{"asset_state": state})
+
+    def get_asset_state(self):
+        return self.retrieve("asset_state") or None
+
+    def get_ref_states(self):
+        return self.retrieve("ref_states") or {}
+
+    def set_commit_message(self, x):
+        self.update(**{"commit_message": x})
+
+    def get_commit_message(self):
+        return self.retrieve("commit_message")
+
+    def add_object_states(self, **kwargs):
+        """Adds to existing list of stats
+
+        Parameters
+        ----------
+        kwargs : dict
+            key:value pairs of id: state
+        """
+        self.update(**{"object_states": kwargs})
+
+    def add_refs_states(self, **kwargs):
+        """Adds to existing list of stats
+
+        Parameters
+        ----------
+        kwargs : dict
+            key:value pairs of id: state
+        """
+        self.update(**{"ref_states": kwargs})
+
+    def get_object_states(self) -> dict:
+        """Returns assets and their md5 hashes"""
+        return self.retrieve("object_states") or {}
+
+    def get_content_states(self) -> dict:
+        return self.retrieve("content_states") or {}
+
+    def add_content_states(self, **kwargs):
+        self.update(**{"content_states": kwargs})
+
+    def remove_content_states(self, ids: list):
+        data = self.data()
+        states = data.get("content_states", {})
+        for id in ids:
+            deleted = states.pop(id, None)
+            if deleted:
+                pass
+                # logger.info("asset {} removed.".format(id))
+            else:
+                logger.warning("content not added yet, ignoring remove for:{}".format(id))
+
+        data["content_states"] = states
+        self._write_to_file(data=data)
+
+    def remove_object_states(self, ids: list):
+        """Deletes specific assets from the list
+
+        Parameters
+        ----------
+        ids : list
+            url ids of asset objects
+        """
+        data = self.data()
+        states = data.get("object_states", {})
+        for id in ids:
+            deleted = states.pop(id, None)
+            if deleted:
+                pass
+                # logger.info("asset {} removed.".format(id))
+            else:
+                logger.warning("asset not added yet, ignoring remove for:{}".format(id))
+
+        data["object_states"] = states
+        self._write_to_file(data=data)
+
+    def remove_ref_states(self, keys: list):
+        """Deletes specific assets_refs from the list
+
+        Parameters
+        ----------
+        keys : list
+            asset ids
+        """
+        data = self.data()
+        states = data.get("ref_states", {})
+        for key in keys:
+            _ = states.pop(key, None)
+
+        data["ref_states"] = states
+        self._write_to_file(data=data)
+
+    def data(self):
+        if not os.path.exists(self.path):
+            # nothing stored yet
+            return {"object_states": {}, "asset_state": None}
+        return self._read_from_file() or {}

amapy-db-1.0.0/asset_db/store_db.py

@@ -0,0 +1,91 @@
+import logging
+
+from asset_db.db import Database
+
+logger = logging.getLogger(__name__)
+
+
+class StoreDB(Database):
+    """
+    Stores Assets Meta information
+    """
+    type = "store-db"
+
+    def add(self, **kwargs):
+        """Adds to existing list of objects"""
+        self.update(**kwargs)
+
+    def get(self, key) -> dict:
+        """Returns objects from yaml file"""
+        return self.retrieve(key=key)
+
+    def add_repo(self, repo_id, data: dict):
+        # pop any existing repos that point to the same path
+        # those are not valid anymore since 2 repos will never share the same path
+        existing = self.data()
+        repos = existing.get("repos") or {}
+        ids = list(repos.keys())
+        for id in ids:
+            if repos[id].get('path') == data.get('path'):
+                del repos[id]
+        repos[repo_id] = data
+        existing["repos"] = repos
+        self._write_to_file(data=existing)
+
+    def get_repos(self):
+        return self.retrieve("repos") or {}
+
+    def remove_repo(self, *repo_ids):
+        if not repo_ids:
+            return
+        data = self.data()
+        repos = data.get("repos")
+        for repo_id in repo_ids:
+            if repo_id in repos:
+                del repos[repo_id]
+        data["repos"] = repos
+        self._write_to_file(data=data)
+
+    def add_temp_asset(self, class_name, asset_data: dict) -> None:
+        temp_assets = self.retrieve("temp_assets") or {}
+        class_assets = temp_assets.get(class_name) or {}
+        id = asset_data.get("seq_id")
+        class_assets[id] = asset_data
+        temp_assets[class_name] = class_assets
+        self.update(temp_assets=temp_assets)
+
+    def remove_temp_asset(self, class_name, seq_id):
+        data = self.data()
+        temp_assets = data.get("temp_assets") or {}
+        class_assets = temp_assets.get(class_name, {})
+        if seq_id in class_assets:
+            del class_assets[seq_id]
+        temp_assets[class_name] = class_assets
+        data["temp_assets"] = temp_assets
+        self._write_to_file(data=data)
+
+    def list_temp_assets(self, class_name):
+        temp_assets = self.retrieve("temp_assets") or {}
+        return temp_assets.get(class_name, {})
+
+    def remove(self, keys: list):
+        """Deletes specific assets from the list
+
+        Parameters
+        ----------
+        keys : list
+            asset ids
+        """
+        data = self.data()
+        for id in keys:
+            deleted = data.pop(id, None)
+            if deleted:
+                pass
+                # logger.info("asset {} removed.".format(id))
+            else:
+                logger.info("asset not added yet, ignoring remove for:{}".format(id))
+        self._write_to_file(data)
+
+    def clear(self):
+        """Clears all assets"""
+        self._write_to_file(data={})

amapy-db-1.0.0/asset_db/store_file_db.py

@@ -0,0 +1,51 @@
+import logging
+import os
+
+from asset_db.db import Database
+
+logger = logging.getLogger(__name__)
+
+
+class StoreFileDB(Database):
+    """Stats db that holds file information to manage
+
+    Important: Stats db is a global db across the entire repo for all assets
+    whereas other dbs are asset specific
+    """
+
+    type = "store-file-stats"
+
+    def add_stats(self, **kwargs):
+        """Adds to existing list of stats"""
+        self.update(**{"stats": kwargs})
+
+    def get_stats(self) -> dict:
+        """Returns assets and their md5 hashes"""
+        return self.retrieve("stats") or {}
+
+    def remove_stats(self, ids: list):
+        """Deletes specific assets from the list
+
+        Parameters
+        ----------
+        ids : list
+            url ids of asset objects
+        """
+        data = self.data()
+        stats = data.get("stats", {})
+        for id in ids:
+            deleted = stats.pop(id, None)
+            if deleted:
+                pass
+                # logger.info("object {} removed.".format(id))
+            else:
+                logger.warning("object not added yet, ignoring remove for:{}".format(id))
+
+        data["stats"] = stats
+        self._write_to_file(data=data)
+
+    def data(self):
+        if not os.path.exists(self.path):
+            # nothing stored yet
+            return {"stats": {}}
+        return self._read_from_file()

amapy-db-1.0.0/asset_db/tests/test_assets_db.py

@@ -0,0 +1,38 @@
+import os
+
+import pytest
+
+from asset_db.assets_db import AssetsDB
+
+
+@pytest.fixture(scope="module")
+def db(asset_root) -> AssetsDB:
+    db_path = os.path.join(asset_root, "database_test", "meta_hashes.json")
+    os.makedirs(os.path.dirname(db_path), exist_ok=True)
+    return AssetsDB(path=db_path)
+
+
+def test_db_create(db: AssetsDB):
+    """make sure metadb got created"""
+    assert db is not None  # db instance got created
+    assert db.path and os.path.exists(db.path)
+
+
+def test_add_file_hash(db: AssetsDB):
+    path = "test/path"
+    hash = "abc123"
+    db.add_file_hash(path, hash)
+    assert db.file_hashes[path] == hash, "File hash should be added to the database"
+
+
+def test_remove_files(db: AssetsDB):
+    # Setup - add some file hashes
+    paths = ["test/path1", "test/path2"]
+    hashes = ["abc123", "def456"]
+    for path, hash in zip(paths, hashes):
+        db.add_file_hash(path, hash)
+
+    # Test removing one file
+    db.remove_files(paths[0])
+    assert paths[0] not in db.file_hashes, "File should be removed from the database"
+    assert paths[1] in db.file_hashes, "Other files should remain in the database"

amapy-db-1.0.0/asset_db/tests/test_db.py

@@ -0,0 +1,53 @@
+import os
+
+import pytest
+
+from asset_db import AssetsDB, FileDB
+from asset_db.db import Database
+
+
+def test_path_singleton():
+    """
+    Test that each path can only have one instance of a db class
+    """
+    db1 = AssetsDB(path="/tmp/test_1.yaml")
+    db2 = AssetsDB(path="/tmp/test_2.yaml")
+    assert db1 != db2
+
+    db3 = AssetsDB(path="/tmp/test_1.yaml")
+    assert db3 == db1
+    assert db3 != db2
+
+    with pytest.raises(Exception) as e:
+        FileDB(path="/tmp/test_2.yaml")
+    assert str(e.value) == "Instance already exists with same path but different class type"
+
+
+@pytest.fixture(scope="module")
+def db(asset_root) -> Database:
+    db_path = os.path.join(asset_root, "database_test", "db_file.json")
+    os.makedirs(os.path.dirname(db_path), exist_ok=True)
+    return Database(path=db_path)
+
+
+def test_db_create(db: Database):
+    """make sure metadb got created"""
+    assert db is not None  # db instance got created
+    assert db.path and os.path.exists(db.path)
+
+
+def test_update(db):
+    db.update(key="value")
+    assert db.retrieve("key") == "value"
+
+
+def test_write_and_read(db):
+    Database.write({"key": "value"}, db.path)
+    data = Database.read(db.path)
+    assert data == {"key": "value"}
+
+
+def test_retrieve(db):
+    db.update(key1="value1", key2="value2")
+    assert db.retrieve("key1") == "value1"
+    assert db.retrieve(["key1", "key2"]) == {"key1": "value1", "key2": "value2"}

amapy-db-1.0.0/asset_db/tests/test_file_db.py

@@ -0,0 +1,32 @@
+import os
+
+import pytest
+
+from asset_db.file_db import FileDB
+
+
+@pytest.fixture(scope="module")
+def db(asset_root) -> FileDB:
+    db_path = os.path.join(asset_root, "database_test", "file_stats.json")
+    os.makedirs(os.path.dirname(db_path), exist_ok=True)
+    return FileDB(path=db_path)
+
+
+def test_db_create(db: FileDB):
+    """make sure metadb got created"""
+    assert db is not None  # db instance got created
+    assert db.path and os.path.exists(db.path)
+
+
+def test_add_and_get_stats(db):
+    db.add_stats(file1="hash1", file2="hash2")
+    stats = db.get_stats()
+    assert stats == {"file1": "hash1", "file2": "hash2"}, "Stats should be correctly added and retrieved"
+
+
+def test_remove_stats(db):
+    db.add_stats(file1="hash1", file2="hash2")
+    db.remove_stats(["file1"])
+    stats = db.get_stats()
+    assert "file1" not in stats, "file1 should be removed from stats"
+    assert "file2" in stats, "file2 should remain in stats"

amapy-db-1.0.0/asset_db/tests/test_manifest_db.py

@@ -0,0 +1,56 @@
+import os
+
+import pytest
+
+from asset_db import ManifestDB
+
+
+@pytest.fixture(scope="module")
+def db(asset_root) -> ManifestDB:
+    db_path = os.path.join(asset_root, "database_test", "manifest.yaml")
+    os.makedirs(os.path.dirname(db_path), exist_ok=True)
+    return ManifestDB(path=db_path)
+
+
+def test_db_create(db: ManifestDB):
+    """make sure metadb got created"""
+    assert db is not None  # db instance got created
+    assert db.path and os.path.exists(db.path)
+
+
+def test_add_objects(db: ManifestDB):
+    objects = {"1": {"name": "test_object1"}, "2": {"name": "test_object2"}}
+    db.add_objects(**objects)
+    result = db.get_objects()
+    assert result == objects
+
+
+def test_get_objects(db: ManifestDB):
+    objects = {"objects": {"1": {"name": "test_object1"}, "2": {"name": "test_object2"}}}
+    db.update(**objects)
+    result = db.get_objects()
+    assert result == objects["objects"]
+
+
+def test_get(db: ManifestDB):
+    data = {"key1": "value1", "key2": "value2"}
+    db.update(**data)
+    result = db.get("key1")
+    assert result == "value1"
+
+
+def test_remove_objects(db: ManifestDB):
+    objects = {"objects": {"1": {"name": "test_object1"}, "2": {"name": "test_object2"}}}
+    db.update(**objects)
+    db.remove_objects(["1"])
+    result = db.get_objects()
+    assert "1" not in result
+    assert "2" in result
+
+
+def test_clear_objects(db: ManifestDB):
+    objects = {"objects": {"1": {"name": "test_object1"}, "2": {"name": "test_object2"}}}
+    db.update(**objects)
+    db.clear_objects()
+    result = db.get_objects()
+    assert result == {}

amapy-db-1.0.0/asset_db/tests/test_states_db.py

@@ -0,0 +1,40 @@
+import os
+
+import pytest
+
+from asset_db import StatesDB
+
+DATA = {
+    "test_data/2.txt": "pending",
+    "test_data/imgs/photo-1493976040374-85c8e12f0c0e.jpg": "uploaded",
+    "test_data/imgs/photo-1504198453319-5ce911bafcde.jpg": "pending",
+    "test_data/imgs/photo-1507143550189-fed454f93097.jpg": "uploading",
+    "test_data/imgs/photo-1513938709626-033611b8cc03.jpg": "version",
+    "test_data/imgs/photo-1516117172878-fd2c41f4a759.jpg": "versioned",
+    "test_data/imgs/photo-1516972810927-80185027ca84.jpg": "committing",
+    "test_data/imgs/photo-1522364723953-452d3431c267.jpg": "committed",
+    "test_data/imgs/photo-1524429656589-6633a470097c.jpg": "renaming",
+    "test_data/imgs/photo-1530122037265-a5f1f91d3b99.jpg": "renamed",
+}
+
+
+@pytest.fixture(scope="module")
+def states_db(asset_root):
+    os.makedirs(os.path.join(asset_root, "states_db_test"))
+    return StatesDB(path=os.path.join(asset_root, "states_db_test", "states.yaml"))
+
+
+def test_add_states(states_db):
+    # create a directory
+    states_db.add_object_states(**DATA)
+    stored = states_db.get_object_states()
+    assert stored == DATA
+
+
+def test_remove_states(states_db):
+    data = DATA.copy()
+    states_db.add_object_states(**data)
+    data.pop("test_data/2.txt")
+    states_db.remove_object_states(["test_data/2.txt"])
+    stored = states_db.get_object_states()
+    assert data == stored

amapy-db-1.0.0/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools>=68.0.0,<69.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "amapy-db"
+version = "1.0.0"
+description = "The Database plugin for asset-manager project. It provides functionalities to manage and store metadata related to digital assets, such as storing MD5 hashes of asset files and managing these records efficiently."
+readme = "README.md"
+license = { text = "Copyright (c) 2024 Roche Diagnostics Computation Science & Informatics" }
+requires-python = ">=3.10,<3.11"  # "==3.10.*"
+dependencies = [
+    "amapy-pluggy==1.*",
+    "amapy-utils==1.*"
+]
+
+[tool.setuptools.packages.find]
+include = ["*"]
+namespaces = false

amapy-db-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+