jupyter-analysis-tools 1.5.1__py3-none-any.whl → 1.6.1__py3-none-any.whl

jupyter_analysis_tools/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # __init__.py
 
-__version__ = "1.5.1"
+__version__ = "1.6.1"
 
 from .binning import reBin
 from .git import checkRepo, isNBstripoutActivated, isNBstripoutInstalled, isRepo

jupyter_analysis_tools/datastore.py

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+# datastore.py
+
+import filecmp
+import getpass
+import tempfile
+import warnings
+from pathlib import Path
+
+from pybis import Openbis
+
+
+class DataStore:
+    url = None
+    _availObj = None
+    _userspace = None
+
+    def __init__(self, url, username=None):
+        self.url = url
+        self.username = username
+        if self.username is None:
+            self.username = getpass.getuser()
+        print(f"Working as user '{self.username}'.")
+        # to generate PAT you need to login normally
+        self.ds = Openbis(url=self.url, verify_certificates=True)
+        # arg. *save_token* saves the openBIS token to ~/.pybis permanently
+        self.ds.login(
+            self.username,
+            getpass.getpass(prompt=f"Password for {self.username}: "),
+            save_token=False,
+        )
+        # create the PAT with the given name, don't store it
+        self.ds.get_or_create_personal_access_token("test-session")
+
+    @property
+    def userspace(self):
+        uspace = self._userspace
+        if uspace is None:
+            allspaces = self.ds.get_spaces()
+            uspace = allspaces.df[
+                allspaces.df.code.str.endswith(self.username.upper())
+            ].code.values[0]
+            self._userspace = uspace
+        return uspace
+
+    @userspace.setter
+    def userspace(self, name):
+        name = name.upper()
+        if name in self.ds.get_spaces().df.code.values:
+            self._userspace = name
+
+    @staticmethod
+    def identifier(objects, code):
+        return objects[objects.code == code].identifier.tolist()[0]
+
+    def createProject(self, projectName, space, spacePrefix=None):
+        """Finds the requested project in the DataStore.
+        Matching project names can be limited to a given *spacePrefix*.
+        If the project is not found, a new project with the given code in the given space
+        is created."""
+        # get available projects, accessible by the current user
+        projectsAvail = self.ds.get_projects()
+        if spacePrefix:
+            projectsAvail = [prj for prj in projectsAvail if f"/{spacePrefix}_" in prj.identifier]
+        projects = [prj for prj in projectsAvail if prj.code == projectName]
+        assert len(projects) <= 1, f"Multiple projects found for '{projectName}'"
+        dsProject = None
+        if len(projects):  # get the existing object
+            dsProject = projects[0]
+        else:  # create it, if not found
+            print(f"Creating project '{projectName}'")
+            dsProject = self.ds.new_project(code=projectName, space=space)
+            dsProject.save()
+        assert dsProject
+        return dsProject
+
+    def createCollection(self, collName, projectObj, defaultObjType=None):
+        collections = self.ds.get_collections(project=projectObj)
+        dsColl = [coll for coll in collections if coll.code == collName.upper()]
+        if len(dsColl):
+            dsColl = dsColl[0]
+        else:  # create it, if not found
+            print(f"Creating collection '{collName}'")
+            dsColl = self.ds.new_collection(
+                code=collName, type="COLLECTION", project=projectObj, props={"$name": collName}
+            )
+            dsColl.save()
+        assert dsColl
+        # update properties (name, default view and object type) if not set)
+        props = dsColl.props.all()  # props as dict
+        propKey = "$name"
+        if propKey in props and props[propKey] is None:
+            props[propKey] = collName
+        propKey = "$default_collection_view"
+        if propKey in props.keys() and props[propKey] is None:
+            propVal = [
+                item
+                for item in self.ds.get_vocabulary(propKey + "s").get_terms().df.code
+                if "list" in item.lower()
+            ]
+            assert len(propVal)
+            props[propKey] = propVal[0]
+        if defaultObjType:
+            propKey = "$default_object_type"
+            if propKey in props.keys() and props[propKey] is None:
+                props[propKey] = defaultObjType
+        # print(f"Setting '{collName}' properties:\n  {props}")
+        dsColl.set_props(props)
+        dsColl.save()
+        return dsColl
+
+    def createObject(
+        self,
+        projectName: str,
+        collectionName: str = None,
+        space: str = None,
+        spacePrefix: str = None,
+        objType: str = None,
+        props: dict = None,
+    ):
+        dsProject = self.createProject(projectName, space, spacePrefix=spacePrefix)
+        dsColl = None
+        if collectionName is None:  # collectionName is required
+            return None
+        dsColl = self.createCollection(collectionName, dsProject, defaultObjType=objType)
+        obj = self.ds.get_objects(type=objType, where={"$name": props["$name"]}).objects
+        if len(obj):
+            obj = obj[0]
+            prefix = objType
+            msg = "'{}' exists already in {}! Updating ...".format(
+                obj.props["$name"], obj.project.identifier
+            )
+            warnings.warn_explicit(msg, UserWarning, prefix, 0)
+        else:  # does not exist yet
+            objName = f" '{props['$name']}'" if len(props.get("$name", "")) else ""
+            print(f"Creating new {objType}{objName} in {dsColl.identifier}")
+            obj = self.ds.new_object(type=objType, props=props, collection=dsColl)
+        obj.set_props(props)
+        return obj
+
+    def findObjects(self, *args, **kwargs):
+        return self.ds.get_objects(**kwargs)
+
+    def uploadDataset(self, obj, datasetType, fpaths=[]):
+        def _checkFile(localPath, remoteFiles):
+            remoteFile = [f for f in remoteFiles if f.name == localPath.name]
+            if not len(remoteFile):  # file exists in the dataset as well
+                return False
+            return filecmp.cmp(localPath, remoteFile[0], shallow=False)
+
+        if not len(fpaths):
+            return  # nothing to do
+        for dataset in obj.get_datasets(type=datasetType):
+            with tempfile.TemporaryDirectory() as tempdir:
+                dataset.download(destination=tempdir)
+                dsFiles = [f for f in Path(tempdir).rglob("*") if f.is_file()]
+                if len(fpaths) == len(dsFiles):
+                    if all([_checkFile(fpath, dsFiles) for fpath in fpaths]):
+                        print(
+                            f"All local files of {datasetType} match files in dataset, "
+                            "not updating."
+                        )
+                        continue  # skip deletion below
+            print(f"Dataset {datasetType} needs update, deleting existing dataset:")
+            dataset.delete("Needs update")
+        if not len(obj.get_datasets(type=datasetType)):  # didn't exist yet or all deleted
+            dataset = self.ds.new_dataset(
+                type=datasetType, collection=obj.collection, object=obj, files=fpaths
+            )
+            dataset.save()

{jupyter_analysis_tools-1.5.1.dist-info → jupyter_analysis_tools-1.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jupyter-analysis-tools
-Version: 1.5.1
+Version: 1.6.1
 Summary: Yet another Python library with helpers and utilities for data analysis and processing.
 Author-email: Ingo Breßler <ingo.bressler@bam.de>, "Brian R. Pauw" <brian.pauw@bam.de>
 License-Expression: MIT
@@ -35,10 +35,10 @@ Requires-Dist: matplotlib
 Requires-Dist: ipywidgets
 Dynamic: license-file
 
-# Jupyter Analysis Tools (v1.5.1)
+# Jupyter Analysis Tools (v1.6.1)
 
 [![PyPI Package latest release](https://img.shields.io/pypi/v/jupyter-analysis-tools.svg)](https://pypi.org/project/jupyter-analysis-tools)
-[![Commits since latest release](https://img.shields.io/github/commits-since/BAMresearch/jupyter-analysis-tools/v1.5.1.svg)](https://github.com/BAMresearch/jupyter-analysis-tools/compare/v1.5.1...main)
+[![Commits since latest release](https://img.shields.io/github/commits-since/BAMresearch/jupyter-analysis-tools/v1.6.1.svg)](https://github.com/BAMresearch/jupyter-analysis-tools/compare/v1.6.1...main)
 [![License](https://img.shields.io/pypi/l/jupyter-analysis-tools.svg)](https://en.wikipedia.org/wiki/MIT_license)
 [![Supported versions](https://img.shields.io/pypi/pyversions/jupyter-analysis-tools.svg)](https://pypi.org/project/jupyter-analysis-tools)
 [![PyPI Wheel](https://img.shields.io/pypi/wheel/jupyter-analysis-tools.svg)](https://pypi.org/project/jupyter-analysis-tools#files)
@@ -97,6 +97,28 @@ are installed:
 
 # CHANGELOG
 
+## v1.6.1 (2025-09-19)
+
+### Bug fixes
+
+* **DataStore**: regression of handling of existing objects ([`28bd44c`](https://github.com/BAMresearch/jupyter-analysis-tools/commit/28bd44cf019aea0203ad252fd434860f45a1e54a))
+
+* **DataStore**: proper forwarding of spacePrefix for filtering relevant spaces ([`c4117f3`](https://github.com/BAMresearch/jupyter-analysis-tools/commit/c4117f3b04b9f6237f6d8471bef2d76c379e6f11))
+
+* **DataStore**: use the URL provided by constructor ([`fcc46fb`](https://github.com/BAMresearch/jupyter-analysis-tools/commit/fcc46fb941006b489054dc492f433e62d77cb605))
+
+## v1.6.0 (2025-09-19)
+
+### Bug fixes
+
+* **DataStore**: f-string syntax ([`9166382`](https://github.com/BAMresearch/jupyter-analysis-tools/commit/916638264be58e75fdfba15d9c6a6584ace92199))
+
+* **Tests**: pybis module required for collecting in new datastore module ([`ea6a21d`](https://github.com/BAMresearch/jupyter-analysis-tools/commit/ea6a21df3656dcc5f926aa7ff67a7136806ded3b))
+
+### Features
+
+* **DataStore**: new module for managing objects in OpenBIS ([`cdf0a27`](https://github.com/BAMresearch/jupyter-analysis-tools/commit/cdf0a27c0ae1412acd5329532ec8ec1fa7e6be94))
+
 ## v1.5.1 (2025-08-04)
 
 ### Bug fixes

{jupyter_analysis_tools-1.5.1.dist-info → jupyter_analysis_tools-1.6.1.dist-info}/RECORD

@@ -1,7 +1,8 @@
-jupyter_analysis_tools/__init__.py,sha256=wJgXbkvBGUcYsY4GmME-p9uQjFMk9TvzAfbOjOl_1zc,398
+jupyter_analysis_tools/__init__.py,sha256=A7AiYc6fGhMxcTkYmkh7R7xKIa3N_CWhHrkssN-4HEY,398
 jupyter_analysis_tools/analysis.py,sha256=AiAvUO648f0PYXqLfal1kDH926neasE5c1RYFu9wtYg,1768
 jupyter_analysis_tools/binning.py,sha256=d6eXRC3IOnnJIF25OfEASyWedT71EX2nF7jAgGJ9suQ,14536
 jupyter_analysis_tools/datalocations.py,sha256=BakfiZOMcBwp-_DAn7l57lGWZmZGNnk0j73V75nLBUA,4322
+jupyter_analysis_tools/datastore.py,sha256=JZqEr-taNmNLbkDOoaRVP5m-D9t4h1nUTcpkil-7H64,6903
 jupyter_analysis_tools/distrib.py,sha256=uyh2jXDdXR6dfd36CAoE5_psoFF0bfA6l1wletPD7Xo,16515
 jupyter_analysis_tools/git.py,sha256=mqSk5nnAFrmk1_2KFuKVrDWOkRbGbAQOq2N1DfxhNpg,2216
 jupyter_analysis_tools/plotting.py,sha256=X5Orrwiof-9MuYMKDJEXIlIt0K6bQT6ktFFjXKIVApI,1962
@@ -10,10 +11,10 @@ jupyter_analysis_tools/ssfz2json.py,sha256=aEJo8No_PZ021RJGqDz9g2uZVh9y2G-wNvUB7
 jupyter_analysis_tools/ssfz_compare.py,sha256=__6qXALyX5pdUBYSEjzNoVHa470QX8Cg_LASpahtAGI,1557
 jupyter_analysis_tools/utils.py,sha256=c8q2-0v7wEjJ_3w5YTZdjFSf-RP1gPUpMJpv5KUyilU,8800
 jupyter_analysis_tools/widgets.py,sha256=rA8qPvY9nS1OtykZwXtCTG29K-N_MYFVb5Aj8yK40_s,2996
-jupyter_analysis_tools-1.5.1.dist-info/licenses/AUTHORS.rst,sha256=-twUESsY0XqFQ0MIC0ylKhglNwL8lyHmGXriM3RF-2s,93
-jupyter_analysis_tools-1.5.1.dist-info/licenses/LICENSE,sha256=jRVl3hmCq0Qv1wifm-EelEKhFWecdoWdhcxSte4a1_c,1125
-jupyter_analysis_tools-1.5.1.dist-info/METADATA,sha256=hagcHmj9cDY740-W1ENigKPUMzIHNIX-CZzr2kCjh2Y,45652
-jupyter_analysis_tools-1.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-jupyter_analysis_tools-1.5.1.dist-info/entry_points.txt,sha256=-LU146dufa_JTwarciGzC6bjsl8pqY_8Z49ODYQ4lPY,124
-jupyter_analysis_tools-1.5.1.dist-info/top_level.txt,sha256=ei_0x-BF85FLoJ_h67ySwDFowtqus_gI4_0GR466PEU,23
-jupyter_analysis_tools-1.5.1.dist-info/RECORD,,
+jupyter_analysis_tools-1.6.1.dist-info/licenses/AUTHORS.rst,sha256=-twUESsY0XqFQ0MIC0ylKhglNwL8lyHmGXriM3RF-2s,93
+jupyter_analysis_tools-1.6.1.dist-info/licenses/LICENSE,sha256=jRVl3hmCq0Qv1wifm-EelEKhFWecdoWdhcxSte4a1_c,1125
+jupyter_analysis_tools-1.6.1.dist-info/METADATA,sha256=wRW9Y1cn-hXYw8PhmvszNUpwZ2H-tt8oYB52q34WJH4,46812
+jupyter_analysis_tools-1.6.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+jupyter_analysis_tools-1.6.1.dist-info/entry_points.txt,sha256=-LU146dufa_JTwarciGzC6bjsl8pqY_8Z49ODYQ4lPY,124
+jupyter_analysis_tools-1.6.1.dist-info/top_level.txt,sha256=ei_0x-BF85FLoJ_h67ySwDFowtqus_gI4_0GR466PEU,23
+jupyter_analysis_tools-1.6.1.dist-info/RECORD,,