kosh 3.0__tar.gz → 3.2.0__tar.gz

kosh-3.2.0/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.1
+Name: kosh
+Version: 3.2.0
+Summary: Manages Data Store with External Bulk Data
+Home-page: https://github.com/LLNL/Kosh
+Author: Charles Doutriaux, Renee Olson
+Author-email: doutriaux1@llnl.gov
+License: MIT
+Description: ![Kosh Logo](share/icons/png/Kosh_Logo_Blue.png)
+        # Overview
+        
+        Kosh allows codes to store, query, share data via an easy-to-use Python API. Kosh lies on top of Sina and as a result can use any database backend supported by Sina.
+        
+        In addition Kosh aims to make data access and sharing as simple as possible.
+        
+        Via "loaders", "transformers" and "operators" Kosh can access and process data in a consistent fashion, decoupled from data format and location. 
+        
+        Kosh is a Hindi word that means *treasury*, which is derived from *Kosha*, a Sanskrit word that means container in either a direct or metaphorical sense. A fairly good translation would be *repository*.
+        
+        # Getting Started
+        
+        
+        To get the latest public version:
+        
+        ```bash
+        pip install kosh
+        ```
+        
+        To get the latest stable, from a cloned repo simply run:
+        
+        ```bash
+        pip install .
+        ```
+        
+        Alternatively  add the path to this repo to your `PYTHONPATH` environment variable, or in your code with:
+        
+        ```python
+        import sys
+        sys.path.append(path_to_kosh_repo)
+        ```
+        
+        For more details look into the [installation doc](INSTALL.md)
+        
+        # First steps
+        
+        See [this file](docs/source/users/index.md)
+        
+        # Getting Involved
+        Kosh is user-oriented, and users' questions, comments, and contributions help guide its evolution. We welcome involvement and feedbacks.
+        
+        # Contact Info
+        You can reach our team at kosh-support@llnl.gov.
+        Kosh main developer can be reached at: doutriaux1@llnl.gov
+        
+        # Contributing
+        Contributions should be submitted as a pull request pointing to the develop branch, and must pass Kosh's CI process; to run the same checks locally follow the instructions in the tests directory.
+        
+        Contributions must be made under the same license as Kosh (see the bottom of this file).
+        
+        # Release and License
+        Kosh is distributed under the terms of the MIT license; new contributions must be made under this license.
+        
+        SPDX-License-Identifier: MIT
+        
+        ``LLNL-CODE-814755``
+Platform: UNKNOWN
+Classifier: Programming Language :: Python
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Description-Content-Type: text/markdown

{kosh-3.0 → kosh-3.2.0}/README.md

@@ -45,10 +45,7 @@ You can reach our team at kosh-support@llnl.gov.
 Kosh main developer can be reached at: doutriaux1@llnl.gov
 
 # Contributing
-Contributions should be submitted as a pull request pointing to the develop branch, and must pass Kosh's CI process; to run the same checks locally, use:
-```
-pytest tests/test_kosh*py
-```
+Contributions should be submitted as a pull request pointing to the develop branch, and must pass Kosh's CI process; to run the same checks locally follow the instructions in the tests directory.
 
 Contributions must be made under the same license as Kosh (see the bottom of this file).
 

{kosh-3.0 → kosh-3.2.0}/kosh/__init__.py

@@ -1,3 +1,4 @@
+import importlib.metadata
 from .current_version import current_version
 import os
 # import .sampling_methods  # noqa
@@ -6,18 +7,17 @@ from .loaders import KoshLoader, KoshSinaLoader  # noqa
 from .utils import create_new_db, walk_dictionary_keys, version  # noqa
 from .schema import KoshSchema  # noqa
 from .operators import KoshOperator  # noqa
-import pkg_resources  # noqa
 from .store import KoshStore, connect  # noqa
 from .dataset import KoshDataset  # noqa
 from .transformers import typed_transformer, numpy_transformer, typed_transformer_with_format  # noqa
 from .transformers import KoshTransformer  # noqa
 from .operators import KoshOperator  # noqa
 from .operators import typed_operator_with_kwargs, typed_operator, numpy_operator  # noqa
+from .kosh_command import KoshCmd  # noqa
 
 try:
-    d = pkg_resources.get_distribution("kosh")
-    __version__ = d.version
-    metadata = list(d._get_metadata(d.PKG_INFO))
+    metadata = importlib.metadata.metadata("kosh")
+    __version__ = metadata["Version"]
     __sha__ = None
     for meta in metadata:
         if "Summary:" in meta:

{kosh-3.0 → kosh-3.2.0}/kosh/core_sina.py

@@ -1,25 +1,32 @@
 import uuid
 import warnings
 import time
+import reprlib
 from .schema import KoshSchema
-from sina.model import Record
-from sina import get_version
-import pickle
+from .utils import KoshPickler
+from . import lock_strategies
 
 
-sina_version = float(".".join(get_version().split(".")[:2]))
+kosh_pickler = KoshPickler()
+
+
+def __getattr__(name):
+    if name == "sina_version":
+        from sina import get_version
+        return float(".".join(get_version().split(".")[:2]))
 
 
 class KoshSinaObject(object):
     """KoshSinaObject Base class for sina objects
     """
-
+    @lock_strategies.lock_method
     def get_record(self):
         return self.__store__.get_record(self.id)
 
     def __init__(self, Id, store, kosh_type,
                  record_handler, protected=[], metadata={}, schema=None,
                  record=None):
+        from sina.model import Record
         """__init__ sina object base class
 
         :param Id: id to use for unique identification, if None is passed set for you via uui4()
@@ -37,56 +44,59 @@ class KoshSinaObject(object):
         :param record: sina record to prevent looking it up again and again in sina
         :type record: Record
         """
-        self.__dict__["__store__"] = store
-        self.__dict__["__schema__"] = schema
-        self.__dict__["__record_handler__"] = record_handler
-        self.__dict__["__protected__"] = [
-            "id", "__type__", "__protected__",
-            "__record_handler__", "__store__", "id", "__schema__"] + protected
-        self.__dict__["__type__"] = kosh_type
-        if Id is None:
-            Id = uuid.uuid4().hex
-            record = Record(id=Id, type=kosh_type)
-            if store.__sync__:
-                store.lock()
-                store.__record_handler__.insert(record)
-                store.unlock()
-            else:
-                record["user_defined"]["last_update_from_db"] = time.time()
-                self.__store__.__sync__dict__[Id] = record
-            self.__dict__["id"] = Id
-        else:
-            self.__dict__["id"] = Id
-            if record is None:
-                try:
-                    record = self.get_record()
-                except BaseException:  # record exists nowhere
-                    record = Record(id=Id, type=kosh_type)
-                    if store.__sync__:
-                        store.lock()
-                        store.__record_handler__.insert(record)
-                        store.unlock()
-                    else:
-                        self.__store__.__sync__dict__[Id] = record
-                        record["user_defined"]["last_update_from_db"] = time.time()
+        with store.lock_strategy:
+            self.__dict__["__store__"] = store
+            self.__dict__["__schema__"] = schema
+            self.__dict__["__record_handler__"] = record_handler
+            self.__dict__["__protected__"] = [
+                "id", "__type__", "__protected__",
+                "__record_handler__", "__store__", "id", "__schema__"] + protected
+            self.__dict__["__type__"] = kosh_type
+            self.__dict__["lock_strategy"] = store.lock_strategy
+            if Id is None:
+                Id = uuid.uuid4().hex
+                record = Record(id=Id, type=kosh_type, user_defined={'kosh_information': {}})
+                if store.__sync__:
+                    store.lock()
+                    store.__record_handler__.insert(record)
+                    store.unlock()
+                else:
+                    record["user_defined"]['kosh_information']["last_update_from_db"] = time.time()
+                    self.__store__.__sync__dict__[Id] = record
+                self.__dict__["id"] = Id
             else:
-                deleted_items = False
-                for att in self.__dict__["__protected__"]:
-                    if att in record["data"]:
-                        del record["data"][att]
-                        deleted_items = True
-                if deleted_items:
-                    if store.__sync__:
-                        self._update_record(record)
-                    else:
-                        self.__store__.__sync__dict__[Id] = record
-
-        metadata_copy = metadata.copy()
-        for key in metadata:
-            if att in self.__dict__["__protected__"]:
-                del metadata_copy[key]
-        self.update(metadata_copy)
-
+                self.__dict__["id"] = Id
+                if record is None:
+                    try:
+                        record = self.get_record()
+                    except BaseException:  # record exists nowhere
+                        record = Record(id=Id, type=kosh_type, user_defined={'kosh_information': {}})
+                        if store.__sync__:
+                            store.lock()
+                            store.__record_handler__.insert(record)
+                            store.unlock()
+                        else:
+                            self.__store__.__sync__dict__[Id] = record
+                            record["user_defined"]['kosh_information']["last_update_from_db"] = time.time()
+                else:
+                    deleted_items = False
+                    for att in self.__dict__["__protected__"]:
+                        if att in record["data"]:
+                            del record["data"][att]
+                            deleted_items = True
+                    if deleted_items:
+                        if store.__sync__:
+                            self._update_record(record)
+                        else:
+                            self.__store__.__sync__dict__[Id] = record
+
+            metadata_copy = metadata.copy()
+            for key in metadata:
+                if key in self.__dict__["__protected__"]:
+                    del metadata_copy[key]
+            self.update(metadata_copy)
+
+    @lock_strategies.lock_method
     def __getattr__(self, name):
         """__getattr__ get an attribute
 
@@ -105,6 +115,12 @@ class KoshSinaObject(object):
                 rels = self.get_sina_store().relationships.find(
                     None, "is a member of ensemble", self.id)
                 return [str(x.subject_id) for x in rels]
+            if name == "__features__":
+                record = self.get_record()
+                try:
+                    return KoshPickler().loads(record["user_defined"]["__features__"])
+                except Exception:
+                    return {None: {}}
             if name == "_associated_data_":
                 from kosh.dataset import KoshDataset
                 record = self.get_record()
@@ -138,13 +154,19 @@ class KoshSinaObject(object):
         elif name == "schema":
             if self.__dict__[
                     "__schema__"] is None and "schema" in record["data"]:
-                schema = pickle.loads(
-                    record["data"]["schema"]["value"].encode("latin1"))
+                schema = kosh_pickler.loads(record["data"]["schema"]["value"])
                 self.__dict__["__schema__"] = schema
             return self.__dict__["__schema__"]
+        elif name in ['alias_feature', 'loader_kwargs']:
+            if name in record["data"]:
+                return kosh_pickler.loads(record["data"][name]["value"])
+            else:
+                return {}
         if name not in record["data"]:
             if name == "mime_type":
                 return record["type"]
+            elif name == "uri":
+                return ""
             else:
                 raise AttributeError(
                     "Object {} does not have {} attribute".format(self.id,
@@ -154,18 +176,20 @@ class KoshSinaObject(object):
             # old records have user id let's fix this
             if value in self.__store__.__record_handler__.find_with_type(
                     self.__store__._users_type, ids_only=True):
-                value = self.__store__.get_record(
-                    value)["data"]["username"]["value"]
+                value = self.__store__.get_record(value)["data"]["username"]["value"]
         return value
 
+    @lock_strategies.lock_method
     def get_sina_store(self):
         """Returns the sina store object"""
         return self.__store__.get_sina_store()
 
+    @lock_strategies.lock_method
     def get_sina_records(self):
         """Returns sina store's records"""
         return self.__record_handler__
 
+    @lock_strategies.lock_method
     def update(self, attributes):
         """update many attributes at once to limit db writes
         :param: attributes: dictionary with attributes to update
@@ -184,6 +208,7 @@ class KoshSinaObject(object):
                 update_db = False
             rec = self.___setattr___(name, value, rec, update_db=update_db)
 
+    @lock_strategies.lock_method
     def __setattr__(self, name, value):
         """set an attribute
         We are calling the ___setattr___
@@ -191,6 +216,7 @@ class KoshSinaObject(object):
         """
         self.___setattr___(name, value)
 
+    @lock_strategies.lock_method
     def ___setattr___(self, name, value, record=None,
                       update_db=True, force=False):
         """__setattr__ set an attribute on an object
@@ -213,6 +239,14 @@ class KoshSinaObject(object):
         if name == "schema":
             assert isinstance(value, KoshSchema)
             value.validate(self)
+        elif name == "__features__":
+            value = kosh_pickler.dumps(value)
+            record["user_defined"]["__features__"] = value
+        elif name in ['alias_feature', 'loader_kwargs']:
+            if isinstance(value, dict):
+                value = kosh_pickler.dumps(value)
+            else:  # Pre-pickled at dataset level
+                value = value
         elif self.schema is not None:
             self.schema.validate_attribute(name, value)
 
@@ -225,36 +259,45 @@ class KoshSinaObject(object):
                 self.id, self.__store__._ensemble_predicate, None)
             for relationship in relationships:
                 ensemble = self.__store__.open(relationship.object_id)
-                if name in ensemble.list_attributes() and name not in ensemble.__dict__["__ok_duplicates__"]:
-                    raise KeyError(
-                        "The attribute {} is controlled by ensemble: {} and cannot be set here".format(
-                            name, relationship.object_id))
+                ens_tags = self.list_ensemble_tags(ensemble.id, dictionary=True, obscure=False)
+                inherit_attributes = ens_tags.get(f"{ensemble.id}_ENSEMBLE_TAG_INHERIT_ATTRIBUTES", True)
+                if inherit_attributes:
+                    if name in ensemble.list_attributes() and name not in ensemble.__dict__["__ok_duplicates__"]:
+                        if value != getattr(ensemble, name):
+                            raise KeyError(
+                                "The attribute {} is controlled by ensemble: {} and cannot be set here".format(
+                                    name, relationship.object_id))
+                        else:
+                            warnings.warn(
+                                "The attribute {} is controlled by ensemble: {}"
+                                ". You should NOT set this attribute at the dataset level"
+                                ". Values match so we will accept it here".format(
+                                    name, relationship.object_id), UserWarning)
 
         # For Ensembles we need to set it on all members
         from kosh.ensemble import KoshEnsemble
         if isinstance(self, KoshEnsemble):
             # First we make a pass to collect all other ensembles datasets are
             # part of
-            other_ensembles = set()
             for dataset in self.get_members():
-                for e in dataset.get_ensembles():
-                    other_ensembles.add(e)
-            for ensemble in other_ensembles:
-                if ensemble.id == self.id:
-                    continue
-                for att in ensemble.list_attributes():
-                    if att in self.__dict__["__ok_duplicates__"]:
-                        continue
-                    if att == name:
-                        raise NameError("A member of this ensemble belongs to ensemble {} "
-                                        "which already controls attribute {}".format(ensemble.id, att))
-            for dataset in self.get_members():
-                dataset.___setattr___(
-                    name=name,
-                    value=value,
-                    record=None,
-                    update_db=update_db,
-                    force=True)
+                for ensemble in dataset.get_ensembles():
+                    ens_tags = dataset.list_ensemble_tags(ensemble.id, dictionary=True, obscure=False)
+                    inherit_attributes = ens_tags.get(f"{ensemble.id}_ENSEMBLE_TAG_INHERIT_ATTRIBUTES", True)
+                    if inherit_attributes:
+                        if ensemble.id != self.id:
+                            for att in ensemble.list_attributes():
+                                if att in self.__dict__["__ok_duplicates__"]:
+                                    continue
+                                if att == name:
+                                    raise NameError("A member of this ensemble belongs to ensemble {} "
+                                                    "which already controls attribute {}".format(ensemble.id, att))
+
+                        dataset.___setattr___(
+                            name=name,
+                            value=value,
+                            record=None,
+                            update_db=update_db,
+                            force=True)
 
         # Did it change on db since we last read it?
         last_modif_att = "{name}_last_modified".format(name=name)
@@ -265,7 +308,7 @@ class KoshSinaObject(object):
             last = time.time()
         try:
             # Time we last read its value
-            last_db = record["user_defined"][last_modif_att]
+            last_db = record["user_defined"]['kosh_information'][last_modif_att]
         except KeyError:
             last_db = last
         # last time attribute was modified in db
@@ -281,15 +324,16 @@ class KoshSinaObject(object):
         if "{name}_last_modified".format(name=name) not in self.__protected__:
             self.__dict__["__protected__"] += [last_modif_att, ]
         self.__dict__[last_modif_att] = now
-        record["user_defined"][last_modif_att] = now
+        record["user_defined"]['kosh_information'][last_modif_att] = now
         if name == "schema":
             self.__dict__["__schema__"] = value
-            value = pickle.dumps(value).decode("latin1")
+            value = kosh_pickler.dumps(value)
         record["data"][name] = {"value": value}
         if update_db and self.__store__.__sync__:
             self._update_record(record)
         return record
 
+    @lock_strategies.lock_method
     def _update_record(self, record, store=None):
         """Updates a record in the sina store
         :param record: The record to update
@@ -302,23 +346,14 @@ class KoshSinaObject(object):
         id_ = record.id
         rels = store.relationships.find(id_, None, None)
         rels += store.relationships.find(None, None, id_)
-        try:
-            # if rec exists let's get it
-            old_record = store.records.get(id_)
-        except Exception:
-            old_record = None  # new record
-        store.records.delete(id_)
-        try:
+        if not self.__store__.__sync__:
+            store.records.delete(id_)
             store.records.insert(record)
-        except Exception as err:
-            # Let's put back in place the old record
-            if old_record is not None:
-                store.records.insert(old_record)
-            raise err
-
-        store.relationships.insert(rels)
+            store.relationships.insert(rels)
+        store.records.update(record)
         self.__store__.unlock()
 
+    @lock_strategies.lock_method
     def __delattr__(self, name):
         """__delattr__ deletes an attribute
 
@@ -330,7 +365,7 @@ class KoshSinaObject(object):
         record = self.get_record()
         last_modif_att = "{name}_last_modified".format(name=name)
         now = time.time()
-        record["user_defined"][last_modif_att] = now
+        record["user_defined"]['kosh_information'][last_modif_att] = now
         # We need to remember we touched it otherwise
         # if we create it again the db will look
         # out of sync.
@@ -339,36 +374,53 @@ class KoshSinaObject(object):
         if self.__store__.__sync__:
             self._update_record(record)
 
+    @lock_strategies.lock_method
     def sync(self):
         """sync this object with database"""
         self.__store__.sync([self.id, ])
 
-    def list_attributes(self, dictionary=False):
+    @lock_strategies.lock_method
+    def list_attributes(self, dictionary=False, ensemble_id=None):
         __doc__ = self.listattributes.__doc__.replace("listattributes", "list_attributes")  # noqa
-        return self.listattributes(dictionary=dictionary)
+        return self.listattributes(dictionary=dictionary, ensemble_id=ensemble_id)
 
-    def listattributes(self, dictionary=False):
+    @lock_strategies.lock_method
+    def listattributes(self, dictionary=False, ensemble_id=None):
         """listattributes list all non protected attributes
 
-        :parm dictionary: return a dictionary of value/pair rather than just attributes names
+        :param dictionary: return a dictionary of value/pair rather than just attributes names
         :type dictionary: bool
+        :param ensemble_id: Provide ensemble ID(s) to return ensemble tags
+        :type ensemble_id: str, lst
 
         :return: list of attributes set on object
         :rtype: list
         """
         record = self.get_record()
         attributes = list(record["data"].keys()) + ['id', ]
+
+        no_tags = [a for a in attributes if "_ENSEMBLE_TAG_" not in a]  # Remove ensemble tags
+        ens_tags = []
+        if ensemble_id is not None:
+            if isinstance(ensemble_id, str):
+                ensemble_id = [ensemble_id]
+            for ens_id in ensemble_id:
+                ens_tags += sorted([a for a in attributes if f"{ens_id}_ENSEMBLE_TAG_" in a])
+        attributes = sorted(no_tags)
+
         for att in self.__protected__:
             if att in attributes and att != "id":
                 attributes.remove(att)
+        attributes += ens_tags
         if dictionary:
             out = {}
             for att in attributes:
                 out[att] = getattr(self, att)
             return out
         else:
-            return sorted(attributes)
+            return attributes
 
+    @lock_strategies.lock_method
     def __getattributes__(self):
         """__getattributes__ return dictionary with pairs of attribute/value
 
@@ -378,27 +430,39 @@ class KoshSinaObject(object):
         record = self.get_record()
         attributes = {}
         for a in record["data"]:
+            if a in ['alias_feature', 'loader_kwargs']:
+                continue
             attributes[a] = record["data"][a]["value"]
             if a == "creator":
                 # old records have user id let's fix this
                 if attributes[a] in self.__store__.__record_handler__.find_with_type(
                         self.__store__._users_type, ids_only=True):
-                    attributes[a] = self.__store__.get_record(
-                        attributes[a])["data"]["username"]["value"]
+                    attributes[a] = self.__store__.get_record(attributes[a])["data"]["username"]["value"]
         return attributes
 
+    @lock_strategies.lock_method
     def __str__(self):
         """String for printing"""
+        if self.__dict__["__store__"].verbose_attributes:
+            def reprtool(item):
+                return item
+        else:
+            def reprtool(item):
+                if isinstance(item, str):
+                    return reprlib.repr(item)[1:-1]
+                else:
+                    return reprlib.repr(item)
         st = "Id: {}".format(self.id)
         for att in sorted(self.listattributes()):
             if att != 'id':
-                st += "\n\t{}: {}".format(att, getattr(self, att))
+                st += "\n\t{}: {}".format(att, reprtool(getattr(self, att)))
         return st
 
 
 class KoshSinaFile(KoshSinaObject):
     """KoshSinaFile file representation in Kosh via Sina"""
 
+    @lock_strategies.lock_method
     def open(self, *args, **kargs):
         """open opens the file
         :return: handle to file in open mode

kosh-3.2.0/kosh/current_version.py

@@ -0,0 +1 @@
+current_version = "3.2.0"