pandahub 0.3.12__zip → 0.3.14__zip

{pandahub-0.3.12 → pandahub-0.3.14}/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## [0.3.13]
+    - FIXED: fetching integer index on empty collection
+    - IMPROVED: context manager for mongo client /database / collection
+    - IMPROVED: project locking returns True if user already holds a lock on the project
+
 ## [0.3.11]
 
     -  IMPROVED: add lib.database_toolbox.get_mongo_client function

{pandahub-0.3.12 → pandahub-0.3.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pandahub
-Version: 0.3.12
+Version: 0.3.14
 Summary: Data hub for pandapower and pandapipes networks based on MongoDB
 Author-email: Jan Ulffers <jan.ulffers@iee.fraunhofer.de>, Leon Thurner <leon.thurner@retoflow.de>, Jannis Kupka <jannis.kupka@retoflow.de>, Mike Vogt <mike.vogt@iee.fraunhofer.de>, Joschka Thurner <joschka.thurner@retoflow.de>, Alexander Scheidler <alexander.scheidler@iee.fraunhofer.de>
 License: Copyright (c) 2022 by University of Kassel, Fraunhofer Institute for Energy Economics

{pandahub-0.3.12 → pandahub-0.3.14}/pandahub/api/routers/variants.py

@@ -22,7 +22,7 @@ class GetVariantsModel(BaseModel):
 @router.post("/get_variants")
 def get_variants(data: GetVariantsModel, ph=Depends(pandahub)):
     ph.set_active_project_by_id(data.project_id)
-    variants_collection = ph.get_project_database("variant")
+    variants_collection = ph.get_project_collection("variant")
 
     variants = variants_collection.find({"net_id": data.net_id}, projection={"_id": 0})
     return {variant.pop("index"): variant for variant in variants}

{pandahub-0.3.12 → pandahub-0.3.14}/pandahub/lib/PandaHub.py

@@ -17,6 +17,8 @@ from functools import reduce
 from operator import getitem
 from uuid import UUID
 from pymongo import MongoClient, ReplaceOne
+from pymongo.database import Database
+from pymongo.collection import Collection
 from pymongo.errors import ServerSelectionTimeoutError, DuplicateKeyError
 
 import pandapipes as pps
@@ -335,14 +337,23 @@ class PandaHub:
         self.set_active_project(project_name, realm)
 
     def lock_project(self):
-        db = self.mongo_client["user_management"]["projects"]
-        result = db.update_one(
+        coll = self.mongo_client["user_management"]["projects"]
+        result = coll.find_one_and_update(
             {
                 "_id": self.active_project["_id"],
+                "$or": [
+                    {"locked": {"$eq": False}},
+                    {
+                        "$and": [
+                            {"locked": {"$eq": True}},
+                            {"locked_by": str(self.user_id)},
+                        ]
+                    },
+                ],
             },
-            {"$set": {"locked": True, "locked_by": self.user_id}},
+            {"$set": {"locked": True, "locked_by": str(self.user_id)}},
         )
-        return result.acknowledged and result.modified_count > 0
+        return result is not None
 
 
     # def lock_project(self):
@@ -409,26 +420,42 @@ class PandaHub:
         else:
             return project_doc
 
-    def _get_project_database(self) -> MongoClient:
+    def _get_project_database(self) -> Database:
         return self.get_project_database()
 
-    def get_project_database(self, collection: Optional[str] = None) -> MongoClient:
+    def get_project_database(self, collection: Optional[str] = None) -> Database:
+        """
+        Get the database for the current active project.
+
+        Parameters
+        ----------
+        collection or None
+            Name of document collection (DEPRECATED)
+
+        Returns
+        -------
+        pymongo.database.Database
+        """
+        if collection is None:
+            return self.mongo_client[str(self.active_project["_id"])]
+        logger.warning(f"Passing a collection to get_project_database is deprecated. Use get_project_collection instead!")
+        return self.get_project_collection(collection)
+
+    def get_project_collection(self, collection_name: str) -> Collection:
         """
         Get a MongoClient instance connected to the database for the current active project, optionally set to the given collection.
 
         Parameters
         ----------
-        collection
+        collection_name
             Name of document collection
 
         Returns
         -------
-        MongoClient
+        pymongo.collection.Collection
         """
-        project_db = self.mongo_client[str(self.active_project["_id"])]
-        if collection is None:
-            return project_db
-        return project_db[collection]
+        return self.get_project_database()[collection_name]
+
 
     def _get_global_database(self) -> MongoClient:
         if (
@@ -461,7 +488,7 @@ class PandaHub:
         """
         if not self.active_project:
             raise PandaHubError("No project activated!")
-        return self.get_project_database("_networks").find({}, {"_id:": 1}).distinct("_id")
+        return self.get_project_collection("_networks").find({}, {"_id:": 1}).distinct("_id")
 
     def get_project_version(self):
         return self.active_project.get("version", "0.2.2")
@@ -1647,21 +1674,22 @@ class PandaHub:
             var_data = {"var_type": "base", "not_in_var": [], "variant": None}
         else:
             var_data = {"var_type": "addition", "not_in_var": [], "variant": variant}
-
+        net_doc = db["_networks"].find_one({"_id": net_id})
         data = []
         for elm_data in elements_data:
-            self._add_missing_defaults(db, net_id, element_type, elm_data)
+            self._add_missing_defaults(element_type, elm_data, net_doc)
             self._ensure_dtypes(element_type, elm_data)
             data.append({**elm_data, **var_data, "net_id": net_id})
         collection = self._collection_name_of_element(element_type)
         db[collection].insert_many(data, ordered=False)
         return data
 
-    def _add_missing_defaults(self, db, net_id, element_type, element_data):
+    def _add_missing_defaults(self, element_type, element_data, net_doc):
         func_str = f"create_{element_type}"
-        if not hasattr(pp, func_str):
+        package = pp if net_doc['sector'] == 'power' else pps
+        if not hasattr(package, func_str):
             return
-        create_func = getattr(pp, func_str)
+        create_func = getattr(package, func_str)
         sig = signature(create_func)
         params = sig.parameters
 
@@ -1677,7 +1705,6 @@ class PandaHub:
         if element_type in ["line", "trafo", "trafo3w"]:
             # add standard type values
             std_type = element_data["std_type"]
-            net_doc = db["_networks"].find_one({"_id": net_id})
             if net_doc is not None:
                 # std_types = json.loads(net_doc["data"]["std_types"], cls=io_pp.PPJSONDecoder)[element_type]
                 std_types = net_doc["data"]["std_types"]
@@ -1688,6 +1715,9 @@ class PandaHub:
             if element_type == "line":
                 if "g_us_per_km" not in element_data:
                     element_data["g_us_per_km"] = 0
+        if element_type in ['sink', 'source']:
+            if not 'mdot_kg_per_s' in element_data:
+                element_data["mdot_kg_per_s"] = None
 
     def _ensure_dtypes(self, element_type, data):
         dtypes = self._datatypes.get(element_type)
@@ -1751,14 +1781,14 @@ class PandaHub:
             The value of the created index.
         """
         query_filter = query_filter if query_filter is not None else {}
-        coll = self.get_project_database(collection)
+        coll = self.get_project_collection(collection)
         for retry in range(retries):
             try:
-                max_index = next(coll.find(query_filter, projection={index_field: 1})
+                max_index_doc = next(coll.find(query_filter, projection={index_field: 1})
                                  .sort(index_field, -1)
                                  .limit(1),
                                  None)
-                index = 0 if max_index is None else int(max_index[0][index_field]) + 1
+                index = 0 if max_index_doc is None else int(max_index_doc[index_field]) + 1
                 coll.insert_one({index_field: index})
                 break
             except DuplicateKeyError:
@@ -2026,6 +2056,17 @@ class PandaHub:
             metadata = kwargs
             if data_type is not None:
                 metadata["data_type"] = data_type
+            net_id = metadata.get('net_id', None)
+            if net_id is None:
+                net_ids = db["_networks"].distinct("_id")
+                if len(net_ids) == 1:
+                    net_id = net_ids[0]
+                else:
+                    raise ValueError(
+                        "No net_id was provided and multiple networks exist in the database. "
+                        "Please provide a net_id."
+                    )
+            metadata["_id"] = f"{net_id}_{metadata.get('element_type', None)}_{metadata.get('element_index', None)}_{data_type}"
             if isinstance(timeseries, pd.Series):
                 documents = [
                     {"metadata": metadata, "timestamp": idx, "value": value}
@@ -2036,7 +2077,10 @@ class PandaHub:
                     {"metadata": metadata, "timestamp": idx, **row.to_dict()}
                     for idx, row in timeseries.iterrows()
                 ]
-            return db[collection_name].insert_many(documents)
+            db[collection_name].insert_many(documents)
+            if kwargs.get("return_id"):
+                return metadata["_id"]
+            return None
         document = create_timeseries_document(
             timeseries=timeseries,
             data_type=data_type,
@@ -2308,12 +2352,17 @@ class PandaHub:
             pipeline.append({"$match": meta_filter})
             pipeline.append({"$project": {"_id": 0, "metadata": 0}})
             timeseries = db[collection_name].aggregate_pandas_all(pipeline)
+            if len(timeseries) == 0:
+                raise PandaHubError("no documents matching the provided filter found", 404)
             timeseries.set_index("timestamp", inplace=True)
             if include_metadata:
                 raise NotImplementedError(
                     "Not implemented yet for timeseries collections"
                 )
-            return timeseries
+            if len(timeseries.columns) == 1:
+                return timeseries[timeseries.columns[0]]
+            else:
+                return timeseries
         filter_document = {**filter_document, **kwargs}
         pipeline = [{"$match": filter_document}]
         if not compressed_ts_data:
@@ -2400,6 +2449,7 @@ class PandaHub:
         collection_name="timeseries",
         global_database=False,
         project_id=None,
+        timestamp_range=None,
     ):
         """
         Returns a DataFrame, containing all metadata matching the provided filter.
@@ -2440,10 +2490,17 @@ class PandaHub:
                 pipeline.append({"$match": document_filter})
             else:
                 document_filter = {}
+            if timestamp_range is not None:
+                document_filter["timestamp"] = {
+                    "$gte": timestamp_range[0],
+                    "$lt": timestamp_range[1],
+                }
             document = db[collection_name].find_one(
                 document_filter, projection={"timestamp": 0, "_id": 0}
             )
-            value_fields = ["$%s" % field for field in document.keys()]
+            if document is None:
+                return pd.DataFrame()
+            value_fields = ["$%s" % field for field in document.keys() if field != "metadata"]
             group_dict = {
                 "_id": "$metadata._id",
                 "max_value": {"$max": {"$max": value_fields}},
@@ -2568,27 +2625,30 @@ class PandaHub:
                     document_filter,
                     projection={"timestamp": 0, "metadata": 0, "_id": 0},
                 )
-                meta_pipeline = []
-                meta_pipeline.append({"$match": document_filter})
-                value_fields = ["$%s" % field for field in document.keys()]
-                group_dict = {
-                    "_id": "$metadata._id",
-                    "max_value": {"$max": {"$max": value_fields}},
-                    "min_value": {"$min": {"$min": value_fields}},
-                    "first_timestamp": {"$min": "$timestamp"},
-                    "last_timestamp": {"$max": "$timestamp"},
-                }
-                document = db[collection_name].find_one(document_filter)
-                metadata_fields = {
-                    metadata_field: {"$first": "$metadata.%s" % metadata_field}
-                    for metadata_field in document["metadata"].keys()
-                    if metadata_field != "_id"
-                }
-                group_dict.update(metadata_fields)
-                meta_pipeline.append({"$group": group_dict})
-                meta_data = {
-                    d["_id"]: d for d in db[collection_name].aggregate(meta_pipeline)
-                }
+                if document is None:
+                    meta_data = {}
+                else:
+                    meta_pipeline = []
+                    meta_pipeline.append({"$match": document_filter})
+                    value_fields = ["$%s" % field for field in document.keys()]
+                    group_dict = {
+                        "_id": "$metadata._id",
+                        "max_value": {"$max": {"$max": value_fields}},
+                        "min_value": {"$min": {"$min": value_fields}},
+                        "first_timestamp": {"$min": "$timestamp"},
+                        "last_timestamp": {"$max": "$timestamp"},
+                    }
+                    document = db[collection_name].find_one(document_filter)
+                    metadata_fields = {
+                        metadata_field: {"$first": "$metadata.%s" % metadata_field}
+                        for metadata_field in document["metadata"].keys()
+                        if metadata_field != "_id"
+                    }
+                    group_dict.update(metadata_fields)
+                    meta_pipeline.append({"$group": group_dict})
+                    meta_data = {
+                        d["_id"]: d for d in db[collection_name].aggregate(meta_pipeline)
+                    }
             timeseries = []
             ts_all = db[collection_name].aggregate_pandas_all(pipeline)
             if len(ts_all) == 0:
@@ -2934,7 +2994,16 @@ class PandaHub:
         """
         self.check_permission("write")
         db = self._get_project_database()
-
+        if self.collection_is_timeseries(collection_name):
+            meta_filter = {
+                "metadata.element_type": element_type,
+                "metadata.data_type": data_type,
+            }
+            if netname is not None:
+                meta_filter["metadata.netname"] = netname
+            if element_index is not None:
+                meta_filter["metadata.element_index"] = element_index
+            return db[collection_name].delete_many(meta_filter)
         filter_document = {"element_type": element_type, "data_type": data_type}
         if netname is not None:
             filter_document["netname"] = netname

{pandahub-0.3.12 → pandahub-0.3.14}/pandahub/lib/database_toolbox.py

@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
-from typing import Optional
+from contextlib import contextmanager
+from collections.abc import Iterator
 
 import numpy as np
 import pandas as pd
 from pymongo import MongoClient
+from pymongo.database import Database
+from pymongo.collection import Collection
 
 from pandahub.api.internal.settings import MONGODB_URL, MONGODB_USER, MONGODB_PASSWORD
 from pandahub.lib.datatypes import DATATYPES
@@ -188,7 +191,7 @@ def create_timeseries_document(timeseries,
     document = add_timestamp_info_to_document(document, timeseries, ts_format)
     document = {**document, **kwargs}
 
-    if not "_id" in document: # IDs set by users will not be overwritten
+    if "_id" not in document: # IDs set by users will not be overwritten
         document["_id"] = get_document_hash(document)
 
     if compress_ts_data:
@@ -425,8 +428,8 @@ def migrate_userdb_to_beanie(ph):
                  {'$out': 'users'}]
     userdb.aggregate(migration)
 
-def get_mongo_client(connection_url=MONGODB_URL, connection_user=MONGODB_USER,
-                     connection_password=MONGODB_PASSWORD) -> MongoClient:
+def get_mongo_client(connection_url: str = MONGODB_URL, connection_user: str = MONGODB_USER,
+                     connection_password: str = MONGODB_PASSWORD) -> MongoClient:
     mongo_client_args = {
         "host": connection_url,
         "uuidRepresentation": "standard",
@@ -438,3 +441,40 @@ def get_mongo_client(connection_url=MONGODB_URL, connection_user=MONGODB_USER,
             "password": connection_password,
         }
     return MongoClient(**mongo_client_args)
+
+@contextmanager
+def mongo_client(database: str | None = None, collection: str | None = None, connection_url: str = MONGODB_URL,
+                 connection_user: str = MONGODB_USER, connection_password: str = MONGODB_PASSWORD) -> Iterator[MongoClient | Database | Collection]:
+    """Contextmanager for pymongo MongoClient / Database / Collection with close after use.
+
+    Parameters
+    ----------
+    database: str or None
+        The database to connect to
+    collection: str or None
+        The collection to connect to
+    connection_url: str or None
+        Defaults to MONGODB_URL env var
+    connection_user: str or None
+        Defaults to MONGODB_USER env var
+    connection_password: str or None
+        Defaults to MONGODB_PASSWORD env var
+
+    Returns
+    -------
+        Contextmanager yielding MongoClient / Database / Collection
+    """
+    if collection is not None and database is None:
+        raise ValueError("Must specify database to access a collection!")
+    client = get_mongo_client(connection_url, connection_user, connection_password)
+    try:
+        if database is not None and collection is not None:
+            yield client[database][collection]
+        elif database is not None:
+            yield client[database]
+        else:
+            yield client
+    finally:
+        client.close()
+
+

{pandahub-0.3.12 → pandahub-0.3.14}/pandahub.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pandahub
-Version: 0.3.12
+Version: 0.3.14
 Summary: Data hub for pandapower and pandapipes networks based on MongoDB
 Author-email: Jan Ulffers <jan.ulffers@iee.fraunhofer.de>, Leon Thurner <leon.thurner@retoflow.de>, Jannis Kupka <jannis.kupka@retoflow.de>, Mike Vogt <mike.vogt@iee.fraunhofer.de>, Joschka Thurner <joschka.thurner@retoflow.de>, Alexander Scheidler <alexander.scheidler@iee.fraunhofer.de>
 License: Copyright (c) 2022 by University of Kassel, Fraunhofer Institute for Energy Economics

{pandahub-0.3.12 → pandahub-0.3.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pandahub"
-version = "0.3.12"  # File format version '__format_version__' is tracked in __init__.py
+version = "0.3.14"  # File format version '__format_version__' is tracked in __init__.py
 authors=[
     { name = "Jan Ulffers", email = "jan.ulffers@iee.fraunhofer.de" },
     { name = "Leon Thurner", email = "leon.thurner@retoflow.de" },