pandahub 0.3.13__zip → 0.3.15__zip

{pandahub-0.3.13 → pandahub-0.3.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pandahub
-Version: 0.3.13
+Version: 0.3.15
 Summary: Data hub for pandapower and pandapipes networks based on MongoDB
 Author-email: Jan Ulffers <jan.ulffers@iee.fraunhofer.de>, Leon Thurner <leon.thurner@retoflow.de>, Jannis Kupka <jannis.kupka@retoflow.de>, Mike Vogt <mike.vogt@iee.fraunhofer.de>, Joschka Thurner <joschka.thurner@retoflow.de>, Alexander Scheidler <alexander.scheidler@iee.fraunhofer.de>
 License: Copyright (c) 2022 by University of Kassel, Fraunhofer Institute for Energy Economics

{pandahub-0.3.13 → pandahub-0.3.15}/pandahub/lib/PandaHub.py

@@ -38,6 +38,7 @@ from pandahub.lib.database_toolbox import (
     get_dtypes,
     decompress_timeseries_data,
     convert_geojsons,
+    get_metadata_for_timeseries_collections
 )
 from pandahub.lib.mongodb_indexes import MONGODB_INDEXES
 
@@ -1674,21 +1675,22 @@ class PandaHub:
             var_data = {"var_type": "base", "not_in_var": [], "variant": None}
         else:
             var_data = {"var_type": "addition", "not_in_var": [], "variant": variant}
-
+        net_doc = db["_networks"].find_one({"_id": net_id})
         data = []
         for elm_data in elements_data:
-            self._add_missing_defaults(db, net_id, element_type, elm_data)
+            self._add_missing_defaults(element_type, elm_data, net_doc)
             self._ensure_dtypes(element_type, elm_data)
             data.append({**elm_data, **var_data, "net_id": net_id})
         collection = self._collection_name_of_element(element_type)
         db[collection].insert_many(data, ordered=False)
         return data
 
-    def _add_missing_defaults(self, db, net_id, element_type, element_data):
+    def _add_missing_defaults(self, element_type, element_data, net_doc):
         func_str = f"create_{element_type}"
-        if not hasattr(pp, func_str):
+        package = pp if net_doc['sector'] == 'power' else pps
+        if not hasattr(package, func_str):
             return
-        create_func = getattr(pp, func_str)
+        create_func = getattr(package, func_str)
         sig = signature(create_func)
         params = sig.parameters
 
@@ -1704,7 +1706,6 @@ class PandaHub:
         if element_type in ["line", "trafo", "trafo3w"]:
             # add standard type values
             std_type = element_data["std_type"]
-            net_doc = db["_networks"].find_one({"_id": net_id})
             if net_doc is not None:
                 # std_types = json.loads(net_doc["data"]["std_types"], cls=io_pp.PPJSONDecoder)[element_type]
                 std_types = net_doc["data"]["std_types"]
@@ -1715,6 +1716,9 @@ class PandaHub:
             if element_type == "line":
                 if "g_us_per_km" not in element_data:
                     element_data["g_us_per_km"] = 0
+        if element_type in ['sink', 'source']:
+            if not 'mdot_kg_per_s' in element_data:
+                element_data["mdot_kg_per_s"] = None
 
     def _ensure_dtypes(self, element_type, data):
         dtypes = self._datatypes.get(element_type)
@@ -2050,9 +2054,21 @@ class PandaHub:
             self.check_permission("write")
             db = self._get_project_database()
         if self.collection_is_timeseries(collection_name, project_id, global_database):
-            metadata = kwargs
-            if data_type is not None:
-                metadata["data_type"] = data_type
+            # get metadata dictionary based on the input arguments
+            metadata = get_metadata_for_timeseries_collections(db, data_type=data_type, **kwargs)
+            _id =  metadata["_id"]
+
+            # delete overlapping timeseries already in the database
+            filter = {
+                "metadata._id": _id,
+                "timestamp": {
+                    "$gte": timeseries.index.min(),
+                    "$lte":  timeseries.index.max()
+                }
+            }
+            db.timeseries.delete_many(filter)
+
+            # create new timeseries documents
             if isinstance(timeseries, pd.Series):
                 documents = [
                     {"metadata": metadata, "timestamp": idx, "value": value}
@@ -2063,7 +2079,12 @@ class PandaHub:
                     {"metadata": metadata, "timestamp": idx, **row.to_dict()}
                     for idx, row in timeseries.iterrows()
                 ]
-            return db[collection_name].insert_many(documents)
+            db[collection_name].insert_many(documents)
+
+            if kwargs.get("return_id"):
+                return _id
+            return None
+
         document = create_timeseries_document(
             timeseries=timeseries,
             data_type=data_type,
@@ -2328,19 +2349,22 @@ class PandaHub:
             self.check_permission("read")
             db = self._get_project_database()
         if self.collection_is_timeseries(collection_name, project_id, global_database):
-            meta_filter = {
-                "metadata." + key: value for key, value in filter_document.items()
-            }
+            metadata = get_metadata_for_timeseries_collections(db, **kwargs)
             pipeline = []
-            pipeline.append({"$match": meta_filter})
+            pipeline.append({"$match": {"metadata._id": metadata["_id"]}})
             pipeline.append({"$project": {"_id": 0, "metadata": 0}})
             timeseries = db[collection_name].aggregate_pandas_all(pipeline)
+            if len(timeseries) == 0:
+                raise PandaHubError("no documents matching the provided filter found", 404)
             timeseries.set_index("timestamp", inplace=True)
             if include_metadata:
                 raise NotImplementedError(
                     "Not implemented yet for timeseries collections"
                 )
-            return timeseries
+            if len(timeseries.columns) == 1:
+                return timeseries[timeseries.columns[0]]
+            else:
+                return timeseries
         filter_document = {**filter_document, **kwargs}
         pipeline = [{"$match": filter_document}]
         if not compressed_ts_data:
@@ -2427,6 +2451,7 @@ class PandaHub:
         collection_name="timeseries",
         global_database=False,
         project_id=None,
+        timestamp_range=None,
     ):
         """
         Returns a DataFrame, containing all metadata matching the provided filter.
@@ -2467,10 +2492,17 @@ class PandaHub:
                 pipeline.append({"$match": document_filter})
             else:
                 document_filter = {}
+            if timestamp_range is not None:
+                document_filter["timestamp"] = {
+                    "$gte": timestamp_range[0],
+                    "$lt": timestamp_range[1],
+                }
             document = db[collection_name].find_one(
                 document_filter, projection={"timestamp": 0, "_id": 0}
             )
-            value_fields = ["$%s" % field for field in document.keys()]
+            if document is None:
+                return pd.DataFrame()
+            value_fields = ["$%s" % field for field in document.keys() if field != "metadata"]
             group_dict = {
                 "_id": "$metadata._id",
                 "max_value": {"$max": {"$max": value_fields}},
@@ -2595,27 +2627,30 @@ class PandaHub:
                     document_filter,
                     projection={"timestamp": 0, "metadata": 0, "_id": 0},
                 )
-                meta_pipeline = []
-                meta_pipeline.append({"$match": document_filter})
-                value_fields = ["$%s" % field for field in document.keys()]
-                group_dict = {
-                    "_id": "$metadata._id",
-                    "max_value": {"$max": {"$max": value_fields}},
-                    "min_value": {"$min": {"$min": value_fields}},
-                    "first_timestamp": {"$min": "$timestamp"},
-                    "last_timestamp": {"$max": "$timestamp"},
-                }
-                document = db[collection_name].find_one(document_filter)
-                metadata_fields = {
-                    metadata_field: {"$first": "$metadata.%s" % metadata_field}
-                    for metadata_field in document["metadata"].keys()
-                    if metadata_field != "_id"
-                }
-                group_dict.update(metadata_fields)
-                meta_pipeline.append({"$group": group_dict})
-                meta_data = {
-                    d["_id"]: d for d in db[collection_name].aggregate(meta_pipeline)
-                }
+                if document is None:
+                    meta_data = {}
+                else:
+                    meta_pipeline = []
+                    meta_pipeline.append({"$match": document_filter})
+                    value_fields = ["$%s" % field for field in document.keys()]
+                    group_dict = {
+                        "_id": "$metadata._id",
+                        "max_value": {"$max": {"$max": value_fields}},
+                        "min_value": {"$min": {"$min": value_fields}},
+                        "first_timestamp": {"$min": "$timestamp"},
+                        "last_timestamp": {"$max": "$timestamp"},
+                    }
+                    document = db[collection_name].find_one(document_filter)
+                    metadata_fields = {
+                        metadata_field: {"$first": "$metadata.%s" % metadata_field}
+                        for metadata_field in document["metadata"].keys()
+                        if metadata_field != "_id"
+                    }
+                    group_dict.update(metadata_fields)
+                    meta_pipeline.append({"$group": group_dict})
+                    meta_data = {
+                        d["_id"]: d for d in db[collection_name].aggregate(meta_pipeline)
+                    }
             timeseries = []
             ts_all = db[collection_name].aggregate_pandas_all(pipeline)
             if len(ts_all) == 0:
@@ -2961,7 +2996,9 @@ class PandaHub:
         """
         self.check_permission("write")
         db = self._get_project_database()
-
+        if self.collection_is_timeseries(collection_name):
+            metadata = get_metadata_for_timeseries_collections(db, data_type, **kwargs)
+            return db[collection_name].delete_many({"metadata._id": metadata["_id"]})
         filter_document = {"element_type": element_type, "data_type": data_type}
         if netname is not None:
             filter_document["netname"] = netname
@@ -3154,6 +3191,7 @@ class PandaHub:
         return self.get_net_collections(db, with_areas)
 
 
+
 if __name__ == "__main__":
     self = PandaHub()
     project_name = "test_project"

{pandahub-0.3.13 → pandahub-0.3.15}/pandahub/lib/database_toolbox.py

@@ -478,3 +478,22 @@ def mongo_client(database: str | None = None, collection: str | None = None, con
         client.close()
 
 
+def get_metadata_for_timeseries_collections(db, data_type=None, net_id=None, element_type=None, element_index=None, **kwargs):
+        if element_type is None:
+            raise ValueError("element_type needs to be defined for timeseries collections")
+        if element_index is None:
+            raise ValueError("element_index needs to be defined for timeseries collections")
+        if data_type is None:
+            raise ValueError("data_type needs to be defined for timeseries collections")
+        if net_id is None:
+            net_ids = db["_networks"].distinct("_id")
+            if len(net_ids) == 1:
+                net_id = net_ids[0]
+            else:
+                raise ValueError(
+                    "No net_id was provided and multiple networks exist in the database. "
+                    "Please provide a net_id."
+                )
+        metadata = {"data_type": data_type, "net_id": net_id, "element_type": element_type, "element_index": element_index, **kwargs}
+        metadata["_id"] = f"{net_id}_{element_type}_{element_index}_{data_type}"
+        return metadata

{pandahub-0.3.13 → pandahub-0.3.15}/pandahub.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pandahub
-Version: 0.3.13
+Version: 0.3.15
 Summary: Data hub for pandapower and pandapipes networks based on MongoDB
 Author-email: Jan Ulffers <jan.ulffers@iee.fraunhofer.de>, Leon Thurner <leon.thurner@retoflow.de>, Jannis Kupka <jannis.kupka@retoflow.de>, Mike Vogt <mike.vogt@iee.fraunhofer.de>, Joschka Thurner <joschka.thurner@retoflow.de>, Alexander Scheidler <alexander.scheidler@iee.fraunhofer.de>
 License: Copyright (c) 2022 by University of Kassel, Fraunhofer Institute for Energy Economics

{pandahub-0.3.13 → pandahub-0.3.15}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pandahub"
-version = "0.3.13"  # File format version '__format_version__' is tracked in __init__.py
+version = "0.3.15"  # File format version '__format_version__' is tracked in __init__.py
 authors=[
     { name = "Jan Ulffers", email = "jan.ulffers@iee.fraunhofer.de" },
     { name = "Leon Thurner", email = "leon.thurner@retoflow.de" },