pandahub 0.3.2__zip → 0.3.3__zip

{pandahub-0.3.2 → pandahub-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pandahub
-Version: 0.3.2
+Version: 0.3.3
 Summary: Data hub for pandapower and pandapipes networks based on MongoDB
 Home-page: https://github.com/e2nIEE/pandahub
 Author: Jan Ulffers, Leon Thurner, Jannis Kupka, Mike Vogt, Joschka Thurner, Alexander Scheidler

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.3.2"
+__version__ = "0.3.3"
 
 from pandahub.lib.PandaHub import PandaHub, PandaHubError
 from pandahub.client.PandaHubClient import PandaHubClient

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/api/internal/db.py

@@ -9,6 +9,7 @@ from fastapi_users_db_beanie.access_token import (
     BeanieBaseAccessToken,
 )
 
+from pandahub.api.internal.settings import REGISTRATION_ADMIN_APPROVAL
 from pandahub.api.internal import settings
 from pydantic import Field
 
@@ -22,6 +23,7 @@ db = client["user_management"]
 
 class User(BeanieBaseUser, Document):
     id: uuid.UUID = Field(default_factory=uuid.uuid4)
+    is_active: bool = not REGISTRATION_ADMIN_APPROVAL
     class Settings(BeanieBaseUser.Settings):
         name = "users"
 

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/api/internal/schemas.py

@@ -7,8 +7,10 @@ from pandahub.api.internal.settings import REGISTRATION_ADMIN_APPROVAL
 class UserRead(schemas.BaseUser[uuid.UUID]):
     pass
 
+
 class UserCreate(schemas.BaseUserCreate):
     is_active: bool = not REGISTRATION_ADMIN_APPROVAL
 
+
 class UserUpdate(schemas.BaseUserUpdate):
     pass

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/api/internal/settings.py

@@ -47,5 +47,4 @@ SECRET = get_secret("SECRET") or None
 REGISTRATION_ENABLED = settings_bool("REGISTRATION_ENABLED", default=True)
 REGISTRATION_ADMIN_APPROVAL = settings_bool("REGISTRATION_ADMIN_APPROVAL", default=False)
 
-DATATYPES_MODULE = os.getenv("DATATYPES_MODULE") or "pandahub.lib.datatypes"
 CREATE_INDEXES_WITH_PROJECT = settings_bool("CREATE_INDEXES_WITH_PROJECT", default=True)

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/api/routers/variants.py

@@ -21,11 +21,10 @@ class GetVariantsModel(BaseModel):
 
 @router.post("/get_variants")
 def get_variants(data: GetVariantsModel, ph=Depends(pandahub)):
-    project_id = data.project_id
-    ph.set_active_project_by_id(project_id)
-    db = ph._get_project_database()
+    ph.set_active_project_by_id(data.project_id)
+    variants_collection = ph.get_project_database("variant")
 
-    variants = db["variant"].find({"net_id": data.net_id}, projection={"_id": 0})
+    variants = variants_collection.find({"net_id": data.net_id}, projection={"_id": 0})
     response = {}
     for var in variants:
         response[var.pop("index")] = var

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/lib/PandaHub.py

@@ -1,13 +1,11 @@
 # -*- coding: utf-8 -*-
 import builtins
-import importlib
 import json
 import logging
-import traceback
 import warnings
 from inspect import signature, _empty
 from collections.abc import Callable
-from typing import Optional, Union
+from typing import Optional, Union, TypeVar
 
 import numpy as np
 import pandas as pd
@@ -20,10 +18,11 @@ from pymongo import MongoClient, ReplaceOne
 from pymongo.errors import ServerSelectionTimeoutError
 
 import pandapipes as pps
-from pandapipes import from_json_string as from_json_pps
+from pandapipes import from_json_string as from_json_pps, FromSerializableRegistryPpipe
 import pandapower as pp
 import pandapower.io_utils as io_pp
-import pandahub.api.internal.settings as SETTINGS
+from pandahub.api.internal.settings import MONGODB_URL, MONGODB_USER, MONGODB_PASSWORD, MONGODB_GLOBAL_DATABASE_URL, \
+    MONGODB_GLOBAL_DATABASE_USER, MONGODB_GLOBAL_DATABASE_PASSWORD, CREATE_INDEXES_WITH_PROJECT
 from pandahub.lib.database_toolbox import (
     create_timeseries_document,
     convert_timeseries_to_subdocuments,
@@ -34,10 +33,11 @@ from pandahub.lib.database_toolbox import (
     decompress_timeseries_data,
     convert_geojsons,
 )
-from pandahub.lib.mongodb_indexes import mongodb_indexes
+from pandahub.lib.mongodb_indexes import MONGODB_INDEXES
 
 logger = logging.getLogger(__name__)
 from pandahub import __version__
+from pandahub.lib.datatypes import DATATYPES
 from packaging import version
 
 
@@ -60,6 +60,8 @@ class PandaHubError(Exception):
 # PandaHub
 # -------------------------
 
+ProjectID = TypeVar("ProjectID", str, int, ObjectId)
+SettingsValue = TypeVar("SettingsValue", str, int, float, list, dict)
 
 class PandaHub:
     permissions = {
@@ -68,21 +70,19 @@ class PandaHub:
         "user_management": ["owner"],
     }
 
-    _datatypes = getattr(
-        importlib.import_module(SETTINGS.DATATYPES_MODULE), "datatypes"
-    )
-
     # -------------------------
     # Initialization
     # -------------------------
 
     def __init__(
         self,
-        connection_url=SETTINGS.MONGODB_URL,
-        connection_user=SETTINGS.MONGODB_USER,
-        connection_password=SETTINGS.MONGODB_PASSWORD,
+        connection_url=MONGODB_URL,
+        connection_user=MONGODB_USER,
+        connection_password=MONGODB_PASSWORD,
         check_server_available=False,
         user_id=None,
+        datatypes=DATATYPES,
+        mongodb_indexes=MONGODB_INDEXES,
     ):
         mongo_client_args = {
             "host": connection_url,
@@ -94,19 +94,19 @@ class PandaHub:
                 "username": connection_user,
                 "password": connection_password,
             }
+        self._datatypes = datatypes
+        self.mongodb_indexes = mongodb_indexes
         self.mongo_client = MongoClient(**mongo_client_args)
         self.mongo_client_global_db = None
         self.active_project = None
         self.user_id = user_id
         self.base_variant_filter = {
             "$or": [
-                {"var_type": {"$exists": False}},
                 {"var_type": "base"},
                 {"var_type": None},
                 {"var_type": np.nan},
             ]
         }
-        self.mongodb_indexes = mongodb_indexes
         if check_server_available:
             self.server_is_available()
 
@@ -212,6 +212,7 @@ class PandaHub:
         metadata=None,
         project_id=None,
         activate=True,
+        additional_project_data=None,
     ):
         if self.project_exists(name, realm):
             raise PandaHubError("Project already exists")
@@ -219,19 +220,22 @@ class PandaHub:
             settings = {}
         if metadata is None:
             metadata = {}
+        if additional_project_data is None:
+            additional_project_data = {}
         project_data = {
             "name": name,
             "realm": realm,
             "settings": settings,
             "metadata": metadata,
             "version": __version__,
+            **additional_project_data,
         }
         if project_id:
             project_data["_id"] = project_id
         if self.user_id is not None:
             project_data["users"] = {self.user_id: "owner"}
         self.mongo_client["user_management"]["projects"].insert_one(project_data)
-        if SETTINGS.CREATE_INDEXES_WITH_PROJECT:
+        if CREATE_INDEXES_WITH_PROJECT:
             self._create_mongodb_indexes(project_data["_id"])
         if activate:
             self.set_active_project_by_id(project_data["_id"])
@@ -269,6 +273,7 @@ class PandaHub:
                 "settings": p["settings"],
                 "locked": p.get("locked"),
                 "locked_by": p.get("locked_by"),
+                "locked_reason": p.get("locked_reason"),
                 "permissions": self.get_permissions_by_role(
                     p.get("users").get(self.user_id)
                 )
@@ -278,7 +283,7 @@ class PandaHub:
             for p in projects
         ]
 
-    def set_active_project(self, project_name, realm=None):
+    def set_active_project(self, project_name:str, realm=None):
         projects = self.get_projects()
         active_projects = [
             project for project in projects if project["name"] == project_name
@@ -291,7 +296,7 @@ class PandaHub:
             project_id = active_projects[0]["id"]
             self.set_active_project_by_id(project_id)
 
-    def set_active_project_by_id(self, project_id):
+    def set_active_project_by_id(self, project_id:ProjectID):
         try:
             project_id = ObjectId(project_id)
         except InvalidId:
@@ -300,7 +305,7 @@ class PandaHub:
         if self.active_project is None:
             raise PandaHubError("Project not found!", 404)
 
-    def rename_project(self, project_name):
+    def rename_project(self, project_name:str):
         self.has_permission("write")
         project_collection = self.mongo_client["user_management"].projects
         realm = self.active_project["realm"]
@@ -334,6 +339,20 @@ class PandaHub:
         )
         return result.acknowledged and result.modified_count > 0
 
+
+    # def lock_project(self):
+    #     db = self.mongo_client["user_management"]["projects"]
+    #     result = db.update_one(
+    #
+    #     result = db.find_one_and_update(
+    #         {
+    #             "_id": self.active_project["_id"],
+    #             "_id": self.active_project["_id"], "locked": False
+    #         },
+    #         {"$set": {"locked": True, "locked_by": self.user_id}},
+    #     )
+    #     return result.acknowledged and result.modified_count > 0
+
     def unlock_project(self):
         db = self.mongo_client["user_management"]["projects"]
         return db.update_one(
@@ -359,7 +378,7 @@ class PandaHub:
         else:
             raise PandaHubError("You don't have rights to access this project", 403)
 
-    def project_exists(self, project_name=None, realm=None):
+    def project_exists(self, project_name:Optional[str]=None, realm=None):
         project_collection = self.mongo_client["user_management"].projects
         project = project_collection.find_one({"name": project_name, "realm": realm})
         return project is not None
@@ -385,22 +404,40 @@ class PandaHub:
         else:
             return project_doc
 
-    def _get_project_database(self):
-        return self.mongo_client[str(self.active_project["_id"])]
+    def _get_project_database(self) -> MongoClient:
+        return self.get_project_database()
+
+    def get_project_database(self, collection: Optional[str] = None) -> MongoClient:
+        """
+        Get a MongoClient instance connected to the database for the current active project, optionally set to the given collection.
+
+        Parameters
+        ----------
+        collection
+            Name of document collection
+
+        Returns
+        -------
+        MongoClient
+        """
+        project_db = self.mongo_client[str(self.active_project["_id"])]
+        if collection is None:
+            return project_db
+        return project_db[collection]
 
-    def _get_global_database(self):
+    def _get_global_database(self) -> MongoClient:
         if (
             self.mongo_client_global_db is None
-            and SETTINGS.MONGODB_GLOBAL_DATABASE_URL is not None
+            and MONGODB_GLOBAL_DATABASE_URL is not None
         ):
             mongo_client_args = {
-                "host": SETTINGS.MONGODB_GLOBAL_DATABASE_URL,
+                "host": MONGODB_GLOBAL_DATABASE_URL,
                 "uuidRepresentation": "standard",
             }
-            if SETTINGS.MONGODB_GLOBAL_DATABASE_USER:
+            if MONGODB_GLOBAL_DATABASE_USER:
                 mongo_client_args |= {
-                    "username": SETTINGS.MONGODB_GLOBAL_DATABASE_USER,
-                    "password": SETTINGS.MONGODB_GLOBAL_DATABASE_PASSWORD,
+                    "username": MONGODB_GLOBAL_DATABASE_USER,
+                    "password": MONGODB_GLOBAL_DATABASE_PASSWORD,
                 }
             self.mongo_client_global_db = MongoClient(**mongo_client_args)
         if self.mongo_client_global_db is None:
@@ -408,6 +445,19 @@ class PandaHub:
         else:
             return self.mongo_client_global_db["global_data"]
 
+    def get_network_ids(self) -> list[int]:
+        """
+        Retrieve the id's of all networks in the active project.
+
+        Returns
+        -------
+        list
+            network ids
+        """
+        if not self.active_project:
+            raise PandaHubError("No project activated!")
+        return self.get_project_database("_network").find({}, {"_id:": 1}).distinct("_id")
+
     def get_project_version(self):
         return self.active_project.get("version", "0.2.2")
 
@@ -655,7 +705,7 @@ class PandaHub:
         only_tables=None,
         project_id=None,
         geo_mode="string",
-        variants=[],
+        variants=None,
     ):
         if project_id:
             self.set_active_project_by_id(project_id)
@@ -675,7 +725,7 @@ class PandaHub:
         only_tables=None,
         convert=True,
         geo_mode="string",
-        variants=[],
+        variants=None,
     ):
         self.check_permission("read")
         return self._get_net_from_db_by_id(
@@ -694,7 +744,7 @@ class PandaHub:
         only_tables=None,
         convert=True,
         geo_mode="string",
-        variants=[],
+        variants=None,
     ):
         db = self._get_project_database()
         meta = self._get_network_metadata(db, id_)
@@ -725,20 +775,16 @@ class PandaHub:
         return net
 
     def deserialize_and_update_data(self, net, meta):
+        registry = io_pp.FromSerializableRegistry if meta.get("sector", "power") == "power" \
+            else FromSerializableRegistryPpipe
         if version.parse(self.get_project_version()) <= version.parse("0.2.3"):
-            if meta.get("sector", "power") == "power":
-                data = dict(
-                    (k, json.loads(v, cls=io_pp.PPJSONDecoder))
-                    for k, v in meta["data"].items()
-                )
-                net.update(data)
-            else:
-                data = dict((k, from_json_pps(v)) for k, v in meta["data"].items())
-                net.update(data)
+            data = dict((k, json.loads(v, cls=io_pp.PPJSONDecoder, registry_class=registry))
+                        for k, v in meta["data"].items())
+            net.update(data)
         else:
             for key, value in meta["data"].items():
                 if type(value) == str and value.startswith("serialized_"):
-                    value = json.loads(value[11:], cls=io_pp.PPJSONDecoder)
+                    value = json.loads(value[11:], cls=io_pp.PPJSONDecoder, registry_class=registry)
                 net[key] = value
 
     def get_subnet_from_db(
@@ -748,7 +794,7 @@ class PandaHub:
         include_results=True,
         add_edge_branches=True,
         geo_mode="string",
-        variants=[],
+        variants=None,
         additional_filters: dict[
             str, Callable[[pp.auxiliary.pandapowerNet], dict]
         ] = {},
@@ -775,7 +821,7 @@ class PandaHub:
         include_results=True,
         add_edge_branches=True,
         geo_mode="string",
-        variants=[],
+        variants=None,
         ignore_elements=[],
         additional_filters: dict[
             str, Callable[[pp.auxiliary.pandapowerNet], dict]
@@ -804,7 +850,14 @@ class PandaHub:
             )
         buses = net.bus.index.tolist()
 
-        branch_operator = "$or" if add_edge_branches else "$and"
+        if isinstance(add_edge_branches, bool):
+            if add_edge_branches:
+                add_edge_branches = ["line", "trafo", "switch"]
+            else:
+                add_edge_branches = []
+        elif not isinstance(add_edge_branches, list):
+            raise ValueError("add_edge_branches must be a list or a boolean")
+        line_operator = "$or" if "line" in add_edge_branches else "$and"
         # Add branch elements connected to at least one bus
         self._add_element_from_collection(
             net,
@@ -812,7 +865,7 @@ class PandaHub:
             "line",
             net_id,
             {
-                branch_operator: [
+                line_operator: [
                     {"from_bus": {"$in": buses}},
                     {"to_bus": {"$in": buses}},
                 ]
@@ -821,12 +874,13 @@ class PandaHub:
             variants=variants,
             dtypes=dtypes,
         )
+        trafo_operator = "$or" if "trafo" in add_edge_branches else "$and"
         self._add_element_from_collection(
             net,
             db,
             "trafo",
             net_id,
-            {branch_operator: [{"hv_bus": {"$in": buses}}, {"lv_bus": {"$in": buses}}]},
+            {trafo_operator: [{"hv_bus": {"$in": buses}}, {"lv_bus": {"$in": buses}}]},
             geo_mode=geo_mode,
             variants=variants,
             dtypes=dtypes,
@@ -837,7 +891,7 @@ class PandaHub:
             "trafo3w",
             net_id,
             {
-                branch_operator: [
+                trafo_operator: [
                     {"hv_bus": {"$in": buses}},
                     {"mv_bus": {"$in": buses}},
                     {"lv_bus": {"$in": buses}},
@@ -847,7 +901,7 @@ class PandaHub:
             variants=variants,
             dtypes=dtypes,
         )
-
+        switch_operator = "$or" if "switch" in add_edge_branches else "$and"
         self._add_element_from_collection(
             net,
             db,
@@ -857,7 +911,7 @@ class PandaHub:
                 "$and": [
                     {"et": "b"},
                     {
-                        branch_operator: [
+                        switch_operator: [
                             {"bus": {"$in": buses}},
                             {"element": {"$in": buses}},
                         ]
@@ -937,11 +991,9 @@ class PandaHub:
         )
         all_elements = list(set(all_elements) - set(ignore_elements))
 
-        # Add elements for which the user has provided a filter function
-        for element, filter_func in additional_filters.items():
-            if element in ignore_elements:
-                continue
-            element_filter = filter_func(net)
+        # add all node elements that are connected to buses within the network
+        for element in node_elements:
+            element_filter = {"bus": {"$in": buses}}
             self._add_element_from_collection(
                 net,
                 db,
@@ -954,9 +1006,11 @@ class PandaHub:
                 dtypes=dtypes,
             )
 
-        # add all node elements that are connected to buses within the network
-        for element in node_elements:
-            element_filter = {"bus": {"$in": buses}}
+        # Add elements for which the user has provided a filter function
+        for element, filter_func in additional_filters.items():
+            if element in ignore_elements:
+                continue
+            element_filter = filter_func(net)
             self._add_element_from_collection(
                 net,
                 db,
@@ -1015,6 +1069,7 @@ class PandaHub:
         project_id=None,
         metadata=None,
         skip_results=False,
+        net_id=None,
     ):
         if project_id:
             self.set_active_project_by_id(project_id)
@@ -1029,8 +1084,11 @@ class PandaHub:
                 self.delete_net_from_db(name)
             else:
                 raise PandaHubError("Network name already exists")
-        max_id_network = db["_networks"].find_one(sort=[("_id", -1)])
-        _id = 0 if max_id_network is None else max_id_network["_id"] + 1
+
+        if net_id is None:
+            max_id_network = db["_networks"].find_one(sort=[("_id", -1)])
+            net_id = 0 if max_id_network is None else max_id_network["_id"] + 1
+        db["_networks"].insert_one({"_id": net_id})
 
         data = {}
         dtypes = {}
@@ -1045,10 +1103,12 @@ class PandaHub:
                 dtypes[element] = get_dtypes(element_data, self._datatypes.get(element))
                 if element_data.empty:
                     continue
-                # convert pandapower dataframe object to dict and save to db
-                element_data = convert_element_to_dict(
-                    element_data.copy(deep=True), _id, self._datatypes.get(element)
-                )
+                element_data = element_data.copy(deep=True)
+                if "var_type" in element_data:
+                    element_data["var_type"] = element_data["var_type"].fillna("base")
+                else:
+                    element_data["var_type"] = "base"
+                element_data = convert_element_to_dict(element_data, net_id, self._datatypes.get(element))
                 self._write_element_to_db(db, element, element_data)
 
             else:
@@ -1057,17 +1117,16 @@ class PandaHub:
                     data[element] = element_data
 
         # write network metadata
-        net_dict = {
-            "_id": _id,
+        network_data = {
             "name": name,
             "sector": sector,
             "dtypes": dtypes,
             "data": data,
         }
-
         if metadata is not None:
-            net_dict.update(metadata)
-        db["_networks"].insert_one(net_dict)
+            network_data.update(metadata)
+        db["_networks"].update_one({"_id": net_id}, {"$set": network_data})
+        return network_data | {"_id": net_id}
 
     def _write_net_collections_to_db(self, db, collections):
         for element, element_data in collections.items():
@@ -1140,7 +1199,7 @@ class PandaHub:
         include_results=True,
         only_tables=None,
         geo_mode="string",
-        variants=[],
+        variants=None,
         dtypes=None,
     ):
         if only_tables is not None and not element_type in only_tables:
@@ -1350,9 +1409,6 @@ class PandaHub:
         project_id=None,
         **kwargs,
     ):
-        logger.info(
-            f"Setting  {parameter} = {value} in {element_type} with index {element_index} and variant {variant}"
-        )
         if variant is not None:
             variant = int(variant)
         if project_id:
@@ -1650,32 +1706,30 @@ class PandaHub:
     # Variants
     # -------------------------
 
-    def create_variant(self, data):
+    def create_variant(self, data, index: Optional[int] = None):
         db = self._get_project_database()
         net_id = int(data["net_id"])
-        max_index = list(
-            db["variant"]
-            .find({"net_id": net_id}, projection={"_id": 0, "index": 1})
-            .sort("index", -1)
-            .limit(1)
-        )
-        if not max_index:
-            index = 1
-            for coll in self._get_net_collections(db):
-                update = {"$set": {"var_type": "base", "not_in_var": []}}
-                db[coll].update_many({}, update)
-
-        else:
-            index = int(max_index[0]["index"]) + 1
+        if index is None:
+            max_index = list(
+                db["variant"]
+                .find({"net_id": net_id}, projection={"_id": 0, "index": 1})
+                .sort("index", -1)
+                .limit(1)
+            )
+            index = int(max_index[0]["index"]) + 1 if max_index else 1
 
         data["index"] = index
-
         if data.get("default_name") is not None and data.get("name") is None:
             data["name"] = data.pop("default_name") + " " + str(index)
-
         db["variant"].insert_one(data)
         del data["_id"]
 
+        if index == 1:
+            for coll in self._get_net_collections(db):
+                db[coll].update_many(
+                    {"$or": [{"var_type": None}, {"var_type": np.nan}]},
+                    {"$set": {"var_type": "base", "not_in_var": []}}
+                )
         return data
 
     def delete_variant(self, net_id, index):
@@ -1702,7 +1756,7 @@ class PandaHub:
         db = self._get_project_database()
         db["variant"].update_one({"net_id": net_id, "index": index}, {"$set": data})
 
-    def get_variant_filter(self, variants):
+    def get_variant_filter(self, variants: Optional[int]) -> dict:
         """
         Creates a mongodb query filter to retrieve pandapower elements for the given variant(s).
 
@@ -1716,8 +1770,22 @@ class PandaHub:
         dict
             mongodb query filter for the given variant(s)
         """
-        if type(variants) is list and variants:
-            if len(variants) > 1:
+        if isinstance(variants, list):
+            warnings.warn(
+                f"Passing variants as list is deprecated, use None or int instead (variants: {variants})",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            if len(variants) == 0:
+                variants = None
+            elif len(variants) == 1:
+                variants = variants[0]
+            elif len(variants) > 1:
+                warnings.warn(
+                    f"Passing multiple variants is deprecated, use None or int instead (variants: {variants})",
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
                 variants = [
                     int(var) for var in variants
                 ]  # make sure variants are of type int
@@ -1730,8 +1798,6 @@ class PandaHub:
                         },
                     ]
                 }
-            else:
-                variants = variants[0]
         if variants:
             variants = int(variants)
             return {
@@ -2261,7 +2327,8 @@ class PandaHub:
                     pipeline.append({"$project": {"timeseries_data": 1}})
         else:
             if not include_metadata:
-                pipeline.append({"$project": {"timeseries_data": 1}})
+                pipeline.append({"$project": {"timeseries_data": 1,
+                                              "num_timestamps": 1}})
         data = list(db[collection_name].aggregate(pipeline))
         if len(data) == 0:
             raise PandaHubError("no documents matching the provided filter found", 404)
@@ -2270,9 +2337,9 @@ class PandaHub:
         else:
             data = data[0]
         if compressed_ts_data:
-            timeseries_data = decompress_timeseries_data(
-                data["timeseries_data"], ts_format
-            )
+            timeseries_data = decompress_timeseries_data(data["timeseries_data"],
+                                                         ts_format,
+                                                         num_timestamps=data["num_timestamps"])
         else:
             if ts_format == "timestamp_value":
                 timeseries_data = pd.Series(

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/lib/database_toolbox.py

@@ -3,7 +3,7 @@ from typing import Optional
 
 import numpy as np
 import pandas as pd
-from pandahub.api.internal import settings
+from pandahub.lib.datatypes import DATATYPES
 import base64
 import hashlib
 import logging
@@ -114,7 +114,7 @@ def convert_timeseries_to_subdocuments(timeseries):
 def compress_timeseries_data(timeseries_data, ts_format):
     import blosc
     if ts_format == "timestamp_value":
-        timeseries_data = np.array([timeseries_data.index.astype(int),
+        timeseries_data = np.array([timeseries_data.index.astype("int64"),
                                     timeseries_data.values])
         return blosc.compress(timeseries_data.tobytes(),
                               shuffle=blosc.SHUFFLE,
@@ -125,11 +125,11 @@ def compress_timeseries_data(timeseries_data, ts_format):
                               cname="zlib")
 
 
-def decompress_timeseries_data(timeseries_data, ts_format):
+def decompress_timeseries_data(timeseries_data, ts_format, num_timestamps):
     import blosc
     if ts_format == "timestamp_value":
         data = np.frombuffer(blosc.decompress(timeseries_data),
-                             dtype=np.float64).reshape((35040,2),
+                             dtype=np.float64).reshape((num_timestamps, 2),
                                                        order="F")
         return pd.Series(data[:,1], index=pd.to_datetime(data[:,0]))
     elif ts_format == "array":
@@ -233,10 +233,8 @@ def convert_element_to_dict(element_data, net_id, default_dtypes=None):
     load_geojsons(element_data)
     return element_data.to_dict(orient="records")
 
-def convert_dataframes_to_dicts(net, net_id, version_, datatypes=None):
-    if datatypes is None:
-        datatypes = getattr(importlib.import_module(settings.DATATYPES_MODULE), "datatypes")
 
+def convert_dataframes_to_dicts(net, net_id, version_, datatypes=DATATYPES):
     dataframes = {}
     other_parameters = {}
     types = {}

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/lib/datatypes.py

@@ -1,5 +1,5 @@
 
-datatypes = {
+DATATYPES = {
     "bus": {
         "name": str,
         "vn_kv": float,

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub/lib/mongodb_indexes.py

@@ -1,44 +1,58 @@
 from pymongo import DESCENDING, GEOSPHERE, IndexModel
 
-mongodb_indexes = {
+VARIANT_INDEXES = [
+    IndexModel([("variant", DESCENDING)]),
+    IndexModel([("var_type", DESCENDING)]),
+    IndexModel([("not_in_var", DESCENDING)]),
+]
+MONGODB_INDEXES = {
     # pandapower
     "net_bus": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("geo", GEOSPHERE)]),
+        IndexModel([("substation", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_line": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_bus", DESCENDING)]),
         IndexModel([("to_bus", DESCENDING)]),
         IndexModel([("geo", GEOSPHERE)]),
+        *VARIANT_INDEXES,
     ],
     "net_trafo":[
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("hv_bus", DESCENDING)]),
         IndexModel([("lv_bus", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_switch": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
         IndexModel([("element", DESCENDING)]),
         IndexModel([("et", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_load": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_sgen": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_gen": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_ext_grid": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
         IndexModel([("junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_shunt": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
@@ -47,87 +61,105 @@ mongodb_indexes = {
     "net_xward": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_ward": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_motor": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_storage": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("bus", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
 
     # pandapipes
     "net_junction": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("geo", GEOSPHERE)]),
+        *VARIANT_INDEXES,
     ],
     "net_pipe": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_junction", DESCENDING)]),
         IndexModel([("to_junction", DESCENDING)]),
         IndexModel([("geo", GEOSPHERE)]),
+        *VARIANT_INDEXES,
     ],
     "net_valve": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_junction", DESCENDING)]),
         IndexModel([("to_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_sink": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_source": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_water_tank": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_flow_control": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_junction", DESCENDING)]),
         IndexModel([("to_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_press_control": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_junction", DESCENDING)]),
         IndexModel([("to_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_compressor": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_junction", DESCENDING)]),
         IndexModel([("to_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_pump": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_junction", DESCENDING)]),
         IndexModel([("to_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_circ_pump_mass": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("return_junction", DESCENDING)]),
         IndexModel([("flow_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_circ_pump_pressure": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("return_junction", DESCENDING)]),
         IndexModel([("flow_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_heat_exchanger": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_junction", DESCENDING)]),
         IndexModel([("to_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_heat_consumer": [
         IndexModel([("net_id", DESCENDING), ("index", DESCENDING), ("variant", DESCENDING)], unique=True),
         IndexModel([("from_junction", DESCENDING)]),
         IndexModel([("to_junction", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
 
     # others
@@ -147,7 +179,7 @@ mongodb_indexes = {
         IndexModel([("substation_buses", DESCENDING)]),
         IndexModel([("level", DESCENDING)]),
         IndexModel([("geo", GEOSPHERE)]),
-        IndexModel([("variant", DESCENDING)]),
+        *VARIANT_INDEXES,
     ],
     "net_substation": [
         IndexModel(
@@ -158,5 +190,6 @@ mongodb_indexes = {
         IndexModel([("type", DESCENDING)]),
         IndexModel([("level", DESCENDING)]),
         IndexModel([("geo", GEOSPHERE)]),
+        *VARIANT_INDEXES,
     ],
 }

{pandahub-0.3.2 → pandahub-0.3.3}/pandahub.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pandahub
-Version: 0.3.2
+Version: 0.3.3
 Summary: Data hub for pandapower and pandapipes networks based on MongoDB
 Home-page: https://github.com/e2nIEE/pandahub
 Author: Jan Ulffers, Leon Thurner, Jannis Kupka, Mike Vogt, Joschka Thurner, Alexander Scheidler

{pandahub-0.3.2 → pandahub-0.3.3}/setup.py

@@ -36,7 +36,7 @@ setup(
     name='pandahub',
     packages=find_packages(),
     url='https://github.com/e2nIEE/pandahub',
-    version='0.3.2',
+    version='0.3.3',
     include_package_data=True,
     long_description_content_type='text/markdown',
     zip_safe=False,