cerc-persistence 0.1.0.4__tar.gz

cerc-persistence-0.1.0.4/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: cerc-persistence
+Version: 0.1.0.4
+Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Dist: pathlib
+Requires-Dist: python-dotenv
+Requires-Dist: SQLAlchemy
+Requires-Dist: cerc-hub
+Requires-Dist: psycopg2-binary
+Requires-Dist: sqlalchemy_utils
+Requires-Dist: setuptools

cerc-persistence-0.1.0.4/README.md

@@ -0,0 +1,63 @@
+# CERC Persistence
+
+The persistence package includes classes to store different class objects in a Postgres database.
+
+## Models
+
+This defines models for all class objects that we want to persist. It is used for Object Relation Mapping (ORM)
+of the class objects to database table columns
+
+## Repositories
+
+This defines repository classes that contain CRUD methods for database operations. The constructor of all repositories
+requires the database name to connect to and the application environment (PROD or TEST). Tests use a different database
+from the production environment, which is why this is necessary. An example is shown below:
+
+```python
+from cerc_persistence.db_setup import DBSetup
+
+# instantiate city repo for hub production database
+db = DBSetup(db_name='hub', app_env='PROD', dotenv_path='/path/to/.env', admin_password='application_password', application_uuid='UUID')
+```
+
+All database operations are conducted with the production database (*PROD*) named *hub* in the example above.
+
+## config_db
+
+This Python file is a configuration class that contains variables that map to configuration parameters in a .env file.
+It also contains a method ``def conn_string()`` which returns the connection string to a Postgres database.
+
+## Base
+
+This class has a constructor that establishes a database connection and returns a reference for database-related CRUD
+operations.
+
+## Database Configuration Parameter
+
+A .env file (or environment variables) with configuration parameters described below are needed to establish a database
+connection:
+
+```
+# production database credentials
+PROD_DB_USER=postgres-database-user
+PROD_DB_PASSWORD=postgres-database-password
+PROD_DB_HOST=database-host
+PROD_DB_PORT=database-port
+
+# test database credentials
+TEST_DB_USER=postgres-database-user
+TEST_DB_PASSWORD=postgres-database-password
+TEST_DB_HOST=database-host
+TEST_DB_PORT=database-port
+```
+
+## Database Related Unit Test
+
+Unit tests that involve database operations require a Postgres database to be set up.
+The tests connect to the database server using the default postgres user (*postgres*).
+NB: You can provide any credentials for the test to connect to postgres, just make sure
+the credentials are set in your .env file as explained above in *Database Configuration Parameters* section.
+
+When the tests are run, a **test_db** database is created and then the required tables for
+the test. Before the tests run, the *test_db* is deleted to ensure that each test starts
+on a clean slate.

cerc-persistence-0.1.0.4/cerc_persistence/__init__.py

@@ -0,0 +1,3 @@
+"""
+Repositories Package
+"""

cerc-persistence-0.1.0.4/cerc_persistence/configuration.py

@@ -0,0 +1,67 @@
+"""
+Persistence (Postgresql) configuration
+SPDX - License - Identifier: LGPL - 3.0 - or -later
+Copyright © 2022 Concordia CERC group
+Project Coder Peter Yefi peteryefi@gmail.com
+"""
+
+import logging
+import os
+from pathlib import Path
+from dotenv import load_dotenv
+from sqlalchemy.ext.declarative import declarative_base
+
+Models = declarative_base()
+
+
+class Configuration:
+  """
+  Configuration class to hold common persistence configuration
+  """
+
+  def __init__(self, db_name: str, dotenv_path: str, app_env='TEST'):
+    """
+    :param db_name: database name
+    :param app_env: application environment, test or production
+    :param dotenv_path: the absolute path to dotenv file
+    """
+    try:
+      # load environmental variables
+      if not Path(dotenv_path).exists():
+        error_message = f'dotenv file doesn\'t exists at {dotenv_path}'
+        logging.error(error_message)
+        raise FileNotFoundError(error_message)
+      load_dotenv(dotenv_path=dotenv_path)
+
+      self._db_name = db_name
+      self._db_host = os.getenv(f'{app_env}_DB_HOST')
+      self._db_user = os.getenv(f'{app_env}_DB_USER')
+      self._db_pass = os.getenv(f'{app_env}_DB_PASSWORD')
+      self._db_port = os.getenv(f'{app_env}_DB_PORT')
+      self.hub_token = os.getenv('HUB_TOKEN')
+    except KeyError as err:
+      logging.error('Error with credentials: %s', err)
+
+  @property
+  def connection_string(self):
+    """
+    Returns a connection string postgresql
+    :return: connection string
+    """
+    if self._db_pass:
+      return f'postgresql://{self._db_user}:{self._db_pass}@{self._db_host}:{self._db_port}/{self._db_name}'
+    return f'postgresql://{self._db_user}@{self._db_host}:{self._db_port}/{self._db_name}'
+
+  @property
+  def db_user(self):
+    """
+    retrieve the configured username
+    """
+    return self._db_user
+
+  @property
+  def db_name(self):
+    """
+    retrieve the configured database name
+    """
+    return self._db_name

cerc-persistence-0.1.0.4/cerc_persistence/db_control.py

@@ -0,0 +1,241 @@
+"""
+DBFactory performs read related operations
+SPDX - License - Identifier: LGPL - 3.0 - or -later
+Copyright © 2022 Concordia CERC group
+Project CoderPeter Yefi peteryefi@gmail.com
+"""
+from typing import Dict
+
+from cerc_persistence.repositories.application import Application
+from cerc_persistence.repositories.city import City
+from cerc_persistence.repositories.city_object import CityObject
+from cerc_persistence.repositories.simulation_results import SimulationResults
+from cerc_persistence.repositories.user import User
+from cerc_persistence.repositories.user import UserRoles
+
+
+class DBControl:
+  """
+  DBFactory class
+  """
+
+  def __init__(self, db_name, app_env, dotenv_path):
+    self._city = City(db_name=db_name, dotenv_path=dotenv_path, app_env=app_env)
+    self._application = Application(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
+    self._user = User(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
+    self._city_object = CityObject(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
+    self._simulation_results = SimulationResults(db_name=db_name, dotenv_path=dotenv_path, app_env=app_env)
+
+  def application_info(self, application_uuid) -> Application:
+    """
+    Retrieve the application info for the given uuid from the database
+    :param application_uuid: the uuid for the application
+    :return: Application
+    """
+    return self._application.get_by_uuid(application_uuid)
+
+  def user_info(self, name, password, application_id) -> User:
+    """
+    Retrieve the user info for the given name and password and application_id from the database
+    :param name: the username
+    :param password: the user password
+    :param application_id: the application id
+    :return: User
+    """
+    return self._user.get_by_name_application_id_and_password(name, password, application_id)
+
+  def user_login(self, name, password, application_uuid) -> User:
+    """
+    Retrieve the user info from the database
+    :param name: the username
+    :param password: the user password
+    :param application_uuid: the application uuid
+    :return: User
+    """
+    return self._user.get_by_name_application_uuid_and_password(name, password, application_uuid)
+
+  def cities_by_user_and_application(self, user_id, application_id) -> [City]:
+    """
+    Retrieve the cities belonging to the user and the application from the database
+    :param user_id: User id
+    :param application_id: Application id
+    :return: [City]
+    """
+    return self._city.get_by_user_id_and_application_id(user_id, application_id)
+
+  def building(self, name, user_id, application_id, scenario) -> CityObject:
+    """
+    Retrieve the building from the database
+    :param name: Building name
+    :param user_id: User id
+    :param application_id: Application id
+    :param scenario: Scenario
+    :
+    """
+    cities = self._city.get_by_user_id_application_id_and_scenario(user_id, application_id, scenario)
+    city = [city[0].id for city in cities]
+    result = self._city_object.building_in_cities_info(name, city)
+    if result is not None:
+      return result
+    return None
+
+  def building_info(self, name, city_id) -> CityObject:
+    """
+    Retrieve the building info from the database
+    :param name: Building name
+    :param city_id: City ID
+    :return: CityObject
+    """
+    return self._city_object.get_by_name_or_alias_and_city(name, city_id)
+
+  def building_info_in_cities(self, name, cities) -> CityObject:
+    """
+    Retrieve the building info from the database
+    :param name: Building name
+    :param cities: [City ID]
+    :return: CityObject
+    """
+    return self._city_object.get_by_name_or_alias_in_cities(name, cities)
+
+  def results(self, user_id, application_id, request_values, result_names=None) -> Dict:
+    """
+    Retrieve the simulation results for the given cities from the database
+    :param user_id: the user id owning the results
+    :param application_id: the application id owning the results
+    :param request_values: dictionary containing the scenario and building names to grab the results
+    :param result_names: if given, filter the results to the selected names
+    """
+    if result_names is None:
+      result_names = []
+    results = {}
+    for scenario in request_values['scenarios']:
+      for scenario_name in scenario.keys():
+        result_sets = self._city.get_by_user_id_application_id_and_scenario(
+          user_id,
+          application_id,
+          scenario_name
+        )
+        if result_sets is None:
+          continue
+        results[scenario_name] = []
+        city_ids = [r[0].id for r in result_sets]
+        for building_name in scenario[scenario_name]:
+          _building = self._city_object.get_by_name_or_alias_in_cities(building_name, city_ids)
+          if _building is None:
+            continue
+          city_object_id = _building.id
+          _ = self._simulation_results.get_simulation_results_by_city_object_id_and_names(
+            city_object_id,
+            result_names)
+
+          for value in _:
+            values = value.values
+            values["building"] = building_name
+            results[scenario_name].append(values)
+    return results
+
+  def persist_city(self, city: City, pickle_path, scenario, application_id: int, user_id: int):
+    """
+    Creates a city into the database
+    :param city: City to be stored
+    :param pickle_path: Path to save the pickle file
+    :param scenario: Simulation scenario name
+    :param application_id: Application id owning this city
+    :param user_id: User who create the city
+    return identity_id
+    """
+    return self._city.insert(city, pickle_path, scenario,  application_id, user_id)
+
+  def update_city(self, city_id, city):
+    """
+   Update an existing city in the database
+   :param city_id: the id of the city to update
+   :param city: the updated city object
+    """
+    return self._city.update(city_id, city)
+
+  def persist_application(self, name: str, description: str, application_uuid: str):
+    """
+    Creates information for an application in the database
+   :param name: name of application
+   :param description: the description of the application
+   :param application_uuid: the uuid of the application to be created
+    """
+    return self._application.insert(name, description, application_uuid)
+
+  def update_application(self, name: str, description: str, application_uuid: str):
+    """
+    Update the application information stored in the database
+   :param name: name of application
+   :param description: the description of the application
+   :param application_uuid: the uuid of the application to be created
+    """
+    return self._application.update(application_uuid, name, description)
+
+  def add_simulation_results(self, name, values, city_id=None, city_object_id=None):
+    """
+    Add simulation results to the city or to the city_object to the database
+    :param name: simulation and simulation engine name
+    :param values: simulation values in json format
+    :param city_id: city id or None
+    :param city_object_id: city object id or None
+    """
+    return self._simulation_results.insert(name, values, city_id, city_object_id)
+
+  def create_user(self, name: str, application_id: int, password: str, role: UserRoles):
+    """
+    Creates a new user in the database
+    :param name: the name of the user
+    :param application_id: the application id of the user
+    :param password: the password of the user
+    :param role: the role of the user
+    """
+    return self._user.insert(name, password, role, application_id)
+
+  def update_user(self, user_id: int, name: str, password: str, role: UserRoles):
+    """
+    Updates a user in the database
+    :param user_id: the id of the user
+    :param name: the name of the user
+    :param password: the password of the user
+    :param role: the role of the user
+    """
+    return self._user.update(user_id, name, password, role)
+
+  def get_by_name_and_application(self, name: str, application: int):
+    """
+    Retrieve a single user from the database
+    :param name: username
+    :param application: application accessing hub
+    """
+    return self._user.get_by_name_and_application(name, application)
+
+  def delete_user(self, user_id):
+    """
+    Delete a single user from the database
+    :param user_id: the id of the user to delete
+    """
+    self._user.delete(user_id)
+
+  def delete_city(self, city_id):
+    """
+    Deletes a single city from the database
+    :param city_id: the id of the city to get
+    """
+    self._city.delete(city_id)
+
+  def delete_results_by_name(self, name, city_id=None, city_object_id=None):
+    """
+    Deletes city object simulation results from the database
+    :param name: simulation name
+    :param city_id: if given, delete delete the results for the city with id city_id
+    :param city_object_id: if given, delete delete the results for the city object with id city_object_id
+    """
+    self._simulation_results.delete(name, city_id=city_id, city_object_id=city_object_id)
+
+  def delete_application(self, application_uuid):
+    """
+    Deletes a single application from the database
+    :param application_uuid: the id of the application to get
+    """
+    self._application.delete(application_uuid)

cerc-persistence-0.1.0.4/cerc_persistence/db_setup.py

@@ -0,0 +1,70 @@
+"""
+Database setup
+SPDX - License - Identifier: LGPL - 3.0 - or -later
+Copyright © 2022 Concordia CERC group
+Project Coder Peter Yefi peteryefi@gmail.com
+"""
+
+import logging
+from cerc_persistence.repository import Repository
+from cerc_persistence.models import Application
+from cerc_persistence.models import City
+from cerc_persistence.models import CityObject
+from cerc_persistence.models import User
+from cerc_persistence.models import UserRoles
+from cerc_persistence.models import SimulationResults
+from cerc_persistence.repositories.user import User as UserRepository
+from cerc_persistence.repositories.application import Application as ApplicationRepository
+
+
+class DBSetup:
+  """
+  Creates a Persistence database structure
+  """
+
+  def __init__(self, db_name, app_env, dotenv_path, admin_password, application_uuid):
+    """
+    Creates database tables a default admin user and a default admin app with the given password and uuid
+    :param db_name: database name
+    :param app_env: application environment type [TEST|PROD]
+    :param dotenv_path: .env file path
+    :param admin_password: administrator password for the application uuid
+    :application_uuid: application uuid
+    """
+    repository = Repository(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
+
+    # Create the tables using the models
+    Application.__table__.create(bind=repository.engine, checkfirst=True)
+    User.__table__.create(bind=repository.engine, checkfirst=True)
+    City.__table__.create(bind=repository.engine, checkfirst=True)
+    CityObject.__table__.create(bind=repository.engine, checkfirst=True)
+    SimulationResults.__table__.create(bind=repository.engine, checkfirst=True)
+
+    self._user_repo = UserRepository(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
+    self._application_repo = ApplicationRepository(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
+    application_id = self._create_admin_app(self._application_repo, application_uuid)
+    self._create_admin_user(self._user_repo, admin_password, application_id)
+
+  @staticmethod
+  def _create_admin_app(application_repo, application_uuid):
+    name = 'AdminTool'
+    description = 'Admin tool to control city persistence and to test the API v1.4'
+    logging.info('Creating default admin tool application...')
+    application = application_repo.insert(name, description, application_uuid)
+
+    if isinstance(application, dict):
+      logging.info(application)
+    else:
+      msg = f'Created Admin tool with application_uuid: {application_uuid}'
+      logging.info(msg)
+    return application.id
+
+  @staticmethod
+  def _create_admin_user(user_repo, admin_password, application_id):
+    password = admin_password
+    logging.info('Creating default admin user...')
+    user = user_repo.insert('Administrator', password, UserRoles.Admin, application_id)
+    if isinstance(user, dict):
+      logging.info(user)
+    else:
+      logging.info('Created Admin user')

cerc-persistence-0.1.0.4/cerc_persistence/models/__init__.py

@@ -0,0 +1,8 @@
+"""
+Models package
+"""
+from .application import Application
+from .city import City
+from .city_object import CityObject
+from .simulation_results import SimulationResults
+from .user import User, UserRoles

cerc-persistence-0.1.0.4/cerc_persistence/models/application.py

@@ -0,0 +1,32 @@
+"""
+Model representation of an application
+SPDX - License - Identifier: LGPL - 3.0 - or -later
+Copyright © 2022 Concordia CERC group
+Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca
+"""
+
+import datetime
+
+from sqlalchemy import Column, Integer, String, Sequence
+from sqlalchemy import DateTime
+from sqlalchemy.dialects.postgresql import UUID
+
+from cerc_persistence.configuration import Models
+
+
+class Application(Models):
+  """
+  A model representation of an application
+  """
+  __tablename__ = 'application'
+  id = Column(Integer, Sequence('application_id_seq'), primary_key=True)
+  name = Column(String, nullable=False)
+  description = Column(String, nullable=False)
+  application_uuid = Column(UUID(as_uuid=True), nullable=False)
+  created = Column(DateTime, default=datetime.datetime.utcnow)
+  updated = Column(DateTime, default=datetime.datetime.utcnow)
+
+  def __init__(self, name, description, application_uuid):
+    self.name = name
+    self.description = description
+    self.application_uuid = application_uuid

cerc-persistence-0.1.0.4/cerc_persistence/models/city.py

@@ -0,0 +1,36 @@
+"""
+Model representation of a City
+SPDX - License - Identifier: LGPL - 3.0 - or -later
+Copyright © 2022 Concordia CERC group
+Project Coder Peter Yefi peteryefi@gmail.com
+"""
+
+import datetime
+
+from sqlalchemy import Column, Integer, String, Sequence, ForeignKey
+from sqlalchemy import DateTime
+
+from cerc_persistence.configuration import Models
+
+
+class City(Models):
+  """A model representation of a city
+  """
+  __tablename__ = 'city'
+  id = Column(Integer, Sequence('city_id_seq'), primary_key=True)
+  pickle_path = Column(String, nullable=False)
+  name = Column(String, nullable=False)
+  scenario = Column(String, nullable=False)
+  application_id = Column(Integer, ForeignKey('application.id', ondelete='CASCADE'), nullable=False)
+  user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE'), nullable=True)
+  hub_release = Column(String, nullable=False)
+  created = Column(DateTime, default=datetime.datetime.utcnow)
+  updated = Column(DateTime, default=datetime.datetime.utcnow)
+
+  def __init__(self, pickle_path, name, scenario, application_id, user_id, hub_release):
+    self.pickle_path = str(pickle_path)
+    self.name = name
+    self.scenario = scenario
+    self.application_id = application_id
+    self.user_id = user_id
+    self.hub_release = hub_release

cerc-persistence-0.1.0.4/cerc_persistence/models/city_object.py

@@ -0,0 +1,82 @@
+"""
+Model representation of a city object
+SPDX - License - Identifier: LGPL - 3.0 - or -later
+Copyright © 2022 Concordia CERC group
+Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca
+"""
+
+import datetime
+import logging
+
+from sqlalchemy import Column, Integer, String, Sequence, ForeignKey, Float
+from sqlalchemy import DateTime
+
+from hub.city_model_structure.building import Building
+from cerc_persistence.configuration import Models
+
+
+class CityObject(Models):
+  """
+  A model representation of an application
+  """
+  __tablename__ = 'city_object'
+  id = Column(Integer, Sequence('city_object_id_seq'), primary_key=True)
+  city_id = Column(Integer, ForeignKey('city.id', ondelete='CASCADE'), nullable=False)
+  name = Column(String, nullable=False)
+  aliases = Column(String, nullable=True)
+  type = Column(String, nullable=False)
+  year_of_construction = Column(Integer, nullable=True)
+  function = Column(String, nullable=True)
+  usage = Column(String, nullable=True)
+  volume = Column(Float, nullable=False)
+  area = Column(Float, nullable=False)
+  total_heating_area = Column(Float, nullable=False)
+  wall_area = Column(Float, nullable=False)
+  windows_area = Column(Float, nullable=False)
+  roof_area = Column(Float, nullable=False)
+  total_pv_area = Column(Float, nullable=False)
+  system_name = Column(String, nullable=False)
+  created = Column(DateTime, default=datetime.datetime.utcnow)
+  updated = Column(DateTime, default=datetime.datetime.utcnow)
+
+  def __init__(self, city_id, building: Building):
+    self.city_id = city_id
+    self.name = building.name
+    self.aliases = building.aliases
+    self.type = building.type
+    self.year_of_construction = building.year_of_construction
+    self.function = building.function
+    self.usage = building.usages_percentage
+    self.volume = building.volume
+    self.area = building.floor_area
+    self.roof_area = sum(roof.solid_polygon.area for roof in building.roofs)
+    self.total_pv_area = sum(
+      roof.solid_polygon.area * roof.solar_collectors_area_reduction_factor for roof in building.roofs
+    )
+    storeys = building.storeys_above_ground
+    wall_area = 0
+    window_ratio = 0
+    try:
+      if storeys is None:
+        storeys = building.max_height / building.average_storey_height
+      for internal_zone in building.internal_zones:
+        for thermal_zone in internal_zone.thermal_zones_from_internal_zones:
+          for thermal_boundary in thermal_zone.thermal_boundaries:
+            window_ratio = thermal_boundary.window_ratio
+            break
+    except TypeError:
+      storeys = 0
+      logging.warning(
+        'building %s has no storey height so heating area, storeys and window ratio cannot be calculated',
+        self.name
+      )
+    self.total_heating_area = building.floor_area * storeys
+
+    for wall in building.walls:
+      wall_area += wall.solid_polygon.area
+    self.wall_area = wall_area
+    self.windows_area = wall_area * window_ratio
+    system_name = building.energy_systems_archetype_name
+    if system_name is None:
+      system_name = ''
+    self.system_name = system_name

cerc-persistence-0.1.0.4/cerc_persistence/models/simulation_results.py

@@ -0,0 +1,33 @@
+"""
+Model representation of simulation results
+SPDX - License - Identifier: LGPL - 3.0 - or -later
+Copyright © 2022 Concordia CERC group
+Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca
+"""
+
+import datetime
+
+from sqlalchemy import Column, Integer, String, Sequence, ForeignKey
+from sqlalchemy import DateTime
+from sqlalchemy.dialects.postgresql import JSONB
+from cerc_persistence.configuration import Models
+
+
+class SimulationResults(Models):
+  """
+  A model representation of an application
+  """
+  __tablename__ = 'simulation_results'
+  id = Column(Integer, Sequence('simulation_results_id_seq'), primary_key=True)
+  city_id = Column(Integer, ForeignKey('city.id', ondelete='CASCADE'), nullable=True)
+  city_object_id = Column(Integer, ForeignKey('city_object.id', ondelete='CASCADE'), nullable=True)
+  name = Column(String, nullable=False)
+  values = Column(JSONB, nullable=False)
+  created = Column(DateTime, default=datetime.datetime.utcnow)
+  updated = Column(DateTime, default=datetime.datetime.utcnow)
+
+  def __init__(self, name, values, city_id=None, city_object_id=None):
+    self.name = name
+    self.values = values
+    self.city_id = city_id
+    self.city_object_id = city_object_id