cim-loader 0.1.0b0__tar.gz

cim_loader-0.1.0b0/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, PNNL-CIM-Tools
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

cim_loader-0.1.0b0/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.1
+Name: cim-loader
+Version: 0.1.0b0
+Summary: CIM Database Uploade & Downloader
+Author: A. Anderson
+Author-email: 19935503+AAndersn@users.noreply.github.com
+Requires-Python: >=3.8.1,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: gridappsd
+Requires-Dist: SPARQLWrapper (>=2.0.0,<3.0.0)
+Requires-Dist: cim-graph (>=0.1.5a0,<0.2.0)
+Requires-Dist: mysql-connector-python (>=8.0.0,<9.0.0)
+Requires-Dist: neo4j (>=5.10.0,<6.0.0)
+Requires-Dist: rdflib (>=7.0.0,<8.0.0)
+Requires-Dist: rdflib-neo4j (>=0.0.1b6,<0.0.2)
+Requires-Dist: requests (>=2.28.0,<3.0.0)
+Description-Content-Type: text/markdown
+
+# CIM-Loader
+Automated scripts for 
+* uploading and downloading CIM files from various databases
+* converting CIM files between common formats (XML, TTL, etc.)
+
+
+## Installation
+PyPi version coming soon.
+
+Clone the repo and run
+`pip install -e CIM-Loader`
+
+
+## Usage
+
+To use CIM-Loader for bulk upload and download, set the connection parameters with the correct url / host / port / username / password and then invoke the associated upload / download method
+
+```python
+from cimloader.databases import ConnectionParameters
+from cimloader.uploaders import BlazegraphUploader
+params = ConnectionParameters(url = "http://localhost:8889/bigdata/namespace/kb/sparql")
+loader = BlazegraphUploader(params)
+
+loader.upload_from_file(filepath='./test_models', filename='ieee13_seto.xml')
+```
+
+
+## Databases Supported
+Databases to be supported in first full release:
+* Blazegraph
+* Neo4J
+* GraphDB
+* MySQL
+
+Support may be added in the future for:
+* Apache Tinkerpop
+* SQlite
+* AVEVA PI Historian
+* Others as requested
+
+
+## Attribution and Disclaimer
+
+This software was created under a project sponsored by the U.S. Department of Energy’s Office of Electricity, an agency of the United States Government.  Neither the United States Government nor the United States Department of Energy, nor Battelle, nor any of their employees, nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe privately owned rights.
+
+Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or any agency thereof, or Battelle Memorial Institute. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof.
+
+PACIFIC NORTHWEST NATIONAL LABORATORY
+operated by
+BATTELLE
+for the
+UNITED STATES DEPARTMENT OF ENERGY
+under Contract DE-AC05-76RL01830
+
+
+

cim_loader-0.1.0b0/README.md

@@ -0,0 +1,55 @@
+# CIM-Loader
+Automated scripts for 
+* uploading and downloading CIM files from various databases
+* converting CIM files between common formats (XML, TTL, etc.)
+
+
+## Installation
+PyPi version coming soon.
+
+Clone the repo and run
+`pip install -e CIM-Loader`
+
+
+## Usage
+
+To use CIM-Loader for bulk upload and download, set the connection parameters with the correct url / host / port / username / password and then invoke the associated upload / download method
+
+```python
+from cimloader.databases import ConnectionParameters
+from cimloader.uploaders import BlazegraphUploader
+params = ConnectionParameters(url = "http://localhost:8889/bigdata/namespace/kb/sparql")
+loader = BlazegraphUploader(params)
+
+loader.upload_from_file(filepath='./test_models', filename='ieee13_seto.xml')
+```
+
+
+## Databases Supported
+Databases to be supported in first full release:
+* Blazegraph
+* Neo4J
+* GraphDB
+* MySQL
+
+Support may be added in the future for:
+* Apache Tinkerpop
+* SQlite
+* AVEVA PI Historian
+* Others as requested
+
+
+## Attribution and Disclaimer
+
+This software was created under a project sponsored by the U.S. Department of Energy’s Office of Electricity, an agency of the United States Government.  Neither the United States Government nor the United States Department of Energy, nor Battelle, nor any of their employees, nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe privately owned rights.
+
+Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or any agency thereof, or Battelle Memorial Institute. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof.
+
+PACIFIC NORTHWEST NATIONAL LABORATORY
+operated by
+BATTELLE
+for the
+UNITED STATES DEPARTMENT OF ENERGY
+under Contract DE-AC05-76RL01830
+
+

cim_loader-0.1.0b0/cimloader/batch_handlers/blazegraph_to_mysql.py

@@ -0,0 +1,18 @@
+
+# from cimloader.databases.blazegraph import Blazegraph
+from cimloader.databases.mysql import MySQLConnection
+from cimloader.databases import ConnectionInterface, ConnectionParameters, Parameter, QueryResponse
+
+import os
+import logging
+import subprocess
+
+
+_log = logging.getLogger(__name__)
+
+class BlazegraphtoMySQL():
+    def __init__(self, naerm_params:ConnectionInterface, neo4j_params:ConnectionInterface, 
+                 tmp_dir:str, docker_container:str):
+        # self.NaermDownloader = NAERM(naerm_params)
+        # self.Neo4jConnection= Neo4j(neo4j_params)
+        pass

cim_loader-0.1.0b0/cimloader/batch_handlers/naerm_to_neo4j.py

@@ -0,0 +1,80 @@
+from cimloader.web_apis.naerm_api import NAERM
+from cimloader.databases.neo4j import Neo4jConnection
+from cimloader.converters.dss_to_cim import DSStoCIM
+from cimloader.databases import ConnectionInterface, ConnectionParameters, Parameter, QueryResponse
+
+import os
+import logging
+import subprocess
+
+
+_log = logging.getLogger(__name__)
+
+class NAERMtoNeo4j():
+    def __init__(self, naerm_params:ConnectionInterface, neo4j_params:ConnectionInterface, 
+                 tmp_dir:str, docker_container:str):
+        self.NaermDownloader = NAERM(naerm_params)
+        # self.Neo4jConnection= Neo4jConnection(neo4j_params)
+        self.dss_converter = DSStoCIM()
+        self.tmp_dir = tmp_dir
+        self.docker_container = docker_container
+
+    def upload_all_dss(self, limit):
+        # self.Neo4jConnection.connect()
+        case_uuid_list, case_uuid_names = self.NaermDownloader.download_all_dss(destination=self.tmp_dir, limit=limit)
+        for case_uuid in case_uuid_list:
+            self.dss_converter.convert_file(file_path=f"{self.tmp_dir}/{case_uuid}", feeder_mrid=case_uuid, 
+                                            sub_name=case_uuid_names[case_uuid], sub_geo="SFO", geo="CA", uuids_file=f"{self.tmp_dir}/sfo_uuid.dat")
+            if self.docker_container is not None:
+                subprocess.call(["docker", "cp", f"{self.tmp_dir}/{case_uuid}/Master.xml", f"{self.docker_container}:import/{case_uuid}/Master.xml"])
+                self.Neo4jConnection.upload(filepath=f"/import/{case_uuid}", filename="Master.xml", format="RDF/XML")
+                _log.info(f"Successfully uploaded feeder id {case_uuid}")
+        # self.Neo4jConnection.disconnect()
+
+    def upload_case(self, case_uuid):
+        # self.Neo4jConnection.connect()
+        self.NaermDownloader.download_case(case_uuid=case_uuid, destination=self.tmp_dir)
+
+        self.dss_converter.convert_file(file_path=f"{self.tmp_dir}/{case_uuid}", feeder_mrid=case_uuid, sub_geo="SFO", geo="CA", uuids_file=f"{self.tmp_dir}/sfo_uuid.dat")
+        if self.docker_container is not None:
+            subprocess.call(["docker", "cp", f"{self.tmp_dir}/{case_uuid}/Master.xml", f"{self.docker_container}:import/{case_uuid}/Master.xml"])
+            self.Neo4jConnection.upload(filepath=f"/import/{case_uuid}", filename="Master.xml", format="RDF/XML")
+            _log.info(f"Successfully uploaded feeder id {case_uuid}")
+        # self.Neo4jConnection.disconnect()
+
+
+
+
+
+def _main():
+    
+    neo4j_params = ConnectionParameters(url = "neo4j://localhost:7687", database="neo4j", username="neo4j", password="neo4j",
+                                         cim_profile='cimhub_2023', namespace="http://iec.ch/TC57/CIM100#") 
+    
+    naerm_params = ConnectionParameters(url = "https://api.develop.naerm.team/data/bes/case_files")
+
+    tmp_dir = "/home/ande188/naerm"
+
+    sfo_uuid = """
+    Station=Station=1 {ad4e7f6e-b4eb-44cb-a75e-ac7f3e2e41c5}
+    GeoRgn=GeoRgn=1 {731690f4-13e0-4555-8084-3cc05d1000f8}
+    SubGeoRgn=SubGeoRgn=1 {277ebb75-f974-4de6-8a85-8526d0aa3351}"""
+
+    with open(f"{tmp_dir}/sfo_uuid.dat", "w") as fp:
+        fp.write(sfo_uuid)
+
+    docker_container = "gridappsd-docker-neo4j-apoc_1"
+
+    nn4j = NAERMtoNeo4j(naerm_params=naerm_params, neo4j_params=neo4j_params, tmp_dir=tmp_dir, docker_container=docker_container)
+
+    nn4j.upload_all_dss(limit=10)
+
+    # case_uuid = "accdf54c-c90c-4c41-9d7a-5e5d3d17a4e5"
+    case_uuid = "1b5adb54-dcc3-472b-a5fd-aaab635857e1"
+    nn4j.upload_case(case_uuid=case_uuid)
+
+
+
+
+if __name__ == "__main__":
+     _main()

cim_loader-0.1.0b0/cimloader/databases/__init__.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, List
+
+
+
+@dataclass
+class Parameter:
+    key: Any
+    value: Any
+
+
+@dataclass
+class ConnectionParameters:
+    url: str = field(default_factory=str)
+    host: str = field(default_factory=str)
+    port: str = field(default_factory=str)
+    filename: str = field(default_factory=str)
+    username: str = field(default_factory=str)
+    password: str = field(default_factory=str)
+    database: str = field(default_factory=str)
+    container: str = field(default_factory=str)
+    namespace: str = field(default="<http://iec.ch/TC57/CIM100#>")
+    cim_profile: str = field(default_factory=str)
+    iec61970_301: int = field(default=7)
+    
+
+@dataclass
+class QueryResponse:
+    response: Any
+
+
+@dataclass
+class ConnectionInterface:
+    connection_params: ConnectionParameters
+
+    def connect(self):
+        raise RuntimeError("Must have implemented connect in inherited class")
+
+    def disconnect(self):
+        raise RuntimeError("Must have implemented disconnect in inherited class")
+
+    def execute(self, query: str) -> QueryResponse:
+        raise RuntimeError("Must have implemented query in the inherited class")
+
+from cimloader.databases.blazegraph import BlazegraphConnection
+from cimloader.databases.neo4j import Neo4jConnection
+from cimloader.databases.mysql import MySQLConnection

cim_loader-0.1.0b0/cimloader/databases/blazegraph.py

@@ -0,0 +1,45 @@
+import logging
+import subprocess
+
+from cimloader.databases import ConnectionInterface, QueryResponse
+from SPARQLWrapper import JSON, POST, SPARQLWrapper
+
+_log = logging.getLogger(__name__)
+
+class BlazegraphConnection(ConnectionInterface):
+    def __init__(self, connection_params: ConnectionInterface) -> None:
+        self.cim_profile = connection_params.cim_profile
+        # self.cim = importlib.import_module('cimgraph.data_profile.' + self.cim_profile)
+        self.namespace = connection_params.namespace
+        self.iec61970_301 = connection_params.iec61970_301
+        self.url = connection_params.url
+        self.connection_params = connection_params
+        self.sparql_obj = None
+
+    def connect(self):
+        if not self.sparql_obj:
+            self.sparql_obj = SPARQLWrapper(self.connection_params.url)
+            self.sparql_obj.setReturnFormat(JSON)
+
+    def configure(self):
+        pass
+
+    def drop_all(self):
+        self.update('drop all')
+
+    def disconnect(self):
+        self.sparql_obj = None
+        
+    def execute(self, query_message: str) -> QueryResponse:
+        self.connect()
+        self.sparql_obj.setQuery(query_message)
+        self.sparql_obj.setMethod(POST)
+        query_output = self.sparql_obj.query().convert()
+        return query_output
+    
+    def update(self, query_message:str) -> QueryResponse:
+        self.connect()
+        self.sparql_obj.setQuery(query_message)
+        self.sparql_obj.setMethod(POST)
+        query_output = self.sparql_obj.query()
+        return query_output

cim_loader-0.1.0b0/cimloader/databases/mysql.py

@@ -0,0 +1,186 @@
+import re
+import mysql.connector
+import logging
+import importlib
+import json
+import enum
+import time
+
+from cimloader.databases import ConnectionInterface, ConnectionParameters, Parameter, QueryResponse
+from cimgraph.data_profile.known_problem_classes import ClassesWithoutMRID
+from cimgraph.models import GraphModel
+
+_log = logging.getLogger(__name__)
+
+class MySQLConnection(ConnectionInterface):
+    def __init__(self, connection_parameters: ConnectionParameters):
+        self.connection_parameters = connection_parameters
+        self.cim_profile = connection_parameters.cim_profile
+        self.cim = importlib.import_module('cimgraph.data_profile.' + self.cim_profile)
+        self.namespace = connection_parameters.namespace
+        self.host = connection_parameters.host
+        self.port = connection_parameters.port
+        self.username = connection_parameters.username
+        self.password = connection_parameters.password
+        self.database = connection_parameters.database
+        self.connection = None
+        self.cursor = None
+
+    def connect(self):
+        if not self.cursor:
+            if not self.database: # Set database name to CIM Profile name if not specified
+                self.database = self.cim_profile 
+            try:
+                self.connection = mysql.connector.connect(host = self.host, user = self.username, password = self.password, database = self.database)
+                self.cursor = self.connection.cursor(buffered = True)
+            except:
+                _log.error('Could not connect to database')
+
+    def disconnect(self):
+        self.cursor = None
+
+    def execute(self, query_message: str) -> QueryResponse:
+        self.connect()
+        self.cursor.execute(query_message)
+        response = self.cursor.fetchall()
+        return response
+    
+    def create_database(self, database:str = None, overwrite:bool = True):
+        if database is None:
+            if self.database is None:
+                database = self.cim_profile
+            else:
+                database = self.database
+        self.database = database
+        connection = mysql.connector.connect(host = self.host, user = self.username, password = self.password)
+        cursor = connection.cursor(buffered = True)
+        if overwrite:
+            # try:
+            cursor.execute(f"DROP DATABASE {database}")
+            # except:
+            #     _log.error("Unable to connect to database")
+        
+        # try:
+        cursor.execute(f"CREATE DATABASE {database}")
+        # except:
+            # _log.error("Unable to create new database")
+
+    def configure(self, overwrite:bool = True):
+        class_list = self.cim.__all__
+        classes_without_mrid = ClassesWithoutMRID()
+        
+        for class_name in class_list:
+            cim_class = eval(f"self.cim.{class_name}")
+            # print(cim_class.__name__)
+
+            try:
+                self.connect()
+                self.cursor.execute(f"DROP TABLE {cim_class.__name__}")
+            except:
+                pass
+            # Get all attribute types from CIMantic Graphs dataclasses
+            try:
+                if type(cim_class) == enum.EnumMeta:
+                        values = cim_class._member_names_
+                        sql_query = f"CREATE TABLE {cim_class.__name__} (enumeration VARCHAR(255))"
+                        self.cursor.execute(sql_query) # create table
+                        for value in values:
+                            sql_insert = f"""INSERT INTO {cim_class.__name__} (enumeration) VALUES ("{value}")"""
+                            self.cursor.execute(sql_insert)
+                else:
+                    # Check if is class and has class fields
+                    fields = cim_class.__dataclass_fields__
+                    sql_query = f"CREATE TABLE {cim_class.__name__} ("
+                    
+                    sql_query = sql_query + "username VARCHAR(255), "
+                    sql_query = sql_query + "timestamp INT, "
+
+                    # Handling for problem classes that don't inherit from IdentifiedObject
+                    if cim_class in classes_without_mrid.classes:
+                        sql_query = sql_query + "_mRID VARCHAR(255), "
+                    # Iterate through all attributes and add to table
+                    for attr in list(fields.keys()):
+                        attribute_name = fields[attr].type
+                        # Determine type of attribute value and use correct column type in table
+                        if "List" in attribute_name:
+                            sql_query = sql_query + f"_{attr} JSON, "
+                        elif "float" in attribute_name:
+                            sql_query = sql_query + f"_{attr} FLOAT, "
+                        elif 'Optional' in attribute_name: #check if attribute is association to a class object
+                            if '\'' in attribute_name: #handling inconsistent '' marks in data profile
+                                at_cls = re.match(r'Optional\[\'(.*)\']',attribute_name)
+                                attribute_name = at_cls.group(1)
+                            else:        
+                                at_cls = re.match(r'Optional\[(.*)]',attribute_name)
+                                attribute_name = at_cls.group(1)
+                            # If a CIM class, then use JSON-LD
+                            if attribute_name in self.cim.__all__:
+                                attribute_class = eval(f"self.cim.{attribute_name}")
+                                if type(attribute_class) == enum.EnumMeta:
+                                    sql_query = sql_query + f"_{attr} FLOAT, "
+                                else:
+                                    sql_query = sql_query + f"_{attr} JSON, "
+
+                            else:
+                                sql_query = sql_query + f"_{attr} VARCHAR(255), "
+                        else:
+                            sql_query = sql_query + f"_{attr} VARCHAR(255), "
+                    sql_query = sql_query[:-2] # remove extra punctuation
+                    sql_query = sql_query + ")"
+                    # print(sql_query)
+                    self.cursor.execute(sql_query) # create table
+                    _log.info(f"Created table for class {class_name}")
+            except:
+                _log.warning(f"Unable to create table for class {class_name}")
+
+
+
+
+
+
+
+    def upload_from_file(self):
+        pass
+
+    def upload_from_url(self):
+        pass
+
+    def upload_from_rdflib(self, rdflib_graph):
+
+        pass
+
+    def upload_from_cimgraph(self, network:GraphModel):
+        
+        class_list = list(network.graph.keys())
+        for cim_class in class_list:
+            # Get JSON-LD representation of model
+            table = network.cim_dump(cim_class)
+            fields = cim_class.__dataclass_fields__
+            # Insert each power system object in CIMantic Graphs model
+            for obj in table.values():
+                # Create SQL Query
+                sql_insert = f"INSERT INTO {cim_class.__name__} ("
+                sql_values = " VALUES ("
+                sql_params = [self.username, int(time.time())]
+
+                sql_insert = sql_insert + "username, timestamp, "
+                sql_values = sql_values + f"%s, %s, "
+                # Iterate through each attribute
+                for attr in list(obj.keys()):
+                    sql_insert = sql_insert + f"_{attr}, "
+                    sql_values = sql_values + f"%s, "
+                    
+                    if "List" in fields[attr].type:
+                        sql_params.append(json.dumps(obj[attr])) # JSON-LD List of RDF links
+                    elif "float" in fields[attr].type:
+                        sql_params.append(obj[attr]) # float
+                    else:
+                        sql_params.append(str(obj[attr])) # free text
+
+                sql_insert = sql_insert[:-2] # remove extra punctuation
+                sql_insert = sql_insert + ")"
+                sql_values = sql_values[:-2] # remove extra punctuation
+                sql_values = sql_values + ")"
+                sql_query = sql_insert + sql_values # append query message
+                sql_params = tuple(sql_params)
+                self.cursor.execute(sql_query, sql_params) # insert all CIM objects

cim_loader-0.1.0b0/cimloader/databases/neo4j.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+import logging
+import subprocess
+
+from typing import Dict, List, Optional
+
+
+from neo4j import GraphDatabase
+from neo4j.exceptions import DriverError, Neo4jError
+
+from cimloader.databases import ConnectionInterface, ConnectionParameters, Parameter, QueryResponse
+
+import rdflib
+# from rdflib import Graph, Namespace
+from rdflib.namespace import RDF
+
+_log = logging.getLogger(__name__)
+
+class Neo4jConnection(ConnectionInterface):
+    def __init__(self, connection_parameters):
+        self.connection_parameters = connection_parameters
+        self.cim_profile = connection_parameters.cim_profile
+        self.namespace = connection_parameters.namespace
+        self.url = connection_parameters.url
+        self.username = connection_parameters.username
+        self.password = connection_parameters.password
+        self.database = connection_parameters.database
+        self.container = connection_parameters.container
+        self.driver = None
+
+    def connect(self):
+        if not self.driver:
+            self.driver = GraphDatabase.driver(self.url, auth=(self.username, self.password))
+            self.driver.verify_connectivity()
+
+    def disconnect(self):
+        self.driver.close()
+        self.driver = None
+
+    def execute(self, query_message: str) -> QueryResponse:
+        self.connect()
+
+        try:
+            records, summary, keys = self.driver.execute_query(query_message, database_=self.database )
+            return records, summary, keys
+        # Capture any errors along with the query and data for traceability
+        except (DriverError, Neo4jError) as exception:
+            _log.error("%s raised an error: \n%s", query_message, exception)
+
+    def configure(self):
+        if self.cim_profile is not None and self.namespace is not None:
+            # self.execute("CALL n10s.nsprefixes.add(\""+self.cim_profile+"\",\""+self.namespace+"\");")
+            self.execute("CREATE CONSTRAINT n10s_unique_uri FOR (r:Resource) REQUIRE r.uri IS UNIQUE;")
+
+        else:
+            _log.exception("CIM profile and namespace must be defined in ConnectionParameters")
+
+        graph_config = """call n10s.graphconfig.init({
+            handleMultival: "OVERWRITE", 
+            handleVocabUris: "IGNORE",
+            keepCustomDataTypes: true,
+            handleRDFTypes: "LABELS"})"""
+        self.execute(graph_config)
+
+    
+
+    def drop_all(self):
+        self.execute("MATCH (n) DETACH DELETE n")
+        self.execute("DROP CONSTRAINT n10s_unique_uri")

cim_loader-0.1.0b0/cimloader/downloaders/naerm_api.py

@@ -0,0 +1,66 @@
+from cimloader.databases import ConnectionInterface, ConnectionParameters
+
+import io
+import zipfile
+import os
+import requests
+import logging
+
+_log = logging.getLogger(__name__)
+
+class NAERM(ConnectionInterface):
+    def __init__(self, connection_parameters:ConnectionParameters):
+        self.connection_parameters = connection_parameters
+        self.url = connection_parameters.url
+
+    def connect(self):
+        pass
+
+    def disconnect(self):
+        pass
+
+    def download_case(self, case_uuid, destination):
+        casefile_url = self.url + "/uuid/" + case_uuid
+        case_info = requests.get(casefile_url).json()
+        file_url = case_info["file_url"]
+        file_path = f"{destination}/{case_uuid}"
+
+        r = requests.get(file_url)
+        z = zipfile.ZipFile(io.BytesIO(r.content))
+        z.extractall(file_path)
+        
+
+
+    def download_all_dss(self, destination, limit:int = 10000):
+        counter = 0
+        case_list = requests.get(self.url).json()
+        case_uuid_list = []
+        case_uuid_names = {}
+        for case in case_list:
+            if "simulator_type" in case["case"]:
+                if case["case"]["simulator_type"] == "opendss":
+                    counter = counter+1
+                    case_uuid_list.append(case["bes_case_uuid"])
+                    case_uuid_names[case["bes_case_uuid"]] = case["case"]["feeder_name"]
+
+            if counter == limit:
+                break
+
+        counter = 0
+        for case_uuid in case_uuid_list:
+            counter = counter+1
+            casefile_url = self.url + "/uuid/" + case_uuid
+            case_info = requests.get(casefile_url).json()
+            file_url = case_info["file_url"]
+            file_path = f"{destination}/{case_uuid}"
+
+            r = requests.get(file_url)
+            z = zipfile.ZipFile(io.BytesIO(r.content))
+            z.extractall(file_path)
+
+        print(f"Successfully downloaded {counter} case files")
+
+        return case_uuid_list, case_uuid_names
+    
+
+    

cim_loader-0.1.0b0/cimloader/uploaders/__init__.py

@@ -0,0 +1,2 @@
+from cimloader.uploaders.blazegraph import BlazegraphUploader
+from cimloader.uploaders.neo4j import Neo4jUploader

cim_loader-0.1.0b0/cimloader/uploaders/blazegraph.py

@@ -0,0 +1,25 @@
+import logging
+import subprocess
+
+from cimloader.databases import ConnectionInterface, QueryResponse
+from cimloader.databases.blazegraph import BlazegraphConnection
+from SPARQLWrapper import JSON, POST, SPARQLWrapper
+
+_log = logging.getLogger(__name__)
+
+class BlazegraphUploader(ConnectionInterface):
+    def __init__(self, connection_params: ConnectionInterface) -> None:
+        self.url = connection_params.url
+        self.connection_params = connection_params
+        self.connection = BlazegraphConnection(connection_params)
+        self.connection.connect()
+
+    def upload_from_file(self, filepath:str, filename:str) -> None:
+        subprocess.call(["curl", "-s", "-D-", "-H", "Content-Type: application/xml", "--upload-file", f"{filepath}/{filename}", "-X", "Post", self.url])
+        
+    def upload_from_url(self):
+        pass
+
+    def upload_from_rdflib(self, rdflib_graph):
+
+        pass

cim_loader-0.1.0b0/cimloader/uploaders/neo4j.py

@@ -0,0 +1,50 @@
+import logging
+import subprocess
+
+from cimloader.databases import ConnectionInterface, QueryResponse
+from cimloader.databases.neo4j import Neo4jConnection
+from SPARQLWrapper import JSON, POST, SPARQLWrapper
+
+_log = logging.getLogger(__name__)
+
+class Neo4jUploader(ConnectionInterface):
+    def __init__(self, connection_params: ConnectionInterface) -> None:
+        self.url = connection_params.url
+        self.connection_params = connection_params
+        self.container = connection_params.container
+        self.connection = Neo4jConnection(connection_params)
+        self.connection.connect()
+
+
+    def upload_from_file(self, filename, filepath):
+        if '.xml' in filename or '.XML' in filename:
+            format = 'RDF/XML'
+        elif '.ttl' in filename:
+            #TODO
+            pass
+
+        if self.container:
+            subprocess.call(["docker", "cp", f"{filepath}/{filename}", f"{self.container}:/var/lib/neo4j/import/{filename}"])
+            records=self.connection.execute(f"""call n10s.rdf.import.fetch( "file:///var/lib/neo4j/import//{filename}", "{format}"); """) 
+        else:
+            records=self.connection.execute(f"""call n10s.rdf.import.fetch( "file://{filepath}/{filename}", "{format}"); """) 
+        return records
+
+    def upload_from_url(self, url):
+        if '.xml' in url or '.XML' in url:
+            format = 'RDF/XML'
+        elif '.ttl' in url:
+            pass
+        records=self.connection.execute(f'''call n10s.rdf.import.fetch("{url}", "{format}"); ''') 
+        return records
+
+    def upload_from_rdflib(self, rdflib_graph):
+
+        pass
+
+    def upload_from_cimgraph(self):
+        pass
+
+    def upload_from_rdflib(self, rdflib_graph):
+
+        pass

cim_loader-0.1.0b0/pyproject.toml

@@ -0,0 +1,37 @@
+[tool.poetry]
+name = "cim-loader"
+version = "0.1.0b0"
+description = "CIM Database Uploade & Downloader"
+authors = ["A. Anderson <19935503+AAndersn@users.noreply.github.com>"]
+packages = [
+    { include = "cimloader" }
+]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+SPARQLWrapper = "^2.0.0"
+neo4j = "^5.10.0"
+rdflib = "^7.0.0"
+rdflib-neo4j = "^0.0.1b6"
+#OpenDSSDirect.py = "^0.8.2"
+requests = "^2.28.0"
+mysql-connector-python = "^8.0.0"
+
+cim-graph = "^0.1.5a0"
+#git = "https://github.com/PNNL-CIM-Tools/CIM-Graph/tree/main", branch = "v_1_0_0a", optional = true }
+
+[tool.poetry.extras]
+gridappsd = ["gridappsd-python"]
+
+[tool.poetry.group.dev.dependencies]
+pre-commit = "^2.17.0"
+xsdata = {extras = ["cli"], version = "^22.2"}
+xsdata-plantuml = "^21.4"
+graphviz = "^0.19.1"
+ipykernel = "^6.25.1"
+
+
+[build-system]
+requires = ["poetry-core>=1.2.0"]
+build-backend = "poetry.core.masonry.api"