pyneoinstance 2.0.2__tar.gz → 2.2.0__tar.gz

{pyneoinstance-2.0.2/pyneoinstance.egg-info → pyneoinstance-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: pyneoinstance
-Version: 2.0.2
+Version: 2.2.0
 Summary: Handle Neo4j read and write transactions and provide methos for performing graph EDA
 Author-email: "Jose A. Alvarado-Guzman" <jose.alvarado-guzman@neo4j.com>
 License: GNU GENERAL PUBLIC LICENSE
@@ -689,6 +689,7 @@ Requires-Dist: neo4j>=5.0.0
 Requires-Dist: pandas
 Requires-Dist: PyYAML
 Requires-Dist: pyvis
+Dynamic: license-file
 
 # PyNeoInstance Package
 

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0}/pyneoinstance/__init__.py

@@ -1,4 +1,5 @@
 from .database.neo4jdbms import Neo4jInstance
 from .fileload.loadyaml import load_yaml_file
+from .util.functions import get_parallel_info
 
-__version__ = '2.0.2'
+__version__ = '2.2.0'

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0}/pyneoinstance/database/context.py

@@ -3,4 +3,4 @@ import os
 import sys
 sys.path.insert(0, os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..')))
-from pyneoinstance.util.functions import get_logger
+from pyneoinstance.util.functions import get_logger,get_parallel_info,get_columns_diff

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0}/pyneoinstance/database/neo4jdbms.py

@@ -22,6 +22,7 @@ from neo4j.exceptions import ServiceUnavailable
 from neo4j.exceptions import AuthError
 from neo4j.exceptions import ClientError
 from neo4j.exceptions import ConfigurationError
+from .context import get_logger, get_parallel_info, get_columns_diff
 
 def _get_driver(neo_info : Dict[str, str]):
     try:
@@ -169,7 +170,12 @@ class Neo4jInstance:
         self.neo_info['password'] = password
         self.neo_info['encrypted'] = kwargs.get('encrypted') or ''
         self.__results = defaultdict(int)
-
+        self.__logger = get_logger('pyneoinstance', logging.WARNING)
+        stream_handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            '%(levelname)s - %(message)s')
+        stream_handler.setFormatter(formatter)
+        self.__logger.addHandler(stream_handler)
     def execute_read_query(self, query: str,
                            database: Optional[str] = None,
                            parameters: Optional[Dict[str, Any]] = None
@@ -289,7 +295,8 @@ class Neo4jInstance:
         result = self.execute_write_queries([query], database, parameters)
         return  result
 
-    def execute_write_queries_with_data(self, queries: List[str], data: DataFrame,
+    def execute_write_queries_with_data(self, queries: List[str],
+                                        data: DataFrame,
                                         database: Optional[str] = None,
                                         partitions: Optional[int] = 1,
                                         parallel: Optional[bool] = False,
@@ -335,7 +342,7 @@ class Neo4jInstance:
         if partitions > data.shape[0]:
             raise ValueError(
                 "The batch size cannot be greater than the number of rows in the data.")
-        data_chunks = np.array_split(data,partitions)
+        data_chunks = np.array_split(data.index,partitions)
         if parameters:
             params = parameters
         else:
@@ -346,19 +353,27 @@ class Neo4jInstance:
             else:
                 workers_num = mp.cpu_count()
             for query in queries:
+                col_diff = get_columns_diff(query,data.columns)
+                msg = f'This columns are not in your data: {col_diff}'
+                if len(col_diff)!=0:
+                    self.__logger.warning(msg)
                 with mp.Pool(workers_num) as worker_pool:
                     results = worker_pool.starmap(
                         _execute_write_parallel,
-                        [(self.neo_info, database,query, d.fillna(value='').to_dict('records'),
+                        [(self.neo_info, database,query, data.iloc[d].to_dict('records'),
                           params) for d in data_chunks]
                     )
         else:
             results = []
             with _get_driver(self.neo_info) as driver:
                 with _get_session(driver, database) as session:
-                    for rows in data_chunks:
-                        rows_dict = {'rows': rows.fillna(value="").to_dict('records')}
+                    for idx in data_chunks:
+                        rows_dict = {'rows': data.iloc[idx].to_dict('records')}
                         for query in queries:
+                            col_diff = get_columns_diff(query,data.columns)
+                            msg = f'This columns are not in your data: {col_diff}'
+                            if len(col_diff)!=0:
+                                self.__logger.warning(msg)
                             results.append(_execute_write(session,
                                                           query,
                                                           rows_dict['rows'],

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0}/pyneoinstance/util/__init__.py

@@ -1,3 +1,5 @@
 from .functions import get_file_name
 from .functions import is_reachable_url
 from .functions import get_logger
+from .functions import get_parallel_info
+from .functions import get_columns_diff

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0}/pyneoinstance/util/functions.py

@@ -4,6 +4,7 @@
 import functools
 import time
 import logging
+import re
 from datetime import datetime
 from typing import Tuple, Any, Callable, List
 from urllib.request import urlopen
@@ -94,3 +95,45 @@ def timing(function : Callable) -> Tuple[Any, float]:
         elapsed_time = end - start * 1.0
         return result, elapsed_time
     return wrap
+
+def get_parallel_info(num_rows: int, partition_rows=100_000) -> Tuple[int,bool]:
+    """Get the number of partitions and whether to execute execute
+    the load in parallel based on the number of rows of a Data Frame.
+
+    Parameters
+    ----------
+    num_rows : int
+        Number of rows of the Data Frame
+    partition_rows : int
+        Number of records per partition
+    Returns
+    -------
+    Tuble[int, bool]
+        Tuple containing the number of partitions and parallelims.
+    """
+    partitions = 1
+    parallel = False
+    batches = int(num_rows/partition_rows)
+    if batches > 1:
+        partitions = batches
+        parallel = True
+    return (partitions, parallel)
+
+def get_columns_diff(query: str, columns: List[str]) -> List[str]:
+    """Get a list of the columns that a Cypher query is trying to access that are
+    not in the list of columns provided.
+
+    Parameters
+    ----------
+    query : str
+       Cypher query
+    columns : List[str]
+        List containing the column names of a Pandas Data Frame
+    Returns
+    -------
+    List[str]
+       List containing the columns that Cypher is trying to access that are not
+       included in the columns names of the data frame.
+    """
+    cypher_attributes = set([w.split('.')[1] for w in re.findall(r'\brow\.\w+',query)])
+    return [a for a in cypher_attributes if a not in columns]

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0/pyneoinstance.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: pyneoinstance
-Version: 2.0.2
+Version: 2.2.0
 Summary: Handle Neo4j read and write transactions and provide methos for performing graph EDA
 Author-email: "Jose A. Alvarado-Guzman" <jose.alvarado-guzman@neo4j.com>
 License: GNU GENERAL PUBLIC LICENSE
@@ -689,6 +689,7 @@ Requires-Dist: neo4j>=5.0.0
 Requires-Dist: pandas
 Requires-Dist: PyYAML
 Requires-Dist: pyvis
+Dynamic: license-file
 
 # PyNeoInstance Package
 

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyneoinstance"
-version = "2.0.2"
+version = "2.2.0"
 description = "Handle Neo4j read and write transactions and provide methos for performing graph EDA"
 readme = "README.md"
 authors = [{ name = "Jose A. Alvarado-Guzman",email = "jose.alvarado-guzman@neo4j.com" }]

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0}/test/test_neo4jdb.py

@@ -9,6 +9,7 @@
 import os
 import unittest
 import pandas as pd
+import numpy as np
 import pandas.testing as pd_testing
 from neo4j.exceptions import ClientError
 from .context import database
@@ -121,7 +122,7 @@ class TestNeo4jInstance(unittest.TestCase):
         # Test get_node_labels_freq
         print('Get node labels freq')
         role_file = os.path.join(test_dir,'roles.csv')
-        role_df = pd.read_csv(role_file)
+        role_df = pd.read_csv(role_file).replace(np.NaN,'')
         result = self.graph.get_node_label_freq(
                 self.queries['database'])
         solution = pd.DataFrame([
@@ -194,7 +195,7 @@ class TestNeo4jInstance(unittest.TestCase):
             {'sourceLabel':'Person','relationshipType':'ACTED_IN','targetLabel':'','frequency':56914}
         ]
         )
-        solution.index = [2,0,1]
+        #solution.index = [2,0,1]
         self.assertEqual(result,solution)
 
 if __name__ == '__main__':

{pyneoinstance-2.0.2 → pyneoinstance-2.2.0}/test/test_util_functions.py

@@ -2,11 +2,13 @@
 """Module use to test the utility functions"""
 
 import os
+import pandas as pd
 import unittest
 from datetime import datetime
 from .context import util
+from .context import fileload
 
-class TestGetFileName(unittest.TestCase):
+class TestUtilFunctions(unittest.TestCase):
     """Test the util function to generate file names.
 
         Methods
@@ -36,3 +38,25 @@ class TestGetFileName(unittest.TestCase):
         url = url_scheme + '://' + url_loc + '/' + file_name
         result =  util.is_reachable_url(url)
         self.assertTrue(result)
+
+    def test_parallel_info(self):
+        print('Parallel Info')
+        """Test the get_parallel_function."""
+        result = util.get_parallel_info(1000)
+        solution = (1,False)
+        self.assertEqual(solution,result)
+        result = util.get_parallel_info(1000000)
+        solution = (10,True)
+        self.assertEqual(solution,result)
+
+    def test_get_colums_diff(self):
+        print('Columns Diff')
+        data_file_name = 'movies.csv'
+        cypher_file_name = 'cypher_queries.yaml'
+        file_location = os.path.abspath(os.path.dirname(__file__))
+        cypher_file = os.path.join(file_location,cypher_file_name)
+        query = fileload.load_yaml_file(cypher_file)['queries']['load_movie']
+        columns = pd.read_csv(os.path.join(file_location,data_file_name)).columns
+        result = util.get_columns_diff(query, columns)
+        solution = []
+        self.assertEqual(solution,result)