pyneoinstance 1.1.2__tar.gz → 2.2.0__tar.gz

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/MANIFEST.in

@@ -1,2 +1,3 @@
 include test/*.yaml
 include test/*.csv
+recursive-exclude images

{pyneoinstance-1.1.2/pyneoinstance.egg-info → pyneoinstance-2.2.0}/PKG-INFO

@@ -1,7 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: pyneoinstance
-Version: 1.1.2
-Summary: Handle Neo4j read and write transactions
+Version: 2.2.0
+Summary: Handle Neo4j read and write transactions and provide methos for performing graph EDA
 Author-email: "Jose A. Alvarado-Guzman" <jose.alvarado-guzman@neo4j.com>
 License: GNU GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
@@ -688,19 +688,31 @@ License-File: LICENSE
 Requires-Dist: neo4j>=5.0.0
 Requires-Dist: pandas
 Requires-Dist: PyYAML
+Requires-Dist: pyvis
+Dynamic: license-file
 
 # PyNeoInstance Package
 
 ## What is it?
 
-PyNeoInstance is a Python package that provides a user friendly API for submitting Cypher queries to Neo4j.
+PyNeoInstance is a Python package that provides a user friendly API for submitting Cypher queries to Neo4j and performing Exploratory Data Analysis (EDA) of your graph.
+
 It supports the following features:
 
-- __Read query__: Submit a Cypher read query, with or without parameters, and get a Pandas Data Frame with the results.
-- __Write query__: Submit a write Cypher query, with or without parameters, to update an existing database.
-- __Write queries__: Submit a list of Cypher queries to update an existing database.
-- __Write query with data__: Update a database based on a Pandas Data Frame and a Cypher query.
-- __Write queries with data__: Update a database based on a Pandas Data Frame and a list of Cypher queries.
+- __<u>Read query</u>__: Submit a Cypher read query, with or without parameters, and get a Pandas Data Frame with the results.
+- __<u>Write query</u>__: Submit a write Cypher query, with or without parameters, to update an existing database.
+- __<u>Write queries</u>__: Submit a list of Cypher queries to update an existing database.
+- __<u>Write query with data</u>__: Update a database based on a Pandas Data Frame and a Cypher query.
+- __<u>Write queries with data</u>__: Update a database based on a Pandas Data Frame and a list of Cypher queries.
+- __<u>Node label frequency</u>__: Get the distribution of the graph node labels.
+- __<u>Multi label node frequency</u>__: Get the distribution of the graph multi label nodes.
+- __<u>Relationship type frequency</u>__: Get the distribution of the graph relationship types.
+- __<u>Relationship type source-target frequency</u>__: Get the distribution of relationship source and target.
+- __<u>Node and relationship properties</u>__: Get information of all properties in the graph.
+- __<u>Constraints</u>__: Get information of all constraints in the graph.
+- __<u>Indexes</u>__: Get information of all indexes in the graph.
+- __<u>Schema Visualization</u>__: Visualize the graph schema.
+
 
 ## How to install it?
 
@@ -921,7 +933,69 @@ person_load_result = graph.execute_write_query_with_data(
     workers = 5
     )
 ```
+## Graph EDA
+
+### Node Label Frequency
+The get_node_label_freq method provides the frequency and relative frequency of all nodes in the graph. This method consider all node labels as single labels, if you want the distribution considering multiple labels use the get_node_multilabel_freq method.
+```python
+graph.get_node_label_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/nodefreq.png" width="250">
+
+### Relationship Type Frequency
+The get_rela_type_freq method provides the frequency and relative frequency of all relationship types in the graph.
+```python
+graph.get_rela_type_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/relaFreq.png" width="300">
+
+### Relationship Source-Target Frequency
+The get_rela_source_target_freq method provides the frequency and relative frequency of all relationship types considering the node label of the source and targets of the relationships.
+```python
+graph.get_rela_type_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/relaSourceTargetFreq.png" width="300">
+
+### Properties
+The get_properties method provides information about all the nodes and relationships properties.
+```python
+graph.graph.get_properties(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/properties.png" width="350" hight="300">
+
+### Constraints
+The get_constraints method provides information about all the graph constraints.
+```python
+graph.get_constraints(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/constraints.png" width="350" hight="300">
+
+### Indexes
+The get_indexes method provides information about all the graph indexes.
+```python
+graph.get_indexes(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/indexes.png" width="450" hight="400">
+
+### Schema Visualization
+The graph.get_schema_visualization provides an interactive UI with the visualization of the graph schema with the ability to filter the schema. This UI can be access via a Notebook or HTML page. This visualization displays the node labels as string close to nodes, relationship types are only display on mouse hover.
+
+```python
+schema = graph.get_schema_visualization(database='mydbname')
+schema.show('schema.html')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/schema.png" width="450" hight="400">
+
+To access the UI via HTTML page just set the parameter notebook to false in the show method and open the html file in your browser.
+
+```python
+schema = graph.get_schema_visualization(database='mydbname')
+schema.show('schema.html',notebook=False)
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/schemaHtml.png" width="450" hight="400">
 
 ## More detailed examples
 
-For more detailed examples please referred to this [article](https://medium.com/@a-gilmore/pyneoinstance-a-user-friendly-python-library-for-neo4j-dbefa3117bb2).
+For more detailed examples please refer to the following articles:
+* Data ingestion: [PyNeoInstance: A User-Friendly Python Library for Neo4j](https://medium.com/@a-gilmore/pyneoinstance-a-user-friendly-python-library-for-neo4j-dbefa3117bb2).
+* Graph EDA: [Performing graph EDA with the PyNeoInstance Python Package](https://medium.com/@jose.alvarado-guzman/0c0f6f230aa9)

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/README.md

@@ -2,14 +2,24 @@
 
 ## What is it?
 
-PyNeoInstance is a Python package that provides a user friendly API for submitting Cypher queries to Neo4j.
+PyNeoInstance is a Python package that provides a user friendly API for submitting Cypher queries to Neo4j and performing Exploratory Data Analysis (EDA) of your graph.
+
 It supports the following features:
 
-- __Read query__: Submit a Cypher read query, with or without parameters, and get a Pandas Data Frame with the results.
-- __Write query__: Submit a write Cypher query, with or without parameters, to update an existing database.
-- __Write queries__: Submit a list of Cypher queries to update an existing database.
-- __Write query with data__: Update a database based on a Pandas Data Frame and a Cypher query.
-- __Write queries with data__: Update a database based on a Pandas Data Frame and a list of Cypher queries.
+- __<u>Read query</u>__: Submit a Cypher read query, with or without parameters, and get a Pandas Data Frame with the results.
+- __<u>Write query</u>__: Submit a write Cypher query, with or without parameters, to update an existing database.
+- __<u>Write queries</u>__: Submit a list of Cypher queries to update an existing database.
+- __<u>Write query with data</u>__: Update a database based on a Pandas Data Frame and a Cypher query.
+- __<u>Write queries with data</u>__: Update a database based on a Pandas Data Frame and a list of Cypher queries.
+- __<u>Node label frequency</u>__: Get the distribution of the graph node labels.
+- __<u>Multi label node frequency</u>__: Get the distribution of the graph multi label nodes.
+- __<u>Relationship type frequency</u>__: Get the distribution of the graph relationship types.
+- __<u>Relationship type source-target frequency</u>__: Get the distribution of relationship source and target.
+- __<u>Node and relationship properties</u>__: Get information of all properties in the graph.
+- __<u>Constraints</u>__: Get information of all constraints in the graph.
+- __<u>Indexes</u>__: Get information of all indexes in the graph.
+- __<u>Schema Visualization</u>__: Visualize the graph schema.
+
 
 ## How to install it?
 
@@ -230,7 +240,69 @@ person_load_result = graph.execute_write_query_with_data(
     workers = 5
     )
 ```
+## Graph EDA
+
+### Node Label Frequency
+The get_node_label_freq method provides the frequency and relative frequency of all nodes in the graph. This method consider all node labels as single labels, if you want the distribution considering multiple labels use the get_node_multilabel_freq method.
+```python
+graph.get_node_label_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/nodefreq.png" width="250">
+
+### Relationship Type Frequency
+The get_rela_type_freq method provides the frequency and relative frequency of all relationship types in the graph.
+```python
+graph.get_rela_type_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/relaFreq.png" width="300">
+
+### Relationship Source-Target Frequency
+The get_rela_source_target_freq method provides the frequency and relative frequency of all relationship types considering the node label of the source and targets of the relationships.
+```python
+graph.get_rela_type_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/relaSourceTargetFreq.png" width="300">
+
+### Properties
+The get_properties method provides information about all the nodes and relationships properties.
+```python
+graph.graph.get_properties(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/properties.png" width="350" hight="300">
+
+### Constraints
+The get_constraints method provides information about all the graph constraints.
+```python
+graph.get_constraints(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/constraints.png" width="350" hight="300">
+
+### Indexes
+The get_indexes method provides information about all the graph indexes.
+```python
+graph.get_indexes(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/indexes.png" width="450" hight="400">
+
+### Schema Visualization
+The graph.get_schema_visualization provides an interactive UI with the visualization of the graph schema with the ability to filter the schema. This UI can be access via a Notebook or HTML page. This visualization displays the node labels as string close to nodes, relationship types are only display on mouse hover.
+
+```python
+schema = graph.get_schema_visualization(database='mydbname')
+schema.show('schema.html')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/schema.png" width="450" hight="400">
+
+To access the UI via HTTML page just set the parameter notebook to false in the show method and open the html file in your browser.
+
+```python
+schema = graph.get_schema_visualization(database='mydbname')
+schema.show('schema.html',notebook=False)
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/schemaHtml.png" width="450" hight="400">
 
 ## More detailed examples
 
-For more detailed examples please referred to this [article](https://medium.com/@a-gilmore/pyneoinstance-a-user-friendly-python-library-for-neo4j-dbefa3117bb2).
+For more detailed examples please refer to the following articles:
+* Data ingestion: [PyNeoInstance: A User-Friendly Python Library for Neo4j](https://medium.com/@a-gilmore/pyneoinstance-a-user-friendly-python-library-for-neo4j-dbefa3117bb2).
+* Graph EDA: [Performing graph EDA with the PyNeoInstance Python Package](https://medium.com/@jose.alvarado-guzman/0c0f6f230aa9)

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyneoinstance/__init__.py

@@ -1,4 +1,5 @@
 from .database.neo4jdbms import Neo4jInstance
 from .fileload.loadyaml import load_yaml_file
+from .util.functions import get_parallel_info
 
-__version__ = '1.1.2'
+__version__ = '2.2.0'

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyneoinstance/database/context.py

@@ -3,4 +3,4 @@ import os
 import sys
 sys.path.insert(0, os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..')))
-from pyneoinstance.util.functions import get_logger
+from pyneoinstance.util.functions import get_logger,get_parallel_info,get_columns_diff

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyneoinstance/database/neo4jdbms.py

@@ -13,13 +13,16 @@ from typing import Optional, List, Dict, Any
 from collections import defaultdict
 import multiprocessing as mp
 import logging
+import re
 import numpy as np
 from pandas import DataFrame
 from neo4j import GraphDatabase
+from pyvis.network import Network
 from neo4j.exceptions import ServiceUnavailable
 from neo4j.exceptions import AuthError
 from neo4j.exceptions import ClientError
 from neo4j.exceptions import ConfigurationError
+from .context import get_logger, get_parallel_info, get_columns_diff
 
 def _get_driver(neo_info : Dict[str, str]):
     try:
@@ -167,8 +170,12 @@ class Neo4jInstance:
         self.neo_info['password'] = password
         self.neo_info['encrypted'] = kwargs.get('encrypted') or ''
         self.__results = defaultdict(int)
-
-
+        self.__logger = get_logger('pyneoinstance', logging.WARNING)
+        stream_handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            '%(levelname)s - %(message)s')
+        stream_handler.setFormatter(formatter)
+        self.__logger.addHandler(stream_handler)
     def execute_read_query(self, query: str,
                            database: Optional[str] = None,
                            parameters: Optional[Dict[str, Any]] = None
@@ -288,7 +295,8 @@ class Neo4jInstance:
         result = self.execute_write_queries([query], database, parameters)
         return  result
 
-    def execute_write_queries_with_data(self, queries: List[str], data: DataFrame,
+    def execute_write_queries_with_data(self, queries: List[str],
+                                        data: DataFrame,
                                         database: Optional[str] = None,
                                         partitions: Optional[int] = 1,
                                         parallel: Optional[bool] = False,
@@ -334,7 +342,7 @@ class Neo4jInstance:
         if partitions > data.shape[0]:
             raise ValueError(
                 "The batch size cannot be greater than the number of rows in the data.")
-        data_chunks = np.array_split(data,partitions)
+        data_chunks = np.array_split(data.index,partitions)
         if parameters:
             params = parameters
         else:
@@ -345,19 +353,27 @@ class Neo4jInstance:
             else:
                 workers_num = mp.cpu_count()
             for query in queries:
+                col_diff = get_columns_diff(query,data.columns)
+                msg = f'This columns are not in your data: {col_diff}'
+                if len(col_diff)!=0:
+                    self.__logger.warning(msg)
                 with mp.Pool(workers_num) as worker_pool:
                     results = worker_pool.starmap(
                         _execute_write_parallel,
-                        [(self.neo_info, database,query, d.fillna(value='').to_dict('records'),
+                        [(self.neo_info, database,query, data.iloc[d].to_dict('records'),
                           params) for d in data_chunks]
                     )
         else:
             results = []
             with _get_driver(self.neo_info) as driver:
                 with _get_session(driver, database) as session:
-                    for rows in data_chunks:
-                        rows_dict = {'rows': rows.fillna(value="").to_dict('records')}
+                    for idx in data_chunks:
+                        rows_dict = {'rows': data.iloc[idx].to_dict('records')}
                         for query in queries:
+                            col_diff = get_columns_diff(query,data.columns)
+                            msg = f'This columns are not in your data: {col_diff}'
+                            if len(col_diff)!=0:
+                                self.__logger.warning(msg)
                             results.append(_execute_write(session,
                                                           query,
                                                           rows_dict['rows'],
@@ -421,3 +437,387 @@ class Neo4jInstance:
         result = self.execute_write_queries_with_data(
             [query], data,database, partitions, parallel, workers, params)
         return result
+
+    def get_node_label_freq(self, database: Optional[str] = None) -> DataFrame:
+        """Use to obtain the graph node label frequency.
+
+            Parameters
+            ----------
+            database : str, optional
+                Name of the Neo4j database of which to execute the transaction.
+                If not provided the default database is going to be use.
+
+            Returns
+            -------
+            DataFrame
+                Pandas DataFrame object containing the frequency and relative
+                frequency of node labels.
+
+            Raises
+            ------
+            ClientError
+                When APOC Library is not install correctly in your Neo4j
+                deployment.
+        """
+        error_msg = """
+                APOC Library not detected.
+                Please install it before procceding.
+            """
+        query = """
+            MATCH(n)
+            WITH count(*) AS nodeCount
+            CALL db.labels() YIELD label
+            CALL apoc.cypher.run('MATCH (:`'+label+'`) RETURN count(*) as freq',{}) YIELD value
+            WITH nodeCount,label,value.freq AS freq
+            WITH *, 10^3 AS scaleFactor, toFloat(freq)/toFloat(nodeCount) AS relFreq
+            RETURN label AS nodeLabel,
+                freq AS frequency,
+                round(relFreq*scaleFactor)/scaleFactor AS relativeFrequency
+            ORDER BY freq DESC
+        """
+        with _get_driver(self.neo_info) as driver:
+            with _get_session(driver, database) as session:
+                try:
+                    result = session.execute_read(
+                        _read_transaction_function,query=query)
+                except ServiceUnavailable as exception:
+                    raise ServiceUnavailable() from exception
+                except ClientError:
+                    raise ClientError(error_msg)
+        return result
+
+    def get_node_multilabel_freq(self, database: Optional[str] = None) -> DataFrame:
+        """Use to obtain the graph node multi-label frequency.
+
+            Parameters
+            ----------
+            database : str, optional
+                Name of the Neo4j database of which to execute the transaction.
+                If not provided the default database is going to be use.
+
+            Returns
+            -------
+            DataFrame
+                Pandas DataFrame object containing the frequency and relative
+                frequency of node with multiple labels.
+
+            Raises
+            ------
+            ClientError
+                When APOC Library is not install correctly in your Neo4j
+                deployment.
+        """
+        error_msg = """
+                APOC Library not detected.
+                Please install it before procceding.
+            """
+        query = """
+            MATCH (n)
+            WITH labels(n) as nodeLabels
+            WHERE size(nodeLabels)>1
+            RETURN nodeLabels, count(*) as frequency
+        """
+        with _get_driver(self.neo_info) as driver:
+            with _get_session(driver, database) as session:
+                try:
+                    result = session.execute_read(
+                        _read_transaction_function,query=query)
+                except ServiceUnavailable as exception:
+                    raise ServiceUnavailable() from exception
+                except ClientError:
+                    raise ClientError(error_msg)
+        return result
+
+    def get_rela_type_freq(self, database: Optional[str] = None) -> DataFrame:
+        """Use to obtain the graph relationship type frequency.
+
+            Parameters
+            ----------
+            database : str, optional
+                Name of the Neo4j database of which to execute the transaction.
+                If not provided the default database is going to be use.
+
+            Returns
+            -------
+            DataFrame
+                Pandas DataFrame object containing the frequency and relative
+                frequency of relationship types.
+
+            Raises
+            ------
+            ClientError
+                When APOC Library is not install correctly in your Neo4j
+                deployment.
+        """
+        error_msg = """
+                APOC Library not detected.
+                Please install it before procceding.
+            """
+        query = """
+            MATCH()-[]->()
+            WITH count(*) AS relCount
+            CALL db.relationshipTypes() YIELD relationshipType as type
+            CALL apoc.cypher.run('MATCH ()-[:`'+type+'`]->() RETURN count(*) as freq',{})
+            YIELD value
+            WITH type AS relationshipType, value.freq AS freq,relCount
+            WITH *,3 AS presicion
+            WITH *, 10^presicion AS factor,toFloat(freq)/toFloat(relCount) as relFreq
+            RETURN relationshipType, freq AS frequency,
+            round(relFreq*factor)/factor AS relativeFrequency
+            ORDER BY freq DESC;
+        """
+        with _get_driver(self.neo_info) as driver:
+            with _get_session(driver, database) as session:
+                try:
+                    result = session.execute_read(
+                        _read_transaction_function,query=query)
+                except ServiceUnavailable as exception:
+                    raise ServiceUnavailable() from exception
+                except ClientError:
+                    raise ClientError(error_msg)
+        return result
+
+    def get_properties(self, database: Optional[str] = None) -> DataFrame:
+        """Use to obtain the node and relationship properties.
+
+            Parameters
+            ----------
+            database : str, optional
+                Name of the Neo4j database of which to execute the transaction.
+                If not provided the default database is going to be use.
+
+            Returns
+            -------
+            DataFrame
+                Pandas DataFrame object containing information about all nodes
+                and relationships properties.
+
+            Raises
+            ------
+            ClientError
+                When APOC Library is not install correctly in your Neo4j
+                deployment.
+        """
+        error_msg = """
+                APOC Library not detected.
+                Please install it before procceding.
+            """
+        query = """
+            CALL apoc.meta.data() YIELD label,property,type,elementType
+            WHERE type<>'RELATIONSHIP'
+            RETURN elementType,label,property,type
+            ORDER BY elementType,label,property;
+        """
+        with _get_driver(self.neo_info) as driver:
+            with _get_session(driver, database) as session:
+                try:
+                    result = session.execute_read(
+                        _read_transaction_function,query=query)
+                except ServiceUnavailable as exception:
+                    raise ServiceUnavailable() from exception
+                except ClientError:
+                    raise ClientError(error_msg)
+        return result
+
+    def get_constraints(self, database: Optional[str] = None) -> DataFrame:
+        """Use to obtain the constraints in the graph.
+
+            Parameters
+            ----------
+            database : str, optional
+                Name of the Neo4j database of which to execute the transaction.
+                If not provided the default database is going to be use.
+
+            Returns
+            -------
+            DataFrame
+                Pandas DataFrame object containing information about all the
+                constraints.
+
+            Raises
+            ------
+            ClientError
+                When APOC Library is not install correctly in your Neo4j
+                deployment.
+        """
+        error_msg = """
+                APOC Library not detected.
+                Please install it before procceding.
+            """
+        query = """
+            SHOW CONSTRAINTS
+        """
+        with _get_driver(self.neo_info) as driver:
+            with _get_session(driver, database) as session:
+                try:
+                    result = session.execute_read(
+                        _read_transaction_function,query=query)
+                except ServiceUnavailable as exception:
+                    raise ServiceUnavailable() from exception
+                except ClientError:
+                    raise ClientError(error_msg)
+        return result
+
+    def get_indexes(self, database: Optional[str] = None) -> DataFrame:
+        """Use to obtain the indexes in the graph.
+
+            Parameters
+            ----------
+            database : str, optional
+                Name of the Neo4j database of which to execute the transaction.
+                If not provided the default database is going to be use.
+
+            Returns
+            -------
+            DataFrame
+                Pandas DataFrame object containing information about all the
+                indexes.
+
+            Raises
+            ------
+            ClientError
+                When APOC Library is not install correctly in your Neo4j
+                deployment.
+        """
+        error_msg = """
+                APOC Library not detected.
+                Please install it before procceding.
+            """
+        query = """
+            SHOW INDEX
+        """
+        with _get_driver(self.neo_info) as driver:
+            with _get_session(driver, database) as session:
+                try:
+                    result = session.execute_read(
+                        _read_transaction_function,query=query)
+                except ServiceUnavailable as exception:
+                    raise ServiceUnavailable() from exception
+                except ClientError:
+                    raise ClientError(error_msg)
+        return result
+
+    def get_rela_source_target_freq(self, database: Optional[str] = None) -> DataFrame:
+        """Use to obtain the relationships source and target frequency.
+
+            Parameters
+            ----------
+            database : str, optional
+                Name of the Neo4j database of which to execute the transaction.
+                If not provided the default database is going to be use.
+
+            Returns
+            -------
+            DataFrame
+                Pandas DataFrame object containing the frequency of
+                relationships cource and target frequency.
+
+            Raises
+            ------
+            ClientError
+                When APOC Library is not install correctly in your Neo4j
+                deployment.
+        """
+        error_msg = """
+                APOC Library not detected.
+                Please install it before procceding.
+            """
+        query = """
+            CALL apoc.meta.stats() YIELD relTypes
+        """
+        with _get_driver(self.neo_info) as driver:
+            with _get_session(driver, database) as session:
+                try:
+                    result = session.execute_read(
+                        _read_transaction_function,query=query)
+                except ServiceUnavailable as exception:
+                    raise ServiceUnavailable() from exception
+                except ClientError:
+                    raise ClientError(error_msg)
+        stat_dict = result.iloc[0,0]
+        stats_dicts = []
+        for key,value in stat_dict.items():
+            nodes = re.findall(r"\(:?(\w*)\)",key)
+            relationship = re.findall(r"\[:?(\w*)\]",key)[0]
+            info = {'sourceLabel':nodes[0],'relationshipType':relationship,'targetLabel':nodes[1],
+                    'frequency':value}
+            stats_dicts.append(info)
+        stats_df = DataFrame(stats_dicts)
+        stats_df.sort_values(by=['relationshipType','sourceLabel','targetLabel'],inplace=True)
+        return stats_df
+
+    def get_schema_visualization(self, database: Optional[str] = None) -> Network:
+        """Use to visualize the graph data model (schema).
+
+            Parameters
+            ----------
+            database : str, optional
+                Name of the Neo4j database of which to execute the transaction.
+                If not provided the default database is going to be use.
+
+            Returns
+            -------
+            Network
+                Pyvis Network object containing the visualization information.
+
+            Raises
+            ------
+            ClientError
+                When APOC Library is not install correctly in your Neo4j
+                deployment.
+        """
+        error_msg = """
+                APOC Library not detected.
+                Please install it before procceding.
+            """
+        query = """
+            CALL apoc.meta.schema() YIELD value
+        """
+        with _get_driver(self.neo_info) as driver:
+            with _get_session(driver, database) as session:
+                try:
+                    result = session.execute_read(
+                        _read_transaction_function,query=query)
+                except ServiceUnavailable as exception:
+                    raise ServiceUnavailable() from exception
+                except ClientError:
+                    raise ClientError(error_msg)
+        schema = result.iloc[0,0]
+        network = Network(notebook=True,cdn_resources = "remote",directed=True,
+                          filter_menu=True,height="800px", width="100%")
+        options = """
+        const options = {
+            "physics": {
+                "barnesHut": {
+                  "gravitationalConstant": -13950,
+                  "centralGravity": 6.15,
+                  "springLength": 160,
+                  "damping": 0.23
+                },
+                "minVelocity": 0.75
+              }
+            }
+        """
+        network.set_options(options)
+        nodes = []
+        relationships = set()
+        for key in schema.keys():
+            if schema[key]['type']=='node':
+                nodes.append(key)
+        network.add_nodes(nodes)
+        for node in nodes:
+            for rel in schema[node]['relationships'].keys():
+                title = rel
+                if schema[node]['relationships'][rel]['direction']=='in':
+                    target = node
+                    for label in schema[node]['relationships'][rel]['labels']:
+                        source = label
+                else:
+                    source = node
+                    for label in schema[node]['relationships'][rel]['labels']:
+                        target = label
+                relationships.add(source + "|" + title + "|" + target)
+        for relationship in relationships:
+            rela_parts = relationship.split("|")
+            network.add_edge(rela_parts[0],rela_parts[2],title=rela_parts[1])
+        return network

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyneoinstance/util/__init__.py

@@ -1,3 +1,5 @@
 from .functions import get_file_name
 from .functions import is_reachable_url
 from .functions import get_logger
+from .functions import get_parallel_info
+from .functions import get_columns_diff

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyneoinstance/util/functions.py

@@ -4,6 +4,7 @@
 import functools
 import time
 import logging
+import re
 from datetime import datetime
 from typing import Tuple, Any, Callable, List
 from urllib.request import urlopen
@@ -94,3 +95,45 @@ def timing(function : Callable) -> Tuple[Any, float]:
         elapsed_time = end - start * 1.0
         return result, elapsed_time
     return wrap
+
+def get_parallel_info(num_rows: int, partition_rows=100_000) -> Tuple[int,bool]:
+    """Get the number of partitions and whether to execute execute
+    the load in parallel based on the number of rows of a Data Frame.
+
+    Parameters
+    ----------
+    num_rows : int
+        Number of rows of the Data Frame
+    partition_rows : int
+        Number of records per partition
+    Returns
+    -------
+    Tuble[int, bool]
+        Tuple containing the number of partitions and parallelims.
+    """
+    partitions = 1
+    parallel = False
+    batches = int(num_rows/partition_rows)
+    if batches > 1:
+        partitions = batches
+        parallel = True
+    return (partitions, parallel)
+
+def get_columns_diff(query: str, columns: List[str]) -> List[str]:
+    """Get a list of the columns that a Cypher query is trying to access that are
+    not in the list of columns provided.
+
+    Parameters
+    ----------
+    query : str
+       Cypher query
+    columns : List[str]
+        List containing the column names of a Pandas Data Frame
+    Returns
+    -------
+    List[str]
+       List containing the columns that Cypher is trying to access that are not
+       included in the columns names of the data frame.
+    """
+    cypher_attributes = set([w.split('.')[1] for w in re.findall(r'\brow\.\w+',query)])
+    return [a for a in cypher_attributes if a not in columns]

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0/pyneoinstance.egg-info}/PKG-INFO

@@ -1,7 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: pyneoinstance
-Version: 1.1.2
-Summary: Handle Neo4j read and write transactions
+Version: 2.2.0
+Summary: Handle Neo4j read and write transactions and provide methos for performing graph EDA
 Author-email: "Jose A. Alvarado-Guzman" <jose.alvarado-guzman@neo4j.com>
 License: GNU GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
@@ -688,19 +688,31 @@ License-File: LICENSE
 Requires-Dist: neo4j>=5.0.0
 Requires-Dist: pandas
 Requires-Dist: PyYAML
+Requires-Dist: pyvis
+Dynamic: license-file
 
 # PyNeoInstance Package
 
 ## What is it?
 
-PyNeoInstance is a Python package that provides a user friendly API for submitting Cypher queries to Neo4j.
+PyNeoInstance is a Python package that provides a user friendly API for submitting Cypher queries to Neo4j and performing Exploratory Data Analysis (EDA) of your graph.
+
 It supports the following features:
 
-- __Read query__: Submit a Cypher read query, with or without parameters, and get a Pandas Data Frame with the results.
-- __Write query__: Submit a write Cypher query, with or without parameters, to update an existing database.
-- __Write queries__: Submit a list of Cypher queries to update an existing database.
-- __Write query with data__: Update a database based on a Pandas Data Frame and a Cypher query.
-- __Write queries with data__: Update a database based on a Pandas Data Frame and a list of Cypher queries.
+- __<u>Read query</u>__: Submit a Cypher read query, with or without parameters, and get a Pandas Data Frame with the results.
+- __<u>Write query</u>__: Submit a write Cypher query, with or without parameters, to update an existing database.
+- __<u>Write queries</u>__: Submit a list of Cypher queries to update an existing database.
+- __<u>Write query with data</u>__: Update a database based on a Pandas Data Frame and a Cypher query.
+- __<u>Write queries with data</u>__: Update a database based on a Pandas Data Frame and a list of Cypher queries.
+- __<u>Node label frequency</u>__: Get the distribution of the graph node labels.
+- __<u>Multi label node frequency</u>__: Get the distribution of the graph multi label nodes.
+- __<u>Relationship type frequency</u>__: Get the distribution of the graph relationship types.
+- __<u>Relationship type source-target frequency</u>__: Get the distribution of relationship source and target.
+- __<u>Node and relationship properties</u>__: Get information of all properties in the graph.
+- __<u>Constraints</u>__: Get information of all constraints in the graph.
+- __<u>Indexes</u>__: Get information of all indexes in the graph.
+- __<u>Schema Visualization</u>__: Visualize the graph schema.
+
 
 ## How to install it?
 
@@ -921,7 +933,69 @@ person_load_result = graph.execute_write_query_with_data(
     workers = 5
     )
 ```
+## Graph EDA
+
+### Node Label Frequency
+The get_node_label_freq method provides the frequency and relative frequency of all nodes in the graph. This method consider all node labels as single labels, if you want the distribution considering multiple labels use the get_node_multilabel_freq method.
+```python
+graph.get_node_label_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/nodefreq.png" width="250">
+
+### Relationship Type Frequency
+The get_rela_type_freq method provides the frequency and relative frequency of all relationship types in the graph.
+```python
+graph.get_rela_type_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/relaFreq.png" width="300">
+
+### Relationship Source-Target Frequency
+The get_rela_source_target_freq method provides the frequency and relative frequency of all relationship types considering the node label of the source and targets of the relationships.
+```python
+graph.get_rela_type_freq(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/relaSourceTargetFreq.png" width="300">
+
+### Properties
+The get_properties method provides information about all the nodes and relationships properties.
+```python
+graph.graph.get_properties(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/properties.png" width="350" hight="300">
+
+### Constraints
+The get_constraints method provides information about all the graph constraints.
+```python
+graph.get_constraints(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/constraints.png" width="350" hight="300">
+
+### Indexes
+The get_indexes method provides information about all the graph indexes.
+```python
+graph.get_indexes(database='mydbname')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/indexes.png" width="450" hight="400">
+
+### Schema Visualization
+The graph.get_schema_visualization provides an interactive UI with the visualization of the graph schema with the ability to filter the schema. This UI can be access via a Notebook or HTML page. This visualization displays the node labels as string close to nodes, relationship types are only display on mouse hover.
+
+```python
+schema = graph.get_schema_visualization(database='mydbname')
+schema.show('schema.html')
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/schema.png" width="450" hight="400">
+
+To access the UI via HTTML page just set the parameter notebook to false in the show method and open the html file in your browser.
+
+```python
+schema = graph.get_schema_visualization(database='mydbname')
+schema.show('schema.html',notebook=False)
+```
+<img src="https://s3.amazonaws.com/neo4j-ps-ds-bootcamp/img/schemaHtml.png" width="450" hight="400">
 
 ## More detailed examples
 
-For more detailed examples please referred to this [article](https://medium.com/@a-gilmore/pyneoinstance-a-user-friendly-python-library-for-neo4j-dbefa3117bb2).
+For more detailed examples please refer to the following articles:
+* Data ingestion: [PyNeoInstance: A User-Friendly Python Library for Neo4j](https://medium.com/@a-gilmore/pyneoinstance-a-user-friendly-python-library-for-neo4j-dbefa3117bb2).
+* Graph EDA: [Performing graph EDA with the PyNeoInstance Python Package](https://medium.com/@jose.alvarado-guzman/0c0f6f230aa9)

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyneoinstance.egg-info/SOURCES.txt

@@ -18,6 +18,8 @@ pyneoinstance/fileload/helpers.py
 pyneoinstance/fileload/loadyaml.py
 pyneoinstance/util/__init__.py
 pyneoinstance/util/functions.py
+test/__init__.py
+test/context.py
 test/cypher_queries.yaml
 test/load_files.yaml
 test/movies.csv

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyneoinstance.egg-info/requires.txt

@@ -1,3 +1,4 @@
 neo4j>=5.0.0
 pandas
 PyYAML
+pyvis

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyneoinstance.egg-info/top_level.txt

@@ -1 +1,2 @@
 pyneoinstance
+test

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/pyproject.toml

@@ -6,8 +6,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyneoinstance"
-version = "1.1.2"
-description = "Handle Neo4j read and write transactions"
+version = "2.2.0"
+description = "Handle Neo4j read and write transactions and provide methos for performing graph EDA"
 readme = "README.md"
 authors = [{ name = "Jose A. Alvarado-Guzman",email = "jose.alvarado-guzman@neo4j.com" }]
 license = { file = "LICENSE" }
@@ -20,6 +20,7 @@ keywords = ["neo4j", "graph", "cypher"]
 dependencies = [
     "neo4j>=5.0.0",
     "pandas",
-    "PyYAML"
+    "PyYAML",
+    "pyvis"
 ]
 requires-python = ">=3.8"

pyneoinstance-2.2.0/setup.py

@@ -0,0 +1,5 @@
+import setuptools
+
+setuptools.setup(
+    packages=setuptools.find_packages(exclude=("images"))
+)

pyneoinstance-2.2.0/test/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+test_dir = os.path.abspath(os.path.dirname(__file__))

pyneoinstance-2.2.0/test/context.py

@@ -0,0 +1,8 @@
+"""Use to provide context to the test module"""
+import os
+import sys
+sys.path.insert(0, os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..')))
+from pyneoinstance import fileload
+from pyneoinstance import util
+from pyneoinstance import database

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/test/cypher_queries.yaml

@@ -28,7 +28,7 @@ queries:
       MATCH(p:Person {personId:row.personId}),(m:Movie {movieId:row.movieId})
       MERGE(p)-[r:ACTED_IN]->(m)
         ON CREATE
-        SET r.character=split(row.character,':')
+        SET r.characters=split(row.characters,':')
   create_actor: |
       MERGE(p:Person:Actor {personId:$id})
         ON CREATE

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/test/test_neo4jdb.py

@@ -9,6 +9,9 @@
 import os
 import unittest
 import pandas as pd
+import numpy as np
+import pandas.testing as pd_testing
+from neo4j.exceptions import ClientError
 from .context import database
 from .context import fileload
 from . import test_dir
@@ -24,6 +27,12 @@ class TestNeo4jInstance(unittest.TestCase):
         test_neo4j_instance()
             Test all the methods in the Neo4jInstance class.
     """
+    def assertDataframeEqual(self, a, b, msg):
+        try:
+            pd_testing.assert_frame_equal(a, b)
+        except AssertionError as e:
+            raise self.failureException(msg) from e
+
     def setUp(self):
         """Set authentication variables and create Neo4j driver connection."""
         try:
@@ -37,10 +46,11 @@ class TestNeo4jInstance(unittest.TestCase):
         self.yaml_file = os.path.join(
             test_dir,'cypher_queries.yaml')
         self.queries = fileload.load_yaml_file(self.yaml_file,required_keys)
+        self.addTypeEqualityFunc(pd.DataFrame, self.assertDataframeEqual)
 
     def tearDown(self):
         """Close the neo4j driver and delete the test data."""
-        queries = ["CALL apoc.schema.assert({},{})","MATCH(n) DELETE n"]
+        queries = ["CALL apoc.schema.assert({},{})","MATCH(n) DETACH DELETE n"]
         self.graph.execute_write_queries(queries, self.queries['database'])
 
     def test_neo4j_instance(self):
@@ -109,5 +119,84 @@ class TestNeo4jInstance(unittest.TestCase):
                     'properties_set':4}
         self.assertEqual(solution, result)
 
+        # Test get_node_labels_freq
+        print('Get node labels freq')
+        role_file = os.path.join(test_dir,'roles.csv')
+        role_df = pd.read_csv(role_file).replace(np.NaN,'')
+        result = self.graph.get_node_label_freq(
+                self.queries['database'])
+        solution = pd.DataFrame([
+            {'nodeLabel':'Person','frequency':18727,'relativeFrequency':.75},
+            {'nodeLabel':'Movie','frequency':6231,'relativeFrequency':.25},
+            {'nodeLabel':'Actor','frequency':1,'relativeFrequency':.00}
+            ])
+        self.assertEqual(result,solution)
+
+        # Test get_node_multilabels_freq
+        print('Get node multilabel freq')
+        result = self.graph.get_node_multilabel_freq(
+            self.queries['database'])
+        solution = pd.DataFrame([
+            {'nodeLabels':['Person','Actor'],'frequency':1}])
+        self.assertEqual(result,solution)
+
+        # Test get_rela_labels_freq
+        print('Get relationship type freq')
+        self.graph.execute_write_query_with_data(
+            self.queries['queries']['load_role'],
+            role_df,
+            self.queries['database'])
+        result = self.graph.get_rela_type_freq(
+            self.queries['database'])
+        solution = pd.DataFrame([
+            {'relationshipType':'ACTED_IN','frequency':56914,'relativeFrequency':1.0}])
+        self.assertEqual(result,solution)
+
+        # Test get_properties
+        print('Get properties')
+        query = """
+            CALL apoc.meta.data() YIELD label,property,type,elementType
+            WHERE type<>'RELATIONSHIP'
+            RETURN elementType,label,property,type
+            ORDER BY elementType,label,property;
+        """
+        solution = self.graph.execute_read_query(query,self.queries['database'])
+        result = self.graph.get_properties(
+            self.queries['database'])
+        self.assertEqual(result,solution)
+
+        # Test constraints
+        print('Get constraints')
+        query = """
+            SHOW CONSTRAINTS
+        """
+        solution = self.graph.execute_read_query(query,self.queries['database'])
+        result = self.graph.get_constraints(
+            self.queries['database'])
+        self.assertEqual(result,solution)
+
+        # Test indexes
+        print('Get indexes')
+        query = """
+            SHOW INDEX
+        """
+        solution = self.graph.execute_read_query(query,self.queries['database'])
+        result = self.graph.get_indexes(
+            self.queries['database'])
+        self.assertEqual(result,solution)
+
+        # Test indexes
+        print('Get rela details')
+        result = self.graph.get_rela_source_target_freq(
+            self.queries['database'])
+        solution = pd.DataFrame([
+            {'sourceLabel':'','relationshipType':'ACTED_IN','targetLabel':'','frequency':56914},
+            {'sourceLabel':'','relationshipType':'ACTED_IN','targetLabel':'Movie','frequency':56914},
+            {'sourceLabel':'Person','relationshipType':'ACTED_IN','targetLabel':'','frequency':56914}
+        ]
+        )
+        #solution.index = [2,0,1]
+        self.assertEqual(result,solution)
+
 if __name__ == '__main__':
     unittest.main()

{pyneoinstance-1.1.2 → pyneoinstance-2.2.0}/test/test_util_functions.py

@@ -2,11 +2,13 @@
 """Module use to test the utility functions"""
 
 import os
+import pandas as pd
 import unittest
 from datetime import datetime
 from .context import util
+from .context import fileload
 
-class TestGetFileName(unittest.TestCase):
+class TestUtilFunctions(unittest.TestCase):
     """Test the util function to generate file names.
 
         Methods
@@ -36,3 +38,25 @@ class TestGetFileName(unittest.TestCase):
         url = url_scheme + '://' + url_loc + '/' + file_name
         result =  util.is_reachable_url(url)
         self.assertTrue(result)
+
+    def test_parallel_info(self):
+        print('Parallel Info')
+        """Test the get_parallel_function."""
+        result = util.get_parallel_info(1000)
+        solution = (1,False)
+        self.assertEqual(solution,result)
+        result = util.get_parallel_info(1000000)
+        solution = (10,True)
+        self.assertEqual(solution,result)
+
+    def test_get_colums_diff(self):
+        print('Columns Diff')
+        data_file_name = 'movies.csv'
+        cypher_file_name = 'cypher_queries.yaml'
+        file_location = os.path.abspath(os.path.dirname(__file__))
+        cypher_file = os.path.join(file_location,cypher_file_name)
+        query = fileload.load_yaml_file(cypher_file)['queries']['load_movie']
+        columns = pd.read_csv(os.path.join(file_location,data_file_name)).columns
+        result = util.get_columns_diff(query, columns)
+        solution = []
+        self.assertEqual(solution,result)

pyneoinstance-1.1.2/setup.py

@@ -1,3 +0,0 @@
-from setuptools import setup
-
-setup()