neo4j-etl-lib 0.3.0__tar.gz → 0.3.1__tar.gz

{neo4j_etl_lib-0.3.0 → neo4j_etl_lib-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: neo4j-etl-lib
-Version: 0.3.0
+Version: 0.3.1
 Summary: Building blocks for ETL pipelines.
 Keywords: etl,graph,database
 Author-email: Bert Radke <bert.radke@pm.me>
@@ -14,11 +14,11 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Classifier: Development Status :: 4 - Beta
 License-File: LICENSE
-Requires-Dist: pydantic>=2.10.5; python_version >= '3.8'
-Requires-Dist: neo4j-rust-ext>=5.27.0; python_version >= '3.7'
-Requires-Dist: python-dotenv>=1.0.1; python_version >= '3.8'
-Requires-Dist: tabulate>=0.9.0; python_version >= '3.7'
-Requires-Dist: click>=8.1.8; python_version >= '3.7'
+Requires-Dist: pydantic>=2.10.5; python_version >= '3.10'
+Requires-Dist: neo4j-rust-ext>=5.27.0,<6; python_version >= '3.10'
+Requires-Dist: python-dotenv>=1.0.1; python_version >= '3.10'
+Requires-Dist: tabulate>=0.9.0; python_version >= '3.10'
+Requires-Dist: click>=8.1.8; python_version >= '3.10'
 Requires-Dist: pydantic[email-validator]
 Requires-Dist: pytest>=8.3.0 ; extra == "dev" and ( python_version >= '3.8')
 Requires-Dist: testcontainers[neo4j]==4.9.0 ; extra == "dev" and ( python_version >= '3.9' and python_version < '4.0')
@@ -35,11 +35,13 @@ Requires-Dist: sphinx-autoapi ; extra == "dev"
 Requires-Dist: sqlalchemy ; extra == "dev"
 Requires-Dist: psycopg2-binary ; extra == "dev"
 Requires-Dist: graphdatascience>=1.13 ; extra == "gds" and ( python_version >= '3.9')
+Requires-Dist: nox>=2024.0.0 ; extra == "nox"
 Requires-Dist: sqlalchemy ; extra == "sql"
 Project-URL: Documentation, https://neo-technology-field.github.io/python-etl-lib/index.html
 Project-URL: Home, https://github.com/neo-technology-field/python-etl-lib
 Provides-Extra: dev
 Provides-Extra: gds
+Provides-Extra: nox
 Provides-Extra: sql
 
 # Neo4j ETL Toolbox

{neo4j_etl_lib-0.3.0 → neo4j_etl_lib-0.3.1}/pyproject.toml

@@ -22,11 +22,11 @@ dynamic = ["version", "description"]
 keywords = ["etl", "graph", "database"]
 
 dependencies = [
-    "pydantic>=2.10.5; python_version >= '3.8'",
-    "neo4j-rust-ext>=5.27.0; python_version >= '3.7'",
-    "python-dotenv>=1.0.1; python_version >= '3.8'",
-    "tabulate>=0.9.0; python_version >= '3.7'",
-    "click>=8.1.8; python_version >= '3.7'",
+    "pydantic>=2.10.5; python_version >= '3.10'",
+    "neo4j-rust-ext>=5.27.0,<6; python_version >= '3.10'",
+    "python-dotenv>=1.0.1; python_version >= '3.10'",
+    "tabulate>=0.9.0; python_version >= '3.10'",
+    "click>=8.1.8; python_version >= '3.10'",
     "pydantic[email_validator]"
 ]
 
@@ -41,6 +41,11 @@ dev = [
 gds = ["graphdatascience>=1.13; python_version >= '3.9'"]
 sql = ["sqlalchemy"]
 
+# Local-only multy-version testing, install via `pip install ".[dev,nox]"`
+nox = [
+    "nox>=2024.0.0"
+]
+
 [project.urls]
 Home = "https://github.com/neo-technology-field/python-etl-lib"
 Documentation = "https://neo-technology-field.github.io/python-etl-lib/index.html"

{neo4j_etl_lib-0.3.0 → neo4j_etl_lib-0.3.1}/src/etl_lib/__init__.py

@@ -1,4 +1,4 @@
 """
 Building blocks for ETL pipelines.
 """
-__version__ = "0.3.0"
+__version__ = "0.3.1"

{neo4j_etl_lib-0.3.0 → neo4j_etl_lib-0.3.1}/src/etl_lib/core/ProgressReporter.py

@@ -45,7 +45,7 @@ class ProgressReporter:
             The task that was provided.
         """
         task.start_time = datetime.now()
-        self.logger.info(f"{'\t' * task.depth}starting {task.task_name()}")
+        self.logger.info(f"{' ' * (4 * task.depth)}starting {task.task_name()}")
         return task
 
     def finished_task(self, task: Task, result: TaskReturn) -> Task:

{neo4j_etl_lib-0.3.0 → neo4j_etl_lib-0.3.1}/src/etl_lib/data_source/CSVBatchSource.py

@@ -30,7 +30,7 @@ class CSVBatchSource(BatchProcessor):
         self.csv_file = csv_file
         self.kwargs = kwargs
 
-    def get_batch(self, max_batch__size: int) -> Generator[BatchResults]:
+    def get_batch(self, max_batch__size: int) -> Generator[BatchResults, None, None]:
         for batch_size, chunks_ in self.__read_csv(self.csv_file, batch_size=max_batch__size, **self.kwargs):
             yield BatchResults(chunk=chunks_, statistics={"csv_lines_read": batch_size}, batch_size=batch_size)
 

{neo4j_etl_lib-0.3.0 → neo4j_etl_lib-0.3.1}/src/etl_lib/task/GDSTask.py

@@ -28,11 +28,14 @@ class GDSTask(Task):
         Function that uses the gds client to perform tasks. See the following example:
 
         def gds_fun(etl_context):
-            with etl_context.neo4j.gds() as gds:
-                gds.graph.drop("neo4j-offices", failIfMissing=False)
-                g_office, project_result = gds.graph.project("neo4j-offices", "City", "FLY_TO")
-                mutate_result = gds.pageRank.mutate(g_office, tolerance=0.5, mutateProperty="rank")
-                return TaskReturn(success=True, summery=transform_dict(mutate_result.to_dict()))
+            gds =  etl_context.neo4j.gds
+            gds.graph.drop("neo4j-offices", failIfMissing=False)
+            g_office, project_result = gds.graph.project("neo4j-offices", "City", "FLY_TO")
+            mutate_result = gds.pageRank.write(g_office, tolerance=0.5, writeProperty="rank")
+            return TaskReturn(success=True, summery=transform_dict(mutate_result.to_dict()))
+
+        Notes: Do *NOT* use `etl_context.neo4j.gds` with a context manager. The GDS client closes the underlying
+            connection when exiting the context.
 
         :param context: The ETLContext to use. Provides the gds client to the func via `etl_context.neo4j.gds()`
         :param func: a function that expects a param `etl_context` and returns a `TaskReturn` object.

{neo4j_etl_lib-0.3.0 → neo4j_etl_lib-0.3.1}/src/etl_lib/test_utils/utils.py

@@ -7,7 +7,7 @@ from _pytest.tmpdir import tmp_path
 from neo4j import Driver
 from neo4j.time import Date
 
-from etl_lib.core.ETLContext import QueryResult, Neo4jContext, ETLContext, SQLContext, gds
+from etl_lib.core.ETLContext import ETLContext, Neo4jContext, QueryResult, SQLContext, gds
 from etl_lib.core.Task import Task
 
 
@@ -96,7 +96,7 @@ def get_database_name():
             raise Exception("define NEO4J_TEST_DATABASE environment variable")
 
 
-class TestNeo4jContext(Neo4jContext):
+class MockNeo4jContext(Neo4jContext):
 
     def __init__(self, driver: Driver):
         self.logger = logging.getLogger(self.__class__.__name__)
@@ -105,29 +105,32 @@ class TestNeo4jContext(Neo4jContext):
         self.gds = gds(self)
 
 
-class TestETLContext(ETLContext):
+class MockETLContext(ETLContext):
 
     def __init__(self, driver: Driver, tmp_path):
         self.logger = logging.getLogger(self.__class__.__name__)
         self.__env_vars = {"ETL_ERROR_PATH": tmp_path}
-        self.neo4j = TestNeo4jContext(driver)
+        self.neo4j = MockNeo4jContext(driver)
         self.reporter = DummyReporter()
 
     def env(self, key: str) -> Any:
         if key in self.__env_vars:
             return self.__env_vars[key]
 
-class TestSQLETLContext(ETLContext):
+
+class MockSQLETLContext(ETLContext):
 
     def __init__(self, sql_uri):
         self.logger = logging.getLogger(self.__class__.__name__)
         self.reporter = DummyReporter()
+        self.__env_vars = {}
         self.sql = SQLContext(sql_uri)
 
     def env(self, key: str) -> Any:
         if key in self.__env_vars:
             return self.__env_vars[key]
 
+
 class DummyReporter:
 
     def register_tasks(self, main: Task):
@@ -163,6 +166,7 @@ class DummyContext:
     def env(self, key: str) -> Any:
         pass
 
+
 class DummyPredecessor:
     def __init__(self, batches):
         self.batches = batches