neo4j-etl-lib 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

etl_lib/__init__.py

@@ -1,4 +1,4 @@
 """
 Building blocks for ETL pipelines.
 """
-__version__ = "0.3.0"
+__version__ = "0.3.1"

etl_lib/core/ProgressReporter.py

@@ -45,7 +45,7 @@ class ProgressReporter:
             The task that was provided.
         """
         task.start_time = datetime.now()
-        self.logger.info(f"{'\t' * task.depth}starting {task.task_name()}")
+        self.logger.info(f"{' ' * (4 * task.depth)}starting {task.task_name()}")
         return task
 
     def finished_task(self, task: Task, result: TaskReturn) -> Task:

etl_lib/data_source/CSVBatchSource.py

@@ -30,7 +30,7 @@ class CSVBatchSource(BatchProcessor):
         self.csv_file = csv_file
         self.kwargs = kwargs
 
-    def get_batch(self, max_batch__size: int) -> Generator[BatchResults]:
+    def get_batch(self, max_batch__size: int) -> Generator[BatchResults, None, None]:
         for batch_size, chunks_ in self.__read_csv(self.csv_file, batch_size=max_batch__size, **self.kwargs):
             yield BatchResults(chunk=chunks_, statistics={"csv_lines_read": batch_size}, batch_size=batch_size)
 

etl_lib/task/GDSTask.py

@@ -28,11 +28,14 @@ class GDSTask(Task):
         Function that uses the gds client to perform tasks. See the following example:
 
         def gds_fun(etl_context):
-            with etl_context.neo4j.gds() as gds:
-                gds.graph.drop("neo4j-offices", failIfMissing=False)
-                g_office, project_result = gds.graph.project("neo4j-offices", "City", "FLY_TO")
-                mutate_result = gds.pageRank.mutate(g_office, tolerance=0.5, mutateProperty="rank")
-                return TaskReturn(success=True, summery=transform_dict(mutate_result.to_dict()))
+            gds =  etl_context.neo4j.gds
+            gds.graph.drop("neo4j-offices", failIfMissing=False)
+            g_office, project_result = gds.graph.project("neo4j-offices", "City", "FLY_TO")
+            mutate_result = gds.pageRank.write(g_office, tolerance=0.5, writeProperty="rank")
+            return TaskReturn(success=True, summery=transform_dict(mutate_result.to_dict()))
+
+        Notes: Do *NOT* use `etl_context.neo4j.gds` with a context manager. The GDS client closes the underlying
+            connection when exiting the context.
 
         :param context: The ETLContext to use. Provides the gds client to the func via `etl_context.neo4j.gds()`
         :param func: a function that expects a param `etl_context` and returns a `TaskReturn` object.

etl_lib/test_utils/utils.py

@@ -7,7 +7,7 @@ from _pytest.tmpdir import tmp_path
 from neo4j import Driver
 from neo4j.time import Date
 
-from etl_lib.core.ETLContext import QueryResult, Neo4jContext, ETLContext, SQLContext, gds
+from etl_lib.core.ETLContext import ETLContext, Neo4jContext, QueryResult, SQLContext, gds
 from etl_lib.core.Task import Task
 
 
@@ -96,7 +96,7 @@ def get_database_name():
             raise Exception("define NEO4J_TEST_DATABASE environment variable")
 
 
-class TestNeo4jContext(Neo4jContext):
+class MockNeo4jContext(Neo4jContext):
 
     def __init__(self, driver: Driver):
         self.logger = logging.getLogger(self.__class__.__name__)
@@ -105,29 +105,32 @@ class TestNeo4jContext(Neo4jContext):
         self.gds = gds(self)
 
 
-class TestETLContext(ETLContext):
+class MockETLContext(ETLContext):
 
     def __init__(self, driver: Driver, tmp_path):
         self.logger = logging.getLogger(self.__class__.__name__)
         self.__env_vars = {"ETL_ERROR_PATH": tmp_path}
-        self.neo4j = TestNeo4jContext(driver)
+        self.neo4j = MockNeo4jContext(driver)
         self.reporter = DummyReporter()
 
     def env(self, key: str) -> Any:
         if key in self.__env_vars:
             return self.__env_vars[key]
 
-class TestSQLETLContext(ETLContext):
+
+class MockSQLETLContext(ETLContext):
 
     def __init__(self, sql_uri):
         self.logger = logging.getLogger(self.__class__.__name__)
         self.reporter = DummyReporter()
+        self.__env_vars = {}
         self.sql = SQLContext(sql_uri)
 
     def env(self, key: str) -> Any:
         if key in self.__env_vars:
             return self.__env_vars[key]
 
+
 class DummyReporter:
 
     def register_tasks(self, main: Task):
@@ -163,6 +166,7 @@ class DummyContext:
     def env(self, key: str) -> Any:
         pass
 
+
 class DummyPredecessor:
     def __init__(self, batches):
         self.batches = batches

{neo4j_etl_lib-0.3.0.dist-info → neo4j_etl_lib-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: neo4j-etl-lib
-Version: 0.3.0
+Version: 0.3.1
 Summary: Building blocks for ETL pipelines.
 Keywords: etl,graph,database
 Author-email: Bert Radke <bert.radke@pm.me>
@@ -14,11 +14,11 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Classifier: Development Status :: 4 - Beta
 License-File: LICENSE
-Requires-Dist: pydantic>=2.10.5; python_version >= '3.8'
-Requires-Dist: neo4j-rust-ext>=5.27.0; python_version >= '3.7'
-Requires-Dist: python-dotenv>=1.0.1; python_version >= '3.8'
-Requires-Dist: tabulate>=0.9.0; python_version >= '3.7'
-Requires-Dist: click>=8.1.8; python_version >= '3.7'
+Requires-Dist: pydantic>=2.10.5; python_version >= '3.10'
+Requires-Dist: neo4j-rust-ext>=5.27.0,<6; python_version >= '3.10'
+Requires-Dist: python-dotenv>=1.0.1; python_version >= '3.10'
+Requires-Dist: tabulate>=0.9.0; python_version >= '3.10'
+Requires-Dist: click>=8.1.8; python_version >= '3.10'
 Requires-Dist: pydantic[email-validator]
 Requires-Dist: pytest>=8.3.0 ; extra == "dev" and ( python_version >= '3.8')
 Requires-Dist: testcontainers[neo4j]==4.9.0 ; extra == "dev" and ( python_version >= '3.9' and python_version < '4.0')
@@ -35,11 +35,13 @@ Requires-Dist: sphinx-autoapi ; extra == "dev"
 Requires-Dist: sqlalchemy ; extra == "dev"
 Requires-Dist: psycopg2-binary ; extra == "dev"
 Requires-Dist: graphdatascience>=1.13 ; extra == "gds" and ( python_version >= '3.9')
+Requires-Dist: nox>=2024.0.0 ; extra == "nox"
 Requires-Dist: sqlalchemy ; extra == "sql"
 Project-URL: Documentation, https://neo-technology-field.github.io/python-etl-lib/index.html
 Project-URL: Home, https://github.com/neo-technology-field/python-etl-lib
 Provides-Extra: dev
 Provides-Extra: gds
+Provides-Extra: nox
 Provides-Extra: sql
 
 # Neo4j ETL Toolbox

{neo4j_etl_lib-0.3.0.dist-info → neo4j_etl_lib-0.3.1.dist-info}/RECORD

@@ -1,11 +1,11 @@
-etl_lib/__init__.py,sha256=FyaTAuElsn3y3j1g15X141PnLTYxPrSUVU_YaVmiyPs,65
+etl_lib/__init__.py,sha256=x6coFV38ytJ_wPhR3c0UEzX65oTz2ouKwygkC_tyRLM,65
 etl_lib/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 etl_lib/cli/run_tools.py,sha256=KIar-y22P4kKm-yoJjecYsPwqC7U76M71dEgFO5-ZBo,8561
 etl_lib/core/BatchProcessor.py,sha256=mRpdxZ6ZMKI8XsY3TPuy4dVcvRqLKCO-p63KeOhFyKE,3417
 etl_lib/core/ClosedLoopBatchProcessor.py,sha256=WzML1nldhZRbP8fhlD6utuK5SBYRl1cJgEobVDIdBP4,1626
 etl_lib/core/ETLContext.py,sha256=wmEnbs3n_80B6La9Py_-MHG8BN0FajE9MjGPej0A3To,8045
 etl_lib/core/ParallelBatchProcessor.py,sha256=jNo1Xv1Ts34UZIseoQLDZOhHOVeEr8dUibKUt0FJ4Hw,7318
-etl_lib/core/ProgressReporter.py,sha256=UvWAPCuOrqyUcb5_kosIsCg1dyVQL-tnjgqnzs2cwZA,9372
+etl_lib/core/ProgressReporter.py,sha256=tkE-W6qlR25nU8nUoECcxZDnjnG8AtQH9s9s5WBh_-Q,9377
 etl_lib/core/SplittingBatchProcessor.py,sha256=OIRMUVFpUoZc0w__JJjUr7B9QC3sBlqQp41xghrQzC0,11616
 etl_lib/core/Task.py,sha256=muQFY5qj2n-ZVV8F6vlHqo2lVSvB3wtGdIgkSXVpOFM,9365
 etl_lib/core/ValidationBatchProcessor.py,sha256=U1M2Qp9Ledt8qFiHAg8zMxE9lLRkBrr51NKs_Y8skK8,3400
@@ -15,13 +15,13 @@ etl_lib/data_sink/CSVBatchSink.py,sha256=oq4VJwnA4WSyJzdvwstGv73vOEuWmPSfCynhVmx
 etl_lib/data_sink/CypherBatchSink.py,sha256=nBH4bzN1IvdSFcKgiAIrAY5IauB565sdyVrnRc1hg_4,1566
 etl_lib/data_sink/SQLBatchSink.py,sha256=vyGrrxpdmCLUZMI2_W2ORej3FLGbwN9-b2GMYHd-k9g,1451
 etl_lib/data_sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etl_lib/data_source/CSVBatchSource.py,sha256=HILkaQAFua1OM7xgSNKY6teXpcJjWUPaS4Aol-GLYL8,2767
+etl_lib/data_source/CSVBatchSource.py,sha256=0q1XdPhAIKw1HcTpnp_F4WxRUzk-24Q8Qd-WeIo5OZ0,2779
 etl_lib/data_source/CypherBatchSource.py,sha256=06WuW11BqYjAXBZqL96Qr9MR8JrcjujDpxXe8cI-SYY,2238
 etl_lib/data_source/SQLBatchSource.py,sha256=O3ZA2GXvo5j_KGwOILzguYZMPY_FJkV5j8FIa3-d9oM,4067
 etl_lib/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 etl_lib/task/CreateReportingConstraintsTask.py,sha256=nTcHLBIgXz_h2OQg-SHjQr68bhH974u0MwrtWPnVwng,762
 etl_lib/task/ExecuteCypherTask.py,sha256=thE8YTZzv1abxNhhDcb4p4ke6qmI6kWR4XQ-GrCBBBU,1284
-etl_lib/task/GDSTask.py,sha256=X1E83wYa-N7AXy43WPEqIy77d__z-2wpBjWNhGNXJzA,1781
+etl_lib/task/GDSTask.py,sha256=UP_NMvdeQ9ueLUFlREfe0q3XhFHTCaMvXArSpvxZNiQ,1918
 etl_lib/task/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 etl_lib/task/data_loading/CSVLoad2Neo4jTask.py,sha256=9XiVdJHpABE-Hx1bsvTKLJWtChc8XMwXeO5RicaHDUo,3873
 etl_lib/task/data_loading/ParallelCSVLoad2Neo4jTask.py,sha256=2xN-5bHV9XgoaJLbbTEBuJFoZHV_CYi_hg6M1HQ-ffA,4030
@@ -29,8 +29,8 @@ etl_lib/task/data_loading/ParallelSQLLoad2Neo4jTask.py,sha256=9xpCW5i8yGnUHyg475
 etl_lib/task/data_loading/SQLLoad2Neo4jTask.py,sha256=HR3DcjOUkQN4SbCkgQYzljQCYhOcb-x2-DR5dBdapzU,2953
 etl_lib/task/data_loading/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 etl_lib/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etl_lib/test_utils/utils.py,sha256=kxWJqdRf1pg-4ByMfrtW3HDbgXIvyVtLndGDVvMCmoI,5641
-neo4j_etl_lib-0.3.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-neo4j_etl_lib-0.3.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-neo4j_etl_lib-0.3.0.dist-info/METADATA,sha256=GJcjdPvmzjEUq0pLndSSVzOg3c7CR6bIWz3sB_9tkVY,2506
-neo4j_etl_lib-0.3.0.dist-info/RECORD,,
+etl_lib/test_utils/utils.py,sha256=CgYOCXcUyndOdRAmGyPLoCIuEik0yzy6FLV2k16cpDM,5673
+neo4j_etl_lib-0.3.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+neo4j_etl_lib-0.3.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+neo4j_etl_lib-0.3.1.dist-info/METADATA,sha256=Pm921qyxL36Ed_Ppp2cW3OFPxUGMv7IyRTmtba3n96o,2580
+neo4j_etl_lib-0.3.1.dist-info/RECORD,,