neo4j-etl-lib 0.0.2__py3-none-any.whl → 0.1.0__py3-none-any.whl

etl_lib/__init__.py

@@ -1,4 +1,4 @@
 """
 Building blocks for ETL pipelines.
 """
-__version__ = "0.0.2"
+__version__ = "0.1.0"

etl_lib/cli/run_tools.py

@@ -98,7 +98,7 @@ def cli(ctx, neo4j_uri, neo4j_user, neo4j_password, log_file, database_name):
 @click.pass_context
 def query(ctx, number_runs):
     """
-    Retrieve the list of the last x etl runs from the database and display them.
+    Retrieve the list of the last x ETL runs from the database and display them.
     """
     print(f"Listing runs in database '{ctx.obj['database_name']}'")
     with __driver(ctx) as driver:
@@ -155,7 +155,7 @@ def detail(ctx, run_id, details):
                 "status": record["status"],
                 "batches": record["batches"],
                 "duration": __duration_from_start_end(record["startTime"], record["endTime"]),
-                "changes": sum(record.get("stats", {}).values())
+                "changes": record.get("changes", 0)
             }
             for record in records
         ]

etl_lib/core/BatchProcessor.py

@@ -4,7 +4,6 @@ import sys
 from dataclasses import dataclass, field
 from typing import Generator
 
-from etl_lib.core.ETLContext import ETLContext
 from etl_lib.core.Task import Task
 from etl_lib.core.utils import merge_summery
 
@@ -53,7 +52,7 @@ class BatchProcessor:
     and returned in batches to the caller. Usage of `Generators` ensure that not all data must be loaded at once.
     """
 
-    def __init__(self, context: ETLContext, task: Task, predecessor=None):
+    def __init__(self, context, task: Task = None, predecessor=None):
         """
         Constructs a new :py:class:`etl_lib.core.BatchProcessor` instance.
 

etl_lib/core/ETLContext.py

@@ -2,7 +2,7 @@ import logging
 from typing import NamedTuple, Any
 
 from graphdatascience import GraphDataScience
-from neo4j import Driver, GraphDatabase, WRITE_ACCESS, SummaryCounters
+from neo4j import GraphDatabase, WRITE_ACCESS, SummaryCounters
 
 from etl_lib.core.ProgressReporter import get_reporter
 
@@ -20,18 +20,19 @@ def append_results(r1: QueryResult, r2: QueryResult) -> QueryResult:
 
 
 class Neo4jContext:
-    uri: str
-    auth: (str, str)
-    driver: Driver
-    database: str
+    """
+    Holds the connection to the neo4j database and provides facilities to execute queries.
+    """
 
     def __init__(self, env_vars: dict):
         """
         Create a new Neo4j context.
+
         Reads the following env_vars keys:
         - `NEO4J_URI`,
         - `NEO4J_USERNAME`,
         - `NEO4J_PASSWORD`.
+        - `NEO4J_DATABASE`,
         """
         self.logger = logging.getLogger(self.__class__.__name__)
         self.uri = env_vars["NEO4J_URI"]
@@ -43,6 +44,10 @@ class Neo4jContext:
     def query_database(self, session, query, **kwargs) -> QueryResult:
         """
         Executes a Cypher query on the Neo4j database.
+
+        Args:
+            session: Neo4j database session.
+            query: Cypher query either as a single query or as a list.
         """
         if isinstance(query, list):
             results = []
@@ -78,12 +83,33 @@ class Neo4jContext:
         }
 
     def session(self, database=None):
+        """
+        Create a new Neo4j session in write mode, caller is responsible to close the session.
+
+        Args:
+            database: name of the database to use for this session. If not provided, the database name provided during
+                construction will be used.
+
+        Returns:
+            newly created Neo4j session.
+
+        """
         if database is None:
             return self.driver.session(database=self.database, default_access_mode=WRITE_ACCESS)
         else:
             return self.driver.session(database=database, default_access_mode=WRITE_ACCESS)
 
     def gds(self, database=None) -> GraphDataScience:
+        """
+        Creates a new GraphDataScience client.
+
+        Args:
+            database: Name of the database to use for this dgs client.
+                If not provided, the database name provided during construction will be used.
+
+        Returns:
+            gds client.
+        """
         if database is None:
             return GraphDataScience.from_neo4j_driver(driver=self.driver, database=self.database)
         else:
@@ -104,8 +130,6 @@ class ETLContext:
     Will be passed to all :py:class:`etl_lib.core.Task` to provide access to environment variables and functionally
     deemed general enough that all parts of the ETL pipeline would need it.
     """
-    neo4j: Neo4jContext
-    __env_vars: dict
 
     def __init__(self, env_vars: dict):
         """

etl_lib/core/ProgressReporter.py

@@ -66,7 +66,7 @@ class ProgressReporter:
         task.success = success
         task.summery = summery
 
-        report = f"{'\t' * task.depth}finished {task.task_name()} with success: {success}"
+        report = f"{'\t' * task.depth} finished {task.task_name()} in {task.end_time - task.start_time} with success: {success}"
         if error is not None:
             report += f", error: \n{error}"
         else:
@@ -197,10 +197,10 @@ def get_reporter(context) -> ProgressReporter:
     """
     Returns a ProgressReporter instance.
 
-    If the :py:class:`ETLContext <etl_lib.core.ETLContext>` env holds the key `REPORTER_DATABASE` then
-    a :py:class:`Neo4jProgressReporter` instance is created with the given database name.
+    If the :class:`ETLContext <etl_lib.core.ETLContext>` env holds the key `REPORTER_DATABASE` then
+    a :class:`Neo4jProgressReporter` instance is created with the given database name.
 
-    Otherwise, a  :py:class:`ProgressReporter` (no logging to database) instance will be created.
+    Otherwise, a  :class:`ProgressReporter` (no logging to database) instance will be created.
     """
 
     db = context.env("REPORTER_DATABASE")

etl_lib/core/Task.py

@@ -78,9 +78,6 @@ class Task:
         """Time when the :py:func:`~execute` has finished., `None` before."""
         self.success: bool
         """True if the task has finished successful. False otherwise, `None` before the task has finished."""
-        self.summery: dict  # TODO: still in use?
-        """Summery statistics about the task performed, such as rows inserted, updated."""
-        self.error: str  # TODO: still in use?
         self.depth: int = 0
         """Level or depth of the task in the hierarchy. The root task is depth 0. Updated by the Reporter"""
 

etl_lib/core/ValidationBatchProcessor.py

@@ -47,7 +47,7 @@ class ValidationBatchProcessor(BatchProcessor):
             for row in batch.chunk:
                 try:
                     # Validate and transform the row
-                    validated_row = self.model(**row).model_dump()
+                    validated_row = json.loads(self.model(**row).model_dump_json())
                     valid_rows.append(validated_row)
                 except ValidationError as e:
                     # Collect invalid rows with errors

etl_lib/core/utils.py

@@ -1,3 +1,6 @@
+import logging
+
+
 def merge_summery(summery_1: dict, summery_2: dict) -> dict:
     """
     Helper function to merge dicts. Assuming that values are numbers.
@@ -5,3 +8,21 @@ def merge_summery(summery_1: dict, summery_2: dict) -> dict:
     """
     return {i: summery_1.get(i, 0) + summery_2.get(i, 0)
             for i in set(summery_1).union(summery_2)}
+
+
+def setup_logging(log_file=None):
+    """
+    Set up logging to console and optionally to a log file.
+
+    :param log_file: Path to the log file
+    :type log_file: str, optional
+    """
+    handlers = [logging.StreamHandler()]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file))
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        handlers=handlers
+    )

etl_lib/data_sink/CSVBatchSink.py

@@ -0,0 +1,57 @@
+import csv
+from pathlib import Path
+from typing import Generator
+
+from etl_lib.core.ETLContext import ETLContext
+from etl_lib.core.BatchProcessor import BatchProcessor, BatchResults, append_result
+from etl_lib.core.Task import Task
+
+
+class CSVBatchSink(BatchProcessor):
+    """
+    BatchProcessor to write batches of data to a CSV file.
+    """
+
+    def __init__(self, context: ETLContext, task: Task, predecessor: BatchProcessor, file_path: Path, **kwargs):
+        """
+        Constructs a new CSVBatchSink.
+
+        Args:
+            context: :class:`etl_lib.core.ETLContext.ETLContext` instance.
+            task: :class:`etl_lib.core.Task.Task` instance owning this batchProcessor.
+            predecessor: BatchProcessor which :func:`~get_batch` function will be called to receive batches to process.
+            file_path: Path to the CSV file where data will be written. If the file exists, data will be appended.
+            **kwargs: Additional arguments passed to `csv.DictWriter` to allow tuning the csv creation.
+        """
+        super().__init__(context, task, predecessor)
+        self.file_path = file_path
+        self.file_initialized = False
+        self.csv_kwargs = kwargs
+
+    def get_batch(self, batch_size: int) -> Generator[BatchResults, None, None]:
+        assert self.predecessor is not None
+
+        for batch_result in self.predecessor.get_batch(batch_size):
+            self._write_to_csv(batch_result.chunk)
+            yield append_result(batch_result, {"rows_written": len(batch_result.chunk)})
+
+    def _write_to_csv(self, data: list[dict]):
+        """
+        Writes a batch of data to the CSV file.
+
+        Args:
+            data: A list of dictionaries representing rows of data.
+        """
+        if not data:
+            return
+
+        fieldnames = data[0].keys()
+        write_header = not self.file_initialized or not self.file_path.exists()
+
+        with self.file_path.open(mode="a", newline="", encoding="utf-8") as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames, **self.csv_kwargs)
+            if write_header:
+                writer.writeheader()
+            writer.writerows(data)
+
+        self.file_initialized = True

etl_lib/data_sink/{CypherBatchProcessor.py → CypherBatchSink.py}

@@ -5,19 +5,19 @@ from etl_lib.core.BatchProcessor import BatchProcessor, BatchResults, append_res
 from etl_lib.core.Task import Task
 
 
-class CypherBatchProcessor(BatchProcessor):
+class CypherBatchSink(BatchProcessor):
     """
     BatchProcessor to write batches of data to a Neo4j database.
     """
 
     def __init__(self, context: ETLContext, task: Task, predecessor: BatchProcessor, query: str):
         """
-        Constructs a new CypherBatchProcessor.
+        Constructs a new CypherBatchSink.
 
         Args:
-            context: :py:class:`etl_lib.core.ETLContext.ETLContext` instance.
-            task: :py:class:`etl_lib.core.Task.Task` instance owning this batchProcessor.
-            predecessor: BatchProcessor which :py:func:`~get_batch` function will be called to receive batches to process.
+            context: :class:`etl_lib.core.ETLContext.ETLContext` instance.
+            task: :class:`etl_lib.core.Task.Task` instance owning this batchProcessor.
+            predecessor: BatchProcessor which :func:`~get_batch` function will be called to receive batches to process.
             query: Cypher to write the query to Neo4j.
                 Data will be passed as `batch` parameter.
                 Therefor, the query should start with a `UNWIND $batch AS row`.

etl_lib/data_source/{CSVBatchProcessor.py → CSVBatchSource.py}

@@ -4,11 +4,10 @@ from pathlib import Path
 from typing import Generator
 
 from etl_lib.core.BatchProcessor import BatchProcessor, BatchResults
-from etl_lib.core.ETLContext import ETLContext
 from etl_lib.core.Task import Task
 
 
-class CSVBatchProcessor(BatchProcessor):
+class CSVBatchSource(BatchProcessor):
     """
     BatchProcessor that reads a CSV file using the `csv` package.
 
@@ -17,13 +16,13 @@ class CSVBatchProcessor(BatchProcessor):
     starting with 0.
     """
 
-    def __init__(self, csv_file: Path, context: ETLContext, task: Task, **kwargs):
+    def __init__(self, csv_file: Path, context, task: Task = None, **kwargs):
         """
-        Constructs a new CSVBatchProcessor.
+        Constructs a new CSVBatchSource.
 
         Args:
             csv_file: Path to the CSV file.
-            context: :py:class:`etl_lib.core.ETLContext.ETLContext` instance.
+            context: :class:`etl_lib.core.ETLContext.ETLContext` instance.
             kwargs: Will be passed on to the `csv.DictReader` providing a way to customise the reading to different
                 csv formats.
         """
@@ -32,10 +31,10 @@ class CSVBatchProcessor(BatchProcessor):
         self.kwargs = kwargs
 
     def get_batch(self, max_batch__size: int) -> Generator[BatchResults]:
-        for batch_size, chunks_ in self.read_csv(self.csv_file, batch_size=max_batch__size, **self.kwargs):
+        for batch_size, chunks_ in self.__read_csv(self.csv_file, batch_size=max_batch__size, **self.kwargs):
             yield BatchResults(chunk=chunks_, statistics={"csv_lines_read": batch_size}, batch_size=batch_size)
 
-    def read_csv(self, file: Path, batch_size: int, **kwargs):
+    def __read_csv(self, file: Path, batch_size: int, **kwargs):
         if file.suffix == ".gz":
             with gzip.open(file, "rt", encoding='utf-8-sig') as f:
                 yield from self.__parse_csv(batch_size, file=f, **kwargs)
@@ -44,30 +43,23 @@ class CSVBatchProcessor(BatchProcessor):
                 yield from self.__parse_csv(batch_size, file=f, **kwargs)
 
     def __parse_csv(self, batch_size, file, **kwargs):
-        csv_file = csv.DictReader(file, **kwargs)
-        yield from self.__split_to_batches(csv_file, batch_size)
+        """Read CSV in batches without loading the entire file at once."""
+        csv_reader = csv.DictReader(file, **kwargs)
 
-    def __split_to_batches(self, source: [dict], batch_size):
-        """
-        Splits the provided source into batches.
-
-        Args:
-            source: Anything that can be loop over, ideally, this should also be a generator
-            batch_size: desired batch size
-
-        Returns:
-            generator object to loop over the batches. Each batch is an Array.
-        """
         cnt = 0
         batch_ = []
-        for i in source:
-            i["_row"] = cnt
+
+        for row in csv_reader:
+            row["_row"] = cnt
             cnt += 1
-            batch_.append(self.__clean_dict(i))
+            batch_.append(self.__clean_dict(row))
+
             if len(batch_) == batch_size:
                 yield len(batch_), batch_
                 batch_ = []
-        if len(batch_) > 0:
+
+        # Yield any remaining data
+        if batch_:
             yield len(batch_), batch_
 
     def __clean_dict(self, input_dict):

etl_lib/data_source/CypherBatchSource.py

@@ -0,0 +1,47 @@
+from typing import Generator
+
+from etl_lib.core.BatchProcessor import BatchResults, BatchProcessor
+from etl_lib.core.ETLContext import ETLContext
+from etl_lib.core.Task import Task
+
+
+class CypherBatchSource(BatchProcessor):
+
+    def __init__(self, context: ETLContext, task: Task, query: str, **kwargs):
+        """
+       Constructs a new CypherBatchSource.
+
+       Args:
+           context: :class:`etl_lib.core.ETLContext.ETLContext` instance.
+           task: :class:`etl_lib.core.Task.Task` instance owning this batchProcessor.
+           query: Cypher query to execute.
+           kwargs: Arguments passed as parameters with the query.
+       """
+        super().__init__(context, task)
+        self.query = query
+        self.kwargs = kwargs
+
+    def __read_records(self, tx, batch_size):
+        batch_ = []
+        result = tx.run(self.query, **self.kwargs)
+
+        for record in result:
+            batch_.append(record.data())
+            if len(batch_) == batch_size:
+                yield batch_
+                batch_ = []
+
+        if batch_:
+            yield batch_
+
+    def get_batch(self, max_batch_size: int) -> Generator[BatchResults, None, None]:
+        # not using managed tx on purpose. First of, we want to keep the tx open while delivering batches
+        # automatic retry logic would help, as we do not want to start the query again
+        with self.context.neo4j.session() as session:
+            with session.begin_transaction() as tx:
+                for chunk in self.__read_records(tx, max_batch_size):
+                    yield BatchResults(
+                        chunk=chunk,
+                        statistics={"cypher_rows_read": len(chunk)},
+                        batch_size=len(chunk)
+                    )

etl_lib/task/CreateReportingConstraintsTask.py

@@ -0,0 +1,17 @@
+from etl_lib.core.Task import Task, TaskReturn
+
+
+class CreateReportingConstraintsTask(Task):
+    """Creates the constraint in the REPORTER_DATABASE database."""
+
+    def __init__(self, config):
+        super().__init__(config)
+
+    def run_internal(self, **kwargs) -> TaskReturn:
+        database = self.context.env("REPORTER_DATABASE")
+        assert database is not None, "REPORTER_DATABASE needs to be set in order to run this task"
+
+        with self.context.neo4j.session(database) as session:
+            result = self.context.neo4j.query_database(session=session,
+                                                       query="CREATE CONSTRAINT IF NOT EXISTS FOR (n:ETLTask) REQUIRE n.uuid IS UNIQUE")
+            return TaskReturn(True, result.summery)

etl_lib/task/ExecuteCypherTask.py

@@ -6,7 +6,12 @@ from etl_lib.core.utils import merge_summery
 
 
 class ExecuteCypherTask(Task):
+    """
+    Execute cypher (write) as a Task.
 
+    This task is for data refinement jobs, as it does not return cypher results.
+    Parameters can be passed as keyword arguments to the constructor and will be available as parameters inside cypher.
+    """
     def __init__(self, context: ETLContext):
         super().__init__(context)
         self.context = context

etl_lib/task/data_loading/CSVLoad2Neo4jTask.py

@@ -9,12 +9,24 @@ from etl_lib.core.ETLContext import ETLContext
 from etl_lib.core.ClosedLoopBatchProcessor import ClosedLoopBatchProcessor
 from etl_lib.core.Task import Task, TaskReturn
 from etl_lib.core.ValidationBatchProcessor import ValidationBatchProcessor
-from etl_lib.data_sink.CypherBatchProcessor import CypherBatchProcessor
-from etl_lib.data_source.CSVBatchProcessor import CSVBatchProcessor
+from etl_lib.data_sink.CypherBatchSink import CypherBatchSink
+from etl_lib.data_source.CSVBatchSource import CSVBatchSource
 
 
-class CSVLoad2Neo4jTasks(Task):
+class CSVLoad2Neo4jTask(Task):
+    """
+    Loads the specified CSV file to Neo4j.
 
+    Uses BatchProcessors to read, validate and write to Neo4j.
+    The validation step is using pydantic, hence a Pydantic model needs to be provided.
+    Rows that fail the validation, will be written to en error file. The location of the error file is determined as
+    follows:
+
+    If the context env vars hold an entry `ETL_ERROR_PATH` the file will be place there, with the name set to name
+    of the provided filename appended with `.error.json`
+
+    If  `ETL_ERROR_PATH` is not set, the file will be placed in the same directory as the CSV file.
+    """
     def __init__(self, context: ETLContext, model: Type[BaseModel], file: Path, batch_size: int = 5000):
         super().__init__(context)
         self.batch_size = batch_size
@@ -23,11 +35,15 @@ class CSVLoad2Neo4jTasks(Task):
         self.file = file
 
     def run_internal(self, **kwargs) -> TaskReturn:
-        error_file = self.file.with_suffix(".error.json")
+        error_path = self.context.env("ETL_ERROR_PATH")
+        if error_path is None:
+            error_file = self.file.with_suffix(".error.json")
+        else:
+            error_file = error_path / self.file.with_name(self.file.stem + ".error.json").name
 
-        csv = CSVBatchProcessor(self.file, self.context, self)
+        csv = CSVBatchSource(self.file, self.context, self)
         validator = ValidationBatchProcessor(self.context, self, csv, self.model, error_file)
-        cypher = CypherBatchProcessor(self.context, self, validator, self._query())
+        cypher = CypherBatchSink(self.context, self, validator, self._query())
         end = ClosedLoopBatchProcessor(self.context, self, cypher)
         result = next(end.get_batch(self.batch_size))
 

etl_lib/test_utils/utils.py

@@ -0,0 +1,153 @@
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+from _pytest.tmpdir import tmp_path
+from neo4j import Driver
+from neo4j.time import Date
+
+from etl_lib.core.ETLContext import QueryResult, Neo4jContext, ETLContext
+from etl_lib.core.Task import Task
+
+
+def run_query(driver, query, data):
+    with driver.session(database=get_database_name()) as session:
+        result = session.run(query, data=data)
+        return result.data()
+
+
+def get_node_count(driver, label: str) -> int:
+    """Get the count of nodes with the specified label"""
+    query = f"MATCH (n:{label}) RETURN COUNT(n) AS count"
+    with driver.session(database=get_database_name()) as session:
+        result = session.run(query)
+        return result.single()["count"]
+
+
+def get_relationship_count(driver, rel_type: str) -> int:
+    """Get the count of relationships with the specified type"""
+    query = f"MATCH ()-[r:{rel_type}]->() RETURN COUNT(r) AS count"
+    with driver.session(database=get_database_name()) as session:
+        result = session.run(query)
+        return result.single()["count"]
+
+
+def check_property_exists(driver, label: str, property_name: str) -> bool:
+    """
+    Checks if all nodes with the given label have the given property.
+    :return:
+    """
+    with driver.session(database=get_database_name()) as session:
+        result = session.run(f"MATCH (n:{label}) WHERE n.{property_name} IS NULL"
+                             f" RETURN COUNT(n) = 0 AS exists")
+        return result.single()["exists"]
+
+
+def get_graph(driver):
+    """
+    Return a grap representation of all data in the database.
+    The returned structure is an array of dicts. Each dict has the following keys:
+    `start`, `end`, and `rel` representing each relationship found in the graph.
+    Use the following query to generate this structure from a known good graph:
+
+        MATCH (s)-[r]->(e)
+        WITH {labels:labels(s), props:properties(s)} AS start, {type:type(r), props:properties(r)} AS rel, {labels:labels(e), props:properties(e)} AS end
+        RETURN {start:start, rel:rel, end:end}
+    """
+    with  driver.session(database=get_database_name()) as session:
+        records = session.run(
+            """
+            MATCH (s)-[r]->(e)
+            WITH {labels:labels(s), props:properties(s)} AS start, 
+                {type:type(r), props:properties(r)} AS rel, 
+                {labels:labels(e), props:properties(e)} AS end 
+            RETURN {start:start, rel:rel, end:end} AS graph
+            """
+        )
+        data = [record.data()["graph"] for record in records]
+        return convert_neo4j_date_to_string(data, "%Y-%m-%d")
+
+
+def convert_neo4j_date_to_string(data, date_format):
+    """
+    Recursively converts all neo4j.time.Date instances in a dictionary into strings using the provided format.
+
+    :param data: The input dictionary or list to process.
+    :param date_format: A format string compatible with Python's strftime.
+    :return: The processed dictionary or list with dates converted to strings.
+    """
+    if isinstance(data, dict):
+        return {key: convert_neo4j_date_to_string(value, date_format) for key, value in data.items()}
+    elif isinstance(data, list):
+        return [convert_neo4j_date_to_string(item, date_format) for item in data]
+    elif isinstance(data, Date):
+        return data.to_native().strftime(date_format)
+    else:
+        return data
+
+
+def get_database_name():
+    if os.getenv("NEO4J_TEST_CONTAINER") is None:
+        # not running with test containers. expect test db to be set
+        if os.getenv("NEO4J_TEST_DATABASE") is not None:
+            return os.getenv("NEO4J_TEST_DATABASE")
+        else:
+            raise Exception("define NEO4J_TEST_DATABASE environment variable")
+
+
+class TestNeo4jContext(Neo4jContext):
+
+    def __init__(self, driver: Driver):
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.driver = driver
+        self.database = get_database_name()
+
+
+class TestETLContext(ETLContext):
+
+    def __init__(self, driver: Driver, tmp_path):
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.__env_vars = {"ETL_ERROR_PATH": tmp_path}
+        self.neo4j = TestNeo4jContext(driver)
+        self.reporter = DummyReporter()
+
+    def env(self, key: str) -> Any:
+        if key in self.__env_vars:
+            return self.__env_vars[key]
+
+
+class DummyReporter:
+
+    def register_tasks(self, main: Task):
+        pass
+
+    def started_task(self, task: Task) -> Task:
+        pass
+
+    def finished_task(self, task, success: bool, summery: dict, error: str = None) -> Task:
+        pass
+
+    def report_progress(self, task, batches: int, expected_batches: int, stats: dict) -> None:
+        pass
+
+
+class DummyNeo4jContext:
+
+    def query_database(self, session, query, **kwargs) -> QueryResult:
+        return QueryResult([], {})
+
+    def session(self, database=None):
+        return None
+
+
+class DummyContext:
+    neo4j: DummyNeo4jContext
+    __env_vars: dict
+    path_error: Path
+    path_import: Path
+    path_processed: Path
+    reporter = DummyReporter()
+
+    def env(self, key: str) -> Any:
+        pass

neo4j_etl_lib-0.1.0.dist-info/METADATA

@@ -0,0 +1,54 @@
+Metadata-Version: 2.4
+Name: neo4j-etl-lib
+Version: 0.1.0
+Summary: Building blocks for ETL pipelines.
+Keywords: etl,graph,database
+Author-email: Bert Radke <bert.radke@pm.me>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Development Status :: 4 - Beta
+License-File: LICENSE
+Requires-Dist: pydantic>=2.10.5; python_version >= '3.8'
+Requires-Dist: neo4j>=5.27.0; python_version >= '3.7'
+Requires-Dist: python-dotenv>=1.0.1; python_version >= '3.8'
+Requires-Dist: tabulate>=0.9.0; python_version >= '3.7'
+Requires-Dist: click>=8.1.8; python_version >= '3.7'
+Requires-Dist: pytest>=8.3.0 ; extra == "dev" and ( python_version >= '3.8')
+Requires-Dist: testcontainers[neo4j]==4.9.0 ; extra == "dev" and ( python_version >= '3.9' and python_version < '4.0')
+Requires-Dist: pytest-cov ; extra == "dev"
+Requires-Dist: bumpver ; extra == "dev"
+Requires-Dist: isort ; extra == "dev"
+Requires-Dist: pip-tools ; extra == "dev"
+Requires-Dist: sphinx ; extra == "dev"
+Requires-Dist: sphinx-rtd-theme ; extra == "dev"
+Requires-Dist: pydata-sphinx-theme ; extra == "dev"
+Requires-Dist: sphinx-autodoc-typehints ; extra == "dev"
+Requires-Dist: sphinxcontrib-napoleon ; extra == "dev"
+Requires-Dist: sphinx-autoapi ; extra == "dev"
+Requires-Dist: graphdatascience>=1.13 ; extra == "gds" and ( python_version >= '3.9')
+Project-URL: Documentation, https://neo-technology-field.github.io/python-etl-lib/index.html
+Project-URL: Home, https://github.com/neo-technology-field/python-etl-lib
+Provides-Extra: dev
+Provides-Extra: gds
+
+# Neo4j ETL Toolbox
+
+A Python library of building blocks to assemble etl pipelines.
+
+Complete documentation can be found on https://neo-technology-field.github.io/python-etl-lib/index.html
+
+See https://github.com/neo-technology-field/python-etl-lib/tree/main/examples/gtfs for an example project.
+
+
+The library can be installed via 
+
+```bash
+pip install neo4j-etl-lib
+```
+

neo4j_etl_lib-0.1.0.dist-info/RECORD

@@ -0,0 +1,29 @@
+etl_lib/__init__.py,sha256=xZKM1gxoW-QX6-igG9rff50v2lL6MgnuhzOOxOORaUI,65
+etl_lib/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+etl_lib/cli/run_tools.py,sha256=YMpa-WICon6mjuPuwyrtdBM9IiHdO9UPBYtA_y6UA0Y,7292
+etl_lib/core/BatchProcessor.py,sha256=6quNPE9Dp8hYJDQDTqxQtxbQ3KCmb56Mko34EIsNhyI,3352
+etl_lib/core/ClosedLoopBatchProcessor.py,sha256=unlx_A339oi2nOOXF0irrVf8j_GFhwcTuk_w5liqbWc,1321
+etl_lib/core/ETLContext.py,sha256=ZTk_IDILpjUji0DphPUzTNx8k_2hZRxy37mqIcEA-kM,5641
+etl_lib/core/ProgressReporter.py,sha256=z5aVBjDJZSNGr6zmY8DsMC6dzEcnhAV7RboHWJdl49g,8557
+etl_lib/core/Task.py,sha256=qhCRYEJciYdaYzMurUTTzGQgm7UeKe0Ik37Fp-qAgr8,9256
+etl_lib/core/ValidationBatchProcessor.py,sha256=EhO6PFQB-4PZgIOTXP4PwkbAl5HRK0zgTeKMseRU5QU,3261
+etl_lib/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+etl_lib/core/utils.py,sha256=wwfyvy78fL6sqHdV0IFqAVyEkp6vo5Yo8gRZua2dulw,816
+etl_lib/data_sink/CSVBatchSink.py,sha256=oq4VJwnA4WSyJzdvwstGv73vOEuWmPSfCynhVmxBByU,2204
+etl_lib/data_sink/CypherBatchSink.py,sha256=RMuelUat55ojLQMRYmoiXG0D_fgWH0RLbmUd01UMv_c,1511
+etl_lib/data_sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+etl_lib/data_source/CSVBatchSource.py,sha256=HILkaQAFua1OM7xgSNKY6teXpcJjWUPaS4Aol-GLYL8,2767
+etl_lib/data_source/CypherBatchSource.py,sha256=Umyr5-eQ5vI7EFqjDhUTgSGzuUkglGKjYIWLpijdGrU,1752
+etl_lib/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+etl_lib/task/CreateReportingConstraintsTask.py,sha256=aV5i1EwjfuG-eEGoNaB-NcaPhyu0NgdVhmZr5MIv8ak,760
+etl_lib/task/ExecuteCypherTask.py,sha256=wpPF-bbawRiNS1cCXLhIwuXROAcXsv3OfdKc6DH5q2o,1252
+etl_lib/task/GDSTask.py,sha256=X1E83wYa-N7AXy43WPEqIy77d__z-2wpBjWNhGNXJzA,1781
+etl_lib/task/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+etl_lib/task/data_loading/CSVLoad2Neo4jTask.py,sha256=US9Sa6ytPPEa6BSVUBttlWdKzqyxlF-09If5XCf-LIE,2277
+etl_lib/task/data_loading/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+etl_lib/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+etl_lib/test_utils/utils.py,sha256=G_qT2WHrBAnNNCmAjCZAgqPP0NseJzBDyBttYmSshQU,5150
+neo4j_etl_lib-0.1.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+neo4j_etl_lib-0.1.0.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
+neo4j_etl_lib-0.1.0.dist-info/METADATA,sha256=nk13cf2M1ErdY9fL0T2leYjJlkdXZOtnKpa-XMu8ifE,2210
+neo4j_etl_lib-0.1.0.dist-info/RECORD,,

{neo4j_etl_lib-0.0.2.dist-info → neo4j_etl_lib-0.1.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.10.1
+Generator: flit 3.11.0
 Root-Is-Purelib: true
 Tag: py3-none-any

neo4j_etl_lib-0.0.2.dist-info/METADATA

@@ -1,126 +0,0 @@
-Metadata-Version: 2.3
-Name: neo4j-etl-lib
-Version: 0.0.2
-Summary: Building blocks for ETL pipelines.
-Keywords: etl,graph,database
-Author-email: Bert Radke <bert.radke@pm.me>
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Intended Audience :: Developers
-Classifier: Programming Language :: Python
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Classifier: Programming Language :: Python :: 3
-Classifier: Topic :: Database
-Classifier: Development Status :: 4 - Beta
-Requires-Dist: pydantic>=2.10.5; python_version >= '3.8'
-Requires-Dist: neo4j>=5.27.0; python_version >= '3.7'
-Requires-Dist: python-dotenv>=1.0.1; python_version >= '3.8'
-Requires-Dist: tabulate>=0.9.0; python_version >= '3.7'
-Requires-Dist: click>=8.1.8; python_version >= '3.7'
-Requires-Dist: pytest>=8.3.0 ; extra == "dev" and ( python_version >= '3.8')
-Requires-Dist: testcontainers[neo4j]==4.9.0 ; extra == "dev" and ( python_version >= '3.9' and python_version < '4.0')
-Requires-Dist: pytest-cov ; extra == "dev"
-Requires-Dist: bumpver ; extra == "dev"
-Requires-Dist: isort ; extra == "dev"
-Requires-Dist: pip-tools ; extra == "dev"
-Requires-Dist: sphinx ; extra == "dev"
-Requires-Dist: sphinx-rtd-theme ; extra == "dev"
-Requires-Dist: pydata-sphinx-theme ; extra == "dev"
-Requires-Dist: sphinx-autodoc-typehints ; extra == "dev"
-Requires-Dist: sphinxcontrib-napoleon ; extra == "dev"
-Requires-Dist: graphdatascience>=1.13 ; extra == "gds" and ( python_version >= '3.9')
-Project-URL: Documentation, https://neo-technology-field.github.io/python-etl-lib/index.html
-Project-URL: Home, https://github.com/neo-technology-field/python-etl-lib
-Provides-Extra: dev
-Provides-Extra: gds
-
-# Python ETL Toolbox
-
-Complete documentation can be found on https://neo-technology-field.github.io/python-etl-lib/index.html
-
-A library of building blocks to assemble etl pipelines.
-
-So, instead of providing yet another etl tool, the aim is to provide quality building blocks for the usual etl task. These building blocks should (do) meet the following functional requirements:
-
-* logging (of tasks performed including times, errors, and statistics)
-* error handling 
-* validation of data (currently via Pydantic)
-* batching and streaming
-* optionally record the information about performed tasks and provide means (NeoDash, console) to review past etl runs.
-
-While this library currently focuses on Neo4j databases, it can be extended to other sources and sinks as needed. 
-
-It does not provide a CLI out of the box, but contains a set of functions to list and manage past runs (if they are stored in a database). In addition, the provided example illustrates how to assemble a etl pipeline and run it from a CLI.
-
-## Quick guide
-
-### Installation
-
-Package is available on PyPi and can be installed (for development) via:
-
-```bash
-python3 -m venv venv
-source venv/bin/activate
-python -m pip install pip-tools
-pip-compile --extra dev pyproject.toml
-pip-sync
-```
-
-### Usage
-
-The below shows a minimalistic etl pipeline to a single CSV file (look at the GTFS example to see more details)
-
-```python
-
-class LoadAgenciesTask(CSVLoad2Neo4jTasks):
-    
-    class Agency(BaseModel):
-        """ Define the Pydantic model for data validation. """
-        id: str = Field(alias="agency_id", default="generic")
-        name: str = Field(alias="agency_name")
-        url: str = Field(alias="agency_url")
-        timezone: str = Field(alias="agency_timezone")
-        lang: str = Field(alias="agency_lang")
-
-    def __init__(self, context: ETLContext, file:Path):
-        super().__init__(context, LoadAgenciesTask.Agency, file)
-
-    def task_name(self) -> str:
-        return f"{self.__class__.__name__}('{self.file}')"
-
-    def _query(self):
-        """Load the data into Neo4j."""
-        return """ UNWIND $batch AS row
-        MERGE (a:Agency {id: row.id})
-            SET a.name= row.name, 
-            a.url= row.url, 
-            a.timezone= row.timezone, 
-            a.lang= row.lang
-        """
-
-    @classmethod
-    def file_name(cls):
-        return "agency.txt"
-
-context = ETLContext(env_vars=dict(os.environ))
-
-schema = SchemaTask(context=context)
-init_group = TaskGroup(context=context, tasks=[schema], name="schema-init")
-
-tasks = [
-    LoadAgenciesTask(context=context, file=input_directory / LoadAgenciesTask.file_name()),
-]
-csv_group = TaskGroup(context=context, tasks=tasks, name="csv-loading")
-
-all_group = TaskGroup(context=context, tasks=[init_group, csv_group], name="main")
-
-context.reporter.register_tasks(all_group)
-
-all_group.execute()
-
-```
-See the provided [example](examples/gtfs/README.md) for a more realistic pipeline and how the logging and reporting would look like.
-
-With the above, all lines in the input file `agency.txt` that do not fit the Pydantic model, would be sent to an json file, containing the error data and a description of why it could not be loaded.
-

neo4j_etl_lib-0.0.2.dist-info/RECORD

@@ -1,24 +0,0 @@
-etl_lib/__init__.py,sha256=pM_SQG2bJ2wlVTBzzIFY5eGB_DuAgTVBlDwZse7DPR0,65
-etl_lib/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etl_lib/cli/run_tools.py,sha256=Z93IZ5WZFSg-DhMANGOgOK3UrZy8s75xwJPAOW9HmTo,7305
-etl_lib/core/BatchProcessor.py,sha256=cAOl8m5Hh0wjGgWpe1NKbN-v-_9JT9B4uQs9guPMlzs,3404
-etl_lib/core/ClosedLoopBatchProcessor.py,sha256=unlx_A339oi2nOOXF0irrVf8j_GFhwcTuk_w5liqbWc,1321
-etl_lib/core/ETLContext.py,sha256=9oeBUntFlIyAODmJopm4rv2HgTH5JLIGCR-fo9VA854,4850
-etl_lib/core/ProgressReporter.py,sha256=SeuA6SxjcFWyKyfKaRkjkeZBFFkty2ZjvCVVGO1cfJY,8528
-etl_lib/core/Task.py,sha256=vrVQTfLfRZJ3GmDDtXwBlb9Cu_oJ32VjNEvhpmw5wiQ,9444
-etl_lib/core/ValidationBatchProcessor.py,sha256=23uYtIVAtR755h34vkrN_QepFJnYroFKVO4xuHF922Y,3244
-etl_lib/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etl_lib/core/utils.py,sha256=Ba3yVA7brtSFiT0z6svq0L1QUnM26QOXGvffogmyGzA,351
-etl_lib/data_sink/CypherBatchProcessor.py,sha256=VrnTOoMJMEjy0CeMPm-Q5p0mMlc4JnwrLDK-QrvEj9U,1530
-etl_lib/data_sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etl_lib/data_source/CSVBatchProcessor.py,sha256=HNOj8YxKM9Sh2sl2V16gK5PkfIK40OYNaLXmkQGj9vs,3156
-etl_lib/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etl_lib/task/ExecuteCypherTask.py,sha256=KPZ42YrSsLJSW2zzEqjG6i_ANJC_VsX94WaWe9MdcZE,997
-etl_lib/task/GDSTask.py,sha256=X1E83wYa-N7AXy43WPEqIy77d__z-2wpBjWNhGNXJzA,1781
-etl_lib/task/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-etl_lib/task/data_loading/CSVLoad2Neo4jTask.py,sha256=aagJCszCvfYEwvivTMX7L56cfL1uF3G9c47ZiAvYUPM,1488
-etl_lib/task/data_loading/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-neo4j_etl_lib-0.0.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-neo4j_etl_lib-0.0.2.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-neo4j_etl_lib-0.0.2.dist-info/METADATA,sha256=U9PM4oS-oFNwzd_PHc9BPY1wr5ScSuXxbxEcxOHDkn8,4961
-neo4j_etl_lib-0.0.2.dist-info/RECORD,,