neo4j-etl-lib 0.0.2__tar.gz → 0.0.3__tar.gz

neo4j_etl_lib-0.0.3/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.3
+Name: neo4j-etl-lib
+Version: 0.0.3
+Summary: Building blocks for ETL pipelines.
+Keywords: etl,graph,database
+Author-email: Bert Radke <bert.radke@pm.me>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Development Status :: 4 - Beta
+Requires-Dist: pydantic>=2.10.5; python_version >= '3.8'
+Requires-Dist: neo4j>=5.27.0; python_version >= '3.7'
+Requires-Dist: python-dotenv>=1.0.1; python_version >= '3.8'
+Requires-Dist: tabulate>=0.9.0; python_version >= '3.7'
+Requires-Dist: click>=8.1.8; python_version >= '3.7'
+Requires-Dist: pytest>=8.3.0 ; extra == "dev" and ( python_version >= '3.8')
+Requires-Dist: testcontainers[neo4j]==4.9.0 ; extra == "dev" and ( python_version >= '3.9' and python_version < '4.0')
+Requires-Dist: pytest-cov ; extra == "dev"
+Requires-Dist: bumpver ; extra == "dev"
+Requires-Dist: isort ; extra == "dev"
+Requires-Dist: pip-tools ; extra == "dev"
+Requires-Dist: sphinx ; extra == "dev"
+Requires-Dist: sphinx-rtd-theme ; extra == "dev"
+Requires-Dist: pydata-sphinx-theme ; extra == "dev"
+Requires-Dist: sphinx-autodoc-typehints ; extra == "dev"
+Requires-Dist: sphinxcontrib-napoleon ; extra == "dev"
+Requires-Dist: sphinx-autoapi ; extra == "dev"
+Requires-Dist: graphdatascience>=1.13 ; extra == "gds" and ( python_version >= '3.9')
+Project-URL: Documentation, https://neo-technology-field.github.io/python-etl-lib/index.html
+Project-URL: Home, https://github.com/neo-technology-field/python-etl-lib
+Provides-Extra: dev
+Provides-Extra: gds
+
+# Neo4j ETL Toolbox
+
+A Python library of building blocks to assemble etl pipelines.
+
+Complete documentation can be found on https://neo-technology-field.github.io/python-etl-lib/index.html
+
+See https://github.com/neo-technology-field/python-etl-lib/tree/main/examples/gtfs for an example project.
+
+
+The library can be installed via 
+
+```bash
+pip install neo4j-etl-lib
+```
+

neo4j_etl_lib-0.0.3/README.md

@@ -0,0 +1,14 @@
+# Neo4j ETL Toolbox
+
+A Python library of building blocks to assemble etl pipelines.
+
+Complete documentation can be found on https://neo-technology-field.github.io/python-etl-lib/index.html
+
+See https://github.com/neo-technology-field/python-etl-lib/tree/main/examples/gtfs for an example project.
+
+
+The library can be installed via 
+
+```bash
+pip install neo4j-etl-lib
+```

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/pyproject.toml

@@ -34,7 +34,8 @@ dev = [
     "pytest>=8.3.0; python_version >= '3.8'",
     "testcontainers[neo4j]==4.9.0; python_version >= '3.9' and python_version < '4.0'",
     "pytest-cov", "bumpver", "isort", "pip-tools",
-    "sphinx", "sphinx-rtd-theme", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-napoleon"
+    "sphinx", "sphinx-rtd-theme", "pydata-sphinx-theme", "sphinx-autodoc-typehints",
+    "sphinxcontrib-napoleon", "sphinx-autoapi"
 ]
 gds = ["graphdatascience>=1.13; python_version >= '3.9'"]
 

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/__init__.py

@@ -1,4 +1,4 @@
 """
 Building blocks for ETL pipelines.
 """
-__version__ = "0.0.2"
+__version__ = "0.0.3"

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/cli/run_tools.py

@@ -155,7 +155,7 @@ def detail(ctx, run_id, details):
                 "status": record["status"],
                 "batches": record["batches"],
                 "duration": __duration_from_start_end(record["startTime"], record["endTime"]),
-                "changes": sum(record.get("stats", {}).values())
+                "changes": record.get("changes", 0)
             }
             for record in records
         ]

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/core/BatchProcessor.py

@@ -4,7 +4,6 @@ import sys
 from dataclasses import dataclass, field
 from typing import Generator
 
-from etl_lib.core.ETLContext import ETLContext
 from etl_lib.core.Task import Task
 from etl_lib.core.utils import merge_summery
 
@@ -53,7 +52,7 @@ class BatchProcessor:
     and returned in batches to the caller. Usage of `Generators` ensure that not all data must be loaded at once.
     """
 
-    def __init__(self, context: ETLContext, task: Task, predecessor=None):
+    def __init__(self, context, task: Task = None, predecessor=None):
         """
         Constructs a new :py:class:`etl_lib.core.BatchProcessor` instance.
 

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/core/ETLContext.py

@@ -2,7 +2,7 @@ import logging
 from typing import NamedTuple, Any
 
 from graphdatascience import GraphDataScience
-from neo4j import Driver, GraphDatabase, WRITE_ACCESS, SummaryCounters
+from neo4j import GraphDatabase, WRITE_ACCESS, SummaryCounters
 
 from etl_lib.core.ProgressReporter import get_reporter
 
@@ -20,18 +20,19 @@ def append_results(r1: QueryResult, r2: QueryResult) -> QueryResult:
 
 
 class Neo4jContext:
-    uri: str
-    auth: (str, str)
-    driver: Driver
-    database: str
+    """
+    Holds the connection to the neo4j database and provides facilities to execute queries.
+    """
 
     def __init__(self, env_vars: dict):
         """
         Create a new Neo4j context.
+
         Reads the following env_vars keys:
         - `NEO4J_URI`,
         - `NEO4J_USERNAME`,
         - `NEO4J_PASSWORD`.
+        - `NEO4J_DATABASE`,
         """
         self.logger = logging.getLogger(self.__class__.__name__)
         self.uri = env_vars["NEO4J_URI"]
@@ -43,6 +44,10 @@ class Neo4jContext:
     def query_database(self, session, query, **kwargs) -> QueryResult:
         """
         Executes a Cypher query on the Neo4j database.
+
+        Args:
+            session: Neo4j database session.
+            query: Cypher query either as a single query or as a list.
         """
         if isinstance(query, list):
             results = []
@@ -78,12 +83,33 @@ class Neo4jContext:
         }
 
     def session(self, database=None):
+        """
+        Create a new Neo4j session in write mode, caller is responsible to close the session.
+
+        Args:
+            database: name of the database to use for this session. If not provided, the database name provided during
+                construction will be used.
+
+        Returns:
+            newly created Neo4j session.
+
+        """
         if database is None:
             return self.driver.session(database=self.database, default_access_mode=WRITE_ACCESS)
         else:
             return self.driver.session(database=database, default_access_mode=WRITE_ACCESS)
 
     def gds(self, database=None) -> GraphDataScience:
+        """
+        Creates a new GraphDataScience client.
+
+        Args:
+            database: Name of the database to use for this dgs client.
+                If not provided, the database name provided during construction will be used.
+
+        Returns:
+            gds client.
+        """
         if database is None:
             return GraphDataScience.from_neo4j_driver(driver=self.driver, database=self.database)
         else:

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/core/ProgressReporter.py

@@ -66,7 +66,7 @@ class ProgressReporter:
         task.success = success
         task.summery = summery
 
-        report = f"{'\t' * task.depth}finished {task.task_name()} with success: {success}"
+        report = f"{'\t' * task.depth} finished {task.task_name()} in {task.end_time - task.start_time} with success: {success}"
         if error is not None:
             report += f", error: \n{error}"
         else:
@@ -197,10 +197,10 @@ def get_reporter(context) -> ProgressReporter:
     """
     Returns a ProgressReporter instance.
 
-    If the :py:class:`ETLContext <etl_lib.core.ETLContext>` env holds the key `REPORTER_DATABASE` then
-    a :py:class:`Neo4jProgressReporter` instance is created with the given database name.
+    If the :class:`ETLContext <etl_lib.core.ETLContext>` env holds the key `REPORTER_DATABASE` then
+    a :class:`Neo4jProgressReporter` instance is created with the given database name.
 
-    Otherwise, a  :py:class:`ProgressReporter` (no logging to database) instance will be created.
+    Otherwise, a  :class:`ProgressReporter` (no logging to database) instance will be created.
     """
 
     db = context.env("REPORTER_DATABASE")

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/core/Task.py

@@ -78,9 +78,6 @@ class Task:
         """Time when the :py:func:`~execute` has finished., `None` before."""
         self.success: bool
         """True if the task has finished successful. False otherwise, `None` before the task has finished."""
-        self.summery: dict  # TODO: still in use?
-        """Summery statistics about the task performed, such as rows inserted, updated."""
-        self.error: str  # TODO: still in use?
         self.depth: int = 0
         """Level or depth of the task in the hierarchy. The root task is depth 0. Updated by the Reporter"""
 

neo4j_etl_lib-0.0.3/src/etl_lib/core/utils.py

@@ -0,0 +1,28 @@
+import logging
+
+
+def merge_summery(summery_1: dict, summery_2: dict) -> dict:
+    """
+    Helper function to merge dicts. Assuming that values are numbers.
+    If a key exists in both dicts, then the result will contain a key with the added values.
+    """
+    return {i: summery_1.get(i, 0) + summery_2.get(i, 0)
+            for i in set(summery_1).union(summery_2)}
+
+
+def setup_logging(log_file=None):
+    """
+    Set up logging to console and optionally to a log file.
+
+    :param log_file: Path to the log file
+    :type log_file: str, optional
+    """
+    handlers = [logging.StreamHandler()]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file))
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        handlers=handlers
+    )

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/data_source/CSVBatchProcessor.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from typing import Generator
 
 from etl_lib.core.BatchProcessor import BatchProcessor, BatchResults
-from etl_lib.core.ETLContext import ETLContext
 from etl_lib.core.Task import Task
 
 
@@ -17,7 +16,7 @@ class CSVBatchProcessor(BatchProcessor):
     starting with 0.
     """
 
-    def __init__(self, csv_file: Path, context: ETLContext, task: Task, **kwargs):
+    def __init__(self, csv_file: Path, context, task: Task = None, **kwargs):
         """
         Constructs a new CSVBatchProcessor.
 
@@ -32,10 +31,10 @@ class CSVBatchProcessor(BatchProcessor):
         self.kwargs = kwargs
 
     def get_batch(self, max_batch__size: int) -> Generator[BatchResults]:
-        for batch_size, chunks_ in self.read_csv(self.csv_file, batch_size=max_batch__size, **self.kwargs):
+        for batch_size, chunks_ in self.__read_csv(self.csv_file, batch_size=max_batch__size, **self.kwargs):
             yield BatchResults(chunk=chunks_, statistics={"csv_lines_read": batch_size}, batch_size=batch_size)
 
-    def read_csv(self, file: Path, batch_size: int, **kwargs):
+    def __read_csv(self, file: Path, batch_size: int, **kwargs):
         if file.suffix == ".gz":
             with gzip.open(file, "rt", encoding='utf-8-sig') as f:
                 yield from self.__parse_csv(batch_size, file=f, **kwargs)
@@ -44,30 +43,23 @@ class CSVBatchProcessor(BatchProcessor):
                 yield from self.__parse_csv(batch_size, file=f, **kwargs)
 
     def __parse_csv(self, batch_size, file, **kwargs):
-        csv_file = csv.DictReader(file, **kwargs)
-        yield from self.__split_to_batches(csv_file, batch_size)
+        """Read CSV in batches without loading the entire file at once."""
+        csv_reader = csv.DictReader(file, **kwargs)
 
-    def __split_to_batches(self, source: [dict], batch_size):
-        """
-        Splits the provided source into batches.
-
-        Args:
-            source: Anything that can be loop over, ideally, this should also be a generator
-            batch_size: desired batch size
-
-        Returns:
-            generator object to loop over the batches. Each batch is an Array.
-        """
         cnt = 0
         batch_ = []
-        for i in source:
-            i["_row"] = cnt
+
+        for row in csv_reader:
+            row["_row"] = cnt
             cnt += 1
-            batch_.append(self.__clean_dict(i))
+            batch_.append(self.__clean_dict(row))
+
             if len(batch_) == batch_size:
                 yield len(batch_), batch_
                 batch_ = []
-        if len(batch_) > 0:
+
+        # Yield any remaining data
+        if batch_:
             yield len(batch_), batch_
 
     def __clean_dict(self, input_dict):

neo4j_etl_lib-0.0.3/src/etl_lib/task/CreateReportingConstraintsTask.py

@@ -0,0 +1,17 @@
+from etl_lib.core.Task import Task, TaskReturn
+
+
+class CreateReportingConstraintsTask(Task):
+    """Creates the constraint in the REPORTER_DATABASE database."""
+
+    def __init__(self, config):
+        super().__init__(config)
+
+    def run_internal(self, **kwargs) -> TaskReturn:
+        database = self.context.env("REPORTER_DATABASE")
+        assert database is not None, "REPORTER_DATABASE needs to be set in order to run this task"
+
+        with self.context.neo4j.session(database) as session:
+            result = self.context.neo4j.query_database(session=session,
+                                                       query="CREATE CONSTRAINT IF NOT EXISTS FOR (n:ETLTask) REQUIRE n.uuid IS UNIQUE")
+            return TaskReturn(True, result.summery)

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/task/ExecuteCypherTask.py

@@ -6,7 +6,12 @@ from etl_lib.core.utils import merge_summery
 
 
 class ExecuteCypherTask(Task):
+    """
+    Execute cypher (write) as a Task.
 
+    This task is for data refinement jobs, as it does not return cypher results.
+    Parameters can be passed as keyword arguments to the constructor and will be available as parameters inside cypher.
+    """
     def __init__(self, context: ETLContext):
         super().__init__(context)
         self.context = context

{neo4j_etl_lib-0.0.2 → neo4j_etl_lib-0.0.3}/src/etl_lib/task/data_loading/CSVLoad2Neo4jTask.py

@@ -13,7 +13,7 @@ from etl_lib.data_sink.CypherBatchProcessor import CypherBatchProcessor
 from etl_lib.data_source.CSVBatchProcessor import CSVBatchProcessor
 
 
-class CSVLoad2Neo4jTasks(Task):
+class CSVLoad2Neo4jTask(Task):
 
     def __init__(self, context: ETLContext, model: Type[BaseModel], file: Path, batch_size: int = 5000):
         super().__init__(context)

neo4j_etl_lib-0.0.2/.env.sample

@@ -1,7 +0,0 @@
-NEO4J_URI=neo4j://localhost:7687
-NEO4J_USERNAME=neo4j
-NEO4J_PASSWORD=secret
-NEO4J_DATABASE=neo4j
-
-NEO4J_TEST_DATABASE=berttest
-#NEO4J_TEST_CONTAINER=neo4j:5.26.0-enterprise

neo4j_etl_lib-0.0.2/.gitignore

@@ -1,9 +0,0 @@
-/.env
-/tests/data/customers.error.json
-**/__pycache__/
-**/.env
-/requirements.txt
-/dist/
-/.idea/workspace.xml
-/docs/_build/
-/docs/_autosummary/

neo4j_etl_lib-0.0.2/PKG-INFO

@@ -1,126 +0,0 @@
-Metadata-Version: 2.3
-Name: neo4j-etl-lib
-Version: 0.0.2
-Summary: Building blocks for ETL pipelines.
-Keywords: etl,graph,database
-Author-email: Bert Radke <bert.radke@pm.me>
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Intended Audience :: Developers
-Classifier: Programming Language :: Python
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Classifier: Programming Language :: Python :: 3
-Classifier: Topic :: Database
-Classifier: Development Status :: 4 - Beta
-Requires-Dist: pydantic>=2.10.5; python_version >= '3.8'
-Requires-Dist: neo4j>=5.27.0; python_version >= '3.7'
-Requires-Dist: python-dotenv>=1.0.1; python_version >= '3.8'
-Requires-Dist: tabulate>=0.9.0; python_version >= '3.7'
-Requires-Dist: click>=8.1.8; python_version >= '3.7'
-Requires-Dist: pytest>=8.3.0 ; extra == "dev" and ( python_version >= '3.8')
-Requires-Dist: testcontainers[neo4j]==4.9.0 ; extra == "dev" and ( python_version >= '3.9' and python_version < '4.0')
-Requires-Dist: pytest-cov ; extra == "dev"
-Requires-Dist: bumpver ; extra == "dev"
-Requires-Dist: isort ; extra == "dev"
-Requires-Dist: pip-tools ; extra == "dev"
-Requires-Dist: sphinx ; extra == "dev"
-Requires-Dist: sphinx-rtd-theme ; extra == "dev"
-Requires-Dist: pydata-sphinx-theme ; extra == "dev"
-Requires-Dist: sphinx-autodoc-typehints ; extra == "dev"
-Requires-Dist: sphinxcontrib-napoleon ; extra == "dev"
-Requires-Dist: graphdatascience>=1.13 ; extra == "gds" and ( python_version >= '3.9')
-Project-URL: Documentation, https://neo-technology-field.github.io/python-etl-lib/index.html
-Project-URL: Home, https://github.com/neo-technology-field/python-etl-lib
-Provides-Extra: dev
-Provides-Extra: gds
-
-# Python ETL Toolbox
-
-Complete documentation can be found on https://neo-technology-field.github.io/python-etl-lib/index.html
-
-A library of building blocks to assemble etl pipelines.
-
-So, instead of providing yet another etl tool, the aim is to provide quality building blocks for the usual etl task. These building blocks should (do) meet the following functional requirements:
-
-* logging (of tasks performed including times, errors, and statistics)
-* error handling 
-* validation of data (currently via Pydantic)
-* batching and streaming
-* optionally record the information about performed tasks and provide means (NeoDash, console) to review past etl runs.
-
-While this library currently focuses on Neo4j databases, it can be extended to other sources and sinks as needed. 
-
-It does not provide a CLI out of the box, but contains a set of functions to list and manage past runs (if they are stored in a database). In addition, the provided example illustrates how to assemble a etl pipeline and run it from a CLI.
-
-## Quick guide
-
-### Installation
-
-Package is available on PyPi and can be installed (for development) via:
-
-```bash
-python3 -m venv venv
-source venv/bin/activate
-python -m pip install pip-tools
-pip-compile --extra dev pyproject.toml
-pip-sync
-```
-
-### Usage
-
-The below shows a minimalistic etl pipeline to a single CSV file (look at the GTFS example to see more details)
-
-```python
-
-class LoadAgenciesTask(CSVLoad2Neo4jTasks):
-    
-    class Agency(BaseModel):
-        """ Define the Pydantic model for data validation. """
-        id: str = Field(alias="agency_id", default="generic")
-        name: str = Field(alias="agency_name")
-        url: str = Field(alias="agency_url")
-        timezone: str = Field(alias="agency_timezone")
-        lang: str = Field(alias="agency_lang")
-
-    def __init__(self, context: ETLContext, file:Path):
-        super().__init__(context, LoadAgenciesTask.Agency, file)
-
-    def task_name(self) -> str:
-        return f"{self.__class__.__name__}('{self.file}')"
-
-    def _query(self):
-        """Load the data into Neo4j."""
-        return """ UNWIND $batch AS row
-        MERGE (a:Agency {id: row.id})
-            SET a.name= row.name, 
-            a.url= row.url, 
-            a.timezone= row.timezone, 
-            a.lang= row.lang
-        """
-
-    @classmethod
-    def file_name(cls):
-        return "agency.txt"
-
-context = ETLContext(env_vars=dict(os.environ))
-
-schema = SchemaTask(context=context)
-init_group = TaskGroup(context=context, tasks=[schema], name="schema-init")
-
-tasks = [
-    LoadAgenciesTask(context=context, file=input_directory / LoadAgenciesTask.file_name()),
-]
-csv_group = TaskGroup(context=context, tasks=tasks, name="csv-loading")
-
-all_group = TaskGroup(context=context, tasks=[init_group, csv_group], name="main")
-
-context.reporter.register_tasks(all_group)
-
-all_group.execute()
-
-```
-See the provided [example](examples/gtfs/README.md) for a more realistic pipeline and how the logging and reporting would look like.
-
-With the above, all lines in the input file `agency.txt` that do not fit the Pydantic model, would be sent to an json file, containing the error data and a description of why it could not be loaded.
-

neo4j_etl_lib-0.0.2/README.md

@@ -1,88 +0,0 @@
-# Python ETL Toolbox
-
-Complete documentation can be found on https://neo-technology-field.github.io/python-etl-lib/index.html
-
-A library of building blocks to assemble etl pipelines.
-
-So, instead of providing yet another etl tool, the aim is to provide quality building blocks for the usual etl task. These building blocks should (do) meet the following functional requirements:
-
-* logging (of tasks performed including times, errors, and statistics)
-* error handling 
-* validation of data (currently via Pydantic)
-* batching and streaming
-* optionally record the information about performed tasks and provide means (NeoDash, console) to review past etl runs.
-
-While this library currently focuses on Neo4j databases, it can be extended to other sources and sinks as needed. 
-
-It does not provide a CLI out of the box, but contains a set of functions to list and manage past runs (if they are stored in a database). In addition, the provided example illustrates how to assemble a etl pipeline and run it from a CLI.
-
-## Quick guide
-
-### Installation
-
-Package is available on PyPi and can be installed (for development) via:
-
-```bash
-python3 -m venv venv
-source venv/bin/activate
-python -m pip install pip-tools
-pip-compile --extra dev pyproject.toml
-pip-sync
-```
-
-### Usage
-
-The below shows a minimalistic etl pipeline to a single CSV file (look at the GTFS example to see more details)
-
-```python
-
-class LoadAgenciesTask(CSVLoad2Neo4jTasks):
-    
-    class Agency(BaseModel):
-        """ Define the Pydantic model for data validation. """
-        id: str = Field(alias="agency_id", default="generic")
-        name: str = Field(alias="agency_name")
-        url: str = Field(alias="agency_url")
-        timezone: str = Field(alias="agency_timezone")
-        lang: str = Field(alias="agency_lang")
-
-    def __init__(self, context: ETLContext, file:Path):
-        super().__init__(context, LoadAgenciesTask.Agency, file)
-
-    def task_name(self) -> str:
-        return f"{self.__class__.__name__}('{self.file}')"
-
-    def _query(self):
-        """Load the data into Neo4j."""
-        return """ UNWIND $batch AS row
-        MERGE (a:Agency {id: row.id})
-            SET a.name= row.name, 
-            a.url= row.url, 
-            a.timezone= row.timezone, 
-            a.lang= row.lang
-        """
-
-    @classmethod
-    def file_name(cls):
-        return "agency.txt"
-
-context = ETLContext(env_vars=dict(os.environ))
-
-schema = SchemaTask(context=context)
-init_group = TaskGroup(context=context, tasks=[schema], name="schema-init")
-
-tasks = [
-    LoadAgenciesTask(context=context, file=input_directory / LoadAgenciesTask.file_name()),
-]
-csv_group = TaskGroup(context=context, tasks=tasks, name="csv-loading")
-
-all_group = TaskGroup(context=context, tasks=[init_group, csv_group], name="main")
-
-context.reporter.register_tasks(all_group)
-
-all_group.execute()
-
-```
-See the provided [example](examples/gtfs/README.md) for a more realistic pipeline and how the logging and reporting would look like.
-
-With the above, all lines in the input file `agency.txt` that do not fit the Pydantic model, would be sent to an json file, containing the error data and a description of why it could not be loaded.