neo4j-etl-lib 0.0.2__py3-none-any.whl

etl_lib/__init__.py

@@ -0,0 +1,4 @@
+"""
+Building blocks for ETL pipelines.
+"""
+__version__ = "0.0.2"

etl_lib/cli/run_tools.py

@@ -0,0 +1,189 @@
+from datetime import datetime
+
+import click
+import neo4j
+from neo4j import GraphDatabase
+from neo4j.time import DateTime
+from tabulate import tabulate
+
+
+def __convert_date_time(input_date_time) -> datetime | None:
+    if input_date_time is None:
+        return None
+    return input_date_time.to_native().strftime("%Y-%m-%d %H:%M")
+
+
+def __duration_from_start_end(start_time: DateTime | None, end_time: DateTime | None) -> str | None:
+    if start_time is None or end_time is None:
+        return None
+
+    # Convert neo4j.time.DateTime to native Python datetime
+    start_time = start_time.to_native()
+    end_time = end_time.to_native()
+
+    # Calculate the duration as a timedelta
+    duration = end_time - start_time
+
+    # Extract hours, minutes, and seconds
+    total_seconds = int(duration.total_seconds())
+    hours = total_seconds // 3600
+    minutes = (total_seconds % 3600) // 60
+    seconds = total_seconds % 60
+
+    # Format as HH:MM:SS
+    return f"{hours}:{minutes:02}:{seconds:02}"
+
+
+def __print_details(driver, run_id):
+    records, _, _ = driver.execute_query("""
+    MATCH (:ETLRun {uuid : $id})-[:HAS_SUB_TASK*]->(task)-[:HAS_STATS]->(stats)
+    WITH task, stats ORDER BY task.order ASC
+    RETURN task.task AS task, task.status AS status, properties(stats) AS stats
+    """, id=run_id, routing_=neo4j.RoutingControl.READ)
+
+    print("Showing detailed stats for each task. Task without non-zero stats are omitted.")
+    for record in records:
+        rows = [(key, value) for key, value in record["stats"].items() if value != 0]
+        if rows:
+            print(f"Showing statistics for Task '{record['task']}' with status '{record['status']}'")
+            print(tabulate(rows, headers=["Name", "Value"], tablefmt='psql'))
+
+
+def __driver(ctx):
+    neo4j_uri = ctx.obj["neo4j_uri"]
+    neo4j_user = ctx.obj["neo4j_user"]
+    database_name = ctx.obj["database_name"]
+    neo4j_password = ctx.obj["neo4j_password"]
+    return GraphDatabase.driver(neo4j_uri, auth=(neo4j_user, neo4j_password), database=database_name,
+                                notifications_min_severity="OFF", user_agent="ETL CLI 0.1")
+
+
+@click.group()
+@click.option('--neo4j-uri', envvar='NEO4J_URI', help='Neo4j database URI')
+@click.option('--neo4j-user', envvar='NEO4J_USERNAME', help='Neo4j username')
+@click.option('--neo4j-password', envvar='NEO4J_PASSWORD', help='Neo4j password')
+@click.option('--log-file', envvar='LOG_FILE', help='Path to the log file', default=None)
+@click.option('--database-name', envvar='DATABASE_NAME', default='neo4j', help='Neo4j database name (default: neo4j)')
+@click.pass_context
+def cli(ctx, neo4j_uri, neo4j_user, neo4j_password, log_file, database_name):
+    """
+        Command-line tool to process files in INPUT_DIRECTORY.
+
+        Environment variables can be configured via a .env file or overridden via CLI options:
+
+        \b
+        - NEO4J_URI: Neo4j database URI
+        - NEO4J_USERNAME: Neo4j username
+        - NEO4J_PASSWORD: Neo4j password
+        - LOG_FILE: Path to the log file
+        - DATABASE_NAME: Neo4j database name (default: neo4j)
+        """
+
+    # Validate Neo4j connection details
+    if not neo4j_uri or not neo4j_user or not neo4j_password:
+        print(
+            "Neo4j connection details are incomplete. Please provide NEO4J_URL, NEO4J_USER, and NEO4J_PASSWORD.")
+        return
+
+    ctx.ensure_object(dict)
+    ctx.obj['neo4j_uri'] = neo4j_uri
+    ctx.obj['neo4j_user'] = neo4j_user
+    ctx.obj['neo4j_password'] = neo4j_password
+    ctx.obj['database_name'] = database_name
+    ctx.obj['log_file'] = log_file
+
+
+@cli.command()
+@click.option("--number-runs", default=10, help="Number of rows to process, defaults to 10", type=int)
+@click.pass_context
+def query(ctx, number_runs):
+    """
+    Retrieve the list of the last x etl runs from the database and display them.
+    """
+    print(f"Listing runs in database '{ctx.obj['database_name']}'")
+    with __driver(ctx) as driver:
+        records, _, _ = driver.execute_query("""
+            MATCH (r:ETLRun:ETLTask) 
+            WITH r, r.name AS name, r.uuid AS id, r.startTime AS startTime, r.endTime AS endTime
+            CALL (r) {
+              MATCH (r)-[:HAS_STATS]->(stats)
+              WITH [k IN keys(stats) | stats[k] ] AS stats
+              UNWIND stats AS stat
+              RETURN sum(stat) AS changes
+              }
+            ORDER BY startTime DESC LIMIT $number_runs
+            RETURN name, id, startTime, endTime, changes
+        """, number_runs=number_runs, routing_=neo4j.RoutingControl.READ)
+        data = [
+            {
+                "name": record["name"], "ID": record["id"],
+                "startTime": __convert_date_time(record["startTime"]),
+                "endTime": __convert_date_time(record["endTime"]),
+                "changes": record["changes"]
+            } for record in records]
+
+        print(tabulate(data, headers='keys', tablefmt='psql'))
+
+
+@cli.command()
+@click.argument('run-id', required=True)
+@click.option("--details", default=False, is_flag=True, help="Show stats for each task", type=bool)
+@click.pass_context
+def detail(ctx, run_id, details):
+    """
+    Show a breakdown of the task for the specified run, including statistics.
+    """
+    print(f"Showing details for run ID: {run_id}")
+    with __driver(ctx) as driver:
+        records, _, _ = driver.execute_query("""
+        MATCH (r:ETLRun {uuid : $id})-[:HAS_SUB_TASK*]->(task) 
+        WITH task ORDER BY task.order ASC
+        CALL (task) {
+          MATCH (task)-[:HAS_STATS]->(stats)
+          WITH [k IN keys(stats) | stats[k] ] AS stats
+          UNWIND stats AS stat
+          RETURN sum(stat) AS changes
+        }
+        RETURN 
+          task.task AS task, task.status AS status,
+          task.batches + ' / ' + coalesce(task.expected_batches, '-') AS batches, 
+          task.startTime AS startTime,  task.endTime AS endTime, changes
+        """, id=run_id, routing_=neo4j.RoutingControl.READ)
+        data = [
+            {
+                "task": record["task"],
+                "status": record["status"],
+                "batches": record["batches"],
+                "duration": __duration_from_start_end(record["startTime"], record["endTime"]),
+                "changes": sum(record.get("stats", {}).values())
+            }
+            for record in records
+        ]
+
+        print(tabulate(data, headers='keys', tablefmt='psql'))
+        if details:
+            __print_details(driver, run_id)
+
+
+@cli.command()
+@click.option('--run-id', required=False, help='Run ID to delete')
+@click.option('--since', help='Delete runs since a specific date')
+@click.option('--older', help='Delete runs older than a specific date')
+@click.pass_context
+def delete(ctx, run_id, since, older):
+    """
+    Delete runs based on run ID, date, or age. One and only one of --run-id, --since, or --older must be provided.
+    """
+    # Ensure mutual exclusivity
+    options = [run_id, since, older]
+    if sum(bool(opt) for opt in options) != 1:
+        print("You must specify exactly one of --run-id, --since, or --older.")
+        return
+
+    if run_id:
+        print(f"Deleting run ID: {run_id}")
+    elif since:
+        print(f"Deleting runs since: {since}")
+    elif older:
+        print(f"Deleting runs older than: {older}")
+    # Implement delete logic here

etl_lib/core/BatchProcessor.py

@@ -0,0 +1,88 @@
+import abc
+import logging
+import sys
+from dataclasses import dataclass, field
+from typing import Generator
+
+from etl_lib.core.ETLContext import ETLContext
+from etl_lib.core.Task import Task
+from etl_lib.core.utils import merge_summery
+
+
+@dataclass
+class BatchResults:
+    """
+    Return object of the :py:func:`~BatchProcessor.get_batch` method, wrapping a batched data together with meta information.
+    """
+    chunk: []
+    """The batch of data."""
+    statistics: dict = field(default_factory=dict)
+    """`dict` of statistic information, such as row processed, nodes writen, .."""
+    batch_size: int = field(default=sys.maxsize)
+    """size of the batch."""
+
+
+def append_result(org: BatchResults, stats: dict) -> BatchResults:
+    """
+    Appends the stats dict to the provided `org`.
+
+    Args:
+        org: The original `BatchResults` object.
+        stats: dict containing statistics to be added to the org object.
+
+    Returns:
+        New `BatchResults` object, where the :py:attr:`~BatchResults.statistics` attribute is the merged result of the
+            provided parameters. Values in the dicts with the same key are added.
+
+    """
+    return BatchResults(chunk=org.chunk, statistics=merge_summery(org.statistics, stats),
+                        batch_size=org.batch_size)
+
+
+class BatchProcessor:
+    """
+    Allows assembly of :py:class:`etl_lib.core.Task.Task` out of smaller building blocks.
+
+    This way, functionally such as reading from a CSV file, writing to a database or validation
+    can be implemented and tested independently and re-used.
+
+    BatchProcessors form, a linked list, where each processor only knows about its predecessor.
+
+    BatchProcessors process data in batches. A batch of data is requested from the provided predecessors
+    :py:func:`~get_batch`
+    and returned in batches to the caller. Usage of `Generators` ensure that not all data must be loaded at once.
+    """
+
+    def __init__(self, context: ETLContext, task: Task, predecessor=None):
+        """
+        Constructs a new :py:class:`etl_lib.core.BatchProcessor` instance.
+
+        Args:
+            context: :py:class:`etl_lib.core.ETLContext.ETLContext` instance. Will be available to subclasses.
+            task: :py:class:`etl_lib.core.Task.Task` this processor is part of.
+                Needed for status reporting only.
+            predecessor: Source of batches for this processor.
+                Can be `None` of no predecessor is needed (such as when this processor is the start of the queue.
+        """
+        self.context = context
+        """:py:class:`etl_lib.core.ETLContext.ETLContext` instance. Providing access to general facilities."""
+        self.predecessor = predecessor
+        """Predecessor, used as a source of batches."""
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.task = task
+        """The :py:class:`etl_lib.core.Task.Task` owning instance."""
+
+    @abc.abstractmethod
+    def get_batch(self, max_batch__size: int) -> Generator[BatchResults, None, None]:
+        """
+        Provides a batch of data to the caller.
+
+        The batch itself could be called and processed from the provided predecessor or generated from other sources.
+
+        Args:
+            max_batch__size: The max size of the batch the caller expects to receive.
+
+        Returns
+            A generator that yields batches.
+        """
+        pass

etl_lib/core/ClosedLoopBatchProcessor.py

@@ -0,0 +1,30 @@
+from typing import Generator
+
+from etl_lib.core.ETLContext import ETLContext
+from etl_lib.core.BatchProcessor import BatchProcessor, BatchResults, append_result
+from etl_lib.core.Task import Task
+
+
+class ClosedLoopBatchProcessor(BatchProcessor):
+    """
+    Reporting implementation of a BatchProcessor.
+
+    Meant to be the last entry in the list of :py:class:`etl_lib.core.BatchProcessor` driving the processing and
+    reporting updates of the processed batches using the :py:class:`etl_lib.core.ProgressReporter` from the context.
+    """
+
+    def __init__(self, context: ETLContext, task: Task, predecessor: BatchProcessor, expected_rows: int = None):
+        super().__init__(context, task, predecessor)
+        self.expected_rows = expected_rows
+
+    def get_batch(self, max_batch__size: int) -> Generator[BatchResults, None, None]:
+        assert self.predecessor is not None
+        batch_cnt = 0
+        result = BatchResults(chunk=[], statistics={}, batch_size=max_batch__size)
+        for batch in self.predecessor.get_batch(max_batch__size):
+            result = append_result(result, batch.statistics)
+            batch_cnt += 1
+            self.context.reporter.report_progress(self.task, batch_cnt, self.expected_rows, result.statistics)
+
+        self.logger.debug(result.statistics)
+        yield result

etl_lib/core/ETLContext.py

@@ -0,0 +1,136 @@
+import logging
+from typing import NamedTuple, Any
+
+from graphdatascience import GraphDataScience
+from neo4j import Driver, GraphDatabase, WRITE_ACCESS, SummaryCounters
+
+from etl_lib.core.ProgressReporter import get_reporter
+
+
+class QueryResult(NamedTuple):
+    """Result of a query against the neo4j database."""
+    data: []
+    """Data as returned from the query."""
+    summery: {}
+    """Counters as reported by neo4j. Contains entries such as `nodes_created`, `nodes_deleted`, etc."""
+
+
+def append_results(r1: QueryResult, r2: QueryResult) -> QueryResult:
+    return QueryResult(r1.data + r2.data, r1.summery + r2.summery)
+
+
+class Neo4jContext:
+    uri: str
+    auth: (str, str)
+    driver: Driver
+    database: str
+
+    def __init__(self, env_vars: dict):
+        """
+        Create a new Neo4j context.
+        Reads the following env_vars keys:
+        - `NEO4J_URI`,
+        - `NEO4J_USERNAME`,
+        - `NEO4J_PASSWORD`.
+        """
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.uri = env_vars["NEO4J_URI"]
+        self.auth = (env_vars["NEO4J_USERNAME"],
+                     env_vars["NEO4J_PASSWORD"])
+        self.database = env_vars["NEO4J_DATABASE"]
+        self.__neo4j_connect()
+
+    def query_database(self, session, query, **kwargs) -> QueryResult:
+        """
+        Executes a Cypher query on the Neo4j database.
+        """
+        if isinstance(query, list):
+            results = []
+            for single_query in query:
+                result = self.query_database(session, single_query, **kwargs)
+                results = append_results(results, result)
+            return results
+        else:
+            try:
+                res = session.run(query, **kwargs)
+                counters = res.consume().counters
+
+                return QueryResult(res, self.__counters_2_dict(counters))
+
+            except Exception as e:
+                self.logger.error(e)
+                raise e
+
+    @staticmethod
+    def __counters_2_dict(counters: SummaryCounters):
+        return {
+            "constraints_added": counters.constraints_added,
+            "constraints_removed": counters.constraints_removed,
+            "indexes_added": counters.indexes_added,
+            "indexes_removed": counters.indexes_removed,
+            "labels_added": counters.labels_added,
+            "labels_removed": counters.labels_removed,
+            "nodes_created": counters.nodes_created,
+            "nodes_deleted": counters.nodes_deleted,
+            "properties_set": counters.properties_set,
+            "relationships_created": counters.relationships_created,
+            "relationships_deleted": counters.relationships_deleted,
+        }
+
+    def session(self, database=None):
+        if database is None:
+            return self.driver.session(database=self.database, default_access_mode=WRITE_ACCESS)
+        else:
+            return self.driver.session(database=database, default_access_mode=WRITE_ACCESS)
+
+    def gds(self, database=None) -> GraphDataScience:
+        if database is None:
+            return GraphDataScience.from_neo4j_driver(driver=self.driver, database=self.database)
+        else:
+            return GraphDataScience.from_neo4j_driver(driver=self.driver, database=database)
+
+    def __neo4j_connect(self):
+        self.driver = GraphDatabase.driver(uri=self.uri, auth=self.auth,
+                                           notifications_min_severity="OFF")
+        self.driver.verify_connectivity()
+        self.logger.info(
+            f"driver connected to instance at {self.uri} with username {self.auth[0]} and database {self.database}")
+
+
+class ETLContext:
+    """
+    General context information.
+
+    Will be passed to all :py:class:`etl_lib.core.Task` to provide access to environment variables and functionally
+    deemed general enough that all parts of the ETL pipeline would need it.
+    """
+    neo4j: Neo4jContext
+    __env_vars: dict
+
+    def __init__(self, env_vars: dict):
+        """
+        Create a new ETLContext.
+
+        Args:
+            env_vars: Environment variables. Stored internally and can be accessed via :py:func:`~env` .
+
+        The context created will contain an :py:class:`~Neo4jContext` and a :py:class:`ProgressReporter`.
+        See there for keys used from the provided `env_vars` dict.
+        """
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.neo4j = Neo4jContext(env_vars)
+        self.__env_vars = env_vars
+        self.reporter = get_reporter(self)
+
+    def env(self, key: str) -> Any:
+        """
+        Returns the value of an entry in the `env_vars` dict.
+
+        Args:
+            key: name of the entry to read.
+
+        Returns:
+            va  lue of the entry, or None if the key is not in the dict.
+        """
+        if key in self.__env_vars:
+            return self.__env_vars[key]

etl_lib/core/ProgressReporter.py

@@ -0,0 +1,210 @@
+import logging
+from datetime import datetime
+
+from tabulate import tabulate
+
+from etl_lib.core.Task import Task, TaskGroup
+
+
+class ProgressReporter:
+    """
+    Responsible for reporting progress of :py:class:`etl_lib.core.Task` .
+
+    This specific implementation uses the python logging module to log progress.
+    Non-error logging is using the INFO level.
+    """
+    start_time: datetime
+    end_time: datetime
+
+    def __init__(self, context):
+        self.context = context
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def register_tasks(self, main: Task):
+        """
+        Registers a :py:class:`etl_lib.core.Task` with this reporter.
+
+        Needs to be called once with the root task. The function will walk the tree of tasks and register them in turn.
+
+        Args:
+            main: Root of the task tree.
+        """
+        self.logger.info("\n" + self.__print_tree(main))
+
+    def started_task(self, task: Task) -> Task:
+        """
+        Marks the task as started.
+
+        Start the time keeping for this task and performs logging.
+
+        Args:
+            task: Task to be marked as started.
+
+        Returns:
+            The task that was provided.
+        """
+        task.start_time = datetime.now()
+        self.logger.info(f"{'\t' * task.depth}starting {task.task_name()}")
+        return task
+
+    def finished_task(self, task: Task, success: bool, summery: dict, error: str = None) -> Task:
+        """
+        Marks the task as finished.
+
+        Stops the time recording for the tasks and performs logging. Logging will include details from the provided summery.
+
+        Args:
+            task: Task to be marked as finished.
+            success: True if the task has successfully finished.
+            summery: statistics for this task (such as `nodes_created`)
+            error: If an exception occurred, the exception text should be provided here.
+
+        Returns:
+            Task to be marked as started.
+        """
+        task.end_time = datetime.now()
+        task.success = success
+        task.summery = summery
+
+        report = f"{'\t' * task.depth}finished {task.task_name()} with success: {success}"
+        if error is not None:
+            report += f", error: \n{error}"
+        else:
+            # for the logger, remove entries with 0, but keep them in the original for reporting
+            cleaned_summery = {key: value for key, value in summery.items() if value != 0}
+            if len(cleaned_summery) > 0:
+                report += f"\n{tabulate([cleaned_summery], headers='keys', tablefmt='psql')}"
+        self.logger.info(report)
+        return task
+
+    def report_progress(self, task: Task, batches: int, expected_batches: int, stats: dict) -> None:
+        """
+        Optionally provide updates during execution of a task, such as batches processed so far.
+
+        This is an optional call, as not all :py:class:`etl_lib.core.Task` need batching.
+
+        Args:
+            task: Task reporting updates.
+            batches: Number of batches processed so far.
+            expected_batches: Number of expected batches. Can be `None` if the overall number of
+                batches is not know before execution.
+            stats: dict of statistics so far (such as `nodes_created`).
+        """
+        pass
+
+    def __print_tree(self, task: Task, last=True, header='') -> str:
+        """Generates a tree view of the task tree."""
+        elbow = "└──"
+        pipe = "│  "
+        tee = "├──"
+        blank = "   "
+        tree_string = header + (elbow if last else tee) + task.task_name() + "\n"
+        if isinstance(task, TaskGroup):
+            children = list(task.sub_tasks())
+            for i, c in enumerate(children):
+                tree_string += self.__print_tree(c, header=header + (blank if last else pipe),
+                                                 last=i == len(children) - 1)
+        return tree_string
+
+
+class Neo4jProgressReporter(ProgressReporter):
+    """
+    Extends the ProgressReporter to additionally write the status updates from the tasks to a Neo4j database.
+    """
+
+    def __init__(self, context, database: str):
+        """
+        Creates a new Neo4j progress reporter.
+
+        Args:
+            context: :py:class:`etl_lib.core.ETLContext` containing a Neo4jConnection instance.
+            database: Name of the database to write the status updates to.
+        """
+        super().__init__(context)
+        self.database = database
+        self.logger.info(f"progress reporting to database: {self.database}")
+        self.__create_constraints()
+
+    def register_tasks(self, root: Task, **kwargs):
+        super().register_tasks(root)
+
+        with self.context.neo4j.session(self.database) as session:
+            order = 0
+            session.run(
+                "CREATE (t:ETLTask:ETLRun {uuid:$id, task:$task, order:$order, name:$name, status: 'open'}) SET t +=$other",
+                id=root.uuid, order=order, task=root.__repr__(), name=root.task_name(), other=kwargs)
+            self.__persist_task(session, root, order)
+
+    def __persist_task(self, session, task: Task | TaskGroup, order: int) -> int:
+        """Writes task information to the database."""
+
+        if type(task) is Task:
+            order += 1
+            session.run(
+                """
+                MERGE (t:ETLTask { uuid: $id })
+                    SET t.task=$task, t.order=$order, t.name=$name, t.status='open'
+                """,
+                id=task.uuid, task=task.__repr__(), order=order, name=task.task_name())
+        else:
+            for child in task.sub_tasks():
+                order += 1
+                session.run(
+                    """
+                    MATCH (p:ETLTask { uuid: $parent_id }) SET p.type='TaskGroup'
+                    CREATE (t:ETLTask { uuid:$id, task:$task, order:$order, name:$name, status: 'open' })
+                    CREATE (p)-[:HAS_SUB_TASK]->(t)
+                    """,
+                    parent_id=task.uuid, id=child.uuid, task=child.__repr__(), order=order, name=child.task_name())
+                if isinstance(child, TaskGroup):
+                    order = self.__persist_task(session, child, order)
+        return order
+
+    def started_task(self, task: Task) -> Task:
+        super().started_task(task=task)
+        with self.context.neo4j.session(self.database) as session:
+            session.run("MATCH (t:ETLTask { uuid: $id }) SET t.startTime = $start_time, t.status= 'running'",
+                        id=task.uuid,
+                        start_time=task.start_time)
+        return task
+
+    def finished_task(self, task: Task, success: bool, summery: dict, error: str = None) -> Task:
+        super().finished_task(task=task, success=success, summery=summery, error=error)
+        if success:
+            status = "success"
+        else:
+            status = "failure"
+        with self.context.neo4j.session(self.database) as session:
+            session.run("""
+            MATCH (t:ETLTask {uuid:$id}) SET t.endTime = $end_time, t.status = $status, t.error = $error
+            CREATE (s:ETLStats) SET s=$summery
+            CREATE (t)-[:HAS_STATS]->(s)
+            """, id=task.uuid, end_time=task.end_time, summery=summery, status=status, error=error)
+        return task
+
+    def __create_constraints(self):
+        with self.context.neo4j.session(self.database) as session:
+            session.run("CREATE CONSTRAINT etl_task_unique IF NOT EXISTS FOR (n:ETLTask) REQUIRE n.uuid IS UNIQUE;")
+
+    def report_progress(self, task: Task, batches: int, expected_batches: int, stats: dict) -> None:
+        self.logger.debug(f"{batches=}, {expected_batches=}, {stats=}")
+        with self.context.neo4j.session(self.database) as session:
+            session.run("MATCH (t:ETLTask {uuid:$id}) SET t.batches =$batches, t.expected_batches =$expected_batches",
+                        id=task.uuid, batches=batches, expected_batches=expected_batches)
+
+
+def get_reporter(context) -> ProgressReporter:
+    """
+    Returns a ProgressReporter instance.
+
+    If the :py:class:`ETLContext <etl_lib.core.ETLContext>` env holds the key `REPORTER_DATABASE` then
+    a :py:class:`Neo4jProgressReporter` instance is created with the given database name.
+
+    Otherwise, a  :py:class:`ProgressReporter` (no logging to database) instance will be created.
+    """
+
+    db = context.env("REPORTER_DATABASE")
+    if db is None:
+        return ProgressReporter(context)
+    else:
+        return Neo4jProgressReporter(context, db)