ingestr 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

ingestr/main.py

@@ -1,209 +1,274 @@
 import hashlib
+from datetime import datetime
+from typing import Optional
 
 import dlt
+import humanize
 import typer
-
-from ingestr.src.factory import SourceDestinationFactory
 from rich.console import Console
-from dlt.common.pipeline import LoadInfo
-import humanize
 from typing_extensions import Annotated
 
+from ingestr.src.factory import SourceDestinationFactory
+from ingestr.src.telemetry.event import track
+
 app = typer.Typer(
     name="ingestr",
     help="ingestr is the CLI tool to ingest data from one source to another",
     rich_markup_mode="rich",
 )
 
-
 console = Console()
 print = console.print
 
+DATE_FORMATS = [
+    "%Y-%m-%d",
+    "%Y-%m-%dT%H:%M:%S",
+    "%Y-%m-%dT%H:%M:%S%z",
+    "%Y-%m-%d %H:%M:%S",
+    "%Y-%m-%dT%H:%M:%S.%f",
+    "%Y-%m-%dT%H:%M:%S.%f%z",
+]
+
 
 @app.command()
 def ingest(
-    source_uri: Annotated[str, typer.Option(help="The URI of the [green]source[/green]")],  # type: ignore
-    dest_uri: Annotated[str, typer.Option(help="The URI of the [cyan]destination[/cyan]")],  # type: ignore
-    source_table: Annotated[str, typer.Option(help="The table name in the [green]source[/green] to fetch")],  # type: ignore
-    dest_table: Annotated[str, typer.Option(help="The table in the [cyan]destination[/cyan] to save the data into")] = None,  # type: ignore
-    incremental_key: Annotated[str, typer.Option(help="The incremental key from the table to be used for incremental strategies")] = None,  # type: ignore
-    incremental_strategy: Annotated[str, typer.Option(help="The incremental strategy to use, must be one of 'replace', 'append' or 'merge'")] = "replace",  # type: ignore
-):
-    if not dest_table:
-        print()
-        print(
-            "[yellow]Destination table is not given, defaulting to the source table.[/yellow]"
-        )
-        dest_table = source_table
-
-    factory = SourceDestinationFactory(source_uri, dest_uri)
-    source = factory.get_source()
-    destination = factory.get_destination()
-
-    m = hashlib.sha256()
-    m.update(dest_table.encode("utf-8"))
-    pipeline_name = m.hexdigest()
-    short_pipeline_name = pipeline_name[:8]
-
-    pipeline = dlt.pipeline(
-        pipeline_name=pipeline_name,
-        destination=destination.dlt_dest(
-            uri=dest_uri,
+    source_uri: Annotated[
+        str, typer.Option(help="The URI of the [green]source[/green]")
+    ],  # type: ignore
+    dest_uri: Annotated[
+        str, typer.Option(help="The URI of the [cyan]destination[/cyan]")
+    ],  # type: ignore
+    source_table: Annotated[
+        str, typer.Option(help="The table name in the [green]source[/green] to fetch")
+    ],  # type: ignore
+    dest_table: Annotated[
+        str,
+        typer.Option(
+            help="The table in the [cyan]destination[/cyan] to save the data into"
         ),
-        progress=dlt.progress.log(dump_system_stats=False),
-        pipelines_dir="pipeline_data",
-    )
-
-    print()
-    print(f"[bold green]Initiated the pipeline with the following:[/bold green]")
-    print(f"[bold yellow]  Pipeline ID:[/bold yellow] {short_pipeline_name}")
-    print(
-        f"[bold yellow]  Source:[/bold yellow] {factory.source_scheme} / {source_table}"
-    )
-    print(
-        f"[bold yellow]  Destination:[/bold yellow] {factory.destination_scheme} / {dest_table}"
-    )
-    print(f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy}")
-    print(
-        f"[bold yellow]  Incremental Key:[/bold yellow] {incremental_key if incremental_key else 'None'}"
-    )
-    print()
-
-    continuePipeline = typer.confirm("Are you sure you would like to continue?")
-    if not continuePipeline:
-        raise typer.Abort()
-
-    print()
-    print(f"[bold green]Starting the ingestion...[/bold green]")
-    print()
-
-    incremental = []
-    if incremental_key:
-        incremental = [incremental_key]
-
-    run_info = pipeline.run(
-        source.dlt_source(
-            uri=source_uri,
-            table=source_table,
-            incremental_key=incremental_key,
-            incremental_strategy=incremental_strategy,
+    ] = None,  # type: ignore
+    incremental_key: Annotated[
+        str,
+        typer.Option(
+            help="The incremental key from the table to be used for incremental strategies"
+        ),
+    ] = None,  # type: ignore
+    incremental_strategy: Annotated[
+        str,
+        typer.Option(
+            help="The incremental strategy to use, must be one of 'replace', 'append', 'delete+insert', or 'merge'"
         ),
-        **destination.dlt_run_params(
-            uri=dest_uri,
-            table=dest_table,
+    ] = "replace",  # type: ignore
+    interval_start: Annotated[
+        Optional[datetime],
+        typer.Option(
+            help="The start of the interval the incremental key will cover",
+            formats=DATE_FORMATS,
         ),
-        write_disposition=incremental_strategy,  # type: ignore
-        primary_key=incremental,
+    ] = None,  # type: ignore
+    interval_end: Annotated[
+        Optional[datetime],
+        typer.Option(
+            help="The end of the interval the incremental key will cover",
+            formats=DATE_FORMATS,
+        ),
+    ] = None,  # type: ignore
+    primary_key: Annotated[Optional[list[str]], typer.Option(help="The merge ")] = None,  # type: ignore
+):
+    track(
+        "command_triggered",
+        {
+            "command": "ingest",
+        },
     )
 
-    elapsedHuman = ""
-    if run_info.started_at:
-        elapsed = run_info.finished_at - run_info.started_at
-        elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
+    try:
+        if not dest_table:
+            print()
+            print(
+                "[yellow]Destination table is not given, defaulting to the source table.[/yellow]"
+            )
+            dest_table = source_table
+
+        merge_key = None
+        if incremental_strategy == "delete+insert":
+            merge_key = incremental_key
+            incremental_strategy = "merge"
+
+        factory = SourceDestinationFactory(source_uri, dest_uri)
+        source = factory.get_source()
+        destination = factory.get_destination()
+
+        m = hashlib.sha256()
+        m.update(dest_table.encode("utf-8"))
+
+        pipeline = dlt.pipeline(
+            pipeline_name=m.hexdigest(),
+            destination=destination.dlt_dest(
+                uri=dest_uri,
+            ),
+            progress=dlt.progress.log(dump_system_stats=False),
+            pipelines_dir="pipeline_data",
+            dataset_name="testschema",
+        )
 
-    print(
-        f"[bold green]Successfully finished loading data from '{factory.source_scheme}' to '{factory.destination_scheme}' {elapsedHuman} [/bold green]"
-    )
+        print()
+        print("[bold green]Initiated the pipeline with the following:[/bold green]")
+        print(
+            f"[bold yellow]  Source:[/bold yellow] {factory.source_scheme} / {source_table}"
+        )
+        print(
+            f"[bold yellow]  Destination:[/bold yellow] {factory.destination_scheme} / {dest_table}"
+        )
+        print(
+            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy}"
+        )
+        print(
+            f"[bold yellow]  Incremental Key:[/bold yellow] {incremental_key if incremental_key else 'None'}"
+        )
+        print()
 
-    # printLoadInfo(short_pipeline_name, run_info)
+        continuePipeline = typer.confirm("Are you sure you would like to continue?")
+        if not continuePipeline:
+            track("command_finished", {"command": "ingest", "status": "aborted"})
+            raise typer.Abort()
 
+        print()
+        print("[bold green]Starting the ingestion...[/bold green]")
+        print()
 
-def printLoadInfo(short_pipeline_name: str, info: LoadInfo):
-    if info.started_at:
-        elapsed = info.finished_at - info.started_at
-        print(
-            f"  ├── Pipeline {short_pipeline_name} load step completed in [bold green]{humanize.precisedelta(elapsed)}[/bold green]"
+        run_info = pipeline.run(
+            source.dlt_source(
+                uri=source_uri,
+                table=source_table,
+                incremental_key=incremental_key,
+                merge_key=merge_key,
+                interval_start=interval_start,
+                interval_end=interval_end,
+            ),
+            **destination.dlt_run_params(
+                uri=dest_uri,
+                table=dest_table,
+            ),
+            write_disposition=incremental_strategy,  # type: ignore
+            primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
         )
 
-    connector = "└──"
-    if info.staging_name:
-        connector = "├──"
+        elapsedHuman = ""
+        if run_info.started_at:
+            elapsed = run_info.finished_at - run_info.started_at
+            elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
 
-    print(
-        f"  {connector} {len(info.loads_ids)} load package{'s were' if len(info.loads_ids) > 1 else ' was'} loaded to destination [bold cyan]{info.destination_name}[/bold cyan] and into dataset [bold cyan]{info.dataset_name}[/bold cyan]",
-        highlight=False,
-    )
-    if info.staging_name:
+        print()
         print(
-            f"  └── The [bold cyan]{info.staging_name}[/bold cyan] staging destination used [bold cyan]{info.staging_displayable_credentials}[/bold cyan] location to stage data"
+            f"[bold green]Successfully finished loading data from '{factory.source_scheme}' to '{factory.destination_scheme}' {elapsedHuman} [/bold green]"
         )
+        print()
+        track(
+            "command_finished",
+            {
+                "command": "ingest",
+                "status": "success",
+            },
+        )
+
+    except Exception as e:
+        track(
+            "command_finished",
+            {"command": "ingest", "status": "failed", "error": str(e)},
+        )
+        raise
 
 
 @app.command()
 def example_uris():
+    track(
+        "command_triggered",
+        {
+            "command": "example-uris",
+        },
+    )
+
     print()
     typer.echo(
-        f"Following are some example URI formats for supported sources and destinations:"
+        "Following are some example URI formats for supported sources and destinations:"
     )
 
     print()
     print(
-        f"[bold green]Postgres:[/bold green] [white]postgres://user:password@host:port/dbname?sslmode=require [/white]"
+        "[bold green]Postgres:[/bold green] [white]postgres://user:password@host:port/dbname?sslmode=require [/white]"
     )
     print(
-        f"[white dim]└── https://docs.sqlalchemy.org/en/20/core/engines.html#postgresql[/white dim]"
+        "[white dim]└── https://docs.sqlalchemy.org/en/20/core/engines.html#postgresql[/white dim]"
     )
 
     print()
     print(
-        f"[bold green]BigQuery:[/bold green] [white]bigquery://project-id?credentials_path=/path/to/credentials.json&location=US [/white]"
+        "[bold green]BigQuery:[/bold green] [white]bigquery://project-id?credentials_path=/path/to/credentials.json&location=US [/white]"
     )
     print(
-        f"[white dim]└── https://github.com/googleapis/python-bigquery-sqlalchemy?tab=readme-ov-file#connection-string-parameters[/white dim]"
+        "[white dim]└── https://github.com/googleapis/python-bigquery-sqlalchemy?tab=readme-ov-file#connection-string-parameters[/white dim]"
     )
 
     print()
     print(
-        f"[bold green]Snowflake:[/bold green] [white]snowflake://user:password@account/dbname?warehouse=COMPUTE_WH [/white]"
+        "[bold green]Snowflake:[/bold green] [white]snowflake://user:password@account/dbname?warehouse=COMPUTE_WH [/white]"
     )
     print(
-        f"[white dim]└── https://docs.snowflake.com/en/developer-guide/python-connector/sqlalchemy#connection-parameters"
+        "[white dim]└── https://docs.snowflake.com/en/developer-guide/python-connector/sqlalchemy#connection-parameters"
     )
 
     print()
     print(
-        f"[bold green]Redshift:[/bold green] [white]redshift://user:password@host:port/dbname?sslmode=require [/white]"
+        "[bold green]Redshift:[/bold green] [white]redshift://user:password@host:port/dbname?sslmode=require [/white]"
     )
     print(
-        f"[white dim]└── https://aws.amazon.com/blogs/big-data/use-the-amazon-redshift-sqlalchemy-dialect-to-interact-with-amazon-redshift/[/white dim]"
+        "[white dim]└── https://aws.amazon.com/blogs/big-data/use-the-amazon-redshift-sqlalchemy-dialect-to-interact-with-amazon-redshift/[/white dim]"
     )
 
     print()
     print(
-        f"[bold green]Databricks:[/bold green] [white]databricks://token:<access_token>@<server_hostname>?http_path=<http_path>&catalog=<catalog>&schema=<schema>[/white]"
+        "[bold green]Databricks:[/bold green] [white]databricks://token:<access_token>@<server_hostname>?http_path=<http_path>&catalog=<catalog>&schema=<schema>[/white]"
     )
-    print(f"[white dim]└── https://docs.databricks.com/en/dev-tools/sqlalchemy.html")
+    print("[white dim]└── https://docs.databricks.com/en/dev-tools/sqlalchemy.html")
 
     print()
     print(
-        f"[bold green]Microsoft SQL Server:[/bold green] [white]mssql://user:password@host:port/dbname?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes [/white]"
+        "[bold green]Microsoft SQL Server:[/bold green] [white]mssql://user:password@host:port/dbname?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes [/white]"
     )
     print(
-        f"[white dim]└── https://docs.sqlalchemy.org/en/20/core/engines.html#microsoft-sql-server"
+        "[white dim]└── https://docs.sqlalchemy.org/en/20/core/engines.html#microsoft-sql-server"
     )
 
     print()
     print(
-        f"[bold green]MySQL:[/bold green] [white]mysql://user:password@host:port/dbname [/white]"
+        "[bold green]MySQL:[/bold green] [white]mysql://user:password@host:port/dbname [/white]"
     )
     print(
-        f"[white dim]└── https://docs.sqlalchemy.org/en/20/core/engines.html#mysql[/white dim]"
+        "[white dim]└── https://docs.sqlalchemy.org/en/20/core/engines.html#mysql[/white dim]"
     )
 
     print()
-    print(f"[bold green]DuckDB:[/bold green] [white]duckdb://path/to/database [/white]")
-    print(f"[white dim]└── https://github.com/Mause/duckdb_engine[/white dim]")
+    print("[bold green]DuckDB:[/bold green] [white]duckdb://path/to/database [/white]")
+    print("[white dim]└── https://github.com/Mause/duckdb_engine[/white dim]")
 
     print()
-    print(f"[bold green]SQLite:[/bold green] [white]sqlite://path/to/database [/white]")
+    print("[bold green]SQLite:[/bold green] [white]sqlite://path/to/database [/white]")
     print(
-        f"[white dim]└── https://docs.sqlalchemy.org/en/20/core/engines.html#sqlite[/white dim]"
+        "[white dim]└── https://docs.sqlalchemy.org/en/20/core/engines.html#sqlite[/white dim]"
     )
 
     print()
     typer.echo(
         "These are all coming from SQLAlchemy's URI format, so they should be familiar to most users."
     )
+    track(
+        "command_finished",
+        {
+            "command": "example-uris",
+            "status": "success",
+        },
+    )
 
 
 def main():

ingestr/main_test.py

@@ -0,0 +1,579 @@
+import os
+import shutil
+
+import duckdb
+import pytest
+from typer.testing import CliRunner
+
+from ingestr.main import app
+
+runner = CliRunner()
+
+
+def get_abs_path(relative_path):
+    return os.path.abspath(os.path.join(os.path.dirname(__file__), relative_path))
+
+
+def invoke_ingest_command(
+    source_uri,
+    source_table,
+    dest_uri,
+    dest_table,
+    inc_strategy=None,
+    inc_key=None,
+    primary_key=None,
+    merge_key=None,
+    interval_start=None,
+    interval_end=None,
+):
+    args = [
+        "ingest",
+        "--source-uri",
+        source_uri,
+        "--source-table",
+        source_table,
+        "--dest-uri",
+        dest_uri,
+        "--dest-table",
+        dest_table,
+    ]
+
+    if inc_strategy:
+        args.append("--incremental-strategy")
+        args.append(inc_strategy)
+
+    if inc_key:
+        args.append("--incremental-key")
+        args.append(inc_key)
+
+    if primary_key:
+        args.append("--primary-key")
+        args.append(primary_key)
+
+    if merge_key:
+        args.append("--merge-key")
+        args.append(merge_key)
+
+    if interval_start:
+        args.append("--interval-start")
+        args.append(interval_start)
+
+    if interval_end:
+        args.append("--interval-end")
+        args.append(interval_end)
+
+    result = runner.invoke(
+        app,
+        args,
+        input="y\n",
+        env={"DISABLE_TELEMETRY": "true"},
+    )
+    return result
+
+
+def test_create_replace():
+    abs_db_path = get_abs_path("./testdata/test_create_replace.db")
+    rel_db_path_to_command = "ingestr/testdata/test_create_replace.db"
+
+    conn = duckdb.connect(abs_db_path)
+    conn.execute("DROP SCHEMA IF EXISTS testschema CASCADE")
+    conn.execute("CREATE SCHEMA testschema")
+    conn.execute(
+        "CREATE TABLE testschema.input (id INTEGER, val VARCHAR, updated_at TIMESTAMP)"
+    )
+    conn.execute("INSERT INTO testschema.input VALUES (1, 'val1', '2022-01-01')")
+    conn.execute("INSERT INTO testschema.input VALUES (2, 'val2', '2022-02-01')")
+
+    res = conn.sql("select count(*) from testschema.input").fetchall()
+    assert res[0][0] == 2
+
+    result = invoke_ingest_command(
+        f"duckdb:///{rel_db_path_to_command}",
+        "testschema.input",
+        f"duckdb:///{rel_db_path_to_command}",
+        "testschema.output",
+    )
+
+    assert result.exit_code == 0
+
+    res = conn.sql(
+        "select id, val, strftime(updated_at, '%Y-%m-%d') as updated_at from testschema.output"
+    ).fetchall()
+    assert len(res) == 2
+    assert res[0] == (1, "val1", "2022-01-01")
+    assert res[1] == (2, "val2", "2022-02-01")
+
+
+@pytest.mark.skip(
+    reason="this doesn't work at the moment due to a bug with dlt: https://github.com/dlt-hub/dlt/issues/971"
+)
+def test_append():
+    try:
+        shutil.rmtree(get_abs_path("../pipeline_data"))
+    except Exception:
+        pass
+
+    abs_db_path = get_abs_path("./testdata/test_append.db")
+    rel_db_path_to_command = "ingestr/testdata/test_append.db"
+    uri = f"duckdb:///{rel_db_path_to_command}"
+
+    conn = duckdb.connect(abs_db_path)
+    conn.execute("DROP SCHEMA IF EXISTS testschema_append CASCADE")
+    conn.execute("CHECKPOINT")
+
+    conn.execute("CREATE SCHEMA testschema_append")
+    conn.execute(
+        "CREATE TABLE testschema_append.input (id INTEGER, val VARCHAR, updated_at DATE)"
+    )
+    conn.execute(
+        "INSERT INTO testschema_append.input VALUES (1, 'val1', '2022-01-01'), (2, 'val2', '2022-01-02')"
+    )
+    conn.execute("CHECKPOINT")
+
+    res = conn.sql("select count(*) from testschema_append.input").fetchall()
+    assert res[0][0] == 2
+
+    def run():
+        res = invoke_ingest_command(
+            uri,
+            "testschema_append.input",
+            uri,
+            "testschema_append.output",
+            "append",
+            "updated_at",
+        )
+        assert res.exit_code == 0
+
+    def get_output_table():
+        conn.execute("CHECKPOINT")
+        return conn.sql(
+            "select id, val, strftime(updated_at, '%Y-%m-%d') as updated_at from testschema_append.output"
+        ).fetchall()
+
+    run()
+
+    res = get_output_table()
+    assert len(res) == 2
+    assert res[0] == (1, "val1", "2022-01-01")
+    assert res[1] == (2, "val2", "2022-01-02")
+
+    # # run again, nothing should be inserted into the output table
+    run()
+
+    res = get_output_table()
+    assert len(res) == 2
+    assert res[0] == (1, "val1", "2022-01-01")
+    assert res[1] == (2, "val2", "2022-02-01")
+
+
+def test_merge_with_primary_key():
+    try:
+        shutil.rmtree(get_abs_path("../pipeline_data"))
+    except Exception:
+        pass
+
+    abs_db_path = get_abs_path("./testdata/test_merge_with_primary_key.db")
+    rel_db_path_to_command = "ingestr/testdata/test_merge_with_primary_key.db"
+    uri = f"duckdb:///{rel_db_path_to_command}"
+
+    conn = duckdb.connect(abs_db_path)
+    conn.execute("DROP SCHEMA IF EXISTS testschema_merge CASCADE")
+    conn.execute("CREATE SCHEMA testschema_merge")
+    conn.execute(
+        "CREATE TABLE testschema_merge.input (id INTEGER, val VARCHAR, updated_at TIMESTAMP)"
+    )
+    conn.execute("INSERT INTO testschema_merge.input VALUES (1, 'val1', '2022-01-01')")
+    conn.execute("INSERT INTO testschema_merge.input VALUES (2, 'val2', '2022-02-01')")
+
+    res = conn.sql("select count(*) from testschema_merge.input").fetchall()
+    assert res[0][0] == 2
+
+    def run():
+        res = invoke_ingest_command(
+            uri,
+            "testschema_merge.input",
+            uri,
+            "testschema_merge.output",
+            "merge",
+            "updated_at",
+            "id",
+        )
+        assert res.exit_code == 0
+        return res
+
+    def get_output_rows():
+        conn.execute("CHECKPOINT")
+        return conn.sql(
+            "select id, val, strftime(updated_at, '%Y-%m-%d') as updated_at from testschema_merge.output order by id asc"
+        ).fetchall()
+
+    def assert_output_equals(expected):
+        res = get_output_rows()
+        assert len(res) == len(expected)
+        for i, row in enumerate(expected):
+            assert res[i] == row
+
+    run()
+    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-02-01")])
+
+    first_run_id = conn.sql(
+        "select _dlt_load_id from testschema_merge.output limit 1"
+    ).fetchall()[0][0]
+
+    ##############################
+    # we'll run again, we don't expect any changes since the data hasn't changed
+    run()
+    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-02-01")])
+
+    # we also ensure that the other rows were not touched
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_merge.output group by 1"
+    ).fetchall()
+    assert len(count_by_run_id) == 1
+    assert count_by_run_id[0][1] == 2
+    assert count_by_run_id[0][0] == first_run_id
+    ##############################
+
+    ##############################
+    # now we'll modify the source data but not the updated at, the output table should not be updated
+    conn.execute("UPDATE testschema_merge.input SET val = 'val1_modified' WHERE id = 2")
+
+    run()
+    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-02-01")])
+
+    # we also ensure that the other rows were not touched
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_merge.output group by 1"
+    ).fetchall()
+    assert len(count_by_run_id) == 1
+    assert count_by_run_id[0][1] == 2
+    assert count_by_run_id[0][0] == first_run_id
+    ##############################
+
+    ##############################
+    # now we'll insert a new row but with an old date, the new row will not show up
+    conn.execute("INSERT INTO testschema_merge.input VALUES (3, 'val3', '2022-01-01')")
+
+    run()
+    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-02-01")])
+
+    # we also ensure that the other rows were not touched
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_merge.output group by 1"
+    ).fetchall()
+    assert len(count_by_run_id) == 1
+    assert count_by_run_id[0][1] == 2
+    assert count_by_run_id[0][0] == first_run_id
+    ##############################
+
+    ##############################
+    # now we'll insert a new row but with a new date, the new row will show up
+    conn.execute("INSERT INTO testschema_merge.input VALUES (3, 'val3', '2022-02-02')")
+
+    run()
+    assert_output_equals(
+        [
+            (1, "val1", "2022-01-01"),
+            (2, "val2", "2022-02-01"),
+            (3, "val3", "2022-02-02"),
+        ]
+    )
+
+    # we have a new run that inserted rows to this table, so the run count should be 2
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_merge.output group by 1 order by 2 desc"
+    ).fetchall()
+    assert len(count_by_run_id) == 2
+    assert count_by_run_id[0][1] == 2
+    assert count_by_run_id[0][0] == first_run_id
+    # we don't care about the run ID
+    assert count_by_run_id[1][1] == 1
+    ##############################
+
+    ##############################
+    # lastly, let's try modifying the updated_at of an old column, it should be updated in the output table
+    conn.execute(
+        "UPDATE testschema_merge.input SET val='val2_modified', updated_at = '2022-02-03' WHERE id = 2"
+    )
+
+    run()
+    assert_output_equals(
+        [
+            (1, "val1", "2022-01-01"),
+            (2, "val2_modified", "2022-02-03"),
+            (3, "val3", "2022-02-02"),
+        ]
+    )
+
+    # we have a new run that inserted rows to this table, so the run count should be 2
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_merge.output group by 1 order by 2 desc, 1 asc"
+    ).fetchall()
+    assert len(count_by_run_id) == 3
+    assert count_by_run_id[0][1] == 1
+    assert count_by_run_id[0][0] == first_run_id
+    # we don't care about the rest of the run IDs
+    assert count_by_run_id[1][1] == 1
+    assert count_by_run_id[2][1] == 1
+    ##############################
+
+
+def test_delete_insert_without_primary_key():
+    try:
+        shutil.rmtree(get_abs_path("../pipeline_data"))
+    except Exception:
+        pass
+
+    abs_db_path = get_abs_path("./testdata/test_delete_insert_without_primary_key.db")
+    rel_db_path_to_command = (
+        "ingestr/testdata/test_delete_insert_without_primary_key.db"
+    )
+    uri = f"duckdb:///{rel_db_path_to_command}"
+
+    conn = duckdb.connect(abs_db_path)
+    conn.execute("DROP SCHEMA IF EXISTS testschema_delete_insert CASCADE")
+    conn.execute("CREATE SCHEMA testschema_delete_insert")
+    conn.execute(
+        "CREATE TABLE testschema_delete_insert.input (id INTEGER, val VARCHAR, updated_at TIMESTAMP)"
+    )
+    conn.execute(
+        "INSERT INTO testschema_delete_insert.input VALUES (1, 'val1', '2022-01-01')"
+    )
+    conn.execute(
+        "INSERT INTO testschema_delete_insert.input VALUES (2, 'val2', '2022-02-01')"
+    )
+
+    res = conn.sql("select count(*) from testschema_delete_insert.input").fetchall()
+    assert res[0][0] == 2
+
+    def run():
+        res = invoke_ingest_command(
+            uri,
+            "testschema_delete_insert.input",
+            uri,
+            "testschema_delete_insert.output",
+            inc_strategy="delete+insert",
+            inc_key="updated_at",
+        )
+        assert res.exit_code == 0
+        return res
+
+    def get_output_rows():
+        conn.execute("CHECKPOINT")
+        return conn.sql(
+            "select id, val, strftime(updated_at, '%Y-%m-%d') as updated_at from testschema_delete_insert.output order by id asc"
+        ).fetchall()
+
+    def assert_output_equals(expected):
+        res = get_output_rows()
+        assert len(res) == len(expected)
+        for i, row in enumerate(expected):
+            assert res[i] == row
+
+    run()
+    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-02-01")])
+
+    first_run_id = conn.sql(
+        "select _dlt_load_id from testschema_delete_insert.output limit 1"
+    ).fetchall()[0][0]
+
+    ##############################
+    # we'll run again, since this is a delete+insert, we expect the run ID to change for the last one
+    run()
+    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-02-01")])
+
+    # we ensure that one of the rows is updated with a new run
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_delete_insert.output group by 1 order by 1 asc"
+    ).fetchall()
+    assert len(count_by_run_id) == 2
+    assert count_by_run_id[0][0] == first_run_id
+    assert count_by_run_id[0][1] == 1
+    assert count_by_run_id[1][0] != first_run_id
+    assert count_by_run_id[1][1] == 1
+    ##############################
+
+    ##############################
+    # now we'll insert a few more lines for the same day, the new rows should show up
+    conn.execute(
+        "INSERT INTO testschema_delete_insert.input VALUES (3, 'val3', '2022-02-01'), (4, 'val4', '2022-02-01')"
+    )
+    conn.execute("CHECKPOINT")
+
+    run()
+    assert_output_equals(
+        [
+            (1, "val1", "2022-01-01"),
+            (2, "val2", "2022-02-01"),
+            (3, "val3", "2022-02-01"),
+            (4, "val4", "2022-02-01"),
+        ]
+    )
+
+    # the new rows should have a new run ID, there should be 2 distinct runs now
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_delete_insert.output group by 1 order by 2 desc, 1 asc"
+    ).fetchall()
+    assert len(count_by_run_id) == 2
+    assert count_by_run_id[0][0] != first_run_id
+    assert count_by_run_id[0][1] == 3  # 2 new rows + 1 old row
+    assert count_by_run_id[1][0] == first_run_id
+    assert count_by_run_id[1][1] == 1
+    ##############################
+
+
+def test_delete_insert_with_timerange():
+    try:
+        shutil.rmtree(get_abs_path("../pipeline_data"))
+    except Exception:
+        pass
+
+    abs_db_path = get_abs_path("./testdata/test_delete_insert_with_timerange.db")
+    rel_db_path_to_command = "ingestr/testdata/test_delete_insert_with_timerange.db"
+    uri = f"duckdb:///{rel_db_path_to_command}"
+
+    conn = duckdb.connect(abs_db_path)
+    conn.execute("DROP SCHEMA IF EXISTS testschema_delete_insert_timerange CASCADE")
+    conn.execute("CREATE SCHEMA testschema_delete_insert_timerange")
+    conn.execute(
+        "CREATE TABLE testschema_delete_insert_timerange.input (id INTEGER, val VARCHAR, updated_at TIMESTAMP)"
+    )
+    conn.execute(
+        """INSERT INTO testschema_delete_insert_timerange.input VALUES 
+            (1, 'val1', '2022-01-01'),
+            (2, 'val2', '2022-01-01'),
+            (3, 'val3', '2022-01-02'),
+            (4, 'val4', '2022-01-02'),
+            (5, 'val5', '2022-01-03'),
+            (6, 'val6', '2022-01-03')
+        """
+    )
+
+    res = conn.sql(
+        "select count(*) from testschema_delete_insert_timerange.input"
+    ).fetchall()
+    assert res[0][0] == 6
+
+    def run(start_date: str, end_date: str):
+        res = invoke_ingest_command(
+            uri,
+            "testschema_delete_insert_timerange.input",
+            uri,
+            "testschema_delete_insert_timerange.output",
+            inc_strategy="delete+insert",
+            inc_key="updated_at",
+            interval_start=start_date,
+            interval_end=end_date,
+        )
+        assert res.exit_code == 0
+        return res
+
+    def get_output_rows():
+        conn.execute("CHECKPOINT")
+        return conn.sql(
+            "select id, val, strftime(updated_at, '%Y-%m-%d') as updated_at from testschema_delete_insert_timerange.output order by id asc"
+        ).fetchall()
+
+    def assert_output_equals(expected):
+        res = get_output_rows()
+        assert len(res) == len(expected)
+        for i, row in enumerate(expected):
+            assert res[i] == row
+
+    run(
+        "2022-01-01T00:00:00Z", "2022-01-02T00:00:00Z"
+    )  # dlt runs them with the end date exclusive
+    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-01-01")])
+
+    first_run_id = conn.sql(
+        "select _dlt_load_id from testschema_delete_insert_timerange.output limit 1"
+    ).fetchall()[0][0]
+
+    ##############################
+    # we'll run again, since this is a delete+insert, we expect the run ID to change for the last one
+    run(
+        "2022-01-01T00:00:00Z", "2022-01-02T00:00:00Z"
+    )  # dlt runs them with the end date exclusive
+    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-01-01")])
+
+    # both rows should have a new run ID
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_delete_insert_timerange.output group by 1 order by 1 asc"
+    ).fetchall()
+    assert len(count_by_run_id) == 1
+    assert count_by_run_id[0][0] != first_run_id
+    assert count_by_run_id[0][1] == 2
+    ##############################
+
+    ##############################
+    # now run for the day after, new rows should land
+    run("2022-01-02T00:00:00Z", "2022-01-03T00:00:00Z")
+    assert_output_equals(
+        [
+            (1, "val1", "2022-01-01"),
+            (2, "val2", "2022-01-01"),
+            (3, "val3", "2022-01-02"),
+            (4, "val4", "2022-01-02"),
+        ]
+    )
+
+    # there should be 4 rows with 2 distinct run IDs
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_delete_insert_timerange.output group by 1 order by 1 asc"
+    ).fetchall()
+    assert len(count_by_run_id) == 2
+    assert count_by_run_id[0][1] == 2
+    assert count_by_run_id[1][1] == 2
+    ##############################
+
+    ##############################
+    # let's bring in the rows for the third day
+    run("2022-01-03T00:00:00Z", "2022-01-04T00:00:00Z")
+    assert_output_equals(
+        [
+            (1, "val1", "2022-01-01"),
+            (2, "val2", "2022-01-01"),
+            (3, "val3", "2022-01-02"),
+            (4, "val4", "2022-01-02"),
+            (5, "val5", "2022-01-03"),
+            (6, "val6", "2022-01-03"),
+        ]
+    )
+
+    # there should be 6 rows with 3 distinct run IDs
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_delete_insert_timerange.output group by 1 order by 1 asc"
+    ).fetchall()
+    assert len(count_by_run_id) == 3
+    assert count_by_run_id[0][1] == 2
+    assert count_by_run_id[1][1] == 2
+    assert count_by_run_id[2][1] == 2
+    ##############################
+
+    ##############################
+    # now let's do a backfill for the first day again, the rows should be updated
+    conn.execute(
+        "UPDATE testschema_delete_insert_timerange.input SET val = 'val1_modified' WHERE id = 1"
+    )
+
+    run("2022-01-01T00:00:00Z", "2022-01-02T00:00:00Z")
+    assert_output_equals(
+        [
+            (1, "val1_modified", "2022-01-01"),
+            (2, "val2", "2022-01-01"),
+            (3, "val3", "2022-01-02"),
+            (4, "val4", "2022-01-02"),
+            (5, "val5", "2022-01-03"),
+            (6, "val6", "2022-01-03"),
+        ]
+    )
+
+    # there should still be 6 rows with 3 distinct run IDs
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_delete_insert_timerange.output group by 1 order by 1 asc"
+    ).fetchall()
+    assert len(count_by_run_id) == 3
+    assert count_by_run_id[0][1] == 2
+    assert count_by_run_id[1][1] == 2
+    assert count_by_run_id[2][1] == 2
+    ##############################

ingestr/src/factory.py

@@ -3,6 +3,7 @@ from urllib.parse import urlparse
 
 from ingestr.src.destinations import (
     BigQueryDestination,
+    DatabricksDestination,
     DuckDBDestination,
     MsSQLDestination,
     PostgresDestination,
@@ -59,12 +60,13 @@ class SourceDestinationFactory:
     def get_destination(self) -> DestinationProtocol:
         match: dict[str, DestinationProtocol] = {
             "bigquery": BigQueryDestination(),
+            "databricks": DatabricksDestination(),
+            "duckdb": DuckDBDestination(),
+            "mssql": MsSQLDestination(),
             "postgres": PostgresDestination(),
             "postgresql": PostgresDestination(),
-            "snowflake": SnowflakeDestination(),
             "redshift": RedshiftDestination(),
-            "duckdb": DuckDBDestination(),
-            "mssql": MsSQLDestination(),
+            "snowflake": SnowflakeDestination(),
         }
 
         if self.destination_scheme in match:

ingestr/src/sources.py

@@ -1,7 +1,6 @@
 from typing import Callable
 
 import dlt
-import pendulum
 
 from ingestr.src.sql_database import sql_table
 
@@ -19,10 +18,14 @@ class SqlSource:
 
         incremental = None
         if kwargs.get("incremental_key"):
+            start_value = kwargs.get("interval_start")
+            end_value = kwargs.get("interval_end")
+
             incremental = dlt.sources.incremental(
                 kwargs.get("incremental_key", ""),
-                primary_key=(),
-                initial_value=pendulum.now(),
+                # primary_key=(),
+                initial_value=start_value,
+                end_value=end_value,
             )
 
         table_instance = self.table_builder(
@@ -30,6 +33,7 @@ class SqlSource:
             schema=table_fields[-2],
             table=table_fields[-1],
             incremental=incremental,
+            merge_key=kwargs.get("merge_key"),
         )
 
         return table_instance

ingestr/src/sources_test.py

@@ -22,11 +22,12 @@ class SqlSourceTest(unittest.TestCase):
         table = "schema.table"
 
         # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental):
+        def sql_table(credentials, schema, table, incremental, merge_key):
             self.assertEqual(credentials, uri)
             self.assertEqual(schema, "schema")
             self.assertEqual(table, "table")
             self.assertIsNone(incremental)
+            self.assertIsNone(merge_key)
             return dlt.resource()
 
         source = SqlSource(table_builder=sql_table)
@@ -39,12 +40,13 @@ class SqlSourceTest(unittest.TestCase):
         incremental_key = "id"
 
         # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental):
+        def sql_table(credentials, schema, table, incremental, merge_key):
             self.assertEqual(credentials, uri)
             self.assertEqual(schema, "schema")
             self.assertEqual(table, "table")
             self.assertIsInstance(incremental, dlt.sources.incremental)
             self.assertEqual(incremental.cursor_path, incremental_key)
+            self.assertIsNone(merge_key)
             return dlt.resource()
 
         source = SqlSource(table_builder=sql_table)

ingestr/src/sql_database/__init__.py

@@ -1,6 +1,6 @@
 """Source that loads tables form any SQLAlchemy supported database, supports batching requests and incremental loads."""
 
-from typing import Any, Iterable, List, Optional, Union
+from typing import Any, Optional, Union
 
 import dlt
 from dlt.sources import DltResource
@@ -9,7 +9,6 @@ from sqlalchemy import MetaData, Table
 from sqlalchemy.engine import Engine
 
 from .helpers import (
-    SqlDatabaseTableConfiguration,
     engine_from_credentials,
     get_primary_key,
     table_rows,
@@ -17,51 +16,6 @@ from .helpers import (
 from .schema_types import table_to_columns
 
 
-@dlt.source
-def sql_database(
-    credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value,
-    schema: Optional[str] = dlt.config.value,
-    metadata: Optional[MetaData] = None,
-    table_names: Optional[List[str]] = dlt.config.value,
-    detect_precision_hints: Optional[bool] = dlt.config.value,
-) -> Iterable[DltResource]:
-    """
-    A DLT source which loads data from an SQL database using SQLAlchemy.
-    Resources are automatically created for each table in the schema or from the given list of tables.
-
-    Args:
-        credentials (Union[ConnectionStringCredentials, Engine, str]): Database credentials or an `sqlalchemy.Engine` instance.
-        schema (Optional[str]): Name of the database schema to load (if different from default).
-        metadata (Optional[MetaData]): Optional `sqlalchemy.MetaData` instance. `schema` argument is ignored when this is used.
-        table_names (Optional[List[str]]): A list of table names to load. By default, all tables in the schema are loaded.
-        detect_precision_hints (bool): Set column precision and scale hints for supported data types in the target schema based on the columns in the source tables.
-            This is disabled by default.
-    Returns:
-        Iterable[DltResource]: A list of DLT resources for each table to be loaded.
-    """
-
-    # set up alchemy engine
-    engine = engine_from_credentials(credentials)
-    engine.execution_options(stream_results=True)
-    metadata = metadata or MetaData(schema=schema)
-
-    # use provided tables or all tables
-    if table_names:
-        tables = [Table(name, metadata, autoload_with=engine) for name in table_names]
-    else:
-        metadata.reflect(bind=engine)
-        tables = list(metadata.tables.values())
-
-    for table in tables:
-        yield dlt.resource(
-            table_rows,
-            name=table.name,
-            primary_key=get_primary_key(table),
-            spec=SqlDatabaseTableConfiguration,
-            columns=table_to_columns(table) if detect_precision_hints else None,
-        )(engine, table)
-
-
 def sql_table(
     credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value,
     table: str = dlt.config.value,
@@ -69,6 +23,7 @@ def sql_table(
     metadata: Optional[MetaData] = None,
     incremental: Optional[dlt.sources.incremental[Any]] = None,
     detect_precision_hints: Optional[bool] = dlt.config.value,
+    merge_key: Optional[str] = None,
 ) -> DltResource:
     """
     A dlt resource which loads data from an SQL database table using SQLAlchemy.
@@ -101,4 +56,5 @@ def sql_table(
         name=table_obj.name,
         primary_key=get_primary_key(table_obj),
         columns=table_to_columns(table_obj) if detect_precision_hints else None,
+        merge_key=merge_key,  # type: ignore
     )(engine, table_obj, incremental=incremental)

ingestr/src/sql_database/helpers.py

@@ -63,7 +63,6 @@ class TableLoader:
         query = query.order_by(order_by)
         if self.last_value is None:
             return query
-
         return query.where(filter_op(self.cursor_column, self.last_value))
 
     def load_rows(self) -> Iterator[List[TDataItem]]:

ingestr/src/telemetry/event.py

@@ -0,0 +1,14 @@
+import os
+
+import machineid
+import rudderstack.analytics as rudder_analytics  # type: ignore
+
+rudder_analytics.write_key = "2cUr13DDQcX2x2kAfMEfdrKvrQa"
+rudder_analytics.dataPlaneUrl = "https://getbruinbumlky.dataplane.rudderstack.com"
+
+
+def track(event_name, event_properties):
+    if os.environ.get("DISABLE_TELEMETRY", False):
+        return
+
+    rudder_analytics.track(machineid.hashed_id(), event_name, event_properties)

{ingestr-0.0.3.dist-info → ingestr-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestr
-Version: 0.0.3
+Version: 0.0.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,11 +16,14 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dlt==0.4.3
+Requires-Dist: duckdb-engine==0.11.1
 Requires-Dist: duckdb==0.9.2
+Requires-Dist: google-cloud-bigquery-storage
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2==2.9.9
 Requires-Dist: pyodbc==5.1.0
 Requires-Dist: rich==13.7.0
+Requires-Dist: rudder-sdk-python==2.0.2
 Requires-Dist: snowflake-sqlalchemy==1.5.1
 Requires-Dist: sqlalchemy-bigquery==1.9.0
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
@@ -32,18 +35,19 @@ Description-Content-Type: text/markdown
 <div align="center">
     <img src="./resources/ingestr.svg" width="500" />
     <p>Ingest & copy data from any source to any destination without any code</p>
+    <img src="./resources/demo.gif" width="500" />
 </div>
 
+
 -----
 
 Ingestr is a command-line application that allows you to ingest data from any source into any destination using simple command-line flags, no code necessary.
 
-- ✨ copy data from your Postges / Mongo / BigQuery or any other source into any destination
-- ➕ incremental loading
+- ✨ copy data from your database into any destination
+- ➕ incremental loading: `append`, `merge` or `delete+insert`
 - 🐍 single-command installation
-- 💅 Docker image for easy installation & usage
 
-ingestr takes away the complexity of managing any backend or writing any code for ingesting data, simply run the command and watch the magic.
+ingestr takes away the complexity of managing any backend or writing any code for ingesting data, simply run the command and watch the data land on its destination.
 
 
 ## Installation
@@ -67,6 +71,10 @@ This command will:
 - get the table `public.some_data` from the Postgres instance.
 - upload this data to your BigQuery warehouse under the schema `ingestr` and table `some_data`.
 
+## Documentation
+You can see the full documentation [here](https://bruindata.github.com/ingestr).
+
+
 ## Supported Sources & Destinations
 
 | Database             | Source | Destination |
@@ -79,4 +87,9 @@ This command will:
 | DuckDB               | ✅      | ✅         |
 | Microsoft SQL Server | ✅      | ✅         |
 | SQLite               | ✅      | ❌         |
-| MySQL                | ✅      | ❌         |
+| MySQL                | ✅      | ❌         |
+
+More to come soon!
+
+## Acknowledgements
+This project would not have been possible without the amazing work done by the [SQLAlchemy](https://www.sqlalchemy.org/) and [dlt](https://dlthub.com/) teams. We relied on their work to connect to various sources and destinations, and built `ingestr` as a simple, opinionated wrapper around their work.

ingestr-0.0.4.dist-info/RECORD

@@ -0,0 +1,23 @@
+ingestr/main.py,sha256=fogv6KlxO2Y0fYEl36s0YVTF1S7TCmksQvL_siVI7rE,8703
+ingestr/main_test.py,sha256=IQx2bjTnuixoY9ndgErBTaP3QE5AWvsLtv7ZPi8vJtA,19285
+ingestr/src/destinations.py,sha256=LyA_26S3tHMeJiFwxX3XYV39lLOKqKACL0wWn3IGyP4,2673
+ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
+ingestr/src/factory.py,sha256=iBmp2spbUkkvOfwdRf6uo_5j9fasTTSWdS79Kc4jRQw,2141
+ingestr/src/sources.py,sha256=WdbkY0S54H3rNy8kgOH3VBfE5oB0TSsOCCY5GBDU8Ss,1130
+ingestr/src/sources_test.py,sha256=l_Obs7z3_WulTVa2ZeCDYOTT97Mj1nd1AgA3P5OivV0,1929
+ingestr/src/sql_database/__init__.py,sha256=_hOl_RpOzzd2Sf6r0ETzcjr8gD9fjXRfrgr8QM4pe6w,2606
+ingestr/src/sql_database/helpers.py,sha256=1yw-E9uTr4_6VnFxYpFBZkA76nC3UmXYwFxoxMRnhC0,4441
+ingestr/src/sql_database/schema_types.py,sha256=PCIdLrT5Xc4vmoaf6OJSeXLlyN05alwgcQ-TDXd8hbQ,2153
+ingestr/src/sql_database/settings.py,sha256=PaLPayAb1QGHHcPlrZ7eJ1fonDA6-sOGh-ZiueIFhRg,76
+ingestr/src/telemetry/event.py,sha256=ByPdlu5YYRXKM6hnZa48uAKvvz5jOq4bP-OAn3Nc7bQ,426
+ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
+ingestr/testdata/test_append.db,sha256=OI0K5lwvwJpbTKdv3MkIq1RKiGA4SHclCXThPW0q4Xo,1060864
+ingestr/testdata/test_create_replace.db,sha256=-ByzuQxPW5wa7hiVUHxvckGgEn1NpGZgN2zogsg80-U,536576
+ingestr/testdata/test_delete_insert_with_timerange.db,sha256=ClL0WO4f3lq7kaEDHs8LTKKl5pdJUB9HncvamRZqsjY,1585152
+ingestr/testdata/test_delete_insert_without_primary_key.db,sha256=feNVnNzfJY-DXQF84eBmW62YNKfB8WxXXt66Or9HqZ4,1847296
+ingestr/testdata/test_merge_with_primary_key.db,sha256=t5fm_kKLtl4G2l8PLdXqOs3eAwEv8keS-sD7fwlRQDQ,1847296
+ingestr-0.0.4.dist-info/METADATA,sha256=lgJ-p08vbVwbgCpWPhpWFFJoaTPeKSKrOVqIvlzHjso,3563
+ingestr-0.0.4.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
+ingestr-0.0.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.0.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.0.4.dist-info/RECORD,,

ingestr-0.0.3.dist-info/RECORD

@@ -1,16 +0,0 @@
-ingestr/main.py,sha256=KheVlifq6v4I5KCd_l1fi2mpk8kfYE4delBXHH_SVxY,7537
-ingestr/src/destinations.py,sha256=LyA_26S3tHMeJiFwxX3XYV39lLOKqKACL0wWn3IGyP4,2673
-ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
-ingestr/src/factory.py,sha256=YMIZSWY9ojrrl-M5_VC5eDR7TdBRF9Wem9pDCOw5DbU,2063
-ingestr/src/sources.py,sha256=8UeHfi1XY0C8us0fHQObkjLXDkWA_fAWhfBltUW2hTQ,956
-ingestr/src/sources_test.py,sha256=gcoEFpakLeYSl9cliwoSAakzr1R9pVml5V4H9ibMCZA,1825
-ingestr/src/sql_database/__init__.py,sha256=mLiaDGaz3h5LRCnmjPTUNhp1fIjgU5rCcBOQ2r3Iu1Q,4643
-ingestr/src/sql_database/helpers.py,sha256=r3b5MDpMjpV4U94gCKSOUH9t7YdP0OCAOqgpjbe7iSs,4442
-ingestr/src/sql_database/schema_types.py,sha256=PCIdLrT5Xc4vmoaf6OJSeXLlyN05alwgcQ-TDXd8hbQ,2153
-ingestr/src/sql_database/settings.py,sha256=PaLPayAb1QGHHcPlrZ7eJ1fonDA6-sOGh-ZiueIFhRg,76
-ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
-ingestr-0.0.3.dist-info/METADATA,sha256=EeURFoj26cB2ivj3hfdAu07AL4ZWKwLE70zDosxQjaQ,2972
-ingestr-0.0.3.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
-ingestr-0.0.3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.0.3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.0.3.dist-info/RECORD,,