ingestr 0.6.0__tar.gz → 0.6.2__tar.gz

ingestr-0.6.2/.python-version

@@ -0,0 +1 @@
+3.11

{ingestr-0.6.0 → ingestr-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.0
+Version: 0.6.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,7 +16,7 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==0.4.8
+Requires-Dist: dlt==0.4.12
 Requires-Dist: duckdb-engine==0.11.5
 Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-api-python-client==2.130.0
@@ -27,6 +27,7 @@ Requires-Dist: py-machineid==0.5.1
 Requires-Dist: pymongo==4.6.3
 Requires-Dist: pymysql==1.1.0
 Requires-Dist: pyodbc==5.1.0
+Requires-Dist: pyrate-limiter==3.6.1
 Requires-Dist: redshift-connector==2.1.0
 Requires-Dist: rich==13.7.1
 Requires-Dist: rudder-sdk-python==2.1.0

{ingestr-0.6.0 → ingestr-0.6.2}/docs/.vitepress/config.mjs

@@ -22,6 +22,7 @@ export default defineConfig({
       { text: "Home", link: "/" },
       { text: "Getting started", link: "/getting-started/quickstart.md" },
     ],
+    outline: 'deep',
 
     sidebar: [
       {
@@ -68,6 +69,7 @@ export default defineConfig({
             text: "Platforms",
             collapsed: false,
             items: [
+              { text: "Gorgias", link: "/supported-sources/gorgias.md" },
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },

{ingestr-0.6.0 → ingestr-0.6.2}/docs/index.md

@@ -5,7 +5,7 @@ layout: home
 hero:
   name: "ingestr"
   text: Copy data between any source and any destination
-  tagline: "ingestr is a command-line application that allows ingesting or copying data from any source into any destination database."
+  tagline: "ingestr is a command-line application that allows copying data from any source into any destination database."
   image:
     src: https://github.com/bruin-data/ingestr/blob/main/resources/demo.gif?raw=true
     alt: ingestr logo
@@ -25,4 +25,7 @@ features:
   - title: Incremental Loading
     details: ingestr supports both full-refresh as well as incremental loading modes.
 ---
+<div style="margin-top: 12px; line-height: 2em; text-align: center;">
 
+<Badge type="info" text="Postgres" /> <Badge type="danger" text="BigQuery" /> <Badge type="tip" text="Snowflake" /> <Badge type="warning" text="Redshift" /> <Badge type="info" text="Databricks" /> <Badge type="danger" text="DuckDB" /> <Badge type="tip" text="Microsoft SQL Server" /> <Badge type="warning" text="Local CSV file" /> <Badge type="info" text="MongoDB" /> <Badge type="danger" text="Oracle" /> <Badge type="tip" text="SAP Hana" /> <Badge type="warning" text="SQLite" /> <Badge type="info" text="MySQL" /> <Badge type="danger" text="Google Sheets" /> <Badge type="tip" text="Notion" /> <Badge type="warning" text="Shopify" />
+</div>

ingestr-0.6.2/docs/supported-sources/gorgias.md

@@ -0,0 +1,53 @@
+# Gorgias
+[Gorgias](https://www.gorgias.com/) is a helpdesk for e-commerce merchants, providing customer service via email, social media, SMS, and live chat.
+
+ingestr supports Gorgias as a source.
+
+## URI Format
+The URI format for Gorgias is as follows:
+
+```plaintext
+gorgias://<domain>?api_key=<api-key>&email=<email>
+```
+
+URI parameters:
+- `domain`: the domain of the Gorgias account without the full `gorgias.com`, e.g. `mycompany`
+- `api_key`: the integration token used for authentication with the Gorgias API
+- `email`: the email address of the user to connect to the Gorgias API
+
+The URI is used to connect to the Gorgias API for extracting data.
+
+## Examples
+```bash
+# get all the tickets that are created/updated since 2024-06-19 and write them to `gorgias.ticket_messages` table on BigQuery
+ingestr ingest --source-table 'tickets' --source-uri $GORGIAS_URI --dest-uri $BIGQUERY_URI --interval-start 2024-06-19  --dest-table 'gorgias.ticket_messages' --loader-file-format jsonl
+
+# get all the customers and write them to `gorgias.customers` table on DuckDB
+ingestr ingest --source-table 'customers' --source-uri $GORGIAS_URI --dest-uri duckdb:///gorgias.duckdb --interval-start 2024-01-01  --dest-table 'gorgias.customers'
+```
+
+## Supported Entities
+The Gorgias source supports a growing list of entities, feel free to create an issue if you need more entities to be supported.
+
+### Customers
+Customers are the users who have interacted with the support team. Each customer has a unique ID and contains information such as the name and email.
+
+You can retrieve customers by using `customers` as the source table.
+
+### Tickets
+Tickets are the main entity in Gorgias, representing customer inquiries. Each ticket has a unique ID and contains information such as the customer, status, and messages.
+
+You can retrieve tickets by using `tickets` as the source table.
+
+### Ticket Messages
+Ticket messages are the messages exchanged between the customer and the support agent in a ticket. Each message has a unique ID and contains information such as the sender, content, and timestamp.
+
+You can retrieve ticket messages by using `ticket_messages` as the source table.
+
+### Satisfaction Surveys
+Satisfaction surveys are sent to customers after a ticket is resolved to gather feedback on their experience. Each survey has a unique ID and contains information such as the rating and comments.
+
+You can retrieve satisfaction surveys by using `satisfaction_surveys` as the source table.
+
+
+

{ingestr-0.6.0 → ingestr-0.6.2}/ingestr/main.py

@@ -90,6 +90,7 @@ class IncrementalStrategy(str, Enum):
     append = "append"
     delete_insert = "delete+insert"
     merge = "merge"
+    none = "none"
 
 
 class LoaderFileFormat(str, Enum):
@@ -136,7 +137,7 @@ def ingest(
         ),
     ] = None,  # type: ignore
     incremental_key: Annotated[
-        str,
+        Optional[str],
         typer.Option(
             help="The incremental key from the table to be used for incremental strategies",
             envvar="INCREMENTAL_KEY",
@@ -257,6 +258,16 @@ def ingest(
             full_refresh=full_refresh,
         )
 
+        if source.handles_incrementality():
+            incremental_strategy = IncrementalStrategy.none
+            incremental_key = None
+
+        incremental_strategy_text = (
+            incremental_strategy.value
+            if incremental_strategy.value != IncrementalStrategy.none
+            else "Platform-specific"
+        )
+
         print()
         print("[bold green]Initiated the pipeline with the following:[/bold green]")
         print(
@@ -266,7 +277,7 @@ def ingest(
             f"[bold yellow]  Destination:[/bold yellow] {factory.destination_scheme} / {dest_table}"
         )
         print(
-            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy.value}"
+            f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy_text}"
         )
         print(
             f"[bold yellow]  Incremental Key:[/bold yellow] {incremental_key if incremental_key else 'None'}"
@@ -317,7 +328,9 @@ def ingest(
                 uri=dest_uri,
                 table=dest_table,
             ),
-            write_disposition=incremental_strategy.value,  # type: ignore
+            write_disposition=incremental_strategy.value
+            if incremental_strategy.value != IncrementalStrategy.none
+            else None,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore
             loader_file_format=loader_file_format.value
             if loader_file_format is not None

{ingestr-0.6.0 → ingestr-0.6.2}/ingestr/main_test.py

@@ -1,5 +1,8 @@
+import csv
 import os
+import random
 import shutil
+import string
 
 import duckdb
 from typer.testing import CliRunner
@@ -80,9 +83,17 @@ def invoke_ingest_command(
     return result
 
 
-def test_create_replace():
-    abs_db_path = get_abs_path("./testdata/test_create_replace.db")
-    rel_db_path_to_command = "ingestr/testdata/test_create_replace.db"
+### These are DuckDB-to-DuckDB tests
+def test_create_replace_duckdb_to_duckdb():
+    try:
+        shutil.rmtree(get_abs_path("../pipeline_data"))
+    except Exception:
+        pass
+
+    dbname = f"test_create_replace_{get_random_string(5)}.db"
+
+    abs_db_path = get_abs_path(f"./testdata/{dbname}")
+    rel_db_path_to_command = f"ingestr/testdata/{dbname}"
 
     conn = duckdb.connect(abs_db_path)
     conn.execute("DROP SCHEMA IF EXISTS testschema CASCADE")
@@ -103,6 +114,8 @@ def test_create_replace():
         "testschema.output",
     )
 
+    print(result.stdout)
+
     assert result.exit_code == 0
 
     res = conn.sql(
@@ -112,8 +125,13 @@ def test_create_replace():
     assert res[0] == (1, "val1", "2022-01-01")
     assert res[1] == (2, "val2", "2022-02-01")
 
+    try:
+        os.remove(abs_db_path)
+    except Exception:
+        pass
+
 
-def test_append():
+def test_append_duckdb_to_duckdb():
     try:
         shutil.rmtree(get_abs_path("../pipeline_data"))
     except Exception:
@@ -172,8 +190,13 @@ def test_append():
     assert res[0] == (1, "val1", "2022-01-01")
     assert res[1] == (2, "val2", "2022-01-02")
 
+    try:
+        os.remove(abs_db_path)
+    except Exception:
+        pass
 
-def test_merge_with_primary_key():
+
+def test_merge_with_primary_key_duckdb_to_duckdb():
     try:
         shutil.rmtree(get_abs_path("../pipeline_data"))
     except Exception:
@@ -325,8 +348,13 @@ def test_merge_with_primary_key():
     assert count_by_run_id[2][1] == 1
     ##############################
 
+    try:
+        os.remove(abs_db_path)
+    except Exception:
+        pass
+
 
-def test_delete_insert_without_primary_key():
+def test_delete_insert_without_primary_key_duckdb_to_duckdb():
     try:
         shutil.rmtree(get_abs_path("../pipeline_data"))
     except Exception:
@@ -435,8 +463,13 @@ def test_delete_insert_without_primary_key():
     assert count_by_run_id[1][1] == 1
     ##############################
 
+    try:
+        os.remove(abs_db_path)
+    except Exception:
+        pass
 
-def test_delete_insert_with_timerange():
+
+def test_delete_insert_with_timerange_duckdb_to_duckdb():
     try:
         shutil.rmtree(get_abs_path("../pipeline_data"))
     except Exception:
@@ -593,3 +626,250 @@ def test_delete_insert_with_timerange():
     assert count_by_run_id[1][1] == 2
     assert count_by_run_id[2][1] == 2
     ##############################
+
+    try:
+        os.remove(abs_db_path)
+    except Exception:
+        pass
+
+
+### These are CSV-to-DuckDB tests
+def test_create_replace_csv_to_duckdb():
+    try:
+        shutil.rmtree(get_abs_path("../pipeline_data"))
+    except Exception:
+        pass
+
+    abs_db_path = get_abs_path("./testdata/test_create_replace_csv.db")
+    rel_db_path_to_command = "ingestr/testdata/test_create_replace_csv.db"
+    rel_source_path_to_command = "ingestr/testdata/create_replace.csv"
+
+    conn = duckdb.connect(abs_db_path)
+
+    result = invoke_ingest_command(
+        f"csv://{rel_source_path_to_command}",
+        "testschema.input",
+        f"duckdb:///{rel_db_path_to_command}",
+        "testschema.output",
+    )
+
+    assert result.exit_code == 0
+
+    res = conn.sql(
+        "select symbol, date, is_enabled, name from testschema.output"
+    ).fetchall()
+
+    # read CSV file
+    actual_rows = []
+    with open(get_abs_path("./testdata/create_replace.csv"), "r") as f:
+        reader = csv.reader(f, delimiter=",", quotechar='"')
+        next(reader, None)
+        for row in reader:
+            actual_rows.append(row)
+
+    # compare the CSV file with the DuckDB table
+    assert len(res) == len(actual_rows)
+    for i, row in enumerate(actual_rows):
+        assert res[i] == tuple(row)
+
+    try:
+        os.remove(abs_db_path)
+    except Exception:
+        pass
+
+
+def get_random_string(length):
+    letters = string.ascii_lowercase
+    result_str = "".join(random.choice(letters) for i in range(length))
+    return result_str
+
+
+def test_merge_with_primary_key_csv_to_duckdb():
+    try:
+        shutil.rmtree(get_abs_path("../pipeline_data"))
+    except Exception:
+        pass
+
+    dbname = f"test_merge_with_primary_key_csv{get_random_string(5)}.db"
+    abs_db_path = get_abs_path(f"./testdata/{dbname}")
+    rel_db_path_to_command = f"ingestr/testdata/{dbname}"
+    uri = f"duckdb:///{rel_db_path_to_command}"
+
+    conn = duckdb.connect(abs_db_path)
+
+    def run(source: str):
+        res = invoke_ingest_command(
+            source,
+            "whatever",  # table name doesnt matter for CSV
+            uri,
+            "testschema_merge.output",
+            "merge",
+            "date",
+            "symbol",
+        )
+        assert res.exit_code == 0
+        return res
+
+    def get_output_rows():
+        conn.execute("CHECKPOINT")
+        return conn.sql(
+            "select symbol, date, is_enabled, name from testschema_merge.output order by symbol asc"
+        ).fetchall()
+
+    def assert_output_equals_to_csv(path: str):
+        res = get_output_rows()
+        actual_rows = []
+        with open(get_abs_path(path), "r") as f:
+            reader = csv.reader(f, delimiter=",", quotechar='"')
+            next(reader, None)
+            for row in reader:
+                actual_rows.append(row)
+
+        assert len(res) == len(actual_rows)
+        for i, row in enumerate(actual_rows):
+            assert res[i] == tuple(row)
+
+    run("csv://ingestr/testdata/merge_part1.csv")
+    assert_output_equals_to_csv("./testdata/merge_part1.csv")
+
+    first_run_id = conn.sql(
+        "select _dlt_load_id from testschema_merge.output limit 1"
+    ).fetchall()[0][0]
+
+    ##############################
+    # we'll run again, we don't expect any changes since the data hasn't changed
+    run("csv://ingestr/testdata/merge_part1.csv")
+    assert_output_equals_to_csv("./testdata/merge_part1.csv")
+
+    # we also ensure that the other rows were not touched
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_merge.output group by 1"
+    ).fetchall()
+    assert len(count_by_run_id) == 1
+    assert count_by_run_id[0][1] == 3
+    assert count_by_run_id[0][0] == first_run_id
+    ##############################
+
+    ##############################
+    # now we'll run the same ingestion but with a different file this time
+
+    run("csv://ingestr/testdata/merge_part2.csv")
+    assert_output_equals_to_csv("./testdata/merge_expected.csv")
+
+    # let's check the runs
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema_merge.output group by 1 order by 1 asc"
+    ).fetchall()
+
+    # we expect that there's a new load ID now
+    assert len(count_by_run_id) == 2
+
+    # there should be only one row with the first load ID
+    assert count_by_run_id[0][1] == 1
+    assert count_by_run_id[0][0] == first_run_id
+
+    # there should be a new run with the rest, 2 rows updated + 1 new row
+    assert count_by_run_id[1][1] == 3
+    ##############################
+
+    try:
+        os.remove(abs_db_path)
+    except Exception:
+        pass
+
+
+def test_delete_insert_without_primary_key_csv_to_duckdb():
+    try:
+        shutil.rmtree(get_abs_path("../pipeline_data"))
+    except Exception:
+        pass
+
+    dbname = f"test_merge_with_primary_key_csv{get_random_string(5)}.db"
+    abs_db_path = get_abs_path(f"./testdata/{dbname}")
+    rel_db_path_to_command = f"ingestr/testdata/{dbname}"
+    uri = f"duckdb:///{rel_db_path_to_command}"
+
+    conn = duckdb.connect(abs_db_path)
+
+    def run(source: str):
+        res = invoke_ingest_command(
+            source,
+            "whatever",  # table name doesnt matter for CSV
+            uri,
+            "testschema.output",
+            "delete+insert",
+            "date",
+        )
+        assert res.exit_code == 0
+        return res
+
+    def get_output_rows():
+        conn.execute("CHECKPOINT")
+        return conn.sql(
+            "select symbol, date, is_enabled, name from testschema.output order by symbol asc"
+        ).fetchall()
+
+    def assert_output_equals_to_csv(path: str):
+        res = get_output_rows()
+        actual_rows = []
+        with open(get_abs_path(path), "r") as f:
+            reader = csv.reader(f, delimiter=",", quotechar='"')
+            next(reader, None)
+            for row in reader:
+                actual_rows.append(row)
+
+        assert len(res) == len(actual_rows)
+        for i, row in enumerate(actual_rows):
+            assert res[i] == tuple(row)
+
+    run("csv://ingestr/testdata/delete_insert_part1.csv")
+    assert_output_equals_to_csv("./testdata/delete_insert_part1.csv")
+
+    first_run_id = conn.sql(
+        "select _dlt_load_id from testschema.output limit 1"
+    ).fetchall()[0][0]
+
+    ##############################
+    # we'll run again, we expect the data to be the same, but a new load_id to exist
+    # this is due to the fact that the old data won't be touched, but the ones with the
+    # latest value will be rewritten
+    run("csv://ingestr/testdata/delete_insert_part1.csv")
+    assert_output_equals_to_csv("./testdata/delete_insert_part1.csv")
+
+    # we also ensure that the other rows were not touched
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema.output group by 1 order by 1 asc"
+    ).fetchall()
+
+    assert len(count_by_run_id) == 2
+    assert count_by_run_id[0][1] == 1
+    assert count_by_run_id[0][0] == first_run_id
+    assert count_by_run_id[1][1] == 3
+    ##############################
+
+    ##############################
+    # now we'll run the same ingestion but with a different file this time
+
+    run("csv://ingestr/testdata/delete_insert_part2.csv")
+    assert_output_equals_to_csv("./testdata/delete_insert_expected.csv")
+
+    # let's check the runs
+    count_by_run_id = conn.sql(
+        "select _dlt_load_id, count(*) from testschema.output group by 1 order by 1 asc"
+    ).fetchall()
+
+    # we expect that there's a new load ID now
+    assert len(count_by_run_id) == 2
+
+    # there should be only one row with the first load ID, oldest date
+    assert count_by_run_id[0][1] == 1
+    assert count_by_run_id[0][0] == first_run_id
+
+    # there should be a new run with the rest, 3 rows updated + 1 new row
+    assert count_by_run_id[1][1] == 4
+    ##############################
+
+    try:
+        os.remove(abs_db_path)
+    except Exception:
+        pass

{ingestr-0.6.0 → ingestr-0.6.2}/ingestr/src/factory.py

@@ -16,6 +16,7 @@ from ingestr.src.destinations import (
 )
 from ingestr.src.sources import (
     GoogleSheetsSource,
+    GorgiasSource,
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
@@ -45,6 +46,9 @@ class SourceProtocol(Protocol):
     def dlt_source(self, uri: str, table: str, **kwargs):
         pass
 
+    def handles_incrementality(self) -> bool:
+        pass
+
 
 class DestinationProtocol(Protocol):
     def dlt_dest(self, uri: str, **kwargs) -> Destination:
@@ -94,6 +98,8 @@ class SourceDestinationFactory:
             return GoogleSheetsSource()
         elif self.source_scheme == "shopify":
             return ShopifySource()
+        elif self.source_scheme == "gorgias":
+            return GorgiasSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")