ingestr 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

ingestr/main.py

@@ -347,7 +347,7 @@ def ingest(
                     print(f"[bold red]  {job.job_file_info.job_id()}[/bold red]")
                     print(f"    [bold yellow]Error:[/bold yellow] {job.failed_message}")
 
-                raise typer.Exit()
+                raise typer.Exit(1)
 
         destination.post_load()
 

ingestr/src/factory.py

@@ -30,6 +30,7 @@ SQL_SOURCE_SCHEMES = [
     "mssql",
     "mysql",
     "mysql+pymysql",
+    "mysql+mysqlconnector",
     "postgres",
     "postgresql",
     "redshift",

ingestr/src/gorgias/__init__.py

@@ -181,7 +181,7 @@ def gorgias_source(
             },
             "from_agent": {
                 "data_type": "bool",
-                "nullable": False,
+                "nullable": True,
                 "description": "Indicates if the ticket was created by an agent",
             },
             "customer": {

ingestr/src/sources.py

@@ -12,6 +12,7 @@ from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.shopify import shopify_source
 from ingestr.src.sql_database import sql_table
+from ingestr.src.table_definition import table_string_to_dataclass
 
 
 class SqlSource:
@@ -24,9 +25,7 @@ class SqlSource:
         return False
 
     def dlt_source(self, uri: str, table: str, **kwargs):
-        table_fields = table.split(".")
-        if len(table_fields) != 2:
-            raise ValueError("Table name must be in the format schema.table")
+        table_fields = table_string_to_dataclass(table)
 
         incremental = None
         if kwargs.get("incremental_key"):
@@ -45,8 +44,8 @@ class SqlSource:
 
         table_instance = self.table_builder(
             credentials=uri,
-            schema=table_fields[-2],
-            table=table_fields[-1],
+            schema=table_fields.dataset,
+            table=table_fields.table,
             incremental=incremental,
             merge_key=kwargs.get("merge_key"),
             backend=kwargs.get("sql_backend", "sqlalchemy"),
@@ -65,9 +64,7 @@ class MongoDbSource:
         return False
 
     def dlt_source(self, uri: str, table: str, **kwargs):
-        table_fields = table.split(".")
-        if len(table_fields) != 2:
-            raise ValueError("Table name must be in the format schema.table")
+        table_fields = table_string_to_dataclass(table)
 
         incremental = None
         if kwargs.get("incremental_key"):
@@ -82,8 +79,8 @@ class MongoDbSource:
 
         table_instance = self.table_builder(
             connection_url=uri,
-            database=table_fields[-2],
-            collection=table_fields[-1],
+            database=table_fields.dataset,
+            collection=table_fields.table,
             parallel=True,
             incremental=incremental,
         )
@@ -293,15 +290,10 @@ class GoogleSheetsSource:
                 base64.b64decode(credentials_base64[0]).decode("utf-8")
             )
 
-        table_fields = table.split(".")
-        if len(table_fields) != 2:
-            raise ValueError(
-                "Table name must be in the format <spreadsheet_id>.<sheet_name>"
-            )
-
+        table_fields = table_string_to_dataclass(table)
         return self.table_builder(
             credentials=credentials,
-            spreadsheet_url_or_id=table_fields[0],
-            range_names=[table_fields[1]],
+            spreadsheet_url_or_id=table_fields.table,
+            range_names=[table_fields.dataset],
             get_named_ranges=False,
         )

ingestr/src/sources_test.py

@@ -13,10 +13,6 @@ class SqlSourceTest(unittest.TestCase):
             uri = "bigquery://my-project"
             source.dlt_source(uri, "onetable")
 
-        with pytest.raises(ValueError):
-            uri = "bigquery://my-project"
-            source.dlt_source(uri, "onetable.with.too.many.fields")
-
     def test_table_instance_is_created(self):
         uri = "bigquery://my-project"
         table = "schema.table"
@@ -63,10 +59,6 @@ class MongoDbSourceTest(unittest.TestCase):
             uri = "mongodb://my-project"
             source.dlt_source(uri, "onetable")
 
-        with pytest.raises(ValueError):
-            uri = "mongodb://my-project"
-            source.dlt_source(uri, "onetable.with.too.many.fields")
-
     def test_table_instance_is_created(self):
         uri = "mongodb://my-project"
         table = "schema.table"

ingestr/src/table_definition.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class TableDefinition:
+    dataset: str
+    table: str
+
+
+def table_string_to_dataclass(table: str) -> TableDefinition:
+    table_fields = table.split(".", 1)
+    if len(table_fields) != 2:
+        raise ValueError("Table name must be in the format <schema>.<table>")
+
+    return TableDefinition(dataset=table_fields[0], table=table_fields[1])

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.6.4"
+__version__ = "0.6.6"

{ingestr-0.6.4.dist-info → ingestr-0.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.4
+Version: 0.6.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -21,6 +21,7 @@ Requires-Dist: duckdb-engine==0.11.5
 Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
+Requires-Dist: mysql-connector-python==9.0.0
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.9
 Requires-Dist: py-machineid==0.5.1
@@ -171,6 +172,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <tr>
+        <td>Gorgias</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>Google Sheets</td>
         <td>✅</td>

{ingestr-0.6.4.dist-info → ingestr-0.6.6.dist-info}/RECORD

@@ -1,18 +1,19 @@
-ingestr/main.py,sha256=jok8jEPLCOXlhKyKzR2mWA36A9DauRc4DI4AY_r4ubM,14861
+ingestr/main.py,sha256=PvZtqrlHO3aSFtdhIdLAhzsOLG1hP_ql24vof76SFaI,14862
 ingestr/main_test.py,sha256=MDV2Eo86W_CcxGgEkYYoBc6xIXjVMER4hMhgAdxXYMc,28464
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
 ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
-ingestr/src/factory.py,sha256=7skwetBXFIwAuKyTFfffGPgSo_PRRZ5uEnxOHUv28yQ,3517
+ingestr/src/factory.py,sha256=w6xb8scWa_uG4otoMjqvqcx99hofrN2YcLzaB7QSgYo,3545
 ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
-ingestr/src/sources.py,sha256=v0oFu-Pt_Zv06FypBb9_rHKP_iIMpkgm9MMaVpOUjPg,10313
-ingestr/src/sources_test.py,sha256=t94u1lYAspxzfe-DkxVtq5vw6xrLWphipvwntrwrzqg,3930
-ingestr/src/version.py,sha256=WMmvm2Keb76yMz8OL_h4fKT34Xpi-1BVfCiTn2QGzz4,22
+ingestr/src/sources.py,sha256=sLT8CDiLz6aTekm6QZCXatZAdTyzCabqjJesnEe88GU,10061
+ingestr/src/sources_test.py,sha256=wZopz4tFtFnOiCEO8pZW816Nj86pLlR5fEqfq0TtWMQ,3629
+ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
+ingestr/src/version.py,sha256=I3h5MyD10PkOUQEBnR6L9ja7s4WeTEg8rRjRKTCWYWQ,22
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
 ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
-ingestr/src/gorgias/__init__.py,sha256=iASGYJhdx6yZ0uDgIaQQfW91l3p0XOhowalIByLFkvw,21228
+ingestr/src/gorgias/__init__.py,sha256=BzX9X1Yc_1Mch6NP1pn26hjRIiaadErgHxkdJHw4P3o,21227
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
 ingestr/src/gorgias/helpers_test.py,sha256=kSR2nhB8U8HZ8pgDnd7HvXlzojmBnpOm8fTKHJvvKGY,1580
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
@@ -40,8 +41,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.6.4.dist-info/METADATA,sha256=P9HvhFDdyABsqWup6JK4-LgdaCDH_UBvTByLtFvxbW4,5699
-ingestr-0.6.4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.6.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.6.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.6.4.dist-info/RECORD,,
+ingestr-0.6.6.dist-info/METADATA,sha256=o9eKbLqKlQ-TkYoaLHFLeQAjuraP9UZhM1jsCL2v7Gg,5830
+ingestr-0.6.6.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.6.6.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.6.6.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.6.6.dist-info/RECORD,,