ingestr 0.6.4__tar.gz → 0.6.6__tar.gz

{ingestr-0.6.4 → ingestr-0.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.4
+Version: 0.6.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -21,6 +21,7 @@ Requires-Dist: duckdb-engine==0.11.5
 Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
+Requires-Dist: mysql-connector-python==9.0.0
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.9
 Requires-Dist: py-machineid==0.5.1
@@ -171,6 +172,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <tr>
+        <td>Gorgias</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>Google Sheets</td>
         <td>✅</td>

{ingestr-0.6.4 → ingestr-0.6.6}/README.md

@@ -128,6 +128,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <tr>
+        <td>Gorgias</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>Google Sheets</td>
         <td>✅</td>

{ingestr-0.6.4 → ingestr-0.6.6}/docs/supported-sources/overview.md

@@ -79,6 +79,11 @@ ingestr supports the following sources and destinations:
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <tr>
+        <td>Gorgias</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>Google Sheets</td>
         <td>✅</td>

{ingestr-0.6.4 → ingestr-0.6.6}/ingestr/main.py

@@ -347,7 +347,7 @@ def ingest(
                     print(f"[bold red]  {job.job_file_info.job_id()}[/bold red]")
                     print(f"    [bold yellow]Error:[/bold yellow] {job.failed_message}")
 
-                raise typer.Exit()
+                raise typer.Exit(1)
 
         destination.post_load()
 

{ingestr-0.6.4 → ingestr-0.6.6}/ingestr/src/factory.py

@@ -30,6 +30,7 @@ SQL_SOURCE_SCHEMES = [
     "mssql",
     "mysql",
     "mysql+pymysql",
+    "mysql+mysqlconnector",
     "postgres",
     "postgresql",
     "redshift",

{ingestr-0.6.4 → ingestr-0.6.6}/ingestr/src/gorgias/__init__.py

@@ -181,7 +181,7 @@ def gorgias_source(
             },
             "from_agent": {
                 "data_type": "bool",
-                "nullable": False,
+                "nullable": True,
                 "description": "Indicates if the ticket was created by an agent",
             },
             "customer": {

{ingestr-0.6.4 → ingestr-0.6.6}/ingestr/src/sources.py

@@ -12,6 +12,7 @@ from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.shopify import shopify_source
 from ingestr.src.sql_database import sql_table
+from ingestr.src.table_definition import table_string_to_dataclass
 
 
 class SqlSource:
@@ -24,9 +25,7 @@ class SqlSource:
         return False
 
     def dlt_source(self, uri: str, table: str, **kwargs):
-        table_fields = table.split(".")
-        if len(table_fields) != 2:
-            raise ValueError("Table name must be in the format schema.table")
+        table_fields = table_string_to_dataclass(table)
 
         incremental = None
         if kwargs.get("incremental_key"):
@@ -45,8 +44,8 @@ class SqlSource:
 
         table_instance = self.table_builder(
             credentials=uri,
-            schema=table_fields[-2],
-            table=table_fields[-1],
+            schema=table_fields.dataset,
+            table=table_fields.table,
             incremental=incremental,
             merge_key=kwargs.get("merge_key"),
             backend=kwargs.get("sql_backend", "sqlalchemy"),
@@ -65,9 +64,7 @@ class MongoDbSource:
         return False
 
     def dlt_source(self, uri: str, table: str, **kwargs):
-        table_fields = table.split(".")
-        if len(table_fields) != 2:
-            raise ValueError("Table name must be in the format schema.table")
+        table_fields = table_string_to_dataclass(table)
 
         incremental = None
         if kwargs.get("incremental_key"):
@@ -82,8 +79,8 @@ class MongoDbSource:
 
         table_instance = self.table_builder(
             connection_url=uri,
-            database=table_fields[-2],
-            collection=table_fields[-1],
+            database=table_fields.dataset,
+            collection=table_fields.table,
             parallel=True,
             incremental=incremental,
         )
@@ -293,15 +290,10 @@ class GoogleSheetsSource:
                 base64.b64decode(credentials_base64[0]).decode("utf-8")
             )
 
-        table_fields = table.split(".")
-        if len(table_fields) != 2:
-            raise ValueError(
-                "Table name must be in the format <spreadsheet_id>.<sheet_name>"
-            )
-
+        table_fields = table_string_to_dataclass(table)
         return self.table_builder(
             credentials=credentials,
-            spreadsheet_url_or_id=table_fields[0],
-            range_names=[table_fields[1]],
+            spreadsheet_url_or_id=table_fields.table,
+            range_names=[table_fields.dataset],
             get_named_ranges=False,
         )

{ingestr-0.6.4 → ingestr-0.6.6}/ingestr/src/sources_test.py

@@ -13,10 +13,6 @@ class SqlSourceTest(unittest.TestCase):
             uri = "bigquery://my-project"
             source.dlt_source(uri, "onetable")
 
-        with pytest.raises(ValueError):
-            uri = "bigquery://my-project"
-            source.dlt_source(uri, "onetable.with.too.many.fields")
-
     def test_table_instance_is_created(self):
         uri = "bigquery://my-project"
         table = "schema.table"
@@ -63,10 +59,6 @@ class MongoDbSourceTest(unittest.TestCase):
             uri = "mongodb://my-project"
             source.dlt_source(uri, "onetable")
 
-        with pytest.raises(ValueError):
-            uri = "mongodb://my-project"
-            source.dlt_source(uri, "onetable.with.too.many.fields")
-
     def test_table_instance_is_created(self):
         uri = "mongodb://my-project"
         table = "schema.table"

ingestr-0.6.6/ingestr/src/table_definition.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class TableDefinition:
+    dataset: str
+    table: str
+
+
+def table_string_to_dataclass(table: str) -> TableDefinition:
+    table_fields = table.split(".", 1)
+    if len(table_fields) != 2:
+        raise ValueError("Table name must be in the format <schema>.<table>")
+
+    return TableDefinition(dataset=table_fields[0], table=table_fields[1])

ingestr-0.6.6/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.6.6"

{ingestr-0.6.4 → ingestr-0.6.6}/requirements.txt

@@ -23,3 +23,4 @@ sqlalchemy2-stubs==0.0.2a38
 tqdm==4.66.2
 typer==0.12.3
 sqlalchemy-hana==2.0.0
+mysql-connector-python==9.0.0

ingestr-0.6.4/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.6.4"