ingestr 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

ingestr/src/sources.py

@@ -1,14 +1,16 @@
 import base64
 import csv
 import json
-from typing import Callable
+from typing import Any, Callable, Optional
 from urllib.parse import parse_qs, urlparse
 
 import dlt
 
 from ingestr.src.google_sheets import google_spreadsheet
+from ingestr.src.gorgias import gorgias_source
 from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
+from ingestr.src.shopify import shopify_source
 from ingestr.src.sql_database import sql_table
 
 
@@ -18,6 +20,9 @@ class SqlSource:
     def __init__(self, table_builder=sql_table) -> None:
         self.table_builder = table_builder
 
+    def handles_incrementality(self) -> bool:
+        return False
+
     def dlt_source(self, uri: str, table: str, **kwargs):
         table_fields = table.split(".")
         if len(table_fields) != 2:
@@ -56,6 +61,9 @@ class MongoDbSource:
     def __init__(self, table_builder=mongodb_collection) -> None:
         self.table_builder = table_builder
 
+    def handles_incrementality(self) -> bool:
+        return False
+
     def dlt_source(self, uri: str, table: str, **kwargs):
         table_fields = table.split(".")
         if len(table_fields) != 2:
@@ -84,18 +92,43 @@ class MongoDbSource:
 
 
 class LocalCsvSource:
+    def handles_incrementality(self) -> bool:
+        return False
+
     def dlt_source(self, uri: str, table: str, **kwargs):
-        def csv_file():
+        def csv_file(
+            incremental: Optional[dlt.sources.incremental[Any]] = None,
+        ):
             file_path = uri.split("://")[1]
             myFile = open(file_path, "r")
             reader = csv.DictReader(myFile)
-            print("running resource")
+            if not reader.fieldnames:
+                raise RuntimeError(
+                    "failed to extract headers from the CSV, are you sure the given file contains a header row?"
+                )
+
+            incremental_key = kwargs.get("incremental_key")
+            if incremental_key and incremental_key not in reader.fieldnames:
+                raise ValueError(
+                    f"incremental_key '{incremental_key}' not found in the CSV file"
+                )
 
             page_size = 1000
             page = []
             current_items = 0
             for dictionary in reader:
                 if current_items < page_size:
+                    if incremental_key and incremental and incremental.start_value:
+                        inc_value = dictionary.get(incremental_key)
+                        if inc_value is None:
+                            raise ValueError(
+                                f"incremental_key '{incremental_key}' not found in the CSV file"
+                            )
+
+                        print("BURAYA GELLDIII")
+                        if inc_value < incremental.start_value:
+                            continue
+
                     page.append(dictionary)
                     current_items += 1
                 else:
@@ -109,6 +142,12 @@ class LocalCsvSource:
         return dlt.resource(
             csv_file,
             merge_key=kwargs.get("merge_key"),  # type: ignore
+        )(
+            incremental=dlt.sources.incremental(
+                kwargs.get("incremental_key", ""),
+                initial_value=kwargs.get("interval_start"),
+                end_value=kwargs.get("interval_end"),
+            )
         )
 
 
@@ -118,6 +157,9 @@ class NotionSource:
     def __init__(self, table_builder=notion_databases) -> None:
         self.table_builder = table_builder
 
+    def handles_incrementality(self) -> bool:
+        return True
+
     def dlt_source(self, uri: str, table: str, **kwargs):
         if kwargs.get("incremental_key"):
             raise ValueError("Incremental loads are not supported for Notion")
@@ -134,12 +176,100 @@ class NotionSource:
         )
 
 
+class ShopifySource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Shopify takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        # shopify://shop_url?api_key=private_app_password
+
+        source_fields = urlparse(uri)
+        source_params = parse_qs(source_fields.query)
+        api_key = source_params.get("api_key")
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Shopify")
+
+        date_args = {}
+        if kwargs.get("interval_start"):
+            date_args["start_date"] = kwargs.get("interval_start")
+
+        if kwargs.get("interval_end"):
+            date_args["end_date"] = kwargs.get("interval_end")
+
+        resource = None
+        if table in ["products", "orders", "customers"]:
+            resource = table
+        else:
+            raise ValueError(
+                f"Table name '{table}' is not supported for Shopify source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        return shopify_source(
+            private_app_password=api_key[0],
+            shop_url=f"https://{source_fields.netloc}",
+            **date_args,
+        ).with_resources(resource)
+
+
+class GorgiasSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Gorgias takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        # gorgias://domain?api_key=<api_key>&email=<email>
+
+        source_fields = urlparse(uri)
+        source_params = parse_qs(source_fields.query)
+        api_key = source_params.get("api_key")
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Gorgias")
+
+        email = source_params.get("email")
+        if not email:
+            raise ValueError("email in the URI is required to connect to Gorgias")
+
+        resource = None
+        if table in ["customers", "tickets", "ticket_messages", "satisfaction_surveys"]:
+            resource = table
+        else:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Gorgias source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        date_args = {}
+        if kwargs.get("interval_start"):
+            date_args["start_date"] = kwargs.get("interval_start")
+
+        if kwargs.get("interval_end"):
+            date_args["end_date"] = kwargs.get("interval_end")
+
+        return gorgias_source(
+            domain=source_fields.netloc,
+            email=email[0],
+            api_key=api_key[0],
+            **date_args,
+        ).with_resources(resource)
+
+
 class GoogleSheetsSource:
     table_builder: Callable
 
     def __init__(self, table_builder=google_spreadsheet) -> None:
         self.table_builder = table_builder
 
+    def handles_incrementality(self) -> bool:
+        return False
+
     def dlt_source(self, uri: str, table: str, **kwargs):
         if kwargs.get("incremental_key"):
             raise ValueError("Incremental loads are not supported for Google Sheets")

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.5.1"
+__version__ = "0.6.1"

ingestr/testdata/create_replace.csv

@@ -0,0 +1,21 @@
+"symbol","date","isEnabled","name"
+"A","2024-04-19","True","AGILENT TECHNOLOGIES INC"
+"AA","2024-04-19","True","ALCOA CORP"
+"AAA","2024-04-19","True","ALTERNATIVE ACCESS FIRST PRI"
+"AAAU","2024-04-19","True","GOLDMAN SACHS PHYSICAL GOLD"
+"AACG","2024-04-19","True","ATA CREATIVITY GLOBAL - ADR"
+"AACI","2024-04-19","True","ARMADA ACQUISITION CORP I"
+"AACIU","2024-04-19","True","ARMADA ACQUISITION CORP I"
+"AACIW","2024-04-19","True",""
+"AACT","2024-04-19","True","ARES ACQUISITION CORP II"
+"AACT+","2024-04-19","True",""
+"AACT=","2024-04-19","True","ARES ACQUISITION CORP II"
+"AADI","2024-04-19","True","AADI BIOSCIENCE INC"
+"AADR","2024-04-19","True","ADVISORSHARES DORSEY WRIGHT"
+"AAGR","2024-04-19","True","AFRICAN AGRICULTURE HOLDINGS"
+"AAGRW","2024-04-19","True",""
+"AAL","2024-04-19","True","AMERICAN AIRLINES GROUP INC"
+"AAMC","2024-04-19","True","ALTISOURCE ASSET MANAGEMENT"
+"AAME","2024-04-19","True","ATLANTIC AMERICAN CORP"
+"AAN","2024-04-19","True","AARON'S CO INC/THE"
+"AAOI","2024-04-19","True","APPLIED OPTOELECTRONICS INC"

ingestr/testdata/delete_insert_expected.csv

@@ -0,0 +1,6 @@
+"symbol","date","isEnabled","name"
+"A","2024-04-20","True","AGILENT TECHNOLOGIES INC____updated"
+"AA","2024-04-19","True","ALCOA CORP____updated"
+"AAA","2024-04-21","True","ALTERNATIVE ACCESS FIRST PRI____updated"
+"AAAU","2024-04-22","True","GOLDMAN SACHS PHYSICAL GOLD____updated"
+"B","2024-04-18","True","SOME TECHNOLOGIES INC"

ingestr/testdata/delete_insert_part1.csv

@@ -0,0 +1,5 @@
+"symbol","date","isEnabled","name"
+"A","2024-04-19","True","AGILENT TECHNOLOGIES INC"
+"AA","2024-04-19","True","ALCOA CORP"
+"AAA","2024-04-19","True","ALTERNATIVE ACCESS FIRST PRI"
+"B","2024-04-18","True","SOME TECHNOLOGIES INC"

ingestr/testdata/delete_insert_part2.csv

@@ -0,0 +1,6 @@
+"symbol","date","isEnabled","name"
+"A","2024-04-20","True","AGILENT TECHNOLOGIES INC____updated"
+"AA","2024-04-19","True","ALCOA CORP____updated"
+"AAA","2024-04-21","True","ALTERNATIVE ACCESS FIRST PRI____updated"
+"AAAU","2024-04-22","True","GOLDMAN SACHS PHYSICAL GOLD____updated"
+"BBB","2024-04-18","True","SOME CORP____updated"

ingestr/testdata/merge_expected.csv

@@ -0,0 +1,5 @@
+"symbol","date","isEnabled","name"
+"A","2024-04-20","True","AGILENT TECHNOLOGIES INC____updated"
+"AA","2024-04-19","True","ALCOA CORP"
+"AAA","2024-04-21","True","ALTERNATIVE ACCESS FIRST PRI____updated"
+"AAAU","2024-04-22","True","GOLDMAN SACHS PHYSICAL GOLD____updated"

ingestr/testdata/merge_part1.csv

@@ -0,0 +1,4 @@
+"symbol","date","isEnabled","name"
+"A","2024-04-19","True","AGILENT TECHNOLOGIES INC"
+"AA","2024-04-19","True","ALCOA CORP"
+"AAA","2024-04-19","True","ALTERNATIVE ACCESS FIRST PRI"

ingestr/testdata/merge_part2.csv

@@ -0,0 +1,5 @@
+"symbol","date","isEnabled","name"
+"A","2024-04-20","True","AGILENT TECHNOLOGIES INC____updated"
+"AA","2024-04-19","True","ALCOA CORP____updated"
+"AAA","2024-04-21","True","ALTERNATIVE ACCESS FIRST PRI____updated"
+"AAAU","2024-04-22","True","GOLDMAN SACHS PHYSICAL GOLD____updated"

{ingestr-0.5.1.dist-info → ingestr-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.5.1
+Version: 0.6.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,7 +16,7 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==0.4.8
+Requires-Dist: dlt==0.4.12
 Requires-Dist: duckdb-engine==0.11.5
 Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-api-python-client==2.130.0
@@ -27,6 +27,7 @@ Requires-Dist: py-machineid==0.5.1
 Requires-Dist: pymongo==4.6.3
 Requires-Dist: pymysql==1.1.0
 Requires-Dist: pyodbc==5.1.0
+Requires-Dist: pyrate-limiter==3.6.1
 Requires-Dist: redshift-connector==2.1.0
 Requires-Dist: rich==13.7.1
 Requires-Dist: rudder-sdk-python==2.1.0
@@ -180,6 +181,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Shopify</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
 </table>
 
 More to come soon!

{ingestr-0.5.1.dist-info → ingestr-0.6.1.dist-info}/RECORD

@@ -1,17 +1,20 @@
-ingestr/main.py,sha256=JkjNzsw-Leg0mws9lYNSn4NSFVJ_Svg7ZKvQkzDRE7o,14407
-ingestr/main_test.py,sha256=fXZw1qZd5CElrFSRnsI5003813LcIMGpGCMnGNFVhNI,19946
-ingestr/src/destinations.py,sha256=_PIoAU-_tDEyX_-vDOgGB5eqXoGhPwtCRApHufj1ae4,6350
+ingestr/main.py,sha256=jok8jEPLCOXlhKyKzR2mWA36A9DauRc4DI4AY_r4ubM,14861
+ingestr/main_test.py,sha256=MDV2Eo86W_CcxGgEkYYoBc6xIXjVMER4hMhgAdxXYMc,28464
+ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
 ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
-ingestr/src/factory.py,sha256=ueoGtf6PNn1InyMLW5g-nNENEl6oJSZFKWABiCuOPkk,3257
+ingestr/src/factory.py,sha256=7skwetBXFIwAuKyTFfffGPgSo_PRRZ5uEnxOHUv28yQ,3517
 ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
-ingestr/src/sources.py,sha256=fZlOpduijX2xIkqpx5f-tZpeJ7rKtqOt67ukVV-d65k,5552
+ingestr/src/sources.py,sha256=v0oFu-Pt_Zv06FypBb9_rHKP_iIMpkgm9MMaVpOUjPg,10313
 ingestr/src/sources_test.py,sha256=t94u1lYAspxzfe-DkxVtq5vw6xrLWphipvwntrwrzqg,3930
-ingestr/src/version.py,sha256=eZ1bOun1DDVV0YLOBW4wj2FP1ajReLjbIrGmzN7ASBw,22
+ingestr/src/version.py,sha256=baAcEjLSYFIeNZF51tOMmA_zAMhN8HvKael-UU-Ruec,22
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
 ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
+ingestr/src/gorgias/__init__.py,sha256=el_rJOTurK3P2tw-CtwjTpTcc7q6w2nLlJPXCU0yEqY,21228
+ingestr/src/gorgias/helpers.py,sha256=dQ56CpZJobBVZP-vCM56vKHZdrfCHqla5peUt0bVr1c,4905
+ingestr/src/gorgias/helpers_test.py,sha256=kSR2nhB8U8HZ8pgDnd7HvXlzojmBnpOm8fTKHJvvKGY,1580
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
 ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
@@ -19,6 +22,10 @@ ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMO
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
+ingestr/src/shopify/__init__.py,sha256=EWjpvZz7K6Pms7uUoqqkM4Wj0XeE2NrDvVp4BNM8dPk,9163
+ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
+ingestr/src/shopify/helpers.py,sha256=OO_Tw-HwVLnRhwT3vqUWEQEEcWIS9KWE6VDDe8BCC2w,4972
+ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
 ingestr/src/sql_database/__init__.py,sha256=S5MVJr8juPSs61C2D7pInsTwNEHetChK6RjjhPAD0Lg,8845
 ingestr/src/sql_database/helpers.py,sha256=tbn-GjjBIVu3hVVh5vrUSiZqQ32_Tp0oBP5Fvv_wY4E,8986
 ingestr/src/sql_database/override.py,sha256=xbKGDztCzvrhJ5kJTXERal3LA56bEeVug4_rrTs8DgA,333
@@ -26,13 +33,15 @@ ingestr/src/sql_database/schema_types.py,sha256=foGHh4iGagGLfS7nF3uGYhBjqgX0jlrj
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
-ingestr/testdata/test_append.db,sha256=RubpUvWP7BlfMHAOkgeEiHhzPSMs6-Fc8f9SF9ZXwxA,798720
-ingestr/testdata/test_create_replace.db,sha256=hrAgcjYTnIzuGbJDpMB-txkKXhM_tn8H3sn0Go4G5uU,536576
-ingestr/testdata/test_delete_insert_with_timerange.db,sha256=a7ytHTvLj2P04lHNPxRgtPmkiSS_oUuw3Jo3EWVZ-GQ,1585152
-ingestr/testdata/test_delete_insert_without_primary_key.db,sha256=OaYWvAdrUTVbz3QzGf6e5W2Vk1nVf0n86UgxLhMQ73E,1847296
-ingestr/testdata/test_merge_with_primary_key.db,sha256=7zskkxks61uxsZYH8gNslNE35KyK0_r8UIGQtdph7zc,1585152
-ingestr-0.5.1.dist-info/METADATA,sha256=vX5T3KAA1Udi5qqgeZx5pdSx5WbYdfy0VysVrhEf_B4,5575
-ingestr-0.5.1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-ingestr-0.5.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.5.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.5.1.dist-info/RECORD,,
+ingestr/testdata/create_replace.csv,sha256=TQDbOSkRKq9ZZv1d68Qjwh94aIyUQ-oEwxpJIrd3YK8,1060
+ingestr/testdata/delete_insert_expected.csv,sha256=wbj7uboVWwm3sNMh1n7f4-OKFEQJv1s96snjEHp9nkg,336
+ingestr/testdata/delete_insert_part1.csv,sha256=mdLFGu6ZOU6zwAigLRq4hFkAbVgyIaoEE39UCeeRc7s,234
+ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmaySYCgtLsNqSrk,337
+ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
+ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
+ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
+ingestr-0.6.1.dist-info/METADATA,sha256=SzZWTcIS64RXkeC_qMSPRUrm_PzD9q7e-11bxF0j2F4,5699
+ingestr-0.6.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.6.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.6.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.6.1.dist-info/RECORD,,

{ingestr-0.5.1.dist-info → ingestr-0.6.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.2
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any