ingestr 0.12.5__py3-none-any.whl → 0.12.7__py3-none-any.whl

ingestr/src/stripe_analytics/__init__.py

@@ -84,7 +84,10 @@ def incremental_stripe_source(
     def incremental_resource(
         endpoint: str,
         created: Optional[Any] = dlt.sources.incremental(
-            "created", initial_value=start_date_unix
+            "created",
+            initial_value=start_date_unix,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Generator[Dict[Any, Any], Any, None]:
         start_value = created.last_value

ingestr/src/tiktok_ads/__init__.py

@@ -110,7 +110,12 @@ def tiktok_source(
     )
     def custom_reports(
         datetime=(
-            dlt.sources.incremental(incremental_loading_param, start_date)
+            dlt.sources.incremental(
+                incremental_loading_param,
+                start_date,
+                range_end="closed",
+                range_start="closed",
+            )
             if is_incremental
             else None
         ),

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.12.5"
+__version__ = "0.12.7"

ingestr/src/zendesk/__init__.py

@@ -260,6 +260,8 @@ def zendesk_support(
             initial_value=start_date_ts,
             end_value=end_date_ts,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItem]:
         # URL For ticket events
@@ -294,6 +296,8 @@ def zendesk_support(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItem]:
         """
@@ -340,6 +344,8 @@ def zendesk_support(
             initial_value=start_date_iso_str,
             end_value=end_date_iso_str,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItem]:
         """

{ingestr-0.12.5.dist-info → ingestr-0.12.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.5
+Version: 0.12.7
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -17,6 +17,7 @@ Requires-Python: >=3.9
 Requires-Dist: asana==3.2.3
 Requires-Dist: confluent-kafka>=2.6.1
 Requires-Dist: databricks-sql-connector==2.9.3
+Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: dlt==1.5.0
 Requires-Dist: duckdb-engine==0.13.5
 Requires-Dist: duckdb==1.1.3
@@ -26,6 +27,7 @@ Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: mysql-connector-python==9.1.0
 Requires-Dist: pendulum==3.0.0
+Requires-Dist: psutil==6.1.1
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3

{ingestr-0.12.5.dist-info → ingestr-0.12.7.dist-info}/RECORD

@@ -1,52 +1,57 @@
-ingestr/main.py,sha256=xLQiPHoD7dNvrHfNTwD379wHg6xZGmLxzPzQLq2E1RA,24746
+ingestr/main.py,sha256=fRWnyoPzMvvxTa61EIAP_dsKu0B_0yOwoyt0Slq9WQU,24723
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=zcHJIIHAZmcD9sJomd6G1Bc-1KsxnBD2aByOSV_9L3g,8850
-ingestr/src/factory.py,sha256=aE7TjHzONb4DKYcfh_6-CJJfvs4lmw7iUySvSm4yQbM,4516
+ingestr/src/errors.py,sha256=MrdLY5Gpr3g3qbYjl-U8-m8kxBJQOJo4ZVOsQpQbRR8,447
+ingestr/src/factory.py,sha256=jjxieXpSK02tNcg7f_t5xxqs49EnI739smRLX8qLsUU,4582
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=FXUTmII3DiEANZN37P9-dTFFRzpv0PL8bfaQvr0un8w,50761
+ingestr/src/sources.py,sha256=dMXTfykbAZTN8SNpOWJbtl10krdJfg12S13at3Z4L38,53647
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=QFQfu3CUVe9Ncr8kv3aaBY3oWrZmv8xboen_Uwy7eXU,23
+ingestr/src/version.py,sha256=WvcWmk-c-I-XXiq6oM9Y1VYwNWFe64tXpnG_RI47HhE,23
 ingestr/src/adjust/__init__.py,sha256=NaRNwDhItG8Q7vUHw7zQvyfWjmT32M0CSc5ufjmBM9U,3067
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
+ingestr/src/appstore/__init__.py,sha256=s39r3YUjdfStA6lBcPzqQzestiojC3U41LB3F6Y8gG0,4538
+ingestr/src/appstore/client.py,sha256=qY9nBZPNIAveR-Dn-pW141Mr9xi9LMOz2HHfnfueHvE,3975
+ingestr/src/appstore/errors.py,sha256=KVpPWth5qlv6_QWEm3aJAt3cdf6miPJs0UDzxknx2Ms,481
+ingestr/src/appstore/models.py,sha256=tW1JSATHBIxZ6a77-RTCBQptJk6iRC8fWcmx4NW7SVA,1716
+ingestr/src/appstore/resources.py,sha256=DJxnNrBohVV0uSeruGV-N_e7UHSlhMhjhYNYdBuqECU,5375
 ingestr/src/arrow/__init__.py,sha256=8fEntgHseKjFMiPQIzxYzw_raicNsEgnveLi1IzBca0,2848
-ingestr/src/asana_source/__init__.py,sha256=Y4Ti_876Yong420fQ2o4A97TdgrZNlZVxlTMLyXdSjA,8116
+ingestr/src/asana_source/__init__.py,sha256=QwQTCb5PXts8I4wLHG9UfRP-5ChfjSe88XAVfxMV5Ag,8183
 ingestr/src/asana_source/helpers.py,sha256=PukcdDQWIGqnGxuuobbLw4hUy4-t6gxXg_XywR7Lg9M,375
 ingestr/src/asana_source/settings.py,sha256=-2tpdkwh04RvLKFvwQodnFLYn9MaxOO1hsebGnDQMTU,2829
 ingestr/src/chess/__init__.py,sha256=y0Q8aKBigeKf3N7wuB_gadMQjVJzBPUT8Jhp1ObEWjk,6812
 ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,549
 ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
 ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
-ingestr/src/facebook_ads/__init__.py,sha256=ZZyogV48gmhDcC3CYQEsC4qT3Q6JI9IOnMff2NS1M-A,9207
+ingestr/src/facebook_ads/__init__.py,sha256=reEpSr4BaKA1wO3qVgCH51gW-TgWkbJ_g24UIhJWbac,9286
 ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
 ingestr/src/facebook_ads/helpers.py,sha256=ZLbNHiKer5lPb4g3_435XeBJr57Wv0o1KTyBA1mQ100,9068
 ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_tl6UTFk,4881
-ingestr/src/filesystem/__init__.py,sha256=wHHaKFuAjsR_ZRjl6g_Flf6FhVs9qhwREthTr03_7cc,4162
+ingestr/src/filesystem/__init__.py,sha256=hcN_sO356ChTPyg72AufrikdkFBBIScTdxtGfDm-W0E,4221
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
-ingestr/src/github/__init__.py,sha256=csA2VcjOxXrVrvp7zY-JodO9Lpy98bJ4AqRdHCLTcGM,5838
+ingestr/src/github/__init__.py,sha256=xVijF-Wi4p88hkVJnKH-oTixismjD3aUcGqGa6Wr4e4,5889
 ingestr/src/github/helpers.py,sha256=Tmnik9811zBWNO6cJwV9PFQxEx2j32LHAQCvNbubsEI,6759
 ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
 ingestr/src/github/settings.py,sha256=N5ahWrDIQ_4IWV9i-hTXxyYduqY9Ym2BTwqsWxcDdJ8,258
-ingestr/src/google_analytics/__init__.py,sha256=HjA13wfJm2MGfy3h_DiM5ekkNqM2dgwYCKJ3pprnDtI,2482
-ingestr/src/google_analytics/helpers/__init__.py,sha256=y_q7dinlEwNBEpq6kCzjTa8lAhe2bb23bDPP0fcy7fY,2744
-ingestr/src/google_analytics/helpers/data_processing.py,sha256=fIdEKr9CmZN_s1T2i9BL8IYTPPqNoK6Vaquq2y8StfE,6072
+ingestr/src/google_analytics/__init__.py,sha256=8Evpmoy464YpNbCI_NmvFHIzWCu7J7SjJw-RrPZ6AL8,3674
+ingestr/src/google_analytics/helpers.py,sha256=vLmFyQ_IEJEK5LlxBJQeJw0VHaE5gRRZdBa54U72CaQ,5965
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
 ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
-ingestr/src/gorgias/__init__.py,sha256=LZ3m6aGuhLVI3eNjvQE0rT4o_wbSPkY_SDKsM-g0V5U,21176
+ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=LshHlFzzs8trAOxSg7C9F7zIBakqsg8XfyNBouip09w,9761
+ingestr/src/hubspot/__init__.py,sha256=DXvn1yGToFUKk-1mMqqoN0OCLNpD16-2mPyEmkhyoVY,9876
 ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
 ingestr/src/hubspot/settings.py,sha256=9P1OKiRL88kl_m8n1HhuG-Qpq9VGbqPLn5Q0QYneToU,2193
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
-ingestr/src/klaviyo/_init_.py,sha256=nq2T1p3Xc7yiwGabsZBp2Jy2fa8_n5oxqxBnUGhKOgg,6592
+ingestr/src/klaviyo/_init_.py,sha256=ucWHqBe8DQvXVpbmxKFAV5ljpCFb4ps_2QTD0OSiWxY,7905
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
 ingestr/src/mongodb/__init__.py,sha256=aMr1PFIDUMRv--ne61lR17HudsN-fsrzMeyxe9PqK2s,4335
@@ -56,23 +61,23 @@ ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMO
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
-ingestr/src/shopify/__init__.py,sha256=Hhv84zRfVsqAGP7pz-PmeopeX9CGu7TXSm3PSXHEwIA,62689
+ingestr/src/shopify/__init__.py,sha256=PF_6VQnS065Br1UzSIekTVXBu3WtrMQL_v5CfbfaX5Y,63151
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
 ingestr/src/shopify/helpers.py,sha256=NfHD6lWXe88ybR0ri-FCQuh2Vf8l5WG0a0FVjmdoSC4,6296
 ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
-ingestr/src/slack/__init__.py,sha256=UF-ficQ6K32u1EHytW3P35suACo9wuc6nMrAPViyZL8,9981
+ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM,10113
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
-ingestr/src/stripe_analytics/__init__.py,sha256=VEXH4to2vNojN4rk3qsypR7urtTzaxSBB3IBiD5tuoE,4514
+ingestr/src/stripe_analytics/__init__.py,sha256=0HCL0qsrh_si1RR3a4k9XS94VWQ4v9aG7CqXF-V-57M,4593
 ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
 ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
-ingestr/src/tiktok_ads/__init__.py,sha256=U4ZHPUW0c4LpKx4hjT2Lz5hgWFgwQSbAAkkYIrxYHZo,4469
+ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
 ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=cfdPflCeR_mCk5fxq0v4d7pzlvZDiAoz8bWQJYqKALM,3935
-ingestr/src/zendesk/__init__.py,sha256=C7HkN195DGdOHId2_Sa_kAlcBrUmnVYZUa_tPkiyf1Q,17564
+ingestr/src/zendesk/__init__.py,sha256=tmJ_jdb6kpwmEKpcv6Im71-bOZI6h-Tcofe18OH4I24,17762
 ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
 ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
 ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
@@ -86,8 +91,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.12.5.dist-info/METADATA,sha256=QhFy0K3FUgK2VGdShWUOeTj_HbHElVPD64bAf2k-4G0,7956
-ingestr-0.12.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.12.5.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.12.5.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.12.5.dist-info/RECORD,,
+ingestr-0.12.7.dist-info/METADATA,sha256=5p9Igsh-xTJT1cNM_dsgAAFDTDuUQQSezkL9dz4JO3k,8024
+ingestr-0.12.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.12.7.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.12.7.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.12.7.dist-info/RECORD,,

ingestr/src/google_analytics/helpers/__init__.py

@@ -1,70 +0,0 @@
-"""Google analytics source helpers"""
-
-from typing import Iterator, List
-
-import dlt
-from apiclient.discovery import Resource  # type: ignore
-from dlt.common import logger, pendulum
-from dlt.common.typing import TDataItem
-from google.analytics.data_v1beta.types import (
-    Dimension,
-    Metric,
-)
-from pendulum.datetime import DateTime
-
-from .data_processing import get_report
-
-
-def basic_report(
-    client: Resource,
-    rows_per_page: int,
-    dimensions: List[str],
-    metrics: List[str],
-    property_id: int,
-    resource_name: str,
-    start_date: str,
-    last_date: dlt.sources.incremental[DateTime],
-) -> Iterator[TDataItem]:
-    """
-    Retrieves the data for a report given dimensions, metrics, and filters required for the report.
-
-    Args:
-        client: The Google Analytics client used to make requests.
-        dimensions: Dimensions for the report. See metadata for the full list of dimensions.
-        metrics: Metrics for the report. See metadata for the full list of metrics.
-        property_id: A reference to the Google Analytics project.
-            More info: https://developers.google.com/analytics/devguides/reporting/data/v1/property-id
-        rows_per_page: Controls how many rows are retrieved per page in the reports.
-            Default is 10000, maximum possible is 100000.
-        resource_name: The resource name used to save incremental into dlt state.
-        start_date: Incremental load start_date.
-            Default is taken from dlt state if it exists.
-        last_date: Incremental load end date.
-            Default is taken from dlt state if it exists.
-
-    Returns:
-        Generator of all rows of data in the report.
-    """
-
-    # grab the start time from last dlt load if not filled, if that is also empty then use the first day of the millennium as the start time instead
-    if last_date.last_value:
-        if start_date != "2015-08-14":
-            logger.warning(
-                f"Using the starting date: {last_date.last_value} for incremental report: {resource_name} and ignoring start date passed as argument {start_date}"
-            )
-        start_date = last_date.last_value.to_date_string()
-    else:
-        start_date = start_date or "2015-08-14"
-
-    processed_response = get_report(
-        client=client,
-        property_id=property_id,
-        # fill dimensions and metrics with the proper api client objects
-        dimension_list=[Dimension(name=dimension) for dimension in dimensions],
-        metric_list=[Metric(name=metric) for metric in metrics],
-        limit=rows_per_page,
-        start_date=start_date,
-        # configure end_date to yesterday as a date string
-        end_date=pendulum.now().to_date_string(),
-    )
-    yield from processed_response