ingestr 0.13.76__py3-none-any.whl → 0.13.78__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.76"
+version = "v0.13.78"

ingestr/src/github/__init__.py

@@ -72,7 +72,7 @@ def github_repo_events(
     name: str,
     access_token: str,
     start_date: pendulum.DateTime,
-    end_date: pendulum.DateTime,
+    end_date: Optional[pendulum.DateTime] = None,
 ) -> DltResource:
     """Gets events for repository `name` with owner `owner` incrementally.
 
@@ -91,12 +91,12 @@ def github_repo_events(
     """
 
     # use naming function in table name to generate separate tables for each event
-    @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
+    @dlt.resource(primary_key= "id", table_name=lambda i: i["type"], write_disposition="merge")
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "created_at",
             initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
             last_value_func=max,
             range_end="closed",
             range_start="closed",
@@ -105,7 +105,7 @@ def github_repo_events(
         repos_path = (
             f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
         )
-
+        
         # Get the date range from the incremental state
         start_filter = pendulum.parse(
             last_created_at.last_value or last_created_at.initial_value
@@ -113,9 +113,9 @@ def github_repo_events(
         end_filter = (
             pendulum.parse(last_created_at.end_value)
             if last_created_at.end_value
-            else None
+            else pendulum.now()
         )
-
+        
         for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
             # Filter events by date range
             filtered_events = []

ingestr/src/github/helpers.py

@@ -61,7 +61,6 @@ def get_stargazers(
             page_items,
         )
 
-
 def get_reactions_data(
     node_type: str,
     owner: str,

ingestr/src/mongodb/helpers.py

@@ -1,5 +1,6 @@
 """Mongo database source helpers"""
 
+import json
 from itertools import islice
 from typing import (
     TYPE_CHECKING,
@@ -204,7 +205,14 @@ class CollectionLoader:
         cursor = self._limit(cursor, limit)
 
         while docs_slice := list(islice(cursor, self.chunk_size)):
-            yield map_nested_in_place(convert_mongo_objs, docs_slice)
+            res = map_nested_in_place(convert_mongo_objs, docs_slice)
+            if len(res) > 0 and "_id" in res[0] and isinstance(res[0]["_id"], dict):
+                yield dlt.mark.with_hints(
+                    res,
+                    dlt.mark.make_hints(columns={"_id": {"data_type": "json"} }),
+                )
+            else:
+                yield res
 
 
 class CollectionLoaderParallel(CollectionLoader):
@@ -531,7 +539,7 @@ def collection_documents(
             LoaderClass = CollectionArrowLoader  # type: ignore
         else:
             LoaderClass = CollectionLoader  # type: ignore
-
+    
     loader = LoaderClass(
         client, collection, incremental=incremental, chunk_size=chunk_size
     )

ingestr/src/sources.py

@@ -428,7 +428,7 @@ class MongoDbSource:
             connection_url=uri,
             database=table_fields.dataset,
             collection=table_fields.table,
-            parallel=True,
+            parallel=False,
             incremental=incremental,
         )
         table_instance.max_table_nesting = 1
@@ -1817,7 +1817,7 @@ class GitHubSource:
             start_date = kwargs.get("interval_start") or pendulum.now().subtract(
                 days=30
             )
-            end_date = kwargs.get("interval_end") or pendulum.now()
+            end_date = kwargs.get("interval_end") or None
 
             if isinstance(start_date, str):
                 start_date = pendulum.parse(start_date)

{ingestr-0.13.76.dist-info → ingestr-0.13.78.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.76
+Version: 0.13.78
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.76.dist-info → ingestr-0.13.78.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=QsNVrz5_NgRUkvfExnd-2E02TGmWivPuop5hYinVAjM,26513
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=1TvMupuRuFAY-Dl2dC1yEvrISDUeoNQCh9YqbKHbRwY,21
+ingestr/src/buildinfo.py,sha256=cARFQnpIzB5xD3JEaIPIkee7dO80kbLs4M_XypNnwSI,21
 ingestr/src/destinations.py,sha256=ivTPio0zzqLRx22i597pxZHMNClz-XvYSyCaCPuGd8g,22248
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=q_rSi4gYMfxnGvzhytPRAgC08N40nqDISvXwl7i-E_M,6655
@@ -11,7 +11,7 @@ ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,73
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=Lj6G_dubjx6O6N-JzHlmWtgVWBUCfIQlKOmmI4PDji8,107644
+ingestr/src/sources.py,sha256=1A1tZKA1NUQnHdgvGPKHuRG5o8lNuCe7bIxB0n73eJw,107635
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -54,8 +54,8 @@ ingestr/src/frankfurter/helpers.py,sha256=SyrkRTDqvKdQxRHTV5kcSeVG3FEnaK5zxHyNyq
 ingestr/src/freshdesk/__init__.py,sha256=uFQW_cJyymxtHQiYb_xjzZAklc487L0n9GkgHgC7yAI,2618
 ingestr/src/freshdesk/freshdesk_client.py,sha256=3z5Yc008ADzRcJWtNc00PwjkLzG-RMI8jVIOOyYA-Rw,4088
 ingestr/src/freshdesk/settings.py,sha256=0Wr_OMnUZcTlry7BmALssLxD2yh686JW4moLNv12Jnw,409
-ingestr/src/github/__init__.py,sha256=YmC8qIhm0Fw5QC6eG1JuqWSHuyC-kp77S7zeWGaPPto,7127
-ingestr/src/github/helpers.py,sha256=rpv_3HzuOl4PQ-FUeA66pev-pgze9SaE8RUHIPYfZ_A,6759
+ingestr/src/github/__init__.py,sha256=R71y33KqzxDTvCLSGj2H2EztfGqsWGR9ZgcaurC1-A4,7220
+ingestr/src/github/helpers.py,sha256=hge8orylwiScRcMftlv4oSZ6ORvVANwHCPAGkg95FtI,6758
 ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
 ingestr/src/github/settings.py,sha256=N5ahWrDIQ_4IWV9i-hTXxyYduqY9Ym2BTwqsWxcDdJ8,258
 ingestr/src/google_ads/__init__.py,sha256=bH0TtnRWcOUESezpvoA7VEUHAq_0ITGQeX4GGVBfl1I,3725
@@ -93,7 +93,7 @@ ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnb
 ingestr/src/mixpanel/__init__.py,sha256=s1QtqMP0BTGW6YtdCabJFWj7lEn7KujzELwGpBOQgfs,1796
 ingestr/src/mixpanel/client.py,sha256=c_reouegOVYBOwHLfgYFwpmkba0Sxro1Zkml07NCYf0,3602
 ingestr/src/mongodb/__init__.py,sha256=T-RYPS_skl_2gNVfYWWXan2bVQYmm0bFBcCCqG5ejvg,7275
-ingestr/src/mongodb/helpers.py,sha256=H0GpOK3bPBhFWBEhJZOjywUBdzih6MOpmyVO_cKSN14,24178
+ingestr/src/mongodb/helpers.py,sha256=8pjNYZu4k2rkR9dItTMAnPaRdF1kroqLYX9FZ34RTqo,24491
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
 ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -151,8 +151,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.76.dist-info/METADATA,sha256=w83DBTlRSGkbu5Qk7B7sKzm8ggPO5TNi57E8e32ZpjM,15093
-ingestr-0.13.76.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.76.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.76.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.76.dist-info/RECORD,,
+ingestr-0.13.78.dist-info/METADATA,sha256=Q7ofO2TRuTOUb4fhZvyr_kejvaOM2OwCrq3FnCLEk6U,15093
+ingestr-0.13.78.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.78.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.78.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.78.dist-info/RECORD,,