ingestr 0.13.76__tar.gz → 0.13.78__tar.gz

{ingestr-0.13.76 → ingestr-0.13.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.76
+Version: 0.13.78
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.76 → ingestr-0.13.78}/docs/supported-sources/github.md

@@ -36,14 +36,13 @@ This is a sample command that will copy the data from the GitHub source to DuckD
 ## Tables
 
 GitHub source allows ingesting the following sources into separate tables:
+| Table           | PK | Inc Key | Inc Strategy | Details                                                                                                                                        |
+| --------------- | ----------- | --------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
+| `issues`        | - | –                | replace               | Retrieves GitHub issues along with their comments and reactions. Full reload on each run.                                        |
+| `pull_requests` | - | –                | replace               | Retrieves pull requests with comments and reactions. Full reload on each run.                                                    |
+| `repo_events`   | `id` | `created_at`     | merge  | Retrieves recent repository events. Appends only new events using `created_at` filter. Only events from the past 30 days allowed. |
+| `stargazers`    | - | –                | replace               | Retrieves stargazers. Full reload on each run.                                  |
 
-- `issues`: Retrieves data about issues, their associated comments, and subsequent reactions
-- `pull_requests`: Retrieves all pull requests
-- `repo_events`: Retrieves all the repo events associated with the repository
-- `stargazers`: Retrieves all stargazers
 
 Use these as `--source-table` parameter in the `ingestr ingest` command.
-
-> [!WARNING]
-> GitHub does not support incremental loading for many endpoints in its APIs, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.
  

ingestr-0.13.78/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.78"

{ingestr-0.13.76 → ingestr-0.13.78}/ingestr/src/github/__init__.py

@@ -72,7 +72,7 @@ def github_repo_events(
     name: str,
     access_token: str,
     start_date: pendulum.DateTime,
-    end_date: pendulum.DateTime,
+    end_date: Optional[pendulum.DateTime] = None,
 ) -> DltResource:
     """Gets events for repository `name` with owner `owner` incrementally.
 
@@ -91,12 +91,12 @@ def github_repo_events(
     """
 
     # use naming function in table name to generate separate tables for each event
-    @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
+    @dlt.resource(primary_key= "id", table_name=lambda i: i["type"], write_disposition="merge")
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "created_at",
             initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
             last_value_func=max,
             range_end="closed",
             range_start="closed",
@@ -105,7 +105,7 @@ def github_repo_events(
         repos_path = (
             f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
         )
-
+        
         # Get the date range from the incremental state
         start_filter = pendulum.parse(
             last_created_at.last_value or last_created_at.initial_value
@@ -113,9 +113,9 @@ def github_repo_events(
         end_filter = (
             pendulum.parse(last_created_at.end_value)
             if last_created_at.end_value
-            else None
+            else pendulum.now()
         )
-
+        
         for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
             # Filter events by date range
             filtered_events = []

{ingestr-0.13.76 → ingestr-0.13.78}/ingestr/src/github/helpers.py

@@ -61,7 +61,6 @@ def get_stargazers(
             page_items,
         )
 
-
 def get_reactions_data(
     node_type: str,
     owner: str,

{ingestr-0.13.76 → ingestr-0.13.78}/ingestr/src/mongodb/helpers.py

@@ -1,5 +1,6 @@
 """Mongo database source helpers"""
 
+import json
 from itertools import islice
 from typing import (
     TYPE_CHECKING,
@@ -204,7 +205,14 @@ class CollectionLoader:
         cursor = self._limit(cursor, limit)
 
         while docs_slice := list(islice(cursor, self.chunk_size)):
-            yield map_nested_in_place(convert_mongo_objs, docs_slice)
+            res = map_nested_in_place(convert_mongo_objs, docs_slice)
+            if len(res) > 0 and "_id" in res[0] and isinstance(res[0]["_id"], dict):
+                yield dlt.mark.with_hints(
+                    res,
+                    dlt.mark.make_hints(columns={"_id": {"data_type": "json"} }),
+                )
+            else:
+                yield res
 
 
 class CollectionLoaderParallel(CollectionLoader):
@@ -531,7 +539,7 @@ def collection_documents(
             LoaderClass = CollectionArrowLoader  # type: ignore
         else:
             LoaderClass = CollectionLoader  # type: ignore
-
+    
     loader = LoaderClass(
         client, collection, incremental=incremental, chunk_size=chunk_size
     )

{ingestr-0.13.76 → ingestr-0.13.78}/ingestr/src/sources.py

@@ -428,7 +428,7 @@ class MongoDbSource:
             connection_url=uri,
             database=table_fields.dataset,
             collection=table_fields.table,
-            parallel=True,
+            parallel=False,
             incremental=incremental,
         )
         table_instance.max_table_nesting = 1
@@ -1817,7 +1817,7 @@ class GitHubSource:
             start_date = kwargs.get("interval_start") or pendulum.now().subtract(
                 days=30
             )
-            end_date = kwargs.get("interval_end") or pendulum.now()
+            end_date = kwargs.get("interval_end") or None
 
             if isinstance(start_date, str):
                 start_date = pendulum.parse(start_date)

ingestr-0.13.76/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.76"