ingestr 0.13.76__tar.gz → 0.13.77__tar.gz

{ingestr-0.13.76 → ingestr-0.13.77}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.76
+Version: 0.13.77
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.76 → ingestr-0.13.77}/docs/supported-sources/github.md

@@ -36,14 +36,13 @@ This is a sample command that will copy the data from the GitHub source to DuckD
 ## Tables
 
 GitHub source allows ingesting the following sources into separate tables:
+| Table           | PK | Inc Key | Inc Strategy | Details                                                                                                                                        |
+| --------------- | ----------- | --------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
+| `issues`        | - | –                | replace               | Retrieves GitHub issues along with their comments and reactions. Full reload on each run.                                        |
+| `pull_requests` | - | –                | replace               | Retrieves pull requests with comments and reactions. Full reload on each run.                                                    |
+| `repo_events`   | `id` | `created_at`     | merge  | Retrieves recent repository events. Appends only new events using `created_at` filter. Only events from the past 30 days allowed. |
+| `stargazers`    | - | –                | replace               | Retrieves stargazers. Full reload on each run.                                  |
 
-- `issues`: Retrieves data about issues, their associated comments, and subsequent reactions
-- `pull_requests`: Retrieves all pull requests
-- `repo_events`: Retrieves all the repo events associated with the repository
-- `stargazers`: Retrieves all stargazers
 
 Use these as `--source-table` parameter in the `ingestr ingest` command.
-
-> [!WARNING]
-> GitHub does not support incremental loading for many endpoints in its APIs, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.
  

ingestr-0.13.77/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.77"

{ingestr-0.13.76 → ingestr-0.13.77}/ingestr/src/github/__init__.py

@@ -72,7 +72,7 @@ def github_repo_events(
     name: str,
     access_token: str,
     start_date: pendulum.DateTime,
-    end_date: pendulum.DateTime,
+    end_date: Optional[pendulum.DateTime] = None,
 ) -> DltResource:
     """Gets events for repository `name` with owner `owner` incrementally.
 
@@ -91,12 +91,12 @@ def github_repo_events(
     """
 
     # use naming function in table name to generate separate tables for each event
-    @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
+    @dlt.resource(primary_key= "id", table_name=lambda i: i["type"], write_disposition="merge")
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "created_at",
             initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
             last_value_func=max,
             range_end="closed",
             range_start="closed",
@@ -105,7 +105,7 @@ def github_repo_events(
         repos_path = (
             f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
         )
-
+        
         # Get the date range from the incremental state
         start_filter = pendulum.parse(
             last_created_at.last_value or last_created_at.initial_value
@@ -113,9 +113,9 @@ def github_repo_events(
         end_filter = (
             pendulum.parse(last_created_at.end_value)
             if last_created_at.end_value
-            else None
+            else pendulum.now()
         )
-
+        
         for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
             # Filter events by date range
             filtered_events = []

{ingestr-0.13.76 → ingestr-0.13.77}/ingestr/src/github/helpers.py

@@ -61,7 +61,6 @@ def get_stargazers(
             page_items,
         )
 
-
 def get_reactions_data(
     node_type: str,
     owner: str,

{ingestr-0.13.76 → ingestr-0.13.77}/ingestr/src/sources.py

@@ -1817,7 +1817,7 @@ class GitHubSource:
             start_date = kwargs.get("interval_start") or pendulum.now().subtract(
                 days=30
             )
-            end_date = kwargs.get("interval_end") or pendulum.now()
+            end_date = kwargs.get("interval_end") or None
 
             if isinstance(start_date, str):
                 start_date = pendulum.parse(start_date)

ingestr-0.13.76/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.76"