ingestr 0.12.5__py3-none-any.whl → 0.12.6__py3-none-any.whl

ingestr/main.py

@@ -444,7 +444,7 @@ def ingest(
 
         progressInstance: Collector = SpinnerCollector()
         if progress == Progress.log:
-            progressInstance = LogCollector(dump_system_stats=False)
+            progressInstance = LogCollector()
 
         is_pipelines_dir_temp = False
         if pipelines_dir is None:

ingestr/src/asana_source/__init__.py

@@ -150,7 +150,10 @@ def tasks(
     project_array: t.List[TDataItem],
     access_token: str = dlt.secrets.value,
     modified_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-        "modified_at", initial_value=DEFAULT_START_DATE
+        "modified_at",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
     ),
     fields: Iterable[str] = TASK_FIELDS,
 ) -> Iterable[TDataItem]:

ingestr/src/facebook_ads/__init__.py

@@ -159,7 +159,10 @@ def facebook_insights_source(
     )
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "date_start", initial_value=initial_load_start_date_str
+            "date_start",
+            initial_value=initial_load_start_date_str,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItems]:
         start_date = get_start_date(date_start, attribution_window_days_lag)

ingestr/src/filesystem/__init__.py

@@ -38,7 +38,9 @@ def readers(
     """
     filesystem_resource = filesystem(bucket_url, credentials, file_glob=file_glob)
     filesystem_resource.apply_hints(
-        incremental=dlt.sources.incremental("modification_date")
+        incremental=dlt.sources.incremental("modification_date"),
+        range_end="closed",
+        range_start="closed",
     )
     return (
         filesystem_resource | dlt.transformer(name="read_csv")(_read_csv),

ingestr/src/github/__init__.py

@@ -14,7 +14,7 @@ from .helpers import get_reactions_data, get_rest_pages, get_stargazers
 def github_reactions(
     owner: str,
     name: str,
-    access_token: str = dlt.secrets.value,
+    access_token: str,
     items_per_page: int = 100,
     max_items: Optional[int] = None,
 ) -> Sequence[DltResource]:
@@ -89,7 +89,11 @@ def github_repo_events(
     @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "created_at", initial_value="1970-01-01T00:00:00Z", last_value_func=max
+            "created_at",
+            initial_value="1970-01-01T00:00:00Z",
+            last_value_func=max,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItems]:
         repos_path = (
@@ -114,7 +118,7 @@ def github_repo_events(
 def github_stargazers(
     owner: str,
     name: str,
-    access_token: str = dlt.secrets.value,
+    access_token: str,
     items_per_page: int = 100,
     max_items: Optional[int] = None,
 ) -> Sequence[DltResource]:

ingestr/src/google_analytics/__init__.py

@@ -2,26 +2,32 @@
 Defines all the sources and resources needed for Google Analytics V4
 """
 
-from typing import List, Optional, Union
+from typing import Iterator, List, Optional, Union
 
 import dlt
-from dlt.common.typing import DictStrAny
-from dlt.sources import DltResource
+from dlt.common import pendulum
+from dlt.common.typing import DictStrAny, TDataItem
+from dlt.extract import DltResource
 from dlt.sources.credentials import GcpOAuthCredentials, GcpServiceAccountCredentials
 from google.analytics.data_v1beta import BetaAnalyticsDataClient
+from google.analytics.data_v1beta.types import (
+    Dimension,
+    Metric,
+)
 
-from .helpers import basic_report
+from .helpers import get_report
 
 
 @dlt.source(max_table_nesting=0)
 def google_analytics(
-    datetime: str,
+    datetime_dimension: str,
     credentials: Union[
         GcpOAuthCredentials, GcpServiceAccountCredentials
     ] = dlt.secrets.value,
     property_id: int = dlt.config.value,
     queries: List[DictStrAny] = dlt.config.value,
-    start_date: Optional[str] = "2015-08-14",
+    start_date: Optional[pendulum.DateTime] = pendulum.datetime(2024, 1, 1),
+    end_date: Optional[pendulum.DateTime] = None,
     rows_per_page: int = 10000,
 ) -> List[DltResource]:
     try:
@@ -50,21 +56,51 @@ def google_analytics(
 
     # always add "date" to dimensions so we are able to track the last day of a report
     dimensions = query["dimensions"]
-    resource_name = query["resource_name"]
 
-    res = dlt.resource(
-        basic_report, name="basic_report", merge_key=datetime, write_disposition="merge"
-    )(
-        client=client,
-        rows_per_page=rows_per_page,
-        property_id=property_id,
-        dimensions=dimensions,
-        metrics=query["metrics"],
-        resource_name=resource_name,
-        start_date=start_date,
-        last_date=dlt.sources.incremental(
-            datetime
-        ),  # pass empty primary key to avoid unique checks, a primary key defined by the resource will be used
+    @dlt.resource(
+        name="basic_report",
+        merge_key=datetime_dimension,
+        write_disposition="merge",
     )
+    def basic_report(
+        incremental=dlt.sources.incremental(
+            datetime_dimension,
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterator[TDataItem]:
+        start_date = incremental.last_value
+        end_date = incremental.end_value
+        if start_date is None:
+            start_date = pendulum.datetime(2024, 1, 1)
+        if end_date is None:
+            end_date = pendulum.yesterday()
+        yield from get_report(
+            client=client,
+            property_id=property_id,
+            dimension_list=[Dimension(name=dimension) for dimension in dimensions],
+            metric_list=[Metric(name=metric) for metric in query["metrics"]],
+            per_page=rows_per_page,
+            start_date=start_date,
+            end_date=end_date,
+        )
+
+    # res = dlt.resource(
+    #     basic_report, name="basic_report", merge_key=datetime_dimension, write_disposition="merge"
+    # )(
+    #     client=client,
+    #     rows_per_page=rows_per_page,
+    #     property_id=property_id,
+    #     dimensions=dimensions,
+    #     metrics=query["metrics"],
+    #     resource_name=resource_name,
+    #     last_date=dlt.sources.incremental(
+    #         datetime_dimension,
+    #         initial_value=start_date,
+    #         end_value=end_date,
+    #     ),
+    # )
 
-    return [res]
+    return [basic_report]

ingestr/src/google_analytics/{helpers/data_processing.py → helpers.py}

@@ -57,9 +57,9 @@ def get_report(
     property_id: int,
     dimension_list: List[Dimension],
     metric_list: List[Metric],
-    limit: int,
-    start_date: str,
-    end_date: str,
+    per_page: int,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime,
 ) -> Iterator[TDataItem]:
     """
     Gets all the possible pages of reports with the given query parameters.
@@ -79,30 +79,36 @@ def get_report(
         Generator of all rows of data in the report.
     """
 
-    request = RunReportRequest(
-        property=f"properties/{property_id}",
-        dimensions=dimension_list,
-        metrics=metric_list,
-        limit=limit,
-        date_ranges=[DateRange(start_date=start_date, end_date=end_date)],
+    print(
+        "fetching for daterange", start_date.to_date_string(), end_date.to_date_string()
     )
-    # process request
-    response = client.run_report(request)
-    processed_response_generator = process_report(response=response)
-    yield from processed_response_generator
 
+    offset = 0
+    while True:
+        request = RunReportRequest(
+            property=f"properties/{property_id}",
+            dimensions=dimension_list,
+            metrics=metric_list,
+            limit=per_page,
+            offset=offset,
+            date_ranges=[
+                DateRange(
+                    start_date=start_date.to_date_string(),
+                    end_date=end_date.to_date_string(),
+                )
+            ],
+        )
+        # process request
+        response = client.run_report(request)
+        processed_response_generator = process_report(response=response)
+        # import pdb; pdb.set_trace()
+        yield from processed_response_generator
+        offset += per_page
+        if len(response.rows) < per_page or offset > 1000000:
+            break
 
-def process_report(response: RunReportResponse) -> Iterator[TDataItems]:
-    """
-    Receives a single page for a report response, processes it, and returns a generator for every row of data in the report page.
-
-    Args:
-        response: The API response for a single page of the report.
-
-    Yields:
-        Generator of dictionaries for every row of the report page.
-    """
 
+def process_report(response: RunReportResponse) -> Iterator[TDataItems]:
     metrics_headers = [header.name for header in response.metric_headers]
     dimensions_headers = [header.name for header in response.dimension_headers]
 
@@ -156,16 +162,6 @@ def process_metric_value(metric_type: MetricType, value: str) -> Union[str, int,
 
 
 def _resolve_dimension_value(dimension_name: str, dimension_value: str) -> Any:
-    """
-    Helper function that receives a dimension's name and value and converts it to a datetime object if needed.
-
-    Args:
-        dimension_name: Name of the dimension.
-        dimension_value: Value of the dimension.
-
-    Returns:
-        The value of the dimension with the correct data type.
-    """
     if dimension_name == "date":
         return pendulum.from_format(dimension_value, "YYYYMMDD", tz="UTC")
     elif dimension_name == "dateHour":

ingestr/src/gorgias/__init__.py

@@ -116,7 +116,9 @@ def gorgias_source(
         },
     )
     def customers(
-        updated_datetime=dlt.sources.incremental("updated_datetime", start_date_obj),
+        updated_datetime=dlt.sources.incremental(
+            "updated_datetime", start_date_obj, range_end="closed", range_start="closed"
+        ),
     ) -> Iterable[TDataItem]:
         """
         The resource for customers on your Gorgias domain, supports incremental loading and pagination.
@@ -290,7 +292,9 @@ def gorgias_source(
         },
     )
     def tickets(
-        updated_datetime=dlt.sources.incremental("updated_datetime", start_date_obj),
+        updated_datetime=dlt.sources.incremental(
+            "updated_datetime", start_date_obj, range_end="closed", range_start="closed"
+        ),
     ) -> Iterable[TDataItem]:
         """
         The resource for tickets on your Gorgias domain, supports incremental loading and pagination.
@@ -481,7 +485,9 @@ def gorgias_source(
         },
     )
     def ticket_messages(
-        updated_datetime=dlt.sources.incremental("updated_datetime", start_date_obj),
+        updated_datetime=dlt.sources.incremental(
+            "updated_datetime", start_date_obj, range_end="closed", range_start="closed"
+        ),
     ) -> Iterable[TDataItem]:
         """
         The resource for ticket messages on your Gorgias domain, supports incremental loading and pagination.
@@ -566,7 +572,9 @@ def gorgias_source(
         },
     )
     def satisfaction_surveys(
-        updated_datetime=dlt.sources.incremental("updated_datetime", start_date_obj),
+        updated_datetime=dlt.sources.incremental(
+            "updated_datetime", start_date_obj, range_end="closed", range_start="closed"
+        ),
     ) -> Iterable[TDataItem]:
         """
         The resource for satisfaction surveys on your Gorgias domain, supports incremental loading and pagination.

ingestr/src/hubspot/__init__.py

@@ -278,4 +278,11 @@ def hubspot_events_for_objects(
         write_disposition="append",
         selected=True,
         table_name=lambda e: name + "_" + str(e["eventType"]),
-    )(dlt.sources.incremental("occurredAt", initial_value=start_date.isoformat()))
+    )(
+        dlt.sources.incremental(
+            "occurredAt",
+            initial_value=start_date.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        )
+    )

ingestr/src/klaviyo/_init_.py

@@ -33,7 +33,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def events(
-        datetime=dlt.sources.incremental("datetime", start_date_obj.isoformat()),
+        datetime=dlt.sources.incremental(
+            "datetime",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(datetime.start_value), pendulum.now()
@@ -44,7 +49,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def profiles(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated.start_value), pendulum.now()
@@ -55,7 +65,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def campaigns(
-        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated_at.start_value), pendulum.now()
@@ -69,7 +84,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id")
     def metrics(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_metrics(create_client(), updated.start_value)
 
@@ -83,7 +103,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id", name="catalog-variants")
     def catalog_variants(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_catalog_variant(create_client(), updated.start_value)
 
@@ -91,19 +116,34 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
         write_disposition="merge", primary_key="id", name="catalog-categories"
     )
     def catalog_categories(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_catalog_categories(create_client(), updated.start_value)
 
     @dlt.resource(write_disposition="merge", primary_key="id", name="catalog-items")
     def catalog_items(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_catalog_item(create_client(), updated.start_value)
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def forms(
-        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated_at.start_value), pendulum.now()
@@ -114,13 +154,23 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id")
     def lists(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_lists(create_client(), updated.start_value)
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def images(
-        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated_at.start_value), pendulum.now()
@@ -130,13 +180,23 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id")
     def segments(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_segments(create_client(), updated.start_value)
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def flows(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated.start_value), pendulum.now()
@@ -146,7 +206,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def templates(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated.start_value), pendulum.now()

ingestr/src/shopify/__init__.py

@@ -158,6 +158,8 @@ def shopify_source(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
         created_at_min: pendulum.DateTime = created_at_min_obj,
         items_per_page: int = items_per_page,
@@ -606,6 +608,8 @@ def shopify_source(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
         created_at_min: pendulum.DateTime = created_at_min_obj,
         items_per_page: int = items_per_page,
@@ -640,6 +644,8 @@ def shopify_source(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
         created_at_min: pendulum.DateTime = created_at_min_obj,
         items_per_page: int = items_per_page,
@@ -671,6 +677,8 @@ def shopify_source(
             "created_at",
             initial_value=start_date_obj,
             end_value=end_date_obj,
+            range_end="closed",
+            range_start="closed",
         ),
         items_per_page: int = items_per_page,
     ) -> Iterable[TDataItem]:
@@ -689,6 +697,8 @@ def shopify_source(
             "updated_at",
             initial_value=start_date_obj,
             end_value=end_date_obj,
+            range_end="closed",
+            range_start="closed",
         ),
         items_per_page: int = items_per_page,
     ) -> Iterable[TDataItem]:
@@ -730,6 +740,8 @@ def shopify_source(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
         items_per_page: int = items_per_page,
     ) -> Iterable[TDataItem]:
@@ -1807,6 +1819,8 @@ query discountNodes($after: String, $query: String, $first: Int)  {
             "updatedAt",
             initial_value=start_date_obj,
             end_value=end_date_obj,
+            range_end="closed",
+            range_start="closed",
         ),
         items_per_page: int = items_per_page,
     ) -> Iterable[TDataItem]:

ingestr/src/slack/__init__.py

@@ -175,6 +175,8 @@ def slack_source(
             initial_value=start_dt,
             end_value=end_dt,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterable[TDataItem]:
         """
@@ -198,6 +200,8 @@ def slack_source(
             initial_value=start_dt,
             end_value=end_dt,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterable[TDataItem]:
         """Yield all messages for a given channel as a DLT resource. Keep blocks column without normalization.

ingestr/src/sources.py

@@ -240,6 +240,8 @@ class ArrowMemoryMappedSource:
                 kwargs.get("incremental_key", ""),
                 initial_value=start_value,
                 end_value=end_value,
+                range_end="closed",
+                range_start="closed",
             )
 
         file_path = uri.split("://")[1]
@@ -285,6 +287,8 @@ class MongoDbSource:
                 kwargs.get("incremental_key", ""),
                 initial_value=start_value,
                 end_value=end_value,
+                range_end="closed",
+                range_start="closed",
             )
 
         table_instance = self.table_builder(
@@ -353,6 +357,8 @@ class LocalCsvSource:
                 kwargs.get("incremental_key", ""),
                 initial_value=kwargs.get("interval_start"),
                 end_value=kwargs.get("interval_end"),
+                range_end="closed",
+                range_start="closed",
             )
         )
 
@@ -1311,6 +1317,8 @@ class DynamoDBSource:
                 incremental_key.strip(),
                 initial_value=isotime(kwargs.get("interval_start")),
                 end_value=isotime(kwargs.get("interval_end")),
+                range_end="closed",
+                range_start="closed",
             )
 
         return dynamodb(table, creds, incremental)
@@ -1336,11 +1344,6 @@ class GoogleAnalyticsSource:
         if not property_id:
             raise ValueError("property_id is required to connect to Google Analytics")
 
-        interval_start = kwargs.get("interval_start")
-        start_date = (
-            interval_start.strftime("%Y-%m-%d") if interval_start else "2015-08-14"
-        )
-
         fields = table.split(":")
         if len(fields) != 3:
             raise ValueError(
@@ -1364,10 +1367,19 @@ class GoogleAnalyticsSource:
             {"resource_name": "custom", "dimensions": dimensions, "metrics": metrics}
         ]
 
+        start_date = pendulum.now().subtract(days=30).start_of("day")
+        if kwargs.get("interval_start") is not None:
+            start_date = pendulum.instance(kwargs.get("interval_start"))  # type: ignore
+
+        end_date = pendulum.now()
+        if kwargs.get("interval_end") is not None:
+            end_date = pendulum.instance(kwargs.get("interval_end"))  # type: ignore
+
         return google_analytics(
             property_id=property_id[0],
             start_date=start_date,
-            datetime=datetime,
+            end_date=end_date,
+            datetime_dimension=datetime,
             queries=queries,
             credentials=credentials,
         ).with_resources("basic_report")
@@ -1398,9 +1410,7 @@ class GitHubSource:
                 "repo variable is required to retrieve data for a specific repository from GitHub."
             )
 
-        access_token = source_fields.get("access_token", [None])[0]
-        if not access_token and table not in ["repo_events"]:
-            raise ValueError("access_token is required to connect with GitHub")
+        access_token = source_fields.get("access_token", [""])[0]
 
         if table in ["issues", "pull_requests"]:
             return github_reactions(

ingestr/src/stripe_analytics/__init__.py

@@ -84,7 +84,10 @@ def incremental_stripe_source(
     def incremental_resource(
         endpoint: str,
         created: Optional[Any] = dlt.sources.incremental(
-            "created", initial_value=start_date_unix
+            "created",
+            initial_value=start_date_unix,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Generator[Dict[Any, Any], Any, None]:
         start_value = created.last_value

ingestr/src/tiktok_ads/__init__.py

@@ -110,7 +110,12 @@ def tiktok_source(
     )
     def custom_reports(
         datetime=(
-            dlt.sources.incremental(incremental_loading_param, start_date)
+            dlt.sources.incremental(
+                incremental_loading_param,
+                start_date,
+                range_end="closed",
+                range_start="closed",
+            )
             if is_incremental
             else None
         ),

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.12.5"
+__version__ = "0.12.6"

ingestr/src/zendesk/__init__.py

@@ -260,6 +260,8 @@ def zendesk_support(
             initial_value=start_date_ts,
             end_value=end_date_ts,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItem]:
         # URL For ticket events
@@ -294,6 +296,8 @@ def zendesk_support(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItem]:
         """
@@ -340,6 +344,8 @@ def zendesk_support(
             initial_value=start_date_iso_str,
             end_value=end_date_iso_str,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItem]:
         """

{ingestr-0.12.5.dist-info → ingestr-0.12.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.5
+Version: 0.12.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,6 +26,7 @@ Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: mysql-connector-python==9.1.0
 Requires-Dist: pendulum==3.0.0
+Requires-Dist: psutil==6.1.1
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3

{ingestr-0.12.5.dist-info → ingestr-0.12.6.dist-info}/RECORD

@@ -1,52 +1,51 @@
-ingestr/main.py,sha256=xLQiPHoD7dNvrHfNTwD379wHg6xZGmLxzPzQLq2E1RA,24746
+ingestr/main.py,sha256=fRWnyoPzMvvxTa61EIAP_dsKu0B_0yOwoyt0Slq9WQU,24723
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=zcHJIIHAZmcD9sJomd6G1Bc-1KsxnBD2aByOSV_9L3g,8850
 ingestr/src/factory.py,sha256=aE7TjHzONb4DKYcfh_6-CJJfvs4lmw7iUySvSm4yQbM,4516
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=FXUTmII3DiEANZN37P9-dTFFRzpv0PL8bfaQvr0un8w,50761
+ingestr/src/sources.py,sha256=GIskUoVL82x_mLerU9cgdixBNNhzBnDN-_MDraqK7hY,51166
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=QFQfu3CUVe9Ncr8kv3aaBY3oWrZmv8xboen_Uwy7eXU,23
+ingestr/src/version.py,sha256=vb8hPdq1CrFlRl6aBYGOWE4MPv-N84JJm1f3KFvG8o4,23
 ingestr/src/adjust/__init__.py,sha256=NaRNwDhItG8Q7vUHw7zQvyfWjmT32M0CSc5ufjmBM9U,3067
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
 ingestr/src/arrow/__init__.py,sha256=8fEntgHseKjFMiPQIzxYzw_raicNsEgnveLi1IzBca0,2848
-ingestr/src/asana_source/__init__.py,sha256=Y4Ti_876Yong420fQ2o4A97TdgrZNlZVxlTMLyXdSjA,8116
+ingestr/src/asana_source/__init__.py,sha256=QwQTCb5PXts8I4wLHG9UfRP-5ChfjSe88XAVfxMV5Ag,8183
 ingestr/src/asana_source/helpers.py,sha256=PukcdDQWIGqnGxuuobbLw4hUy4-t6gxXg_XywR7Lg9M,375
 ingestr/src/asana_source/settings.py,sha256=-2tpdkwh04RvLKFvwQodnFLYn9MaxOO1hsebGnDQMTU,2829
 ingestr/src/chess/__init__.py,sha256=y0Q8aKBigeKf3N7wuB_gadMQjVJzBPUT8Jhp1ObEWjk,6812
 ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,549
 ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
 ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
-ingestr/src/facebook_ads/__init__.py,sha256=ZZyogV48gmhDcC3CYQEsC4qT3Q6JI9IOnMff2NS1M-A,9207
+ingestr/src/facebook_ads/__init__.py,sha256=reEpSr4BaKA1wO3qVgCH51gW-TgWkbJ_g24UIhJWbac,9286
 ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
 ingestr/src/facebook_ads/helpers.py,sha256=ZLbNHiKer5lPb4g3_435XeBJr57Wv0o1KTyBA1mQ100,9068
 ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_tl6UTFk,4881
-ingestr/src/filesystem/__init__.py,sha256=wHHaKFuAjsR_ZRjl6g_Flf6FhVs9qhwREthTr03_7cc,4162
+ingestr/src/filesystem/__init__.py,sha256=hcN_sO356ChTPyg72AufrikdkFBBIScTdxtGfDm-W0E,4221
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
-ingestr/src/github/__init__.py,sha256=csA2VcjOxXrVrvp7zY-JodO9Lpy98bJ4AqRdHCLTcGM,5838
+ingestr/src/github/__init__.py,sha256=xVijF-Wi4p88hkVJnKH-oTixismjD3aUcGqGa6Wr4e4,5889
 ingestr/src/github/helpers.py,sha256=Tmnik9811zBWNO6cJwV9PFQxEx2j32LHAQCvNbubsEI,6759
 ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
 ingestr/src/github/settings.py,sha256=N5ahWrDIQ_4IWV9i-hTXxyYduqY9Ym2BTwqsWxcDdJ8,258
-ingestr/src/google_analytics/__init__.py,sha256=HjA13wfJm2MGfy3h_DiM5ekkNqM2dgwYCKJ3pprnDtI,2482
-ingestr/src/google_analytics/helpers/__init__.py,sha256=y_q7dinlEwNBEpq6kCzjTa8lAhe2bb23bDPP0fcy7fY,2744
-ingestr/src/google_analytics/helpers/data_processing.py,sha256=fIdEKr9CmZN_s1T2i9BL8IYTPPqNoK6Vaquq2y8StfE,6072
+ingestr/src/google_analytics/__init__.py,sha256=8Evpmoy464YpNbCI_NmvFHIzWCu7J7SjJw-RrPZ6AL8,3674
+ingestr/src/google_analytics/helpers.py,sha256=vLmFyQ_IEJEK5LlxBJQeJw0VHaE5gRRZdBa54U72CaQ,5965
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
 ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
-ingestr/src/gorgias/__init__.py,sha256=LZ3m6aGuhLVI3eNjvQE0rT4o_wbSPkY_SDKsM-g0V5U,21176
+ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=LshHlFzzs8trAOxSg7C9F7zIBakqsg8XfyNBouip09w,9761
+ingestr/src/hubspot/__init__.py,sha256=DXvn1yGToFUKk-1mMqqoN0OCLNpD16-2mPyEmkhyoVY,9876
 ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
 ingestr/src/hubspot/settings.py,sha256=9P1OKiRL88kl_m8n1HhuG-Qpq9VGbqPLn5Q0QYneToU,2193
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
-ingestr/src/klaviyo/_init_.py,sha256=nq2T1p3Xc7yiwGabsZBp2Jy2fa8_n5oxqxBnUGhKOgg,6592
+ingestr/src/klaviyo/_init_.py,sha256=ucWHqBe8DQvXVpbmxKFAV5ljpCFb4ps_2QTD0OSiWxY,7905
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
 ingestr/src/mongodb/__init__.py,sha256=aMr1PFIDUMRv--ne61lR17HudsN-fsrzMeyxe9PqK2s,4335
@@ -56,23 +55,23 @@ ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMO
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
-ingestr/src/shopify/__init__.py,sha256=Hhv84zRfVsqAGP7pz-PmeopeX9CGu7TXSm3PSXHEwIA,62689
+ingestr/src/shopify/__init__.py,sha256=PF_6VQnS065Br1UzSIekTVXBu3WtrMQL_v5CfbfaX5Y,63151
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
 ingestr/src/shopify/helpers.py,sha256=NfHD6lWXe88ybR0ri-FCQuh2Vf8l5WG0a0FVjmdoSC4,6296
 ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
-ingestr/src/slack/__init__.py,sha256=UF-ficQ6K32u1EHytW3P35suACo9wuc6nMrAPViyZL8,9981
+ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM,10113
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
-ingestr/src/stripe_analytics/__init__.py,sha256=VEXH4to2vNojN4rk3qsypR7urtTzaxSBB3IBiD5tuoE,4514
+ingestr/src/stripe_analytics/__init__.py,sha256=0HCL0qsrh_si1RR3a4k9XS94VWQ4v9aG7CqXF-V-57M,4593
 ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
 ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
-ingestr/src/tiktok_ads/__init__.py,sha256=U4ZHPUW0c4LpKx4hjT2Lz5hgWFgwQSbAAkkYIrxYHZo,4469
+ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
 ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=cfdPflCeR_mCk5fxq0v4d7pzlvZDiAoz8bWQJYqKALM,3935
-ingestr/src/zendesk/__init__.py,sha256=C7HkN195DGdOHId2_Sa_kAlcBrUmnVYZUa_tPkiyf1Q,17564
+ingestr/src/zendesk/__init__.py,sha256=tmJ_jdb6kpwmEKpcv6Im71-bOZI6h-Tcofe18OH4I24,17762
 ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
 ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
 ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
@@ -86,8 +85,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.12.5.dist-info/METADATA,sha256=QhFy0K3FUgK2VGdShWUOeTj_HbHElVPD64bAf2k-4G0,7956
-ingestr-0.12.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.12.5.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.12.5.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.12.5.dist-info/RECORD,,
+ingestr-0.12.6.dist-info/METADATA,sha256=y-o_BL8nj7pVQU3sSaz9UJ9XsNVUi8Rjf5G0vNGi6io,7985
+ingestr-0.12.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.12.6.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.12.6.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.12.6.dist-info/RECORD,,

ingestr/src/google_analytics/helpers/__init__.py

@@ -1,70 +0,0 @@
-"""Google analytics source helpers"""
-
-from typing import Iterator, List
-
-import dlt
-from apiclient.discovery import Resource  # type: ignore
-from dlt.common import logger, pendulum
-from dlt.common.typing import TDataItem
-from google.analytics.data_v1beta.types import (
-    Dimension,
-    Metric,
-)
-from pendulum.datetime import DateTime
-
-from .data_processing import get_report
-
-
-def basic_report(
-    client: Resource,
-    rows_per_page: int,
-    dimensions: List[str],
-    metrics: List[str],
-    property_id: int,
-    resource_name: str,
-    start_date: str,
-    last_date: dlt.sources.incremental[DateTime],
-) -> Iterator[TDataItem]:
-    """
-    Retrieves the data for a report given dimensions, metrics, and filters required for the report.
-
-    Args:
-        client: The Google Analytics client used to make requests.
-        dimensions: Dimensions for the report. See metadata for the full list of dimensions.
-        metrics: Metrics for the report. See metadata for the full list of metrics.
-        property_id: A reference to the Google Analytics project.
-            More info: https://developers.google.com/analytics/devguides/reporting/data/v1/property-id
-        rows_per_page: Controls how many rows are retrieved per page in the reports.
-            Default is 10000, maximum possible is 100000.
-        resource_name: The resource name used to save incremental into dlt state.
-        start_date: Incremental load start_date.
-            Default is taken from dlt state if it exists.
-        last_date: Incremental load end date.
-            Default is taken from dlt state if it exists.
-
-    Returns:
-        Generator of all rows of data in the report.
-    """
-
-    # grab the start time from last dlt load if not filled, if that is also empty then use the first day of the millennium as the start time instead
-    if last_date.last_value:
-        if start_date != "2015-08-14":
-            logger.warning(
-                f"Using the starting date: {last_date.last_value} for incremental report: {resource_name} and ignoring start date passed as argument {start_date}"
-            )
-        start_date = last_date.last_value.to_date_string()
-    else:
-        start_date = start_date or "2015-08-14"
-
-    processed_response = get_report(
-        client=client,
-        property_id=property_id,
-        # fill dimensions and metrics with the proper api client objects
-        dimension_list=[Dimension(name=dimension) for dimension in dimensions],
-        metric_list=[Metric(name=metric) for metric in metrics],
-        limit=rows_per_page,
-        start_date=start_date,
-        # configure end_date to yesterday as a date string
-        end_date=pendulum.now().to_date_string(),
-    )
-    yield from processed_response