ingestr 0.13.38__py3-none-any.whl → 0.13.39__py3-none-any.whl

ingestr/main.py

@@ -34,6 +34,7 @@ PARQUET_SUPPORTED_DESTINATIONS = [
     "snowflake",
     "databricks",
     "synapse",
+    "s3",
 ]
 
 # these sources would return a JSON for sure, which means they cannot be used with Parquet loader for BigQuery
@@ -485,9 +486,7 @@ def ingest(
         print(
             f"[bold yellow]  Primary Key:[/bold yellow] {primary_key if primary_key else 'None'}"
         )
-        print(
-            f"[bold yellow]  Pipeline ID:[/bold yellow] {m.hexdigest()}"
-        )
+        print(f"[bold yellow]  Pipeline ID:[/bold yellow] {m.hexdigest()}")
         print()
 
         if not yes:

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.38"
+version = "v0.13.39"

ingestr/src/destinations.py

@@ -7,11 +7,13 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+import dlt.destinations.impl.filesystem.filesystem
 from dlt.common.configuration.specs import AwsCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
 )
 
+from ingestr.src.errors import MissingValueError
 from ingestr.src.loader import load_dlt_file
 
 
@@ -382,3 +384,71 @@ class ClickhouseDestination:
 
     def post_load(self):
         pass
+
+
+class S3FSClient(dlt.destinations.impl.filesystem.filesystem.FilesystemClient):
+    @property
+    def dataset_path(self):
+        # override to remove dataset path
+        return self.bucket_path
+
+
+class S3FS(dlt.destinations.filesystem):
+    @property
+    def client_class(self):
+        return S3FSClient
+
+
+class S3Destination:
+    def dlt_dest(self, uri: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        access_key_id = params.get("access_key_id", [None])[0]
+        if access_key_id is None:
+            raise MissingValueError("access_key_id", "S3")
+
+        secret_access_key = params.get("secret_access_key", [None])[0]
+        if secret_access_key is None:
+            raise MissingValueError("secret_access_key", "S3")
+
+        endpoint_url = params.get("endpoint_url", [None])[0]
+
+        creds = AwsCredentials(
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            endpoint_url=endpoint_url,
+        )
+
+        dest_table = self.validate_table(kwargs["dest_table"])
+        table_parts = dest_table.split("/")
+        base_path = "/".join(table_parts[:-1])
+
+        opts = {
+            "bucket_url": f"s3://{base_path}",
+            "credentials": creds,
+            # supresses dlt warnings about dataset name normalization.
+            # we don't use dataset names in S3 so it's fine to disable this.
+            "enable_dataset_name_normalization": False,
+        }
+        layout = params.get("layout", [None])[0]
+        if layout is not None:
+            opts["layout"] = layout
+
+        return S3FS(**opts)  # type: ignore
+
+    def validate_table(self, table: str):
+        table = table.strip("/ ")
+        if len(table.split("/")) < 2:
+            raise ValueError("Table name must be in the format {bucket-name}/{path}")
+        return table
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        table = self.validate_table(table)
+        table_parts = table.split("/")
+        return {
+            "table_name": table_parts[-1],
+        }
+
+    def post_load(self) -> None:
+        pass

ingestr/src/factory.py

@@ -13,6 +13,7 @@ from ingestr.src.destinations import (
     MsSQLDestination,
     PostgresDestination,
     RedshiftDestination,
+    S3Destination,
     SnowflakeDestination,
     SynapseDestination,
 )
@@ -29,6 +30,7 @@ from ingestr.src.sources import (
     DynamoDBSource,
     FacebookAdsSource,
     FrankfurterSource,
+    FreshdeskSource,
     GCSSource,
     GitHubSource,
     GoogleAdsSource,
@@ -44,6 +46,7 @@ from ingestr.src.sources import (
     MongoDbSource,
     NotionSource,
     PersonioSource,
+    PhantombusterSource,
     PipedriveSource,
     S3Source,
     SalesforceSource,
@@ -53,8 +56,6 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     ZendeskSource,
-    FreshdeskSource,
-    PhantombusterSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -170,6 +171,7 @@ class SourceDestinationFactory:
         "athena": AthenaDestination,
         "clickhouse+native": ClickhouseDestination,
         "clickhouse": ClickhouseDestination,
+        "s3": S3Destination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

ingestr/src/google_analytics/helpers.py

@@ -149,7 +149,7 @@ def get_report(
 
         # process request
         processed_response_generator = process_report(response=response)
-        
+
         # import pdb; pdb.set_trace()
         yield from processed_response_generator
         offset += per_page
@@ -225,7 +225,9 @@ def _resolve_dimension_value(dimension_name: str, dimension_value: str) -> Any:
         return dimension_value
 
 
-def convert_minutes_ranges_to_minute_range_objects(minutes_ranges: str) -> List[MinuteRange]:
+def convert_minutes_ranges_to_minute_range_objects(
+    minutes_ranges: str,
+) -> List[MinuteRange]:
     minutes_ranges = minutes_ranges.strip()
     minutes = minutes_ranges.replace(" ", "").split(",")
     if minutes == "":
@@ -233,7 +235,6 @@ def convert_minutes_ranges_to_minute_range_objects(minutes_ranges: str) -> List[
             "Invalid input. Minutes range should be startminute-endminute format. For example: 1-2,5-6"
         )
 
-    
     minute_range_objects = []
     for min_range in minutes:
         if "-" not in min_range:
@@ -246,14 +247,16 @@ def convert_minutes_ranges_to_minute_range_objects(minutes_ranges: str) -> List[
             raise ValueError(
                 f"Invalid input '{min_range}'. Both start and end minutes must be digits. For example: 1-2,5-6"
             )
-        
+
         end_minutes_ago = int(parts[0])
         start_minutes_ago = int(parts[1])
-        minute_range_objects.append(MinuteRange(
-            name=f"{end_minutes_ago}-{start_minutes_ago} minutes ago",
-            start_minutes_ago= start_minutes_ago,
-            end_minutes_ago=end_minutes_ago
-        ))
+        minute_range_objects.append(
+            MinuteRange(
+                name=f"{end_minutes_ago}-{start_minutes_ago} minutes ago",
+                start_minutes_ago=start_minutes_ago,
+                end_minutes_ago=end_minutes_ago,
+            )
+        )
 
     return minute_range_objects
 

ingestr/src/phantombuster/__init__.py

@@ -3,11 +3,10 @@ from typing import Iterable, Optional
 import dlt
 import pendulum
 import requests
-from dlt.common.typing import TDataItem, TAnyDateTime
+from dlt.common.typing import TAnyDateTime, TDataItem
 from dlt.sources import DltResource
 from dlt.sources.helpers.requests import Client
 
-
 from ingestr.src.phantombuster.client import PhantombusterClient
 
 
@@ -27,9 +26,13 @@ def create_client() -> requests.Session:
         request_backoff_factor=2,
     ).session
 
+
 @dlt.source(max_table_nesting=0)
-def phantombuster_source(api_key: str, agent_id: str, start_date: TAnyDateTime, end_date: TAnyDateTime | None) -> Iterable[DltResource]:
+def phantombuster_source(
+    api_key: str, agent_id: str, start_date: TAnyDateTime, end_date: TAnyDateTime | None
+) -> Iterable[DltResource]:
     client = PhantombusterClient(api_key)
+
     @dlt.resource(
         write_disposition="merge",
         primary_key="container_id",
@@ -55,6 +58,8 @@ def phantombuster_source(api_key: str, agent_id: str, start_date: TAnyDateTime,
 
         start_dt = dateTime.last_value
 
-        yield client.fetch_containers_result(create_client(), agent_id, start_date=start_dt, end_date=end_dt)
+        yield client.fetch_containers_result(
+            create_client(), agent_id, start_date=start_dt, end_date=end_dt
+        )
 
     return completed_phantoms

ingestr/src/phantombuster/client.py

@@ -14,14 +14,22 @@ class PhantombusterClient:
             "accept": "application/json",
         }
 
-    def fetch_containers_result(self, session: requests.Session, agent_id: str, start_date: pendulum.DateTime, end_date: pendulum.DateTime):
+    def fetch_containers_result(
+        self,
+        session: requests.Session,
+        agent_id: str,
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
+    ):
         url = "https://api.phantombuster.com/api/v2/containers/fetch-all/"
         before_ended_at = None
         limit = 100
 
-        started_at = start_date.int_timestamp * 1000 + int(start_date.microsecond / 1000)
+        started_at = start_date.int_timestamp * 1000 + int(
+            start_date.microsecond / 1000
+        )
         ended_at = end_date.int_timestamp * 1000 + int(end_date.microsecond / 1000)
-       
+
         while True:
             params: dict[str, Union[str, int, float, bytes, None]] = {
                 "agentId": agent_id,
@@ -40,24 +48,33 @@ class PhantombusterClient:
                 container_ended_at = container.get("endedAt")
 
                 if before_ended_at is None or before_ended_at > container_ended_at:
-                        before_ended_at = container_ended_at
-                
+                    before_ended_at = container_ended_at
+
                 if container_ended_at < started_at or container_ended_at > ended_at:
                     continue
-              
+
                 try:
                     result = self.fetch_result_object(session, container["id"])
-                    partition_dt = pendulum.from_timestamp(container_ended_at / 1000, tz="UTC").date()
-                    container_ended_at_datetime = pendulum.from_timestamp(container_ended_at / 1000, tz="UTC")
-                    row = {"container_id": container["id"],"container": container, "result": result, "partition_dt": partition_dt, "ended_at": container_ended_at_datetime}
+                    partition_dt = pendulum.from_timestamp(
+                        container_ended_at / 1000, tz="UTC"
+                    ).date()
+                    container_ended_at_datetime = pendulum.from_timestamp(
+                        container_ended_at / 1000, tz="UTC"
+                    )
+                    row = {
+                        "container_id": container["id"],
+                        "container": container,
+                        "result": result,
+                        "partition_dt": partition_dt,
+                        "ended_at": container_ended_at_datetime,
+                    }
                     yield row
-                    
+
                 except requests.RequestException as e:
                     print(f"Error fetching result for container {container['id']}: {e}")
-         
+
             if data["maxLimitReached"] is False:
                 break
- 
 
     def fetch_result_object(self, session: requests.Session, container_id: str):
         result_url = (

ingestr/src/sources.py

@@ -1492,7 +1492,9 @@ class GoogleAnalyticsSource:
 
         minute_range_objects = []
         if len(fields) == 4:
-            minute_range_objects = helpers.convert_minutes_ranges_to_minute_range_objects(fields[3])
+            minute_range_objects = (
+                helpers.convert_minutes_ranges_to_minute_range_objects(fields[3])
+            )
 
         datetime = ""
         resource_name = fields[0].lower()
@@ -2217,10 +2219,10 @@ class FrankfurterSource:
 
 
 class FreshdeskSource:
-     # freshdesk://domain?api_key=<api_key>
+    # freshdesk://domain?api_key=<api_key>
     def handles_incrementality(self) -> bool:
         return True
-    
+
     def dlt_source(self, uri: str, table: str, **kwargs):
         parsed_uri = urlparse(uri)
         domain = parsed_uri.netloc
@@ -2230,43 +2232,54 @@ class FreshdeskSource:
         if not domain:
             raise MissingValueError("domain", "Freshdesk")
 
-        if '.' in domain:
-            domain = domain.split('.')[0]
-        
+        if "." in domain:
+            domain = domain.split(".")[0]
+
         api_key = params.get("api_key")
         if api_key is None:
             raise MissingValueError("api_key", "Freshdesk")
-        
-        if table not in ["agents", "companies", "contacts", "groups", "roles", "tickets"]:
+
+        if table not in [
+            "agents",
+            "companies",
+            "contacts",
+            "groups",
+            "roles",
+            "tickets",
+        ]:
             raise UnsupportedResourceError(table, "Freshdesk")
-        
+
         from ingestr.src.freshdesk import freshdesk_source
-        return freshdesk_source(api_secret_key=api_key[0], domain=domain).with_resources(table)
+
+        return freshdesk_source(
+            api_secret_key=api_key[0], domain=domain
+        ).with_resources(table)
+
 
 class PhantombusterSource:
     def handles_incrementality(self) -> bool:
         return True
-    
+
     def dlt_source(self, uri: str, table: str, **kwargs):
-        #phantombuster://?api_key=<api_key>
-        #source table = phantom_results:agent_id
+        # phantombuster://?api_key=<api_key>
+        # source table = phantom_results:agent_id
         parsed_uri = urlparse(uri)
         params = parse_qs(parsed_uri.query)
         api_key = params.get("api_key")
         if api_key is None:
             raise MissingValueError("api_key", "Phantombuster")
-        
+
         table_fields = table.replace(" ", "").split(":")
         table_name = table_fields[0]
-        
+
         agent_id = table_fields[1] if len(table_fields) > 1 else None
-        
+
         if table_name not in ["completed_phantoms"]:
             raise UnsupportedResourceError(table_name, "Phantombuster")
-        
+
         if not agent_id:
             raise MissingValueError("agent_id", "Phantombuster")
-        
+
         start_date = kwargs.get("interval_start")
         if start_date is None:
             start_date = ensure_pendulum_datetime("2018-01-01").in_tz("UTC")
@@ -2276,6 +2289,12 @@ class PhantombusterSource:
         end_date = kwargs.get("interval_end")
         if end_date is not None:
             end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
-        
+
         from ingestr.src.phantombuster import phantombuster_source
-        return phantombuster_source(api_key=api_key[0], agent_id=agent_id, start_date=start_date, end_date=end_date).with_resources(table_name)
+
+        return phantombuster_source(
+            api_key=api_key[0],
+            agent_id=agent_id,
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table_name)

{ingestr-0.13.38.dist-info → ingestr-0.13.39.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.38
+Version: 0.13.39
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.38.dist-info → ingestr-0.13.39.dist-info}/RECORD

@@ -1,16 +1,16 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=QHLjpCItCgL5XHcTezBOp6Vdy_VHAPciRjqy63ZIK2s,25285
+ingestr/main.py,sha256=Pe_rzwcDRKIYa7baEVUAAPOHyqQbX29RUexMl0F_S1k,25273
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=iW7GVGJjTCNtuINLzbFuur_d0ALxf_JmZSmuwD9yZ2Y,21
-ingestr/src/destinations.py,sha256=Z79f01BSmEaXnQno2IQVt4Th4dmD-BiOQXlibZJ5sTw,13180
+ingestr/src/buildinfo.py,sha256=edyodue-Rkn4zTwWVR9OU0dSsDXVokKw_KKllrI1amM,21
+ingestr/src/destinations.py,sha256=MctbeJUyNr0DRB0XYt2xAbEKkHZ40-nXXEOYCs4KuoE,15420
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=Mm_Be60PFO4mUIeJLBMDVU_uyH0IeCiZ1dtNDFiDFSo,5463
+ingestr/src/factory.py,sha256=j-FKRBEBZVLT_DEn-SCu9KEvaab3BchEV5hzTjpree8,5511
 ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=q9FJ3vRaeTjbL5sgau_GHPxbn5LfSI4t5TBwycVdwSc,79060
+ingestr/src/sources.py,sha256=vppNI75ucM0EtW2kP5ldKyhc4Pij_hGVmKlZ9DNL4g0,79181
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -57,7 +57,7 @@ ingestr/src/google_ads/metrics.py,sha256=tAqpBpm-8l95oPT9cBxMWaEoDTNHVXnqUphYDHW
 ingestr/src/google_ads/predicates.py,sha256=K4wTuqfmJ9ko1RKeHTBDfQO_mUADVyuRqtywBPP-72w,683
 ingestr/src/google_ads/reports.py,sha256=AVY1pPt5yaIFskQe1k5VW2Dhlux3bzewsHlDrdGEems,12686
 ingestr/src/google_analytics/__init__.py,sha256=8b9CBWJFrBpHVRl993Z7J01sKKbYyXEtngdfEUwqlfE,4343
-ingestr/src/google_analytics/helpers.py,sha256=bUTPp5C-k5wqq-ccEAn-asRH2CLbBS2SOs1v9wiRU6U,10087
+ingestr/src/google_analytics/helpers.py,sha256=tM7h_yughca3l7tnS_2XGIBM37mVm-Uewv7tK7XHVbc,10111
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=CL0HfY74uxX8-ge0ucI0VhWMYZVAfoX7WRPBitRi-CI,6647
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
@@ -87,8 +87,8 @@ ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falP
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
 ingestr/src/personio/__init__.py,sha256=sHYpoV-rg-kA1YsflctChis0hKcTrL6mka9O0CHV4zA,11638
 ingestr/src/personio/helpers.py,sha256=EKmBN0Lf4R0lc3yqqs7D-RjoZ75E8gPcctt59xwHxrY,2901
-ingestr/src/phantombuster/__init__.py,sha256=5XGwbtNYmRxL4lmcC0TmyXgljY_21DvDIdw9zaWmgvI,1757
-ingestr/src/phantombuster/client.py,sha256=WO87AGU3Fphd9DgWTkrSAfldLH1XxcPxvNVmNxQ_5eU,2785
+ingestr/src/phantombuster/__init__.py,sha256=8AQTiA8fp1NT8TellQQqwBCl6vGvGwUBLif6LIzgAik,1786
+ingestr/src/phantombuster/client.py,sha256=9zx58sFunXjUNh6jeEYLNfwNxGxX9odifwAmS0E9AaY,3018
 ingestr/src/pipedrive/__init__.py,sha256=iRrxeMwo8_83ptgGnTFTNHV1nYvIsFfg0a3XzugPYeI,6982
 ingestr/src/pipedrive/settings.py,sha256=q119Fy4C5Ip1rMoCILX2BkHV3bwiXC_dW58KIiDUzsY,708
 ingestr/src/pipedrive/typing.py,sha256=lEMXu4hhAA3XkhVSlBUa-juqyupisd3c-qSQKxFvzoE,69
@@ -127,8 +127,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.38.dist-info/METADATA,sha256=ZknSXuae-_lb60AWDACr731hTr0Q_bX_9sUds6rHM28,13575
-ingestr-0.13.38.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.38.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.38.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.38.dist-info/RECORD,,
+ingestr-0.13.39.dist-info/METADATA,sha256=goY5MW5AzJwYQ0cbwTmlNxZgCP1QRSt6ROmBOImESIM,13575
+ingestr-0.13.39.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.39.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.39.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.39.dist-info/RECORD,,