ingestr 0.13.37__tar.gz → 0.13.39__tar.gz

{ingestr-0.13.37 → ingestr-0.13.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.37
+Version: 0.13.39
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -74,7 +74,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.1
+Requires-Dist: greenlet==3.2.2
 Requires-Dist: grpcio-status==1.62.3
 Requires-Dist: grpcio==1.70.0
 Requires-Dist: hdbcli==2.23.27

{ingestr-0.13.37 → ingestr-0.13.39}/docs/.vitepress/config.mjs

@@ -131,6 +131,7 @@ export default defineConfig({
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Personio", link: "/supported-sources/personio.md" },
+              { text: "PhantomBuster", link: "/supported-sources/phantombuster.md" },
               { text: "Pipedrive", link: "/supported-sources/pipedrive.md" },
               { text: "S3", link: "/supported-sources/s3.md" },
               { text: "Salesforce", link: "/supported-sources/salesforce.md" },

{ingestr-0.13.37 → ingestr-0.13.39}/docs/supported-sources/frankfurter.md

@@ -10,7 +10,7 @@ The `ingestr` command to use the `frankfurter` source is as follows:
 
 ```bash
 ingestr ingest \
---source-uri 'frankfurter://' \
+--source-uri 'frankfurter://?base=IDR' \
 --interval-start '2025-03-27' \     # Optional. See 'exchange_rates'.
 --interval-end '2025-03-28' \       # Optional.
 --source-table '<table_name>' \     # E.g 'currencies', 'latest', 'exchange_rates'. See below.
@@ -25,7 +25,9 @@ ingestr ingest \
 ### **`--source-uri`**
 - **Description**: Specifies the source URI for the Frankfurter API.
 - **Value**: `'frankfurter://'`
-- **Purpose**: Indicates that the data will be fetched from the Frankfurter API.
+- **Purpose**: Indicates that the data will be fetched from the Frankfurter API. 
+  - An optional base currency can be added `?base={base_currency}`.
+  - If no base currency is included, base currency defaults to USD.
 
 ---
 
@@ -89,9 +91,10 @@ ingestr ingest \
 - **Description**: Fetches the latest exchange rates.
 - **Columns**:
   - `date`: The date of the exchange rates.
-  - `currency_name`: The ISO 4217 currency code (e.g., `USD`, `EUR`).
+  - `currency_code`: The ISO 4217 currency code (e.g., `USD`, `EUR`).
   - `rate`: The exchange rate relative to the base currency.
-- **Primary Key**: Composite key of `date` and `currency_name`.
+  - `base_currency`: The base currency used to calculate the exchange rate.
+- **Primary Key**: Composite key of `date`, `currency_code` and `base_currency`.
 - **Notes**:
   - The base currency (e.g., `EUR`) is included with a rate of `1.0`.
 
@@ -101,9 +104,10 @@ ingestr ingest \
 - **Description**: Fetches historical exchange rates for a specified date range.
 - **Columns**:
   - `date`: The date of the exchange rates.
-  - `currency_name`: The ISO 4217 currency code (e.g., `USD`, `EUR`).
+  - `currency_code`: The ISO 4217 currency code (e.g., `USD`, `EUR`).
   - `rate`: The exchange rate relative to the base currency.
-- **Primary Key**: Composite key of `date` and `currency_name`.
+  - `base_currency`: The base currency used to calculate the exchange rate.
+- **Primary Key**: Composite key of `date`, `currency_code` and `base_currency`.
 - **Notes**:
   - An optional start and end date can be added via the arguments `--interval-start` and optionally `--interval-end` to define the date range (see examples below). If no start date is specified, the date will default today's date (and thus return the latest exchange rates).
   - If a start date but no end date is specified, then the end date will default to today's date and ingestr will retrieve data up until the latest published data.
@@ -114,24 +118,24 @@ Here `--interval-start` is set to a weekend date (e.g., `2025-03-29` -- a Saturd
 
 `--interval-start` defaults to the previous Friday (`2025-03-28`) and the next data is from the following Monday (for simplicity, only a subset of currencies is shown below):
 
-| **date**     | **currency_name** | **rate** |
-|--------------|-------------------|----------|
-| 2025-03-28   | EUR               | 1.0      | 
-| 2025-03-28   | USD               | 1.0783   | 
-| 2025-03-28   | GBP               | 0.8571   | 
-| 2025-03-31   | EUR               | 1.0      | 
-| 2025-03-31   | USD               | 1.0783   |
-| 2025-03-31   | GBP               | 0.8571   |
+| **date**     | **currency_code** | **rate** | **base_currency** |
+|--------------|-------------------|----------|-------------------|
+| 2025-03-28   | EUR               | 1.0      | EUR               |
+| 2025-03-28   | USD               | 1.0783   | EUR               |
+| 2025-03-28   | GBP               | 0.8571   | EUR               |
+| 2025-03-31   | EUR               | 1.0      | EUR               |
+| 2025-03-31   | USD               | 1.0783   | EUR               |
+| 2025-03-31   | GBP               | 0.8571   | EUR               |
 
 
 ---
 
 ## **Examples**
 
-### **1. Fetch the Latest Exchange Rates**
+### **1. Fetch the Latest Exchange Rates with GBP as Base Currency**
 ```bash
 ingestr ingest \
---source-uri 'frankfurter://' \
+--source-uri 'frankfurter://?base=GBP' \
 --source-table 'latest' \
 --dest-uri 'duckdb.db' \
 --dest-table 'schema.latest_new_scheme'
@@ -139,7 +143,7 @@ ingestr ingest \
 
 ---
 
-### **2. Fetch Historical Exchange Rates**
+### **2. Fetch Historical Exchange Rates with USD as Default Base Currency**
 ```bash
 ingestr ingest \
 --source-uri 'frankfurter://' \

ingestr-0.13.39/docs/supported-sources/phantombuster.md

@@ -0,0 +1,38 @@
+# PhantomBuster
+[PhantomBuster](https://phantombuster.com/) is a cloud-based data automation and web scraping platform that allows users to extract data from websites, automate actions.
+
+ingestr supports PhantomBuster as a source.
+
+## URI format
+
+The URI format for PhantomBuster is as follows:
+
+```plaintext
+PhantomBuster://?api_key=<api_key>
+```
+
+URI parameters:
+- `api_key`: the API key used for authentication with the PhantomBuster API
+
+## Setting up a PhantomBuster Integration
+
+You can find your PhantomBuster API key by following the guide [here](https://hub.phantombuster.com/docs/api#how-to-find-my-api-key).
+
+Let's say your `api_key` is key_123, here's a sample command that will copy the data from PhantomBuster into a DuckDB database:
+
+
+```bash
+ingestr ingest \
+--source-uri 'PhantomBuster://?api_key=key_123' \
+--source-table 'completed_phantoms:<agent_id>' \
+--dest-uri duckdb:///PhantomBuster.duckdb \
+--dest-table 'dest.result'
+```
+
+
+<img alt="PhantomBuster_img" src="../media/phantombuster.png"/>
+
+
+For now, we only support `completed_phantoms` table followed by an `agent_id`. For example: `completed_phantoms:<agent_id>` Where agent id is a unique identifier for a specific Phantom which can be found in URI of a specific phantom.
+
+Use this as `--source-table` parameter in the `ingestr ingest` command.

{ingestr-0.13.37 → ingestr-0.13.39}/ingestr/main.py

@@ -34,6 +34,7 @@ PARQUET_SUPPORTED_DESTINATIONS = [
     "snowflake",
     "databricks",
     "synapse",
+    "s3",
 ]
 
 # these sources would return a JSON for sure, which means they cannot be used with Parquet loader for BigQuery
@@ -485,6 +486,7 @@ def ingest(
         print(
             f"[bold yellow]  Primary Key:[/bold yellow] {primary_key if primary_key else 'None'}"
         )
+        print(f"[bold yellow]  Pipeline ID:[/bold yellow] {m.hexdigest()}")
         print()
 
         if not yes:

ingestr-0.13.39/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.39"

{ingestr-0.13.37 → ingestr-0.13.39}/ingestr/src/destinations.py

@@ -7,11 +7,13 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+import dlt.destinations.impl.filesystem.filesystem
 from dlt.common.configuration.specs import AwsCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
 )
 
+from ingestr.src.errors import MissingValueError
 from ingestr.src.loader import load_dlt_file
 
 
@@ -382,3 +384,71 @@ class ClickhouseDestination:
 
     def post_load(self):
         pass
+
+
+class S3FSClient(dlt.destinations.impl.filesystem.filesystem.FilesystemClient):
+    @property
+    def dataset_path(self):
+        # override to remove dataset path
+        return self.bucket_path
+
+
+class S3FS(dlt.destinations.filesystem):
+    @property
+    def client_class(self):
+        return S3FSClient
+
+
+class S3Destination:
+    def dlt_dest(self, uri: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        access_key_id = params.get("access_key_id", [None])[0]
+        if access_key_id is None:
+            raise MissingValueError("access_key_id", "S3")
+
+        secret_access_key = params.get("secret_access_key", [None])[0]
+        if secret_access_key is None:
+            raise MissingValueError("secret_access_key", "S3")
+
+        endpoint_url = params.get("endpoint_url", [None])[0]
+
+        creds = AwsCredentials(
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            endpoint_url=endpoint_url,
+        )
+
+        dest_table = self.validate_table(kwargs["dest_table"])
+        table_parts = dest_table.split("/")
+        base_path = "/".join(table_parts[:-1])
+
+        opts = {
+            "bucket_url": f"s3://{base_path}",
+            "credentials": creds,
+            # supresses dlt warnings about dataset name normalization.
+            # we don't use dataset names in S3 so it's fine to disable this.
+            "enable_dataset_name_normalization": False,
+        }
+        layout = params.get("layout", [None])[0]
+        if layout is not None:
+            opts["layout"] = layout
+
+        return S3FS(**opts)  # type: ignore
+
+    def validate_table(self, table: str):
+        table = table.strip("/ ")
+        if len(table.split("/")) < 2:
+            raise ValueError("Table name must be in the format {bucket-name}/{path}")
+        return table
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        table = self.validate_table(table)
+        table_parts = table.split("/")
+        return {
+            "table_name": table_parts[-1],
+        }
+
+    def post_load(self) -> None:
+        pass

{ingestr-0.13.37 → ingestr-0.13.39}/ingestr/src/factory.py

@@ -13,6 +13,7 @@ from ingestr.src.destinations import (
     MsSQLDestination,
     PostgresDestination,
     RedshiftDestination,
+    S3Destination,
     SnowflakeDestination,
     SynapseDestination,
 )
@@ -29,6 +30,7 @@ from ingestr.src.sources import (
     DynamoDBSource,
     FacebookAdsSource,
     FrankfurterSource,
+    FreshdeskSource,
     GCSSource,
     GitHubSource,
     GoogleAdsSource,
@@ -44,6 +46,7 @@ from ingestr.src.sources import (
     MongoDbSource,
     NotionSource,
     PersonioSource,
+    PhantombusterSource,
     PipedriveSource,
     S3Source,
     SalesforceSource,
@@ -53,8 +56,6 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     ZendeskSource,
-    FreshdeskSource,
-    PhantombusterSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -170,6 +171,7 @@ class SourceDestinationFactory:
         "athena": AthenaDestination,
         "clickhouse+native": ClickhouseDestination,
         "clickhouse": ClickhouseDestination,
+        "s3": S3Destination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

{ingestr-0.13.37 → ingestr-0.13.39}/ingestr/src/frankfurter/__init__.py

@@ -1,4 +1,4 @@
-from typing import Any, Iterator
+from typing import Any, Iterator, Optional
 
 import dlt
 from dlt.common.pendulum import pendulum
@@ -15,13 +15,13 @@ from ingestr.src.frankfurter.helpers import get_path_with_retry
 def frankfurter_source(
     start_date: TAnyDateTime,
     end_date: TAnyDateTime,
+    base_currency: str,
 ) -> Any:
     """
     A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
     various types of data: currencies, latest rates, historical rates.
     """
     date_time = dlt.sources.incremental(
-
         "date",
         initial_value=start_date,
         end_value=end_date,
@@ -31,9 +31,10 @@ def frankfurter_source(
 
     return (
         currencies(),
-        latest(),
-        exchange_rates(start_date=date_time, end_date=end_date),
-
+        latest(base_currency=base_currency),
+        exchange_rates(
+            start_date=date_time, end_date=end_date, base_currency=base_currency
+        ),
     )
 
 
@@ -61,29 +62,33 @@ def currencies() -> Iterator[dict]:
         "date": {"data_type": "text"},
         "currency_code": {"data_type": "text"},
         "rate": {"data_type": "double"},
+        "base_currency": {"data_type": "text"},
     },
-    primary_key=["date", "currency_code"],  # Composite primary key
+    primary_key=["date", "currency_code", "base_currency"],
 )
-def latest() -> Iterator[dict]:
+def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
     """
     Fetches the latest exchange rates and yields them as rows.
     """
     # Base URL
     url = "latest?"
 
+    if base_currency:
+        url += f"base={base_currency}"
+
     # Fetch data
     data = get_path_with_retry(url)
 
     # Extract rates and base currency
     rates = data["rates"]
-
     date = pendulum.parse(data["date"])
 
-    # Add the base currency (EUR) with a rate of 1.0
+    # Add the base currency with a rate of 1.0
     yield {
         "date": date,
-        "currency_code": "EUR",
+        "currency_code": base_currency,
         "rate": 1.0,
+        "base_currency": base_currency,
     }
 
     # Add all currencies and their rates
@@ -92,6 +97,7 @@ def latest() -> Iterator[dict]:
             "date": date,
             "currency_code": currency_code,
             "rate": rate,
+            "base_currency": base_currency,
         }
 
 
@@ -101,12 +107,14 @@ def latest() -> Iterator[dict]:
         "date": {"data_type": "text"},
         "currency_code": {"data_type": "text"},
         "rate": {"data_type": "double"},
+        "base_currency": {"data_type": "text"},
     },
-    primary_key=("date", "currency_code"),  # Composite primary key
+    primary_key=("date", "currency_code", "base_currency"),
 )
 def exchange_rates(
     end_date: TAnyDateTime,
     start_date: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental("date"),
+    base_currency: Optional[str] = "",
 ) -> Iterator[dict]:
     """
     Fetches exchange rates for a specified date range.
@@ -124,6 +132,9 @@ def exchange_rates(
     # Compose the URL
     url = f"{start_date_str}..{end_date_str}?"
 
+    if base_currency:
+        url += f"base={base_currency}"
+
     # Fetch data from the API
     data = get_path_with_retry(url)
 
@@ -137,8 +148,9 @@ def exchange_rates(
         # Add the base currency with a rate of 1.0
         yield {
             "date": formatted_date,
-            "currency_code": "EUR",
+            "currency_code": base_currency,
             "rate": 1.0,
+            "base_currency": base_currency,
         }
 
         # Add all other currencies and their rates
@@ -147,4 +159,5 @@ def exchange_rates(
                 "date": formatted_date,
                 "currency_code": currency_code,
                 "rate": rate,
+                "base_currency": base_currency,
             }

{ingestr-0.13.37 → ingestr-0.13.39}/ingestr/src/frankfurter/helpers.py

@@ -30,3 +30,19 @@ def validate_dates(start_date: datetime, end_date: datetime) -> None:
     # Check if start_date is before end_date
     if start_date > end_date:
         raise ValueError("Interval-end cannot be before interval-start.")
+
+
+def validate_currency(currency_code: str) -> bool:
+    url = "https://api.frankfurter.dev/v1/currencies"
+
+    response = requests.get(url, timeout=5)
+    currencies = response.json()
+
+    if currency_code.upper() in currencies:
+        return True
+    else:
+        supported_currencies = list(currencies.keys())
+        print(
+            f"Invalid base currency '{currency_code}'. Supported currencies are: {supported_currencies}"
+        )
+        return False

{ingestr-0.13.37 → ingestr-0.13.39}/ingestr/src/google_analytics/helpers.py

@@ -149,7 +149,7 @@ def get_report(
 
         # process request
         processed_response_generator = process_report(response=response)
-        
+
         # import pdb; pdb.set_trace()
         yield from processed_response_generator
         offset += per_page
@@ -225,7 +225,9 @@ def _resolve_dimension_value(dimension_name: str, dimension_value: str) -> Any:
         return dimension_value
 
 
-def convert_minutes_ranges_to_minute_range_objects(minutes_ranges: str) -> List[MinuteRange]:
+def convert_minutes_ranges_to_minute_range_objects(
+    minutes_ranges: str,
+) -> List[MinuteRange]:
     minutes_ranges = minutes_ranges.strip()
     minutes = minutes_ranges.replace(" ", "").split(",")
     if minutes == "":
@@ -233,7 +235,6 @@ def convert_minutes_ranges_to_minute_range_objects(minutes_ranges: str) -> List[
             "Invalid input. Minutes range should be startminute-endminute format. For example: 1-2,5-6"
         )
 
-    
     minute_range_objects = []
     for min_range in minutes:
         if "-" not in min_range:
@@ -246,14 +247,16 @@ def convert_minutes_ranges_to_minute_range_objects(minutes_ranges: str) -> List[
             raise ValueError(
                 f"Invalid input '{min_range}'. Both start and end minutes must be digits. For example: 1-2,5-6"
             )
-        
+
         end_minutes_ago = int(parts[0])
         start_minutes_ago = int(parts[1])
-        minute_range_objects.append(MinuteRange(
-            name=f"{end_minutes_ago}-{start_minutes_ago} minutes ago",
-            start_minutes_ago= start_minutes_ago,
-            end_minutes_ago=end_minutes_ago
-        ))
+        minute_range_objects.append(
+            MinuteRange(
+                name=f"{end_minutes_ago}-{start_minutes_ago} minutes ago",
+                start_minutes_ago=start_minutes_ago,
+                end_minutes_ago=end_minutes_ago,
+            )
+        )
 
     return minute_range_objects
 

ingestr-0.13.39/ingestr/src/phantombuster/__init__.py

@@ -0,0 +1,65 @@
+from typing import Iterable, Optional
+
+import dlt
+import pendulum
+import requests
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+
+from ingestr.src.phantombuster.client import PhantombusterClient
+
+
+def retry_on_limit(
+    response: Optional[requests.Response], exception: Optional[BaseException]
+) -> bool:
+    if response is not None and response.status_code == 429:
+        return True
+    return False
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+
+
+@dlt.source(max_table_nesting=0)
+def phantombuster_source(
+    api_key: str, agent_id: str, start_date: TAnyDateTime, end_date: TAnyDateTime | None
+) -> Iterable[DltResource]:
+    client = PhantombusterClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key="container_id",
+        columns={
+            "partition_dt": {"data_type": "date", "partition": True},
+        },
+    )
+    def completed_phantoms(
+        dateTime=(
+            dlt.sources.incremental(
+                "ended_at",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterable[TDataItem]:
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+
+        yield client.fetch_containers_result(
+            create_client(), agent_id, start_date=start_dt, end_date=end_dt
+        )
+
+    return completed_phantoms

{ingestr-0.13.37 → ingestr-0.13.39}/ingestr/src/phantombuster/client.py

@@ -14,12 +14,22 @@ class PhantombusterClient:
             "accept": "application/json",
         }
 
-    def fetch_containers_result(self, session: requests.Session, agent_id: str, start_date: pendulum.DateTime, end_date: pendulum.DateTime):
+    def fetch_containers_result(
+        self,
+        session: requests.Session,
+        agent_id: str,
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
+    ):
         url = "https://api.phantombuster.com/api/v2/containers/fetch-all/"
         before_ended_at = None
         limit = 100
-        started_at = start_date.int_timestamp * 1000 + int(start_date.microsecond / 1000)
+
+        started_at = start_date.int_timestamp * 1000 + int(
+            start_date.microsecond / 1000
+        )
         ended_at = end_date.int_timestamp * 1000 + int(end_date.microsecond / 1000)
+
         while True:
             params: dict[str, Union[str, int, float, bytes, None]] = {
                 "agentId": agent_id,
@@ -36,23 +46,35 @@ class PhantombusterClient:
 
             for container in containers:
                 container_ended_at = container.get("endedAt")
-                if before_ended_at is None or before_ended_at > container["endedAt"]:
-                        before_ended_at = container["endedAt"]
-                        
-                if not (started_at <= container_ended_at <= ended_at):
+
+                if before_ended_at is None or before_ended_at > container_ended_at:
+                    before_ended_at = container_ended_at
+
+                if container_ended_at < started_at or container_ended_at > ended_at:
                     continue
+
                 try:
                     result = self.fetch_result_object(session, container["id"])
-                    partition_dt = pendulum.from_timestamp(container_ended_at / 1000, tz="UTC").to_date_string()
-                    row = {"container": container, "result": result, "partition_dt": partition_dt}
+                    partition_dt = pendulum.from_timestamp(
+                        container_ended_at / 1000, tz="UTC"
+                    ).date()
+                    container_ended_at_datetime = pendulum.from_timestamp(
+                        container_ended_at / 1000, tz="UTC"
+                    )
+                    row = {
+                        "container_id": container["id"],
+                        "container": container,
+                        "result": result,
+                        "partition_dt": partition_dt,
+                        "ended_at": container_ended_at_datetime,
+                    }
                     yield row
-                    
+
                 except requests.RequestException as e:
                     print(f"Error fetching result for container {container['id']}: {e}")
-         
+
             if data["maxLimitReached"] is False:
                 break
- 
 
     def fetch_result_object(self, session: requests.Session, container_id: str):
         result_url = (