ingestr 0.13.58__tar.gz → 0.13.60__tar.gz

{ingestr-0.13.58 → ingestr-0.13.60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.58
+Version: 0.13.60
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.58 → ingestr-0.13.60}/docs/.vitepress/config.mjs

@@ -131,12 +131,14 @@ export default defineConfig({
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
               { text: "Gorgias", link: "/supported-sources/gorgias.md" },
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
+              { text: "Internet Society Pulse", link: "/supported-sources/isoc-pulse.md" },
               { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
               { text: "Mixpanel", link: "/supported-sources/mixpanel.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Personio", link: "/supported-sources/personio.md" },
               { text: "PhantomBuster", link: "/supported-sources/phantombuster.md" },
+              { text: "Pinterest", link: "/supported-sources/pinterest.md" },
               { text: "Pipedrive", link: "/supported-sources/pipedrive.md" },
               { text: "QuickBooks", link: "/supported-sources/quickbooks.md" },
               { text: "S3", link: "/supported-sources/s3.md" },

ingestr-0.13.60/docs/supported-sources/isoc-pulse.md

@@ -0,0 +1,147 @@
+# Internet Society Pulse
+
+[Internet Society Pulse](https://pulse.internetsociety.org/) is a platform that monitors the health, availability, and evolution of the Internet, providing metrics on key technologies that contribute to its security, resilience, and trustworthiness.
+
+ingestr supports Internet Society Pulse as a source.
+
+## URI format
+
+The URI format for Internet Society Pulse is as follows:
+
+```plaintext
+isoc-pulse://?token=<your-token>
+```
+
+URI parameters:
+- `token`: The API token used for authentication with the Internet Society Pulse API.
+
+## Setting up an Internet Society Pulse Integration
+
+To use the Internet Society Pulse source, you need to obtain an API token from the Internet Society.
+
+Once you have the token, you can access various metrics from the Pulse platform. Here's a sample command that will copy data from the ISOC Pulse API into a DuckDB database:
+
+```sh
+ingestr ingest \
+  --source-uri 'isoc-pulse://?token=your_token_here' \
+  --source-table 'https' \
+  --dest-uri 'duckdb:///pulse_data.duckdb' \
+  --dest-table 'dest.https_adoption' \
+  --interval-start '2023-01-01' \
+  --interval-end '2023-12-31'
+```
+
+This will retrieve HTTPS adoption metrics for the specified date range and store them in your database.
+
+## Tables
+
+Internet Society Pulse source allows ingesting the following metrics as separate tables:
+
+| Metric | Description | Country Support | Additional Options |
+|--------|-------------|----------------|-------------------|
+| `dnssec_adoption` | DNSSEC adoption metrics for specific domains | No | Domain name |
+| `dnssec_tld_adoption` | DNSSEC adoption metrics for top-level domains | Yes | Country code |
+| `dnssec_validation` | DNSSEC validation metrics | Yes | Country code |
+| `http` | HTTP protocol metrics | No | None |
+| `http3` | HTTP/3 protocol metrics | No | None |
+| `https` | HTTPS adoption metrics | Yes | `topsites`, Country code |
+| `ipv6` | IPv6 adoption metrics | Yes | `topsites`, Country code |
+| `net_loss` | Internet disconnection metrics | Yes | Shutdown type, Country code |
+| `resilience` | Internet resilience metrics | Yes | Country code |
+| `roa` | Route Origin Authorization metrics | Yes | IP version (`4`/`6`), Country code |
+| `rov` | Route Origin Validation metrics | No | None |
+| `tls` | TLS protocol metrics | No | None |
+| `tls13` | TLS 1.3 protocol metrics | No | None |
+
+## Parameter Syntax
+
+Many metrics support additional parameters, including country-specific data. These parameters are specified using colons in the source table name. The general format is:
+
+```
+metric[:option][:country]
+```
+
+Where the options and parameters vary by metric type.
+
+### General Parameter Rules
+
+1. Country codes should follow the ISO 3166-1 alpha-2 format (e.g., US, GB, DE)
+2. The parameters are position-specific – the order matters
+
+## Metric-Specific Parameters
+
+### HTTPS Metrics
+
+- Global data: `https`
+- Country-specific: `https:US` 
+- Top sites data: `https:topsites`
+- Top sites for a specific country: `https:topsites:US`
+
+### IPv6 Metrics
+
+- Global data: `ipv6` 
+- Country-specific: `ipv6:DE` 
+- Top sites data: `ipv6:topsites`
+
+### DNSSEC Metrics
+
+- Global validation: `dnssec_validation`
+- Country-specific validation: `dnssec_validation:SE` 
+- Global TLD adoption: `dnssec_tld_adoption`
+- Country-specific TLD adoption: `dnssec_tld_adoption:JP`
+
+### ROA Metrics (Route Origin Authorization)
+
+- Global for IPv4: `roa:4`
+- Global for IPv6: `roa:6`
+- Country-specific for IPv4: `roa:4:CN`
+- Country-specific for IPv6: `roa:6:BR`
+
+### Net Loss Metrics (Internet Disconnections)
+
+- Country-specific with shutdown type: `net_loss:shutdown_type:country_code`
+- Example for shutdowns in India: `net_loss:shutdown:IN`
+- Example for internet blocking in Japan: `net_loss:blocking:JP`
+
+### Resilience Metrics
+
+- Global resilience: `resilience`
+- Country-specific resilience: `resilience::FR`
+
+## Examples
+
+### Country-specific HTTPS Adoption
+
+```sh
+ingestr ingest \
+  --source-uri 'isoc-pulse://?token=your_token_here' \
+  --source-table 'https::US' \
+  --dest-uri 'duckdb:///pulse_data.duckdb' \
+  --dest-table 'dest.us_https_adoption' \
+  --interval-start '2023-01-01'
+```
+
+### IPv6 Top Sites Data
+
+```sh
+ingestr ingest \
+  --source-uri 'isoc-pulse://?token=your_token_here' \
+  --source-table 'ipv6:topsites' \
+  --dest-uri 'duckdb:///pulse_data.duckdb' \
+  --dest-table 'dest.ipv6_topsites' \
+  --interval-start '2023-01-01'
+```
+
+### ROA IPv4 Data by Country
+
+```sh
+ingestr ingest \
+  --source-uri 'isoc-pulse://?token=your_token_here' \
+  --source-table 'roa:4:US' \
+  --dest-uri 'duckdb:///pulse_data.duckdb' \
+  --dest-table 'dest.us_roa_ipv4' \
+  --interval-start '2023-01-01'
+```
+
+## Further Reading:
+* [Pulse Documentation](https://pulse.internetsociety.org/api/docs)

ingestr-0.13.60/docs/supported-sources/pinterest.md

@@ -0,0 +1,27 @@
+# Pinterest
+
+[Pinterest](https://www.pinterest.com/) is a social media platform for discovering and sharing ideas using visual bookmarks.
+
+ingestr supports Pinterest as a source.
+
+
+## URI Format
+
+The URI format for Pinterest is as follows:
+
+```plaintext
+pinterest://?access_token=<access_token>
+```
+
+URI parameters:
+- `access_token`: The token used for authentication with the Pinterest API. You can obtain an access token from the [official Pinterest documentation](https://developers.pinterest.com/docs/getting-started/connect-app/)
+
+
+
+## Tables
+
+Pinterest source allows ingesting the following sources into separate tables:
+- `pins`: Retrieves a list of pins.
+- `boards`: Retrieves a list of boards.
+
+Use these table names with the `--source-table` parameter in the `ingestr ingest` command.

{ingestr-0.13.58 → ingestr-0.13.60}/ingestr/main.py

@@ -543,6 +543,7 @@ def ingest(
             sql_reflection_level=sql_reflection_level.value,
             sql_limit=sql_limit,
             sql_exclude_columns=sql_exclude_columns,
+            extract_parallelism=extract_parallelism,
         )
 
         resource.for_each(dlt_source, lambda x: x.add_map(cast_set_to_list))

ingestr-0.13.60/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.60"

{ingestr-0.13.58 → ingestr-0.13.60}/ingestr/src/factory.py

@@ -42,6 +42,7 @@ from ingestr.src.sources import (
     GoogleSheetsSource,
     GorgiasSource,
     HubspotSource,
+    IsocPulseSource,
     KafkaSource,
     KinesisSource,
     KlaviyoSource,
@@ -52,6 +53,7 @@ from ingestr.src.sources import (
     NotionSource,
     PersonioSource,
     PhantombusterSource,
+    PinterestSource,
     PipedriveSource,
     QuickBooksSource,
     S3Source,
@@ -172,8 +174,10 @@ class SourceDestinationFactory:
         "attio": AttioSource,
         "solidgate": SolidgateSource,
         "quickbooks": QuickBooksSource,
+        "isoc-pulse": IsocPulseSource,
         "smartsheet": SmartsheetSource,
         "sftp": SFTPSource,
+        "pinterest": PinterestSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr-0.13.60/ingestr/src/isoc_pulse/__init__.py

@@ -0,0 +1,159 @@
+import math
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Dict, Iterable, List, Optional
+
+import dlt
+from dlt.sources.rest_api import RESTAPIConfig, rest_api_resources
+
+METRICS: Dict[str, str] = {
+    "dnssec_adoption": "dnssec/adoption",
+    "dnssec_tld_adoption": "dnssec/adoption",
+    "dnssec_validation": "dnssec/validation",
+    "http": "http",
+    "http3": "http3",
+    "https": "https",
+    "ipv6": "ipv6",
+    "net_loss": "net-loss",
+    "resilience": "resilience",
+    "roa": "roa",
+    "rov": "rov",
+    "tls": "tls",
+    "tls13": "tls13",
+}
+
+
+@dlt.source
+def pulse_source(
+    token: str,
+    start_date: str,
+    metric: str,
+    opts: List[str],
+    end_date: Optional[str] = None,
+) -> Iterable[dlt.sources.DltResource]:
+    validate(metric, opts)
+    cfg = get_metric_cfg(metric, opts, start_date)
+    endpoint: Dict[str, Any] = {
+        "path": cfg.path,
+        "params": {
+            "start_date": "{incremental.start_value}",
+            **cfg.params,
+        },
+        "incremental": {
+            "cursor_path": "date",
+            "start_param": "start_date",
+            "end_param": "end_date",
+            "initial_value": start_date,
+            "end_value": end_date,
+            "range_start": "closed",
+            "range_end": "closed",
+        },
+        "paginator": "single_page",
+    }
+
+    if end_date is not None:
+        endpoint["params"]["end_date"] = end_date
+
+    resources = [
+        {
+            "name": metric,
+            "write_disposition": "merge",
+            "primary_key": "date",
+            "columns": {"date": {"data_type": "date"}},
+            "endpoint": endpoint,
+        }
+    ]
+
+    config: RESTAPIConfig = {
+        "client": {
+            "base_url": "https://pulse.internetsociety.org/api/",
+            "headers": {"Authorization": f"Bearer {token}"},
+        },
+        "resource_defaults": {
+            "write_disposition": "merge",
+            "primary_key": "date",
+        },
+        "resources": resources, # type:ignore
+    }
+    res = rest_api_resources(config)
+    if metric == "net_loss":
+        res[0].add_map(add_date(start_date))
+    yield from res
+
+
+@dataclass
+class MetricCfg:
+    path: str
+    params: Dict[str, Any]
+
+
+def get_metric_cfg(metric: str, opts: List[str], start_date: str) -> MetricCfg:
+    path = METRICS.get(metric)
+    if path is None:
+        raise ValueError(f"Unknown metric '{metric}'.")
+    if len(opts) == 0:
+        return MetricCfg(path=path, params={})
+
+    if metric == "https":
+        return MetricCfg(
+            path=f"{path}/country/{opts[-1]}",
+            params={
+                "topsites": True if "topsites" in opts else False,
+            },
+        )
+    elif metric in ["dnssec_validation", "dnssec_tld_adoption"]:
+        return MetricCfg(path=f"{path}/country/{opts[-1]}", params={})
+    elif metric == "dnssec_adoption":
+        return MetricCfg(path=f"{path}/domains/{opts[-1]}", params={})
+    elif metric == "ipv6":
+        if "topsites" in opts:
+            return MetricCfg(path=path, params={"topsites": True})
+        return MetricCfg(path=f"{path}/country/{opts[-1]}", params={})
+    elif metric == "roa":
+        if len(opts) > 1:
+            return MetricCfg(
+                path=f"{path}/country/{opts[-1]}", params={"ip_version": opts[-2]}
+            )
+        return MetricCfg(path=path, params={"ip_version": opts[-1]})
+    elif metric == "net_loss":
+        return MetricCfg(
+            path=path,
+            params={
+                "country": opts[-1],
+                "shutdown_type": opts[-2],
+            },
+        )
+    elif metric == "resilience":
+        date = datetime.strptime(start_date, "%Y-%m-%d")
+        return MetricCfg(
+            path=path,
+            params={
+                "country": opts[-1],
+                "year": date.year,
+                "quarter": math.floor(date.month / 4) + 1,
+            },
+        )
+    else:
+        raise ValueError(
+            f"Unsupported metric '{metric}' with options {opts}. "
+            "Please check the metric and options."
+        )
+
+
+def add_date(start_date: str):
+    def transform(item: dict):
+        item["date"] = start_date
+        return item
+
+    return transform
+
+
+def validate(metric: str, opts: List[str]) -> None:
+    nopts = len(opts)
+    if metric == "net_loss" and nopts != 2:
+        raise ValueError(
+            "For 'net_loss' metric, two options are required: "
+            "'shutdown_type' and 'country'."
+        )
+    if nopts > 0 and metric in ["http", "http3", "tls", "tls13", "rov"]:
+        raise ValueError(f"metric '{metric}' does not support options. ")

ingestr-0.13.60/ingestr/src/pinterest/__init__.py

@@ -0,0 +1,82 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers import requests
+
+
+@dlt.source(name="pinterest", max_table_nesting=0)
+def pinterest_source(
+    start_date: pendulum.DateTime,
+    access_token: str,
+    page_size: int = 200,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    session = requests.Session()
+    session.headers.update({"Authorization": f"Bearer {access_token}"})
+    base_url = "https://api.pinterest.com/v5"
+
+    def fetch_data(
+        endpoint: str,
+        start_dt: pendulum.DateTime,
+        end_dt: pendulum.DateTime,
+    ) -> Iterable[TDataItem]:
+        url = f"{base_url}/{endpoint}"
+        params = {"page_size": page_size}
+        bookmark = None
+        while True:
+            if bookmark:
+                params["bookmark"] = bookmark
+
+            resp = session.get(url, params=params)
+            resp.raise_for_status()
+            data = resp.json()
+            items = data.get("items") or []
+
+            for item in items:
+                item_created = ensure_pendulum_datetime(item["created_at"])
+                if item_created <= start_dt:
+                    continue
+                if item_created > end_dt:
+                    continue
+                item["created_at"] = item_created
+                yield item
+
+            bookmark = data.get("bookmark")
+            if not bookmark:
+                break
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def pins(
+        datetime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+        ),
+    ) -> Iterable[TDataItem]:
+        _start_date = datetime.last_value or start_date
+        if end_date is None:
+            _end_date = pendulum.now("UTC")
+        else:
+            _end_date = datetime.end_value
+        yield from fetch_data("pins", _start_date, _end_date)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def boards(
+        datetime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+        ),
+    ) -> Iterable[TDataItem]:
+        _start_date = datetime.last_value or start_date
+        if end_date is None:
+            _end_date = pendulum.now("UTC")
+        else:
+            _end_date = datetime.end_value
+        yield from fetch_data("boards", _start_date, _end_date)
+
+    return pins, boards

{ingestr-0.13.58 → ingestr-0.13.60}/ingestr/src/sources.py

@@ -699,9 +699,7 @@ class StripeAnalyticsSource:
             )
 
         if incremental and not sync:
-            raise ValueError(
-                "incremental loads must be used with sync loading"
-            )
+            raise ValueError("incremental loads must be used with sync loading")
 
         if incremental:
             from ingestr.src.stripe_analytics import incremental_stripe_source
@@ -739,6 +737,7 @@ class StripeAnalyticsSource:
                         endpoint,
                     ],
                     stripe_secret_key=api_key[0],
+                    max_workers=kwargs.get("extract_parallelism", 4),
                 ).with_resources(endpoint)
 
         raise ValueError(
@@ -2783,3 +2782,72 @@ class QuickBooksSource:
             minor_version=minor_version[0],
             object=table_name,
         ).with_resources(table_name)
+
+
+class IsocPulseSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        token = params.get("token")
+        if not token or not token[0].strip():
+            raise MissingValueError("token", "Internet Society Pulse")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is None:
+            start_date = pendulum.now().in_tz("UTC").subtract(days=30)
+
+        end_date = kwargs.get("interval_end")
+
+        metric = table
+        opts = []
+        if ":" in metric:
+            metric, *opts = metric.strip().split(":")
+            opts = [opt.strip() for opt in opts]
+
+        from ingestr.src.isoc_pulse import pulse_source
+
+        src = pulse_source(
+            token=token[0],
+            start_date=start_date.strftime("%Y-%m-%d"),
+            end_date=str(end_date) if end_date else None,
+            metric=metric,
+            opts=opts,
+        )
+        return src.with_resources(metric)
+
+
+class PinterestSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        access_token = params.get("access_token")
+
+        if not access_token:
+            raise MissingValueError("access_token", "Pinterest")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date is not None:
+            end_date = end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        from ingestr.src.pinterest import pinterest_source
+
+        if table not in {"pins", "boards"}:
+            raise UnsupportedResourceError(table, "Pinterest")
+
+        return pinterest_source(
+            access_token=access_token[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

{ingestr-0.13.58 → ingestr-0.13.60}/ingestr/src/stripe_analytics/__init__.py

@@ -10,7 +10,6 @@ from pendulum import DateTime
 from .helpers import (
     async_parallel_pagination,
     pagination,
-    parallel_pagination,
     transform_date,
 )
 
@@ -55,53 +54,13 @@ def stripe_source(
         )(endpoint)
 
 
-@dlt.source(max_table_nesting=0)
-def parallel_stripe_source(
-    endpoints: Tuple[str, ...],
-    stripe_secret_key: str = dlt.secrets.value,
-    start_date: Optional[DateTime] = None,
-    end_date: Optional[DateTime] = None,
-    max_workers: int = 12,
-) -> Iterable[DltResource]:
-    """
-    Retrieves data from the Stripe API for the specified endpoints using parallel pagination.
-
-    This source divides the date range across multiple workers to fetch data in parallel,
-    which can significantly speed up data retrieval for large date ranges.
-
-    Args:
-        endpoints (Tuple[str, ...]): A tuple of endpoint names to retrieve data from.
-        stripe_secret_key (str): The API access token for authentication. Defaults to the value in the `dlt.secrets` object.
-        start_date (Optional[DateTime]): An optional start date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Required for parallel processing.
-        end_date (Optional[DateTime]): An optional end date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Required for parallel processing.
-        max_workers (int): Maximum number of worker threads for parallel fetching. Defaults to 4.
-
-    Returns:
-        Iterable[DltResource]: Resources with data that was created during the period greater than or equal to 'start_date' and less than 'end_date'.
-    """
-    stripe.api_key = stripe_secret_key
-    stripe.api_version = "2022-11-15"
-
-    def parallel_stripe_resource(
-        endpoint: str,
-    ) -> Generator[Dict[Any, Any], Any, None]:
-        yield from parallel_pagination(endpoint, start_date, end_date, max_workers)
-
-    for endpoint in endpoints:
-        yield dlt.resource(
-            parallel_stripe_resource,
-            name=endpoint,
-            write_disposition="replace",
-        )(endpoint)
-
-
 @dlt.source(max_table_nesting=0)
 def async_stripe_source(
     endpoints: Tuple[str, ...],
     stripe_secret_key: str = dlt.secrets.value,
     start_date: Optional[DateTime] = None,
     end_date: Optional[DateTime] = None,
-    max_workers: int = 40,
+    max_workers: int = 4,
     rate_limit_delay: float = 0.03,
 ) -> Iterable[DltResource]:
     """