ingestr 0.13.57__py3-none-any.whl → 0.13.59__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.57"
+version = "v0.13.59"

ingestr/src/factory.py

@@ -42,6 +42,7 @@ from ingestr.src.sources import (
     GoogleSheetsSource,
     GorgiasSource,
     HubspotSource,
+    IsocPulseSource,
     KafkaSource,
     KinesisSource,
     KlaviyoSource,
@@ -52,6 +53,7 @@ from ingestr.src.sources import (
     NotionSource,
     PersonioSource,
     PhantombusterSource,
+    PinterestSource,
     PipedriveSource,
     QuickBooksSource,
     S3Source,
@@ -172,8 +174,10 @@ class SourceDestinationFactory:
         "attio": AttioSource,
         "solidgate": SolidgateSource,
         "quickbooks": QuickBooksSource,
+        "isoc-pulse": IsocPulseSource,
         "smartsheet": SmartsheetSource,
         "sftp": SFTPSource,
+        "pinterest": PinterestSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/isoc_pulse/__init__.py

@@ -0,0 +1,159 @@
+import math
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Dict, Iterable, List, Optional
+
+import dlt
+from dlt.sources.rest_api import RESTAPIConfig, rest_api_resources
+
+METRICS: Dict[str, str] = {
+    "dnssec_adoption": "dnssec/adoption",
+    "dnssec_tld_adoption": "dnssec/adoption",
+    "dnssec_validation": "dnssec/validation",
+    "http": "http",
+    "http3": "http3",
+    "https": "https",
+    "ipv6": "ipv6",
+    "net_loss": "net-loss",
+    "resilience": "resilience",
+    "roa": "roa",
+    "rov": "rov",
+    "tls": "tls",
+    "tls13": "tls13",
+}
+
+
+@dlt.source
+def pulse_source(
+    token: str,
+    start_date: str,
+    metric: str,
+    opts: List[str],
+    end_date: Optional[str] = None,
+) -> Iterable[dlt.sources.DltResource]:
+    validate(metric, opts)
+    cfg = get_metric_cfg(metric, opts, start_date)
+    endpoint: Dict[str, Any] = {
+        "path": cfg.path,
+        "params": {
+            "start_date": "{incremental.start_value}",
+            **cfg.params,
+        },
+        "incremental": {
+            "cursor_path": "date",
+            "start_param": "start_date",
+            "end_param": "end_date",
+            "initial_value": start_date,
+            "end_value": end_date,
+            "range_start": "closed",
+            "range_end": "closed",
+        },
+        "paginator": "single_page",
+    }
+
+    if end_date is not None:
+        endpoint["params"]["end_date"] = end_date
+
+    resources = [
+        {
+            "name": metric,
+            "write_disposition": "merge",
+            "primary_key": "date",
+            "columns": {"date": {"data_type": "date"}},
+            "endpoint": endpoint,
+        }
+    ]
+
+    config: RESTAPIConfig = {
+        "client": {
+            "base_url": "https://pulse.internetsociety.org/api/",
+            "headers": {"Authorization": f"Bearer {token}"},
+        },
+        "resource_defaults": {
+            "write_disposition": "merge",
+            "primary_key": "date",
+        },
+        "resources": resources, # type:ignore
+    }
+    res = rest_api_resources(config)
+    if metric == "net_loss":
+        res[0].add_map(add_date(start_date))
+    yield from res
+
+
+@dataclass
+class MetricCfg:
+    path: str
+    params: Dict[str, Any]
+
+
+def get_metric_cfg(metric: str, opts: List[str], start_date: str) -> MetricCfg:
+    path = METRICS.get(metric)
+    if path is None:
+        raise ValueError(f"Unknown metric '{metric}'.")
+    if len(opts) == 0:
+        return MetricCfg(path=path, params={})
+
+    if metric == "https":
+        return MetricCfg(
+            path=f"{path}/country/{opts[-1]}",
+            params={
+                "topsites": True if "topsites" in opts else False,
+            },
+        )
+    elif metric in ["dnssec_validation", "dnssec_tld_adoption"]:
+        return MetricCfg(path=f"{path}/country/{opts[-1]}", params={})
+    elif metric == "dnssec_adoption":
+        return MetricCfg(path=f"{path}/domains/{opts[-1]}", params={})
+    elif metric == "ipv6":
+        if "topsites" in opts:
+            return MetricCfg(path=path, params={"topsites": True})
+        return MetricCfg(path=f"{path}/country/{opts[-1]}", params={})
+    elif metric == "roa":
+        if len(opts) > 1:
+            return MetricCfg(
+                path=f"{path}/country/{opts[-1]}", params={"ip_version": opts[-2]}
+            )
+        return MetricCfg(path=path, params={"ip_version": opts[-1]})
+    elif metric == "net_loss":
+        return MetricCfg(
+            path=path,
+            params={
+                "country": opts[-1],
+                "shutdown_type": opts[-2],
+            },
+        )
+    elif metric == "resilience":
+        date = datetime.strptime(start_date, "%Y-%m-%d")
+        return MetricCfg(
+            path=path,
+            params={
+                "country": opts[-1],
+                "year": date.year,
+                "quarter": math.floor(date.month / 4) + 1,
+            },
+        )
+    else:
+        raise ValueError(
+            f"Unsupported metric '{metric}' with options {opts}. "
+            "Please check the metric and options."
+        )
+
+
+def add_date(start_date: str):
+    def transform(item: dict):
+        item["date"] = start_date
+        return item
+
+    return transform
+
+
+def validate(metric: str, opts: List[str]) -> None:
+    nopts = len(opts)
+    if metric == "net_loss" and nopts != 2:
+        raise ValueError(
+            "For 'net_loss' metric, two options are required: "
+            "'shutdown_type' and 'country'."
+        )
+    if nopts > 0 and metric in ["http", "http3", "tls", "tls13", "rov"]:
+        raise ValueError(f"metric '{metric}' does not support options. ")

ingestr/src/pinterest/__init__.py

@@ -0,0 +1,82 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers import requests
+
+
+@dlt.source(name="pinterest", max_table_nesting=0)
+def pinterest_source(
+    start_date: pendulum.DateTime,
+    access_token: str,
+    page_size: int = 200,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    session = requests.Session()
+    session.headers.update({"Authorization": f"Bearer {access_token}"})
+    base_url = "https://api.pinterest.com/v5"
+
+    def fetch_data(
+        endpoint: str,
+        start_dt: pendulum.DateTime,
+        end_dt: pendulum.DateTime,
+    ) -> Iterable[TDataItem]:
+        url = f"{base_url}/{endpoint}"
+        params = {"page_size": page_size}
+        bookmark = None
+        while True:
+            if bookmark:
+                params["bookmark"] = bookmark
+
+            resp = session.get(url, params=params)
+            resp.raise_for_status()
+            data = resp.json()
+            items = data.get("items") or []
+
+            for item in items:
+                item_created = ensure_pendulum_datetime(item["created_at"])
+                if item_created <= start_dt:
+                    continue
+                if item_created > end_dt:
+                    continue
+                item["created_at"] = item_created
+                yield item
+
+            bookmark = data.get("bookmark")
+            if not bookmark:
+                break
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def pins(
+        datetime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+        ),
+    ) -> Iterable[TDataItem]:
+        _start_date = datetime.last_value or start_date
+        if end_date is None:
+            _end_date = pendulum.now("UTC")
+        else:
+            _end_date = datetime.end_value
+        yield from fetch_data("pins", _start_date, _end_date)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def boards(
+        datetime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+        ),
+    ) -> Iterable[TDataItem]:
+        _start_date = datetime.last_value or start_date
+        if end_date is None:
+            _end_date = pendulum.now("UTC")
+        else:
+            _end_date = datetime.end_value
+        yield from fetch_data("boards", _start_date, _end_date)
+
+    return pins, boards

ingestr/src/sources.py

@@ -16,7 +16,7 @@ from typing import (
     TypeAlias,
     Union,
 )
-from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
+from urllib.parse import ParseResult, parse_qs, urlencode, urlparse
 
 import fsspec  # type: ignore
 import pendulum
@@ -699,9 +699,7 @@ class StripeAnalyticsSource:
             )
 
         if incremental and not sync:
-            raise ValueError(
-                "incremental loads must be used with sync loading"
-            )
+            raise ValueError("incremental loads must be used with sync loading")
 
         if incremental:
             from ingestr.src.stripe_analytics import incremental_stripe_source
@@ -1370,7 +1368,7 @@ class S3Source:
             )
 
         parsed_uri = urlparse(uri)
-        source_fields = parse_qs(quote(parsed_uri.query, safe="=&"))
+        source_fields = parse_qs(parsed_uri.query)
         access_key_id = source_fields.get("access_key_id")
         if not access_key_id:
             raise ValueError("access_key_id is required to connect to S3")
@@ -1599,7 +1597,7 @@ class DynamoDBSource:
         if not region:
             raise ValueError("region is required to connect to Dynamodb")
 
-        qs = parse_qs(quote(parsed_uri.query, safe="=&"))
+        qs = parse_qs(parsed_uri.query)
         access_key = qs.get("access_key_id")
 
         if not access_key:
@@ -2783,3 +2781,72 @@ class QuickBooksSource:
             minor_version=minor_version[0],
             object=table_name,
         ).with_resources(table_name)
+
+
+class IsocPulseSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        token = params.get("token")
+        if not token or not token[0].strip():
+            raise MissingValueError("token", "Internet Society Pulse")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is None:
+            start_date = pendulum.now().in_tz("UTC").subtract(days=30)
+
+        end_date = kwargs.get("interval_end")
+
+        metric = table
+        opts = []
+        if ":" in metric:
+            metric, *opts = metric.strip().split(":")
+            opts = [opt.strip() for opt in opts]
+
+        from ingestr.src.isoc_pulse import pulse_source
+
+        src = pulse_source(
+            token=token[0],
+            start_date=start_date.strftime("%Y-%m-%d"),
+            end_date=str(end_date) if end_date else None,
+            metric=metric,
+            opts=opts,
+        )
+        return src.with_resources(metric)
+
+
+class PinterestSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        access_token = params.get("access_token")
+
+        if not access_token:
+            raise MissingValueError("access_token", "Pinterest")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date is not None:
+            end_date = end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        from ingestr.src.pinterest import pinterest_source
+
+        if table not in {"pins", "boards"}:
+            raise UnsupportedResourceError(table, "Pinterest")
+
+        return pinterest_source(
+            access_token=access_token[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

{ingestr-0.13.57.dist-info → ingestr-0.13.59.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.57
+Version: 0.13.59
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.57.dist-info → ingestr-0.13.59.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=GkC1hdq8AVGrvolc95zMfjmibI95p2pmFkbgCOVf-Og,26311
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=mMPkVD8bGtndOM5rdoqMRQZCk_zPuHM2EG9vRNh5cIk,21
+ingestr/src/buildinfo.py,sha256=RkwXqGVjCaeOi85qaAT-2wI5-IYpZt76x8qkp2dVM1o,21
 ingestr/src/destinations.py,sha256=TcxM2rcwHfgMMP6U0yRNcfWKlEzkBbZbqCIDww7lkTY,16882
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=R7KzGRQ9tYZ_N-daD9OtnEp0K-DrsP8bUyXWdv4LV4A,6200
+ingestr/src/factory.py,sha256=OKqjYqvHhgaOF48-eSNSabcfXt4Gmr1yZ8cFGizXh0g,6319
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=dtr5sXF2ecO54eJR1Mu-dH9YgSxAVx93gjfSwJISI00,96867
+ingestr/src/sources.py,sha256=C2qPplmvRQdm1nzSPvGbMpYG6oGCbGZMTlVtVS48n6k,98977
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -73,6 +73,7 @@ ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOIN
 ingestr/src/hubspot/__init__.py,sha256=wqHefhc_YRI5dNFCcpvH-UUilNThE49sbGouSBiHYsw,11776
 ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
+ingestr/src/isoc_pulse/__init__.py,sha256=WDgKBn15gyQheXE6oJ_2OuMUQwKPbAjflKAsnucu7u8,4647
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=YretSz4F28tbkcPhd55mBp2Xk7XE9unyWx0nmvl8iEc,6235
@@ -96,6 +97,7 @@ ingestr/src/personio/__init__.py,sha256=sHYpoV-rg-kA1YsflctChis0hKcTrL6mka9O0CHV
 ingestr/src/personio/helpers.py,sha256=EKmBN0Lf4R0lc3yqqs7D-RjoZ75E8gPcctt59xwHxrY,2901
 ingestr/src/phantombuster/__init__.py,sha256=8AQTiA8fp1NT8TellQQqwBCl6vGvGwUBLif6LIzgAik,1786
 ingestr/src/phantombuster/client.py,sha256=9zx58sFunXjUNh6jeEYLNfwNxGxX9odifwAmS0E9AaY,3018
+ingestr/src/pinterest/__init__.py,sha256=5xTLNE2Vn_00PXMLKjY41Fh1LsyzM7UnBSKxKPITUl0,2581
 ingestr/src/pipedrive/__init__.py,sha256=iRrxeMwo8_83ptgGnTFTNHV1nYvIsFfg0a3XzugPYeI,6982
 ingestr/src/pipedrive/settings.py,sha256=q119Fy4C5Ip1rMoCILX2BkHV3bwiXC_dW58KIiDUzsY,708
 ingestr/src/pipedrive/typing.py,sha256=lEMXu4hhAA3XkhVSlBUa-juqyupisd3c-qSQKxFvzoE,69
@@ -141,8 +143,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.57.dist-info/METADATA,sha256=k5LnyrqN3QlhkaGqKGtQBD370oNkTTkoEWDAhlCq-3M,14993
-ingestr-0.13.57.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.57.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.57.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.57.dist-info/RECORD,,
+ingestr-0.13.59.dist-info/METADATA,sha256=8yM2vLMiUV_zBq15gg_1Vf6UHgtwoRzawb_tcT_K3Wc,14993
+ingestr-0.13.59.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.59.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.59.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.59.dist-info/RECORD,,