ingestr 0.13.24__py3-none-any.whl → 0.13.26__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.24"
+version = "v0.13.26"

ingestr/src/factory.py

@@ -28,6 +28,7 @@ from ingestr.src.sources import (
     ChessSource,
     DynamoDBSource,
     FacebookAdsSource,
+    FrankfurterSource,
     GCSSource,
     GitHubSource,
     GoogleAdsSource,
@@ -146,6 +147,7 @@ class SourceDestinationFactory:
         "personio": PersonioSource,
         "kinesis": KinesisSource,
         "pipedrive": PipedriveSource,
+        "frankfurter": FrankfurterSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/frankfurter/__init__.py

@@ -0,0 +1,142 @@
+from typing import Any, Iterator, Optional
+
+import dlt
+from dlt.common.pendulum import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+
+from ingestr.src.frankfurter.helpers import get_path_with_retry
+
+
+@dlt.source(
+    name="frankfurter",
+    max_table_nesting=0,
+)
+def frankfurter_source(
+    table: str,
+    start_date: Optional[TAnyDateTime] = None,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Any:
+    """
+    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
+    various types of data: currencies, latest rates, historical rates.
+
+    Returns the appropriate resource based on the provided parameters.
+    """
+    # Determine which resource to return based on the `table` parameter
+    if table == "currencies":
+        return currencies()
+
+    elif table == "latest":
+        return latest()
+
+    elif table == "exchange_rates":
+        return exchange_rates(start_date=start_date, end_date=end_date)
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "currency_code": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+    },
+)
+def currencies() -> Iterator[dict]:
+    """
+    Yields each currency as a separate row with two columns: currency_code and currency_name.
+    """
+    # Retrieve the list of currencies from the API
+    currencies_data = get_path_with_retry("currencies")
+
+    for currency_code, currency_name in currencies_data.items():
+        yield {"currency_code": currency_code, "currency_name": currency_name}
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "date": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+        "rate": {"data_type": "double"},
+    },
+    primary_key=["date", "currency_name"],  # Composite primary key
+)
+def latest() -> Iterator[dict]:
+    """
+    Fetches the latest exchange rates and yields them as rows.
+    """
+    # Base URL
+    url = "latest?"
+
+    # Fetch data
+    latest_data = get_path_with_retry(url)
+
+    # Extract rates and base currency
+    rates = latest_data["rates"]
+
+    # Prepare the date
+    date = pendulum.now().to_date_string()
+
+    # Add the base currency (EUR) with a rate of 1.0
+    yield {
+        "date": date,
+        "currency_name": "EUR",
+        "rate": 1.0,
+    }
+
+    # Add all currencies and their rates
+    for currency_name, rate in rates.items():
+        yield {
+            "date": date,
+            "currency_name": currency_name,
+            "rate": rate,
+        }
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "date": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+        "rate": {"data_type": "double"},
+    },
+    primary_key=["date", "currency_name"],  # Composite primary key
+)
+def exchange_rates(
+    start_date: TAnyDateTime,
+    end_date: TAnyDateTime,
+) -> Iterator[dict]:
+    """
+    Fetches exchange rates for a specified date range.
+    If only start_date is provided, fetches data for that date.
+    If both start_date and end_date are provided, fetches data for each day in the range.
+    """
+    start_date_str = ensure_pendulum_datetime(start_date).format("YYYY-MM-DD")
+    end_date_str = ensure_pendulum_datetime(end_date).format("YYYY-MM-DD")
+
+    # Compose the URL
+    url = f"{start_date_str}..{end_date_str}?"
+
+    # Fetch data from the API
+    data = get_path_with_retry(url)
+
+    # Extract base currency and rates from the API response
+    base_currency = data["base"]
+    rates = data["rates"]
+
+    # Iterate over the rates dictionary (one entry per date)
+    for date, daily_rates in rates.items():
+        # Add the base currency with a rate of 1.0
+        yield {
+            "date": date,
+            "currency_name": base_currency,
+            "rate": 1.0,
+        }
+
+        # Add all other currencies and their rates
+        for currency_name, rate in daily_rates.items():
+            yield {
+                "date": date,
+                "currency_name": currency_name,
+                "rate": rate,
+            }

ingestr/src/frankfurter/helpers.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+
+from dlt.common.pendulum import pendulum
+from dlt.common.typing import StrAny
+from dlt.sources.helpers import requests
+
+FRANKFURTER_API_URL = "https://api.frankfurter.dev/v1/"
+
+
+def get_url_with_retry(url: str) -> StrAny:
+    r = requests.get(url)
+    return r.json()  # type: ignore
+
+
+def get_path_with_retry(path: str) -> StrAny:
+    return get_url_with_retry(f"{FRANKFURTER_API_URL}{path}")
+
+
+def validate_dates(start_date: datetime, end_date: datetime) -> None:
+    current_date = pendulum.now()
+
+    # Check if start_date is in the future
+    if start_date > current_date:
+        raise ValueError("Interval-start cannot be in the future.")
+
+    # Check if end_date is in the future
+    if end_date > current_date:
+        raise ValueError("Interval-end cannot be in the future.")
+
+    # Check if start_date is before end_date
+    if start_date > end_date:
+        raise ValueError("Interval-end cannot be before interval-start.")

ingestr/src/hubspot/__init__.py

@@ -36,6 +36,7 @@ from .helpers import _get_property_names, fetch_data, fetch_property_history
 from .settings import (
     ALL,
     CRM_OBJECT_ENDPOINTS,
+    CRM_SCHEMAS_ENDPOINT,
     DEFAULT_COMPANY_PROPS,
     DEFAULT_CONTACT_PROPS,
     DEFAULT_DEAL_PROPS,
@@ -161,6 +162,13 @@ def hubspot(
             props,
             include_custom_props,
         )
+      
+    @dlt.resource(name="schemas", write_disposition="merge", primary_key="id")
+    def schemas(
+        api_key: str = api_key,
+    ) -> Iterator[TDataItems]:
+        """Hubspot schemas resource"""
+        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key)
 
     @dlt.resource(name="quotes", write_disposition="replace")
     def quotes(
@@ -178,7 +186,7 @@ def hubspot(
             include_custom_props,
         )
 
-    return companies, contacts, deals, tickets, products, quotes
+    return companies, contacts, deals, tickets, products, quotes, schemas
 
 
 def crm_objects(

ingestr/src/hubspot/helpers.py

@@ -132,16 +132,27 @@ def fetch_data(
         if "results" in _data:
             _objects: List[Dict[str, Any]] = []
             for _result in _data["results"]:
-                _obj = _result.get("properties", _result)
-                if "id" not in _obj and "id" in _result:
-                    # Move id from properties to top level
-                    _obj["id"] = _result["id"]
-                if "associations" in _result:
-                    for association in _result["associations"]:
-                        __values = [
-                            {
-                                "value": _obj["hs_object_id"],
-                                f"{association}_id": __r["id"],
+                if endpoint == "/crm/v3/schemas":
+                    _objects.append({
+                        "name": _result["labels"].get("singular", ""),
+                        "objectTypeId": _result.get("objectTypeId", ""),
+                        "id": _result.get("id", ""),
+                        "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
+                        "properties": _result.get("properties", ""),
+                        "createdAt": _result.get("createdAt", ""),
+                        "updatedAt": _result.get("updatedAt", "")
+                    })
+                else:
+                    _obj = _result.get("properties", _result)
+                    if "id" not in _obj and "id" in _result:
+                        # Move id from properties to top level
+                        _obj["id"] = _result["id"]
+                    if "associations" in _result:
+                        for association in _result["associations"]:
+                            __values = [
+                                {
+                                    "value": _obj["hs_object_id"],
+                                    f"{association}_id": __r["id"],
                             }
                             for __r in _result["associations"][association]["results"]
                         ]
@@ -152,7 +163,7 @@ def fetch_data(
                         ]
 
                         _obj[association] = __values
-                _objects.append(_obj)
+                    _objects.append(_obj)
             yield _objects
 
         # Follow pagination links if they exist

ingestr/src/hubspot/settings.py

@@ -14,6 +14,7 @@ CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals"
 CRM_PRODUCTS_ENDPOINT = "/crm/v3/objects/products"
 CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets"
 CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes"
+CRM_SCHEMAS_ENDPOINT = "/crm/v3/schemas"
 
 CRM_OBJECT_ENDPOINTS = {
     "contact": CRM_CONTACTS_ENDPOINT,

ingestr/src/sources.py

@@ -67,6 +67,8 @@ from ingestr.src.errors import (
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
 from ingestr.src.filters import table_adapter_exclude_columns
+from ingestr.src.frankfurter import frankfurter_source
+from ingestr.src.frankfurter.helpers import validate_dates
 from ingestr.src.github import github_reactions, github_repo_events, github_stargazers
 from ingestr.src.google_ads import google_ads
 from ingestr.src.google_analytics import google_analytics
@@ -801,7 +803,7 @@ class HubspotSource:
             raise ValueError("api_key in the URI is required to connect to Hubspot")
 
         endpoint = None
-        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes"]:
+        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
             endpoint = table
         else:
             raise ValueError(
@@ -2041,3 +2043,40 @@ class PipedriveSource:
         return pipedrive_source(
             pipedrive_api_key=api_key, since_timestamp=start_date
         ).with_resources(table)
+
+
+class FrankfurterSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        # start and end dates only assigned and validated for exchange_rates table
+        # Note: if an end date but no start date is provided, start date and end date will be set to current date
+        if table == "exchange_rates":
+            if kwargs.get("interval_start"):
+                start_date = ensure_pendulum_datetime(str(kwargs.get("interval_start")))
+                if kwargs.get("interval_end"):
+                    end_date = ensure_pendulum_datetime(str(kwargs.get("interval_end")))
+                else:
+                    end_date = start_date
+            else:
+                start_date = pendulum.now()
+                end_date = pendulum.now()
+            validate_dates(start_date=start_date, end_date=end_date)
+
+        # For currencies and latest tables, set start and end dates to current date
+        else:
+            start_date = pendulum.now()
+            end_date = pendulum.now()
+
+        # Validate table
+        if table not in ["currencies", "latest", "exchange_rates"]:
+            raise ValueError(
+                f"Table '{table}' is not supported for Frankfurter source."
+            )
+
+        return frankfurter_source(
+            table=table,
+            start_date=start_date,
+            end_date=end_date,
+        )

{ingestr-0.13.24.dist-info → ingestr-0.13.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.24
+Version: 0.13.26
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.24.dist-info → ingestr-0.13.26.dist-info}/RECORD

@@ -2,15 +2,15 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=wvbRCJ2--M0Zw2cYtSH874TxTtlD0wadHREeLG3anOY,25618
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=x-bxDOuFDQ1rgDJf03eHD1bXb9Yfo3wX39XyaBE0LkU,21
+ingestr/src/buildinfo.py,sha256=VzL8INFpli0urCbtG-JAzjTMJgmPVwIKFd9agl-cDsY,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=1jqcLv_QUUGeyg1OYN3ywrRdcDZyDRtMOongwyjDapU,5268
+ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
 ingestr/src/filters.py,sha256=5LNpBgm8FJXdrFHGyM7dLVyphKykSpPk7yuQAZ8GML4,1133
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=kGsgFWf8Ghha0-HlC6PlDIIKX2Lriah4UmAseziGdr4,72035
+ingestr/src/sources.py,sha256=pF0o1xi84hDo_fdM2o_Wmhcrye_mr55dMQ1BvL8sZZA,73582
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -41,6 +41,8 @@ ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_
 ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
+ingestr/src/frankfurter/__init__.py,sha256=xJUicENGYtOPsGznKP8IA_5Jt-_gJP29onrByBgUf-g,4259
+ingestr/src/frankfurter/helpers.py,sha256=RSqI-WAAJfunWnLqiBRmPuonRg7rDOqmY76beb8a6rM,967
 ingestr/src/github/__init__.py,sha256=xVijF-Wi4p88hkVJnKH-oTixismjD3aUcGqGa6Wr4e4,5889
 ingestr/src/github/helpers.py,sha256=rpv_3HzuOl4PQ-FUeA66pev-pgze9SaE8RUHIPYfZ_A,6759
 ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
@@ -59,9 +61,9 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=NYgSIAPXQh2Qp1eKun7TgcerKogq6pWtNkr-_f0FXbI,9464
-ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
-ingestr/src/hubspot/settings.py,sha256=9P1OKiRL88kl_m8n1HhuG-Qpq9VGbqPLn5Q0QYneToU,2193
+ingestr/src/hubspot/__init__.py,sha256=nHjMvqUAUNVy9dX-1YiaY6PeplDnzc_LjQl25DdVrYg,9765
+ingestr/src/hubspot/helpers.py,sha256=vJPaF9qU71-w-LUe4NaRowioIJG3GfX4MObyitHJC6U,7388
+ingestr/src/hubspot/settings.py,sha256=hXnIhE4_FlhtgUiCjMLGGZJ_lBlBQJ0YWi6lHJ6Q3WU,2234
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=u5ThH1y8uObZKXgIo71em1UnX6MsVHWOjcf1jKqKbE8,6205
@@ -119,8 +121,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.24.dist-info/METADATA,sha256=qzKyEOTPIb6cTD49Q6zC-bqSn7ax45JmCKcGh0jtJRw,13659
-ingestr-0.13.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.24.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.24.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.24.dist-info/RECORD,,
+ingestr-0.13.26.dist-info/METADATA,sha256=cPGZ4OGDE-jWfIm8D5gEOLCGyMOioafvTC27zlxeHvo,13659
+ingestr-0.13.26.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.26.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.26.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.26.dist-info/RECORD,,