ingestr 0.13.75__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/fluxx/helpers.py

@@ -0,0 +1,209 @@
+import json
+from typing import Any, Dict, Iterator, List, Optional
+
+import dlt
+import pendulum
+import requests
+
+FLUXX_API_BASE = "https://{instance}.fluxxlabs.com"
+FLUXX_OAUTH_TOKEN_PATH = "/oauth/token"
+FLUXX_API_V2_PATH = "/api/rest/v2"
+
+
+def get_access_token(instance: str, client_id: str, client_secret: str) -> str:
+    """Obtain OAuth access token using client credentials flow."""
+    token_url = f"{FLUXX_API_BASE.format(instance=instance)}{FLUXX_OAUTH_TOKEN_PATH}"
+
+    response = requests.post(
+        token_url,
+        data={
+            "grant_type": "client_credentials",
+            "client_id": client_id,
+            "client_secret": client_secret,
+        },
+    )
+    response.raise_for_status()
+
+    token_data = response.json()
+    return token_data["access_token"]
+
+
+def fluxx_api_request(
+    instance: str,
+    access_token: str,
+    endpoint: str,
+    method: str = "GET",
+    params: Optional[Dict[str, Any]] = None,
+    data: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Make an authenticated request to the Fluxx API."""
+    url = f"{FLUXX_API_BASE.format(instance=instance)}{FLUXX_API_V2_PATH}/{endpoint}"
+
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json",
+    }
+
+    response = requests.request(
+        method=method,
+        url=url,
+        headers=headers,
+        params=params,
+        json=data,
+    )
+    response.raise_for_status()
+
+    if response.text:
+        return response.json()
+    return {}
+
+
+def paginate_fluxx_resource(
+    instance: str,
+    access_token: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    page_size: int = 100,
+) -> Iterator[List[Dict[str, Any]]]:
+    """Paginate through a Fluxx API resource."""
+    if params is None:
+        params = {}
+
+    page = 1
+    params["per_page"] = page_size
+
+    while True:
+        params["page"] = page
+
+        response = fluxx_api_request(
+            instance=instance,
+            access_token=access_token,
+            endpoint=endpoint,
+            params=params,
+        )
+
+        if not response:
+            break
+
+        # Get the first available key from records instead of assuming endpoint name
+        records = response["records"]
+        if records:
+            # Pick the first key available in records
+            first_key = next(iter(records))
+            items = records[first_key]
+        else:
+            items = []
+
+        yield items
+
+        if response["per_page"] is None or len(items) < response["per_page"]:
+            break
+
+        page += 1
+
+
+def get_date_range(updated_at, start_date):
+    """Extract current start and end dates from incremental state."""
+    if updated_at.last_value:
+        current_start_date = pendulum.parse(updated_at.last_value)
+    else:
+        current_start_date = (
+            pendulum.parse(start_date)
+            if start_date
+            else pendulum.now().subtract(days=30)
+        )
+
+    if updated_at.end_value:
+        current_end_date = pendulum.parse(updated_at.end_value)
+    else:
+        current_end_date = pendulum.now(tz="UTC")
+
+    return current_start_date, current_end_date
+
+
+def create_dynamic_resource(
+    resource_name: str,
+    endpoint: str,
+    instance: str,
+    access_token: str,
+    start_date: Optional[pendulum.DateTime] = None,
+    end_date: Optional[pendulum.DateTime] = None,
+    fields_to_extract: Optional[Dict[str, Any]] = None,
+):
+    """Factory function to create dynamic Fluxx resources."""
+
+    # Extract column definitions for DLT resource
+    columns = {}
+    if fields_to_extract:
+        for field_name, field_config in fields_to_extract.items():
+            data_type = field_config.get("data_type")
+            if data_type:
+                columns[field_name] = {"data_type": data_type}
+
+    @dlt.resource(name=resource_name, write_disposition="replace", columns=columns)  # type: ignore
+    def fluxx_resource() -> Iterator[Dict[str, Any]]:
+        params = {}
+        if fields_to_extract:
+            field_names = list(fields_to_extract.keys())
+            params["cols"] = json.dumps(field_names)
+
+        for page in paginate_fluxx_resource(
+            instance=instance,
+            access_token=access_token,
+            endpoint=endpoint,
+            params=params,
+            page_size=100,
+        ):
+            yield [normalize_fluxx_item(item, fields_to_extract) for item in page]  # type: ignore
+
+    return fluxx_resource
+
+
+def normalize_fluxx_item(
+    item: Dict[str, Any], fields_to_extract: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    """
+    Normalize a Fluxx API response item.
+    Handles nested structures and field extraction based on field types.
+    Rounds all decimal/float values to 4 decimal places regardless of field type.
+    """
+    normalized: Dict[str, Any] = {}
+
+    # If no field mapping provided, just return the item as-is
+    if not fields_to_extract:
+        return item
+
+    for field_name, field_config in fields_to_extract.items():
+        if field_name in item:
+            value = item[field_name]
+            field_type = field_config.get("data_type")
+
+            if isinstance(value, float):
+                # Round any numeric value with decimal places
+                normalized[field_name] = round(value, 4)
+            elif field_type == "json":
+                # Handle json fields (arrays/relations)
+                if value is None:
+                    normalized[field_name] = None
+                elif value == "":
+                    normalized[field_name] = None
+                elif isinstance(value, (list, dict)):
+                    normalized[field_name] = value
+                else:
+                    # Single value - wrap in array for json fields
+                    normalized[field_name] = [value]
+            elif field_type in ("date", "timestamp", "datetime", "text"):
+                # Handle text/date fields - convert empty strings to None
+                if value == "":
+                    normalized[field_name] = None
+                else:
+                    normalized[field_name] = value
+            else:
+                # All other field types - pass through as-is
+                normalized[field_name] = value
+
+    # Always include id if present
+    if "id" in item:
+        normalized["id"] = item["id"]
+
+    return normalized

ingestr/src/frankfurter/__init__.py

@@ -1,163 +1,157 @@
-from typing import Any, Iterator, Optional
-
-import dlt
-from dlt.common.pendulum import pendulum
-from dlt.common.time import ensure_pendulum_datetime
-from dlt.common.typing import TAnyDateTime
-
-from ingestr.src.frankfurter.helpers import get_path_with_retry
-
-
-@dlt.source(
-    name="frankfurter",
-    max_table_nesting=0,
-)
-def frankfurter_source(
-    start_date: TAnyDateTime,
-    end_date: TAnyDateTime,
-    base_currency: str,
-) -> Any:
-    """
-    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
-    various types of data: currencies, latest rates, historical rates.
-    """
-    date_time = dlt.sources.incremental(
-        "date",
-        initial_value=start_date,
-        end_value=end_date,
-        range_start="closed",
-        range_end="closed",
-    )
-
-    return (
-        currencies(),
-        latest(base_currency=base_currency),
-        exchange_rates(
-            start_date=date_time, end_date=end_date, base_currency=base_currency
-        ),
-    )
-
-
-@dlt.resource(
-    write_disposition="replace",
-    columns={
-        "currency_code": {"data_type": "text"},
-        "currency_name": {"data_type": "text"},
-    },
-)
-def currencies() -> Iterator[dict]:
-    """
-    Yields each currency as a separate row with two columns: currency_code and currency_name.
-    """
-    # Retrieve the list of currencies from the API
-    currencies_data = get_path_with_retry("currencies")
-
-    for currency_code, currency_name in currencies_data.items():
-        yield {"currency_code": currency_code, "currency_name": currency_name}
-
-
-@dlt.resource(
-    write_disposition="merge",
-    columns={
-        "date": {"data_type": "text"},
-        "currency_code": {"data_type": "text"},
-        "rate": {"data_type": "double"},
-        "base_currency": {"data_type": "text"},
-    },
-    primary_key=["date", "currency_code", "base_currency"],
-)
-def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
-    """
-    Fetches the latest exchange rates and yields them as rows.
-    """
-    # Base URL
-    url = "latest?"
-
-    if base_currency:
-        url += f"base={base_currency}"
-
-    # Fetch data
-    data = get_path_with_retry(url)
-
-    # Extract rates and base currency
-    rates = data["rates"]
-    date = pendulum.parse(data["date"])
-
-    # Add the base currency with a rate of 1.0
-    yield {
-        "date": date,
-        "currency_code": base_currency,
-        "rate": 1.0,
-        "base_currency": base_currency,
-    }
-
-    # Add all currencies and their rates
-    for currency_code, rate in rates.items():
-        yield {
-            "date": date,
-            "currency_code": currency_code,
-            "rate": rate,
-            "base_currency": base_currency,
-        }
-
-
-@dlt.resource(
-    write_disposition="merge",
-    columns={
-        "date": {"data_type": "text"},
-        "currency_code": {"data_type": "text"},
-        "rate": {"data_type": "double"},
-        "base_currency": {"data_type": "text"},
-    },
-    primary_key=("date", "currency_code", "base_currency"),
-)
-def exchange_rates(
-    end_date: TAnyDateTime,
-    start_date: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental("date"),
-    base_currency: Optional[str] = "",
-) -> Iterator[dict]:
-    """
-    Fetches exchange rates for a specified date range.
-    If only start_date is provided, fetches data until now.
-    If both start_date and end_date are provided, fetches data for each day in the range.
-    """
-    # Ensure start_date.last_value is a pendulum.DateTime object
-    start_date_obj = ensure_pendulum_datetime(start_date.last_value)  # type: ignore
-    start_date_str = start_date_obj.format("YYYY-MM-DD")
-
-    # Ensure end_date is a pendulum.DateTime object
-    end_date_obj = ensure_pendulum_datetime(end_date)
-    end_date_str = end_date_obj.format("YYYY-MM-DD")
-
-    # Compose the URL
-    url = f"{start_date_str}..{end_date_str}?"
-
-    if base_currency:
-        url += f"base={base_currency}"
-
-    # Fetch data from the API
-    data = get_path_with_retry(url)
-
-    # Extract base currency and rates from the API response
-    rates = data["rates"]
-
-    # Iterate over the rates dictionary (one entry per date)
-    for date, daily_rates in rates.items():
-        formatted_date = pendulum.parse(date)
-
-        # Add the base currency with a rate of 1.0
-        yield {
-            "date": formatted_date,
-            "currency_code": base_currency,
-            "rate": 1.0,
-            "base_currency": base_currency,
-        }
-
-        # Add all other currencies and their rates
-        for currency_code, rate in daily_rates.items():
-            yield {
-                "date": formatted_date,
-                "currency_code": currency_code,
-                "rate": rate,
-                "base_currency": base_currency,
-            }
+from typing import Any, Iterator, Optional
+
+import dlt
+from dlt.common.pendulum import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+
+from ingestr.src.frankfurter.helpers import get_path_with_retry
+
+
+@dlt.source(
+    name="frankfurter",
+    max_table_nesting=0,
+)
+def frankfurter_source(
+    start_date: TAnyDateTime,
+    end_date: TAnyDateTime | None,
+    base_currency: str,
+) -> Any:
+    """
+    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
+    various types of data: currencies, latest rates, historical rates.
+    """
+
+    @dlt.resource(
+        write_disposition="replace",
+    )
+    def currencies() -> Iterator[dict]:
+        """
+        Yields each currency as a separate row with two columns: currency_code and currency_name.
+        """
+        # Retrieve the list of currencies from the API
+        currencies_data = get_path_with_retry("currencies")
+
+        for currency_code, currency_name in currencies_data.items():
+            yield {"currency_code": currency_code, "currency_name": currency_name}
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=["date", "currency_code", "base_currency"],
+    )
+    def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
+        """
+        Fetches the latest exchange rates and yields them as rows.
+        """
+        # Base URL
+        url = "latest?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data
+        data = get_path_with_retry(url)
+
+        # Extract rates and base currency
+        rates = data["rates"]
+        date = pendulum.parse(data["date"])
+
+        # Add the base currency with a rate of 1.0
+        yield {
+            "date": date,
+            "currency_code": base_currency,
+            "rate": 1.0,
+            "base_currency": base_currency,
+        }
+
+        # Add all currencies and their rates
+        for currency_code, rate in rates.items():
+            yield {
+                "date": date,
+                "currency_code": currency_code,
+                "rate": rate,
+                "base_currency": base_currency,
+            }
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=("date", "currency_code", "base_currency"),
+    )
+    def exchange_rates(
+        date_time=dlt.sources.incremental(
+            "date",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        """
+        Fetches exchange rates for a specified date range.
+        If only start_date is provided, fetches data until now.
+        If both start_date and end_date are provided, fetches data for each day in the range.
+        """
+        if date_time.last_value is not None:
+            start_date = date_time.last_value
+        else:
+            start_date = start_date
+
+        if date_time.end_value is not None:
+            end_date = date_time.end_value
+        else:
+            end_date = pendulum.now()
+
+        # Ensure start_date.last_value is a pendulum.DateTime object
+        start_date_obj = ensure_pendulum_datetime(start_date)  # type: ignore
+        start_date_str = start_date_obj.format("YYYY-MM-DD")
+
+        # Ensure end_date is a pendulum.DateTime object
+        end_date_obj = ensure_pendulum_datetime(end_date)
+        end_date_str = end_date_obj.format("YYYY-MM-DD")
+
+        # Compose the URL
+        url = f"{start_date_str}..{end_date_str}?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data from the API
+        data = get_path_with_retry(url)
+
+        # Extract base currency and rates from the API response
+        rates = data["rates"]
+
+        # Iterate over the rates dictionary (one entry per date)
+        for date, daily_rates in rates.items():
+            formatted_date = pendulum.parse(date)
+
+            # Add the base currency with a rate of 1.0
+            yield {
+                "date": formatted_date,
+                "currency_code": base_currency,
+                "rate": 1.0,
+                "base_currency": base_currency,
+            }
+
+            # Add all other currencies and their rates
+            for currency_code, rate in daily_rates.items():
+                yield {
+                    "date": formatted_date,
+                    "currency_code": currency_code,
+                    "rate": rate,
+                    "base_currency": base_currency,
+                }
+
+    return currencies, latest, exchange_rates

ingestr/src/frankfurter/helpers.py

@@ -16,7 +16,7 @@ def get_path_with_retry(path: str) -> StrAny:
     return get_url_with_retry(f"{FRANKFURTER_API_URL}{path}")
 
 
-def validate_dates(start_date: datetime, end_date: datetime) -> None:
+def validate_dates(start_date: datetime, end_date: datetime | None) -> None:
     current_date = pendulum.now()
 
     # Check if start_date is in the futurep
@@ -24,11 +24,11 @@ def validate_dates(start_date: datetime, end_date: datetime) -> None:
         raise ValueError("Interval-start cannot be in the future.")
 
     # Check if end_date is in the future
-    if end_date > current_date:
+    if end_date is not None and end_date > current_date:
         raise ValueError("Interval-end cannot be in the future.")
 
     # Check if start_date is before end_date
-    if start_date > end_date:
+    if end_date is not None and start_date > end_date:
         raise ValueError("Interval-end cannot be before interval-start.")
 
 

ingestr/src/freshdesk/__init__.py

@@ -4,6 +4,8 @@ etc. to the database"""
 from typing import Any, Dict, Generator, Iterable, List, Optional
 
 import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
 from dlt.sources import DltResource
 
 from .freshdesk_client import FreshdeskClient
@@ -12,10 +14,13 @@ from .settings import DEFAULT_ENDPOINTS
 
 @dlt.source()
 def freshdesk_source(
-    endpoints: Optional[List[str]] = None,
+    domain: str,
+    api_secret_key: str,
+    start_date: pendulum.DateTime,
+    end_date: Optional[pendulum.DateTime] = None,
     per_page: int = 100,
-    domain: str = dlt.secrets.value,
-    api_secret_key: str = dlt.secrets.value,
+    endpoints: Optional[List[str]] = None,
+    query: Optional[str] = None,
 ) -> Iterable[DltResource]:
     """
     Retrieves data from specified Freshdesk API endpoints.
@@ -39,7 +44,11 @@ def freshdesk_source(
     def incremental_resource(
         endpoint: str,
         updated_at: Optional[Any] = dlt.sources.incremental(
-            "updated_at", initial_value="2022-01-01T00:00:00Z"
+            "updated_at",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
         ),
     ) -> Generator[Dict[Any, Any], Any, None]:
         """
@@ -48,15 +57,23 @@ def freshdesk_source(
         to ensure incremental loading.
         """
 
-        # Retrieve the last updated timestamp to fetch only new or updated records.
-        if updated_at is not None:
-            updated_at = updated_at.last_value
+        if updated_at.last_value is not None:
+            start_date = ensure_pendulum_datetime(updated_at.last_value)
+        else:
+            start_date = start_date
+
+        if updated_at.end_value is not None:
+            end_date = ensure_pendulum_datetime(updated_at.end_value)
+        else:
+            end_date = pendulum.now(tz="UTC")
 
         # Use the FreshdeskClient instance to fetch paginated responses
         yield from freshdesk.paginated_response(
             endpoint=endpoint,
             per_page=per_page,
-            updated_at=updated_at,
+            start_date=start_date,
+            end_date=end_date,
+            query=query,
         )
 
     # Set default endpoints if not provided