omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/fluxx/helpers.py

@@ -0,0 +1,233 @@
+import json
+from typing import Any, Dict, Iterator, List, Optional
+from urllib.parse import urlparse
+
+import dlt
+import pendulum
+import requests
+import tldextract
+
+FLUXX_OAUTH_TOKEN_PATH = "/oauth/token"
+FLUXX_API_V2_PATH = "/api/rest/v2"
+
+
+def _get_base_url(instance: str) -> str:
+    """Get the base URL for Fluxx API.
+
+    If instance has a valid TLD (e.g., 'acme.fluxx.io'), use it as full domain.
+    Otherwise, append '.fluxxlabs.com' for backward compatibility.
+    Preserves the original scheme (http/https) if provided, defaults to https.
+
+    Examples:
+        - "mycompany" -> "https://mycompany.fluxxlabs.com"
+        - "mycompany.preprod" -> "https://mycompany.preprod.fluxxlabs.com"
+        - "acme.fluxx.io" -> "https://acme.fluxx.io"
+        - "http://acme.fluxx.io" -> "http://acme.fluxx.io"
+    """
+    parsed = urlparse(instance)
+    scheme = parsed.scheme or "https"
+    host = parsed.netloc or instance
+
+    extracted = tldextract.extract(host)
+    if extracted.suffix:
+        return f"{scheme}://{extracted.fqdn}"
+    return f"{scheme}://{host}.fluxxlabs.com"
+
+
+def get_access_token(instance: str, client_id: str, client_secret: str) -> str:
+    """Obtain OAuth access token using client credentials flow."""
+    token_url = f"{_get_base_url(instance)}{FLUXX_OAUTH_TOKEN_PATH}"
+
+    response = requests.post(
+        token_url,
+        data={
+            "grant_type": "client_credentials",
+            "client_id": client_id,
+            "client_secret": client_secret,
+        },
+    )
+    response.raise_for_status()
+
+    token_data = response.json()
+    return token_data["access_token"]
+
+
+def fluxx_api_request(
+    instance: str,
+    access_token: str,
+    endpoint: str,
+    method: str = "GET",
+    params: Optional[Dict[str, Any]] = None,
+    data: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Make an authenticated request to the Fluxx API."""
+    url = f"{_get_base_url(instance)}{FLUXX_API_V2_PATH}/{endpoint}"
+
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json",
+    }
+
+    response = requests.request(
+        method=method,
+        url=url,
+        headers=headers,
+        params=params,
+        json=data,
+    )
+    response.raise_for_status()
+
+    if response.text:
+        return response.json()
+    return {}
+
+
+def paginate_fluxx_resource(
+    instance: str,
+    access_token: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    page_size: int = 100,
+) -> Iterator[List[Dict[str, Any]]]:
+    """Paginate through a Fluxx API resource."""
+    if params is None:
+        params = {}
+
+    page = 1
+    params["per_page"] = page_size
+
+    while True:
+        params["page"] = page
+
+        response = fluxx_api_request(
+            instance=instance,
+            access_token=access_token,
+            endpoint=endpoint,
+            params=params,
+        )
+
+        if not response:
+            break
+
+        # Get the first available key from records instead of assuming endpoint name
+        records = response["records"]
+        if records:
+            # Pick the first key available in records
+            first_key = next(iter(records))
+            items = records[first_key]
+        else:
+            items = []
+
+        yield items
+
+        if response["per_page"] is None or len(items) < response["per_page"]:
+            break
+
+        page += 1
+
+
+def get_date_range(updated_at, start_date):
+    """Extract current start and end dates from incremental state."""
+    if updated_at.last_value:
+        current_start_date = pendulum.parse(updated_at.last_value)
+    else:
+        current_start_date = (
+            pendulum.parse(start_date)
+            if start_date
+            else pendulum.now().subtract(days=30)
+        )
+
+    if updated_at.end_value:
+        current_end_date = pendulum.parse(updated_at.end_value)
+    else:
+        current_end_date = pendulum.now(tz="UTC")
+
+    return current_start_date, current_end_date
+
+
+def create_dynamic_resource(
+    resource_name: str,
+    endpoint: str,
+    instance: str,
+    access_token: str,
+    start_date: Optional[pendulum.DateTime] = None,
+    end_date: Optional[pendulum.DateTime] = None,
+    fields_to_extract: Optional[Dict[str, Any]] = None,
+):
+    """Factory function to create dynamic Fluxx resources."""
+
+    # Extract column definitions for DLT resource
+    columns = {}
+    if fields_to_extract:
+        for field_name, field_config in fields_to_extract.items():
+            data_type = field_config.get("data_type")
+            if data_type:
+                columns[field_name] = {"data_type": data_type}
+
+    @dlt.resource(name=resource_name, write_disposition="replace", columns=columns)  # type: ignore
+    def fluxx_resource() -> Iterator[Dict[str, Any]]:
+        params = {}
+        if fields_to_extract:
+            field_names = list(fields_to_extract.keys())
+            params["cols"] = json.dumps(field_names)
+
+        for page in paginate_fluxx_resource(
+            instance=instance,
+            access_token=access_token,
+            endpoint=endpoint,
+            params=params,
+            page_size=100,
+        ):
+            yield [normalize_fluxx_item(item, fields_to_extract) for item in page]  # type: ignore
+
+    return fluxx_resource
+
+
+def normalize_fluxx_item(
+    item: Dict[str, Any], fields_to_extract: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    """
+    Normalize a Fluxx API response item.
+    Handles nested structures and field extraction based on field types.
+    Rounds all decimal/float values to 4 decimal places regardless of field type.
+    """
+    normalized: Dict[str, Any] = {}
+
+    # If no field mapping provided, just return the item as-is
+    if not fields_to_extract:
+        return item
+
+    for field_name, field_config in fields_to_extract.items():
+        if field_name in item:
+            value = item[field_name]
+            field_type = field_config.get("data_type")
+
+            if isinstance(value, float):
+                # Round any numeric value with decimal places
+                normalized[field_name] = round(value, 4)
+            elif field_type == "json":
+                # Handle json fields (arrays/relations)
+                if value is None:
+                    normalized[field_name] = None
+                elif value == "":
+                    normalized[field_name] = None
+                elif isinstance(value, (list, dict)):
+                    normalized[field_name] = value
+                else:
+                    # Single value - wrap in array for json fields
+                    normalized[field_name] = [value]
+            elif field_type in ("date", "timestamp", "datetime", "text"):
+                # Handle text/date fields - convert empty strings to None
+                if value == "":
+                    normalized[field_name] = None
+                else:
+                    normalized[field_name] = value
+            else:
+                # All other field types - pass through as-is
+                normalized[field_name] = value
+
+    # Always include id if present
+    if "id" in item:
+        normalized["id"] = item["id"]
+
+    return normalized

omniload/src/frankfurter/__init__.py

@@ -0,0 +1,157 @@
+from typing import Any, Iterator, Optional
+
+import dlt
+from dlt.common.pendulum import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+
+from omniload.src.frankfurter.helpers import get_path_with_retry
+
+
+@dlt.source(
+    name="frankfurter",
+    max_table_nesting=0,
+)
+def frankfurter_source(
+    start_date: TAnyDateTime,
+    end_date: TAnyDateTime | None,
+    base_currency: str,
+) -> Any:
+    """
+    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
+    various types of data: currencies, latest rates, historical rates.
+    """
+
+    @dlt.resource(
+        write_disposition="replace",
+    )
+    def currencies() -> Iterator[dict]:
+        """
+        Yields each currency as a separate row with two columns: currency_code and currency_name.
+        """
+        # Retrieve the list of currencies from the API
+        currencies_data = get_path_with_retry("currencies")
+
+        for currency_code, currency_name in currencies_data.items():
+            yield {"currency_code": currency_code, "currency_name": currency_name}
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=["date", "currency_code", "base_currency"],
+    )
+    def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
+        """
+        Fetches the latest exchange rates and yields them as rows.
+        """
+        # Base URL
+        url = "latest?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data
+        data = get_path_with_retry(url)
+
+        # Extract rates and base currency
+        rates = data["rates"]
+        date = pendulum.parse(data["date"])
+
+        # Add the base currency with a rate of 1.0
+        yield {
+            "date": date,
+            "currency_code": base_currency,
+            "rate": 1.0,
+            "base_currency": base_currency,
+        }
+
+        # Add all currencies and their rates
+        for currency_code, rate in rates.items():
+            yield {
+                "date": date,
+                "currency_code": currency_code,
+                "rate": rate,
+                "base_currency": base_currency,
+            }
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=("date", "currency_code", "base_currency"),
+    )
+    def exchange_rates(
+        date_time=dlt.sources.incremental(
+            "date",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        """
+        Fetches exchange rates for a specified date range.
+        If only start_date is provided, fetches data until now.
+        If both start_date and end_date are provided, fetches data for each day in the range.
+        """
+        if date_time.last_value is not None:
+            start_date = date_time.last_value
+        else:
+            start_date = start_date
+
+        if date_time.end_value is not None:
+            end_date = date_time.end_value
+        else:
+            end_date = pendulum.now()
+
+        # Ensure start_date.last_value is a pendulum.DateTime object
+        start_date_obj = ensure_pendulum_datetime(start_date)  # type: ignore
+        start_date_str = start_date_obj.format("YYYY-MM-DD")
+
+        # Ensure end_date is a pendulum.DateTime object
+        end_date_obj = ensure_pendulum_datetime(end_date)
+        end_date_str = end_date_obj.format("YYYY-MM-DD")
+
+        # Compose the URL
+        url = f"{start_date_str}..{end_date_str}?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data from the API
+        data = get_path_with_retry(url)
+
+        # Extract base currency and rates from the API response
+        rates = data["rates"]
+
+        # Iterate over the rates dictionary (one entry per date)
+        for date, daily_rates in rates.items():
+            formatted_date = pendulum.parse(date)
+
+            # Add the base currency with a rate of 1.0
+            yield {
+                "date": formatted_date,
+                "currency_code": base_currency,
+                "rate": 1.0,
+                "base_currency": base_currency,
+            }
+
+            # Add all other currencies and their rates
+            for currency_code, rate in daily_rates.items():
+                yield {
+                    "date": formatted_date,
+                    "currency_code": currency_code,
+                    "rate": rate,
+                    "base_currency": base_currency,
+                }
+
+    return currencies, latest, exchange_rates

omniload/src/frankfurter/helpers.py

@@ -0,0 +1,48 @@
+from datetime import datetime
+
+from dlt.common.pendulum import pendulum
+from dlt.common.typing import StrAny
+from dlt.sources.helpers import requests
+
+FRANKFURTER_API_URL = "https://api.frankfurter.dev/v1/"
+
+
+def get_url_with_retry(url: str) -> StrAny:
+    r = requests.get(url, timeout=5)
+    return r.json()  # type: ignore
+
+
+def get_path_with_retry(path: str) -> StrAny:
+    return get_url_with_retry(f"{FRANKFURTER_API_URL}{path}")
+
+
+def validate_dates(start_date: datetime, end_date: datetime | None) -> None:
+    current_date = pendulum.now()
+
+    # Check if start_date is in the futurep
+    if start_date > current_date:
+        raise ValueError("Interval-start cannot be in the future.")
+
+    # Check if end_date is in the future
+    if end_date is not None and end_date > current_date:
+        raise ValueError("Interval-end cannot be in the future.")
+
+    # Check if start_date is before end_date
+    if end_date is not None and start_date > end_date:
+        raise ValueError("Interval-end cannot be before interval-start.")
+
+
+def validate_currency(currency_code: str) -> bool:
+    url = "https://api.frankfurter.dev/v1/currencies"
+
+    response = requests.get(url, timeout=5)
+    currencies = response.json()
+
+    if currency_code.upper() in currencies:
+        return True
+    else:
+        supported_currencies = list(currencies.keys())
+        print(
+            f"Invalid base currency '{currency_code}'. Supported currencies are: {supported_currencies}"
+        )
+        return False

omniload/src/freshdesk/__init__.py

@@ -0,0 +1,103 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This source uses Freshdesk API and dlt to load data such as Agents, Companies, Tickets
+etc. to the database"""
+
+from typing import Any, Dict, Generator, Iterable, List, Optional
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource
+
+from .freshdesk_client import FreshdeskClient
+from .settings import DEFAULT_ENDPOINTS
+
+
+@dlt.source()
+def freshdesk_source(
+    domain: str,
+    api_secret_key: str,
+    start_date: pendulum.DateTime,
+    end_date: Optional[pendulum.DateTime] = None,
+    per_page: int = 100,
+    endpoints: Optional[List[str]] = None,
+    query: Optional[str] = None,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from specified Freshdesk API endpoints.
+
+    This source supports pagination and incremental data loading. It fetches data from a list of
+    specified endpoints, or defaults to predefined endpoints in 'settings.py'.
+
+    Args:
+        endpoints: A list of Freshdesk API endpoints to fetch. Deafults to 'settings.py'.
+        per_page: The number of items to fetch per page, with a maximum of 100.
+        domain: The Freshdesk domain from which to fetch the data. Defaults to 'config.toml'.
+        api_secret_key: Freshdesk API key. Defaults to 'secrets.toml'.
+
+    Yields:
+        Iterable[DltResource]: Resources with data updated after the last 'updated_at'
+        timestamp for each endpoint.
+    """
+    # Instantiate FreshdeskClient with the provided domain and API key
+    freshdesk = FreshdeskClient(api_key=api_secret_key, domain=domain)
+
+    def incremental_resource(
+        endpoint: str,
+        updated_at: Optional[Any] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Generator[Dict[Any, Any], Any, None]:
+        """
+        Fetches and yields paginated data from a specified API endpoint.
+        Each page of data is fetched based on the `updated_at` timestamp
+        to ensure incremental loading.
+        """
+
+        if updated_at.last_value is not None:
+            start_date = ensure_pendulum_datetime(updated_at.last_value)
+        else:
+            start_date = start_date
+
+        if updated_at.end_value is not None:
+            end_date = ensure_pendulum_datetime(updated_at.end_value)
+        else:
+            end_date = pendulum.now(tz="UTC")
+
+        # Use the FreshdeskClient instance to fetch paginated responses
+        yield from freshdesk.paginated_response(
+            endpoint=endpoint,
+            per_page=per_page,
+            start_date=start_date,
+            end_date=end_date,
+            query=query,
+        )
+
+    # Set default endpoints if not provided
+    endpoints = endpoints or DEFAULT_ENDPOINTS
+
+    # For each endpoint, create and yield a DLT resource
+    for endpoint in endpoints:
+        yield dlt.resource(
+            incremental_resource,
+            name=endpoint,
+            write_disposition="merge",
+            primary_key="id",
+        )(endpoint=endpoint)

omniload/src/freshdesk/freshdesk_client.py

@@ -0,0 +1,151 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Freshdesk Client for making authenticated requests"""
+
+import logging
+import time
+from typing import Any, Dict, Iterable, Optional
+
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources.helpers import requests
+
+from omniload.src.errors import HTTPError
+
+TICKETS_QUERY_MAX_PAGE = 10
+
+
+class FreshdeskClient:
+    """
+    Client for making authenticated requests to the Freshdesk API. It incorporates API requests with
+    rate limit and pagination.
+
+    Attributes:
+        api_key (str): The API key used for authenticating requests to the Freshdesk API.
+        domain (str): The Freshdesk domain specific to the user, used in constructing the base URL.
+        base_url (str): The base URL constructed from the domain, targeting the Freshdesk API v2.
+    """
+
+    def __init__(self, api_key: str, domain: str):
+        # Initialize the FreshdeskClient instance with API key and domain.
+        # The API key is used for authentication with the Freshdesk API.
+        # The domain specifies the unique Freshdesk domain of the user.
+
+        # Store the API key provided during initialization.
+        self.api_key = api_key
+        # Store the Freshdesk domain provided during initialization.
+        self.domain = domain
+
+        # Construct the base URL for the API requests.
+        # This URL is formed by appending the domain to the standard Freshdesk API base URL format.
+        # All API requests will use this base URL as their starting point.
+        self.base_url = f"https://{domain}.freshdesk.com/api/v2"
+
+    def _request_with_rate_limit(self, url: str, **kwargs: Any) -> requests.Response:
+        """
+        Handles rate limits in HTTP requests and ensures
+        that the client doesn't exceed the limit set by the server.
+        """
+
+        while True:
+            try:
+                response = requests.get(url, **kwargs, auth=(self.api_key, "X"))
+                response.raise_for_status()
+
+                return response
+            except requests.HTTPError as e:
+                if e.response.status_code == 429:
+                    # Get the 'Retry-After' header to know how long to wait
+                    # Fallback to 60 seconds if header is missing
+                    seconds_to_wait = int(e.response.headers.get("Retry-After", 60))
+                    # Log a warning message
+                    logging.warning(
+                        "Rate limited. Waiting to retry after: %s secs", seconds_to_wait
+                    )
+
+                    # Wait for the specified number of seconds before retrying
+                    time.sleep(seconds_to_wait)
+                else:
+                    # If the error is not a rate limit (429), raise the exception to be
+                    # handled elsewhere or stop execution
+                    raise HTTPError(e) from e
+
+    def paginated_response(
+        self,
+        endpoint: str,
+        per_page: int,
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
+        query: Optional[str] = None,
+    ) -> Iterable[TDataItem]:
+        """
+        Fetches a paginated response from a specified endpoint.
+
+        This method will continuously fetch data from the given endpoint,
+        page by page, until no more data is available or until it reaches data
+        updated at the specified timestamp.
+        """
+        page = 1
+        if query is not None:
+            query = query.replace('"', "").strip()
+
+        is_tickets_query = query and endpoint == "tickets"
+
+        while True:
+            # Construct the URL for the specific endpoint
+            url = f"{self.base_url}/{endpoint}"
+
+            params: Dict[str, Any] = {"per_page": per_page, "page": page}
+
+            # Implement date range splitting logic here, if applicable
+            if endpoint in ["tickets", "contacts"]:
+                param_key = (
+                    "updated_since" if endpoint == "tickets" else "_updated_since"
+                )
+
+                params[param_key] = start_date.to_iso8601_string()
+
+            if is_tickets_query:
+                url = f"{self.base_url}/search/tickets"
+                params = {
+                    "query": f'"{query}"',
+                    "page": page,
+                }
+
+            # Handle requests with rate-limiting
+            # A maximum of 300 pages (30000 tickets) will be returned.
+            response = self._request_with_rate_limit(url, params=params)
+            data = response.json()
+
+            if query and endpoint == "tickets":
+                data = data["results"]
+
+            if not data:
+                break  # Stop if no data or max page limit reached
+
+            filtered_data = [
+                item
+                for item in data
+                if "updated_at" in item
+                and pendulum.parse(item["updated_at"]) <= end_date
+            ]
+            if not filtered_data:
+                break
+            yield filtered_data
+            page += 1
+
+            # https://developers.freshdesk.com/api/#filter_tickets
+            if is_tickets_query and page > TICKETS_QUERY_MAX_PAGE:
+                break