ingestr 0.13.85__py3-none-any.whl → 0.13.87__py3-none-any.whl

ingestr/src/fluxx/helpers.py

@@ -0,0 +1,216 @@
+import json
+from typing import Any, Dict, Iterator, List, Optional
+
+import dlt
+import pendulum
+import requests
+
+FLUXX_API_BASE = "https://{instance}.fluxxlabs.com"
+FLUXX_OAUTH_TOKEN_PATH = "/oauth/token"
+FLUXX_API_V2_PATH = "/api/rest/v2"
+
+
+def get_access_token(instance: str, client_id: str, client_secret: str) -> str:
+    """Obtain OAuth access token using client credentials flow."""
+    token_url = f"{FLUXX_API_BASE.format(instance=instance)}{FLUXX_OAUTH_TOKEN_PATH}"
+
+    response = requests.post(
+        token_url,
+        data={
+            "grant_type": "client_credentials",
+            "client_id": client_id,
+            "client_secret": client_secret,
+        },
+    )
+    response.raise_for_status()
+
+    token_data = response.json()
+    return token_data["access_token"]
+
+
+def fluxx_api_request(
+    instance: str,
+    access_token: str,
+    endpoint: str,
+    method: str = "GET",
+    params: Optional[Dict[str, Any]] = None,
+    data: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Make an authenticated request to the Fluxx API."""
+    url = f"{FLUXX_API_BASE.format(instance=instance)}{FLUXX_API_V2_PATH}/{endpoint}"
+
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json",
+    }
+    # print(f"Making request to Fluxx API:")
+    # print(f"  Method: {method}")
+    # print(f"  URL: {url}")
+    # print(f"  Headers: {headers}")
+    # print(f"  Params: {params}")
+    # print(f"  Data: {data}")
+
+    response = requests.request(
+        method=method,
+        url=url,
+        headers=headers,
+        params=params,
+        json=data,
+    )
+    response.raise_for_status()
+
+    if response.text:
+        return response.json()
+    return {}
+
+
+def paginate_fluxx_resource(
+    instance: str,
+    access_token: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    page_size: int = 100,
+) -> Iterator[List[Dict[str, Any]]]:
+    """Paginate through a Fluxx API resource."""
+    if params is None:
+        params = {}
+
+    page = 1
+    params["per_page"] = page_size
+
+    while True:
+        params["page"] = page
+
+        response = fluxx_api_request(
+            instance=instance,
+            access_token=access_token,
+            endpoint=endpoint,
+            params=params,
+        )
+
+        print("resssponse", response)
+        if not response:
+            break
+
+        # Get the first available key from records instead of assuming endpoint name
+        records = response["records"]
+        if records:
+            # Pick the first key available in records
+            first_key = next(iter(records))
+            items = records[first_key]
+        else:
+            items = []
+
+        yield items
+
+        if response["per_page"] is None or len(items) < response["per_page"]:
+            break
+
+        page += 1
+
+
+def get_date_range(updated_at, start_date):
+    """Extract current start and end dates from incremental state."""
+    if updated_at.last_value:
+        current_start_date = pendulum.parse(updated_at.last_value)
+    else:
+        current_start_date = (
+            pendulum.parse(start_date)
+            if start_date
+            else pendulum.now().subtract(days=30)
+        )
+
+    if updated_at.end_value:
+        current_end_date = pendulum.parse(updated_at.end_value)
+    else:
+        current_end_date = pendulum.now(tz="UTC")
+
+    return current_start_date, current_end_date
+
+
+def create_dynamic_resource(
+    resource_name: str,
+    endpoint: str,
+    instance: str,
+    access_token: str,
+    start_date: Optional[pendulum.DateTime] = None,
+    end_date: Optional[pendulum.DateTime] = None,
+    fields_to_extract: Optional[Dict[str, Any]] = None,
+):
+    """Factory function to create dynamic Fluxx resources."""
+
+    # Extract column definitions for DLT resource
+    columns = {}
+    if fields_to_extract:
+        for field_name, field_config in fields_to_extract.items():
+            data_type = field_config.get("data_type")
+            if data_type:
+                columns[field_name] = {"data_type": data_type}
+
+    @dlt.resource(name=resource_name, write_disposition="replace", columns=columns)  # type: ignore
+    def fluxx_resource() -> Iterator[Dict[str, Any]]:
+        params = {}
+        if fields_to_extract:
+            field_names = list(fields_to_extract.keys())
+            params["cols"] = json.dumps(field_names)
+
+        for page in paginate_fluxx_resource(
+            instance=instance,
+            access_token=access_token,
+            endpoint=endpoint,
+            params=params,
+            page_size=100,
+        ):
+            yield [normalize_fluxx_item(item, fields_to_extract) for item in page]  # type: ignore
+
+    return fluxx_resource
+
+
+def normalize_fluxx_item(
+    item: Dict[str, Any], fields_to_extract: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    """
+    Normalize a Fluxx API response item.
+    Handles nested structures and field extraction based on field types.
+    Rounds all decimal/float values to 4 decimal places regardless of field type.
+    """
+    normalized: Dict[str, Any] = {}
+
+    # If no field mapping provided, just return the item as-is
+    if not fields_to_extract:
+        return item
+
+    for field_name, field_config in fields_to_extract.items():
+        if field_name in item:
+            value = item[field_name]
+            field_type = field_config.get("data_type")
+
+            if isinstance(value, float):
+                # Round any numeric value with decimal places
+                normalized[field_name] = round(value, 4)
+            elif field_type == "json":
+                # Handle json fields (arrays/relations)
+                if value is None:
+                    normalized[field_name] = None
+                elif value == "":
+                    normalized[field_name] = None
+                elif isinstance(value, (list, dict)):
+                    normalized[field_name] = value
+                else:
+                    # Single value - wrap in array for json fields
+                    normalized[field_name] = [value]
+            elif field_type in ("date", "timestamp", "datetime", "text"):
+                # Handle text/date fields - convert empty strings to None
+                if value == "":
+                    normalized[field_name] = None
+                else:
+                    normalized[field_name] = value
+            else:
+                # All other field types - pass through as-is
+                normalized[field_name] = value
+
+    # Always include id if present
+    if "id" in item:
+        normalized["id"] = item["id"]
+
+    return normalized

ingestr/src/frankfurter/__init__.py

@@ -14,150 +14,148 @@ from ingestr.src.frankfurter.helpers import get_path_with_retry
 )
 def frankfurter_source(
     start_date: TAnyDateTime,
-    end_date: TAnyDateTime,
+    end_date: TAnyDateTime|None,
     base_currency: str,
 ) -> Any:
     """
     A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
     various types of data: currencies, latest rates, historical rates.
     """
-    date_time = dlt.sources.incremental(
-        "date",
-        initial_value=start_date,
-        end_value=end_date,
-        range_start="closed",
-        range_end="closed",
-    )
+    
 
-    return (
-        currencies(),
-        latest(base_currency=base_currency),
-        exchange_rates(
-            start_date=date_time, end_date=end_date, base_currency=base_currency
-        ),
+    @dlt.resource(
+        write_disposition="replace",
     )
+    def currencies() -> Iterator[dict]:
+        """
+        Yields each currency as a separate row with two columns: currency_code and currency_name.
+        """
+        # Retrieve the list of currencies from the API
+        currencies_data = get_path_with_retry("currencies")
+
+        for currency_code, currency_name in currencies_data.items():
+            yield {"currency_code": currency_code, "currency_name": currency_name}
+
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=["date", "currency_code", "base_currency"],
+    )
+    def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
+        """
+        Fetches the latest exchange rates and yields them as rows.
+        """
+        # Base URL
+        url = "latest?"
 
+        if base_currency:
+            url += f"base={base_currency}"
 
-@dlt.resource(
-    write_disposition="replace",
-    columns={
-        "currency_code": {"data_type": "text"},
-        "currency_name": {"data_type": "text"},
-    },
-)
-def currencies() -> Iterator[dict]:
-    """
-    Yields each currency as a separate row with two columns: currency_code and currency_name.
-    """
-    # Retrieve the list of currencies from the API
-    currencies_data = get_path_with_retry("currencies")
-
-    for currency_code, currency_name in currencies_data.items():
-        yield {"currency_code": currency_code, "currency_name": currency_name}
-
-
-@dlt.resource(
-    write_disposition="merge",
-    columns={
-        "date": {"data_type": "text"},
-        "currency_code": {"data_type": "text"},
-        "rate": {"data_type": "double"},
-        "base_currency": {"data_type": "text"},
-    },
-    primary_key=["date", "currency_code", "base_currency"],
-)
-def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
-    """
-    Fetches the latest exchange rates and yields them as rows.
-    """
-    # Base URL
-    url = "latest?"
-
-    if base_currency:
-        url += f"base={base_currency}"
-
-    # Fetch data
-    data = get_path_with_retry(url)
-
-    # Extract rates and base currency
-    rates = data["rates"]
-    date = pendulum.parse(data["date"])
-
-    # Add the base currency with a rate of 1.0
-    yield {
-        "date": date,
-        "currency_code": base_currency,
-        "rate": 1.0,
-        "base_currency": base_currency,
-    }
-
-    # Add all currencies and their rates
-    for currency_code, rate in rates.items():
-        yield {
-            "date": date,
-            "currency_code": currency_code,
-            "rate": rate,
-            "base_currency": base_currency,
-        }
-
-
-@dlt.resource(
-    write_disposition="merge",
-    columns={
-        "date": {"data_type": "text"},
-        "currency_code": {"data_type": "text"},
-        "rate": {"data_type": "double"},
-        "base_currency": {"data_type": "text"},
-    },
-    primary_key=("date", "currency_code", "base_currency"),
-)
-def exchange_rates(
-    end_date: TAnyDateTime,
-    start_date: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental("date"),
-    base_currency: Optional[str] = "",
-) -> Iterator[dict]:
-    """
-    Fetches exchange rates for a specified date range.
-    If only start_date is provided, fetches data until now.
-    If both start_date and end_date are provided, fetches data for each day in the range.
-    """
-    # Ensure start_date.last_value is a pendulum.DateTime object
-    start_date_obj = ensure_pendulum_datetime(start_date.last_value)  # type: ignore
-    start_date_str = start_date_obj.format("YYYY-MM-DD")
-
-    # Ensure end_date is a pendulum.DateTime object
-    end_date_obj = ensure_pendulum_datetime(end_date)
-    end_date_str = end_date_obj.format("YYYY-MM-DD")
-
-    # Compose the URL
-    url = f"{start_date_str}..{end_date_str}?"
-
-    if base_currency:
-        url += f"base={base_currency}"
-
-    # Fetch data from the API
-    data = get_path_with_retry(url)
-
-    # Extract base currency and rates from the API response
-    rates = data["rates"]
+        # Fetch data
+        data = get_path_with_retry(url)
 
-    # Iterate over the rates dictionary (one entry per date)
-    for date, daily_rates in rates.items():
-        formatted_date = pendulum.parse(date)
+        # Extract rates and base currency
+        rates = data["rates"]
+        date = pendulum.parse(data["date"])
 
         # Add the base currency with a rate of 1.0
         yield {
-            "date": formatted_date,
+            "date": date,
             "currency_code": base_currency,
             "rate": 1.0,
             "base_currency": base_currency,
         }
 
-        # Add all other currencies and their rates
-        for currency_code, rate in daily_rates.items():
+        # Add all currencies and their rates
+        for currency_code, rate in rates.items():
             yield {
-                "date": formatted_date,
+                "date": date,
                 "currency_code": currency_code,
                 "rate": rate,
                 "base_currency": base_currency,
             }
+
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=("date", "currency_code", "base_currency"),
+    )
+    def exchange_rates(
+        date_time = dlt.sources.incremental(
+        "date",
+        initial_value=start_date,
+        end_value=end_date,
+        range_start="closed",
+        range_end="closed",
+    )
+    ) -> Iterator[dict]:
+        """
+        Fetches exchange rates for a specified date range.
+        If only start_date is provided, fetches data until now.
+        If both start_date and end_date are provided, fetches data for each day in the range.
+        """
+        if date_time.last_value is not None:
+            start_date = date_time.last_value
+        else:
+            start_date = start_date
+
+        if date_time.end_value is not None:
+            end_date = date_time.end_value
+        else:
+            end_date = pendulum.now()
+        
+        # Ensure start_date.last_value is a pendulum.DateTime object
+        start_date_obj = ensure_pendulum_datetime(start_date) # type: ignore
+        start_date_str = start_date_obj.format("YYYY-MM-DD")
+
+        # Ensure end_date is a pendulum.DateTime object
+        end_date_obj = ensure_pendulum_datetime(end_date)
+        end_date_str = end_date_obj.format("YYYY-MM-DD")
+
+        # Compose the URL
+        url = f"{start_date_str}..{end_date_str}?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data from the API
+        data = get_path_with_retry(url)
+
+        # Extract base currency and rates from the API response
+        rates = data["rates"]
+
+        # Iterate over the rates dictionary (one entry per date)
+        for date, daily_rates in rates.items():
+            formatted_date = pendulum.parse(date)
+
+            # Add the base currency with a rate of 1.0
+            yield {
+                "date": formatted_date,
+                "currency_code": base_currency,
+                "rate": 1.0,
+                "base_currency": base_currency,
+            }
+
+            # Add all other currencies and their rates
+            for currency_code, rate in daily_rates.items():
+                yield {
+                    "date": formatted_date,
+                    "currency_code": currency_code,
+                    "rate": rate,
+                    "base_currency": base_currency,
+                }
+
+    return currencies, latest, exchange_rates
+    

ingestr/src/frankfurter/helpers.py

@@ -16,19 +16,19 @@ def get_path_with_retry(path: str) -> StrAny:
     return get_url_with_retry(f"{FRANKFURTER_API_URL}{path}")
 
 
-def validate_dates(start_date: datetime, end_date: datetime) -> None:
+def validate_dates(start_date: datetime, end_date: datetime|None) -> None:
     current_date = pendulum.now()
-
+    
     # Check if start_date is in the futurep
     if start_date > current_date:
         raise ValueError("Interval-start cannot be in the future.")
 
     # Check if end_date is in the future
-    if end_date > current_date:
+    if end_date is not None and end_date > current_date:
         raise ValueError("Interval-end cannot be in the future.")
 
     # Check if start_date is before end_date
-    if start_date > end_date:
+    if end_date is not None and start_date > end_date:
         raise ValueError("Interval-end cannot be before interval-start.")
 
 

ingestr/src/klaviyo/__init__.py

@@ -30,7 +30,7 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
     start_date_obj = ensure_pendulum_datetime(start_date)
     client = KlaviyoClient(api_key)
 
-    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def events(
         datetime=dlt.sources.incremental(
             "datetime",
@@ -135,7 +135,7 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
     ) -> Iterable[TDataItem]:
         yield from client.fetch_catalog_item(create_client(), updated.start_value)
 
-    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def forms(
         updated_at=dlt.sources.incremental(
             "updated_at",
@@ -162,7 +162,7 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
     ) -> Iterable[TDataItem]:
         yield from client.fetch_lists(create_client(), updated.start_value)
 
-    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def images(
         updated_at=dlt.sources.incremental(
             "updated_at",
@@ -188,7 +188,7 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
     ) -> Iterable[TDataItem]:
         yield from client.fetch_segments(create_client(), updated.start_value)
 
-    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def flows(
         updated=dlt.sources.incremental(
             "updated",
@@ -203,7 +203,7 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
         for start, end in intervals:
             yield lambda s=start, e=end: client.fetch_flows(create_client(), s, e)
 
-    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def templates(
         updated=dlt.sources.incremental(
             "updated",

ingestr/src/linear/__init__.py

@@ -2,10 +2,13 @@ from typing import Any, Dict, Iterable, Iterator
 
 import dlt
 import pendulum
-import requests
-
-from .helpers import _graphql, normalize_dictionaries, _get_date_range, _create_paginated_resource
 
+from .helpers import (
+    _create_paginated_resource,
+    _get_date_range,
+    _graphql,
+    normalize_dictionaries,
+)
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -249,7 +252,6 @@ query Initiatives($cursor: String) {
 """
 
 
-
 INITIATIVE_TO_PROJECTS_QUERY = """
 query InitiativeToProjects($cursor: String) {
   initiativeToProjects(first: 50, after: $cursor) {
@@ -406,7 +408,6 @@ query ProjectUpdates($cursor: String) {
 """
 
 
-
 TEAM_MEMBERSHIPS_QUERY = """
 query TeamMemberships($cursor: String) {
   teamMemberships(first: 50, after: $cursor) {
@@ -594,14 +595,12 @@ PAGINATED_RESOURCES = [
 ]
 
 
-
 @dlt.source(name="linear", max_table_nesting=0)
 def linear_source(
     api_key: str,
     start_date: pendulum.DateTime,
     end_date: pendulum.DateTime | None = None,
 ) -> Iterable[dlt.sources.DltResource]:
-
     @dlt.resource(name="organization", primary_key="id", write_disposition="merge")
     def organization(
         updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
@@ -623,10 +622,12 @@ def linear_source(
 
     # Create paginated resources dynamically
     paginated_resources = [
-        _create_paginated_resource(resource_name, query, query_field, api_key, start_date, end_date)
+        _create_paginated_resource(
+            resource_name, query, query_field, api_key, start_date, end_date
+        )
         for resource_name, query, query_field in PAGINATED_RESOURCES
     ]
-    
+
     return [
         *paginated_resources,
         organization,

ingestr/src/linear/helpers.py

@@ -1,8 +1,8 @@
 from typing import Any, Dict, Iterator, Optional
 
-import requests
-import pendulum
 import dlt
+import pendulum
+import requests
 
 LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
 
@@ -34,8 +34,6 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
         cursor = data["pageInfo"]["endCursor"]
 
 
-
-
 def _get_date_range(updated_at, start_date):
     """Extract current start and end dates from incremental state."""
     if updated_at.last_value:
@@ -47,11 +45,13 @@ def _get_date_range(updated_at, start_date):
         current_end_date = pendulum.parse(updated_at.end_value)
     else:
         current_end_date = pendulum.now(tz="UTC")
-    
+
     return current_start_date, current_end_date
 
 
-def _paginated_resource(api_key: str, query: str, query_field: str, updated_at, start_date) -> Iterator[Dict[str, Any]]:
+def _paginated_resource(
+    api_key: str, query: str, query_field: str, updated_at, start_date
+) -> Iterator[Dict[str, Any]]:
     """Helper function for paginated resources with date filtering."""
     current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
@@ -61,8 +61,16 @@ def _paginated_resource(api_key: str, query: str, query_field: str, updated_at,
                 yield normalize_dictionaries(item)
 
 
-def _create_paginated_resource(resource_name: str, query: str, query_field: str, api_key: str, start_date, end_date = None):
+def _create_paginated_resource(
+    resource_name: str,
+    query: str,
+    query_field: str,
+    api_key: str,
+    start_date,
+    end_date=None,
+):
     """Factory function to create paginated resources dynamically."""
+
     @dlt.resource(name=resource_name, primary_key="id", write_disposition="merge")
     def paginated_resource(
         updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
@@ -73,9 +81,11 @@ def _create_paginated_resource(resource_name: str, query: str, query_field: str,
             range_end="closed",
         ),
     ) -> Iterator[Dict[str, Any]]:
-        for item in _paginated_resource(api_key, query, query_field, updated_at, start_date):
+        for item in _paginated_resource(
+            api_key, query, query_field, updated_at, start_date
+        ):
             yield normalize_dictionaries(item)
-    
+
     return paginated_resource
 
 
@@ -84,7 +94,7 @@ def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
     Automatically normalize dictionary fields by detecting their structure:
     - Convert nested objects with 'id' field to {field_name}_id
     - Convert objects with 'nodes' field to arrays
- 
+
     """
     normalized_item = item.copy()