ingestr 0.14.93__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/revenuecat/helpers.py

@@ -64,12 +64,9 @@ def _paginate(
     while True:
         data = _make_request(api_key, endpoint, current_params)
 
-        # Yield items from the current page
         if "items" in data and data["items"] is not None:
-            for item in data["items"]:
-                yield item
+            yield data["items"]
 
-        # Check if there's a next page
         if "next_page" not in data:
             break
 
@@ -88,7 +85,6 @@ def convert_timestamps_to_iso(
     """Convert timestamp fields from milliseconds to ISO format."""
     for field in timestamp_fields:
         if field in record and record[field] is not None:
-            # Convert from milliseconds timestamp to ISO datetime string
             timestamp_ms = record[field]
             dt = pendulum.from_timestamp(timestamp_ms / 1000)
             record[field] = dt.to_iso8601_string()
@@ -177,87 +173,37 @@ async def _paginate_async(
     return items
 
 
-async def fetch_and_process_nested_resource_async(
-    session: aiohttp.ClientSession,
-    api_key: str,
-    project_id: str,
-    customer_id: str,
-    customer: Dict[str, Any],
-    resource_name: str,
-    timestamp_fields: Optional[List[str]] = None,
-) -> None:
-    """
-    Fetch and process any nested resource for a customer asynchronously.
-
-    Args:
-        session: aiohttp ClientSession
-        api_key: RevenueCat API key
-        project_id: Project ID
-        customer_id: Customer ID
-        customer: Customer data dictionary to modify
-        resource_name: Name of the nested resource (e.g., 'purchases', 'subscriptions', 'events')
-        timestamp_fields: List of timestamp fields to convert to ISO format
-    """
-    # If resource not included in customer data, fetch separately
-    if resource_name not in customer or customer[resource_name] is None:
-        endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
-        customer[resource_name] = await _paginate_async(session, api_key, endpoint)
-
-    # Convert timestamps if fields specified
-    if (
-        timestamp_fields
-        and resource_name in customer
-        and customer[resource_name] is not None
-    ):
-        for item in customer[resource_name]:
-            convert_timestamps_to_iso(item, timestamp_fields)
-
-
 async def process_customer_with_nested_resources_async(
     session: aiohttp.ClientSession,
     api_key: str,
     project_id: str,
     customer: Dict[str, Any],
 ) -> Dict[str, Any]:
-    """
-    Process a customer and fetch nested resources concurrently.
-
-    Args:
-        session: aiohttp ClientSession
-        api_key: RevenueCat API key
-        project_id: Project ID
-        customer: Customer data to process
-
-    Returns:
-        Customer data with nested resources populated
-    """
     customer_id = customer["id"]
-
-    # Convert customer timestamps
     customer = convert_timestamps_to_iso(customer, ["first_seen_at", "last_seen_at"])
-
-    # Define nested resources to fetch concurrently
     nested_resources = [
         ("subscriptions", ["purchased_at", "expires_at", "grace_period_expires_at"]),
         ("purchases", ["purchased_at", "expires_at"]),
     ]
 
-    # Create concurrent tasks for fetching nested resources
-    tasks = []
-    for resource_name, timestamp_fields in nested_resources:
-        task = fetch_and_process_nested_resource_async(
-            session,
-            api_key,
-            project_id,
-            customer_id,
-            customer,
-            resource_name,
-            timestamp_fields,
-        )
-        tasks.append(task)
-
-    # Wait for all nested resources to be fetched
-    await asyncio.gather(*tasks)
+    async def fetch_and_convert(resource_name, timestamp_fields):
+        if resource_name not in customer or customer[resource_name] is None:
+            endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
+            customer[resource_name] = await _paginate_async(session, api_key, endpoint)
+        if (
+            timestamp_fields
+            and resource_name in customer
+            and customer[resource_name] is not None
+        ):
+            for item in customer[resource_name]:
+                convert_timestamps_to_iso(item, timestamp_fields)
+
+    await asyncio.gather(
+        *[
+            fetch_and_convert(resource_name, timestamp_fields)
+            for resource_name, timestamp_fields in nested_resources
+        ]
+    )
 
     return customer
 

ingestr/src/socrata_source/__init__.py

@@ -0,0 +1,83 @@
+"""A source loading data from Socrata open data platform"""
+
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+
+from .helpers import fetch_data
+
+
+@dlt.source(name="socrata", max_table_nesting=0)
+def source(
+    domain: str,
+    dataset_id: str,
+    app_token: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    incremental: Optional[Any] = None,
+    primary_key: Optional[str] = None,
+    write_disposition: Optional[str] = dlt.config.value,
+):
+    """
+    A dlt source for the Socrata open data platform.
+
+    Supports both full refresh (replace) and incremental loading (merge).
+
+    Args:
+        domain: The Socrata domain (e.g., "evergreen.data.socrata.com")
+        dataset_id: The dataset identifier (e.g., "6udu-fhnu")
+        app_token: Socrata app token for higher rate limits (recommended)
+        username: Username for authentication (if dataset is private)
+        password: Password for authentication (if dataset is private)
+        incremental: DLT incremental object for incremental loading
+        primary_key: Primary key field for merge operations (default: ":id")
+        write_disposition: Write disposition ("replace", "append", "merge").
+            If not provided, automatically determined based on incremental setting.
+
+    Returns:
+        A dlt source with a single "dataset" resource
+    """
+
+    @dlt.resource(
+        write_disposition=write_disposition or "replace",
+        primary_key=primary_key,  # type: ignore[call-overload]
+    )
+    def dataset(
+        incremental: Optional[dlt.sources.incremental] = incremental,  # type: ignore[type-arg]
+    ) -> Iterator[Dict[str, Any]]:
+        """
+        Yields records from a Socrata dataset.
+
+        Supports both full refresh (replace) and incremental loading (merge).
+        When incremental is provided, filters data using SoQL WHERE clause on the server side.
+
+        Yields:
+            Dict[str, Any]: Individual records from the dataset
+        """
+        fetch_kwargs: Dict[str, Any] = {
+            "domain": domain,
+            "dataset_id": dataset_id,
+            "app_token": app_token,
+            "username": username,
+            "password": password,
+        }
+
+        if incremental and incremental.cursor_path:
+            fetch_kwargs["incremental_key"] = incremental.cursor_path
+            fetch_kwargs["start_value"] = (
+                str(incremental.last_value)
+                if incremental.last_value is not None
+                else None
+            )
+            if getattr(incremental, "end_value", None) is not None:
+                ev = incremental.end_value  # type: ignore[attr-defined]
+                fetch_kwargs["end_value"] = (
+                    ev.isoformat()  # type: ignore[union-attr]
+                    if hasattr(ev, "isoformat")
+                    else str(ev)
+                )
+
+        # Fetch and yield records
+        yield from fetch_data(**fetch_kwargs)
+
+    return (dataset,)

ingestr/src/socrata_source/helpers.py

@@ -0,0 +1,85 @@
+"""Socrata API helpers"""
+
+from typing import Any, Dict, Iterator, Optional
+
+from dlt.sources.helpers import requests
+
+from .settings import DEFAULT_PAGE_SIZE, REQUEST_TIMEOUT
+
+
+def fetch_data(
+    domain: str,
+    dataset_id: str,
+    app_token: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    incremental_key: Optional[str] = None,
+    start_value: Optional[str] = None,
+    end_value: Optional[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch records from Socrata dataset with pagination and optional filtering.
+
+    Uses offset-based pagination to get all records, not just first 50000.
+    Supports incremental loading via SoQL WHERE clause for server-side filtering.
+
+    Args:
+        domain: Socrata domain (e.g., "data.seattle.gov")
+        dataset_id: Dataset identifier (e.g., "6udu-fhnu")
+        app_token: Socrata app token for higher rate limits
+        username: Username for authentication
+        password: Password for authentication
+        start_value: Minimum value for incremental_key (inclusive)
+        end_value: Maximum value for incremental_key (exclusive)
+
+    Yields:
+        Lists of records (one list per page)
+
+    Raises:
+        requests.HTTPError: If API request fails
+    """
+    url = f"https://{domain}/resource/{dataset_id}.json"
+
+    headers = {"Accept": "application/json"}
+    if app_token:
+        headers["X-App-Token"] = app_token
+
+    auth = (username, password) if username and password else None
+
+    limit = DEFAULT_PAGE_SIZE
+    offset = 0
+
+    while True:
+        params: Dict[str, Any] = {"$limit": limit, "$offset": offset}
+
+        if incremental_key and start_value:
+            start_value_iso = str(start_value).replace(" ", "T")
+            where_conditions = [f"{incremental_key} >= '{start_value_iso}'"]
+
+            if end_value:
+                end_value_iso = str(end_value).replace(" ", "T")
+                where_conditions.append(f"{incremental_key} < '{end_value_iso}'")
+
+            params["$where"] = " AND ".join(where_conditions)
+            params["$order"] = f"{incremental_key} ASC"
+
+        response = requests.get(
+            url,
+            headers=headers,
+            auth=auth,
+            params=params,
+            timeout=REQUEST_TIMEOUT,
+        )
+        response.raise_for_status()
+
+        data = response.json()
+
+        if not data:
+            break
+
+        yield data
+
+        if len(data) < limit:
+            break
+
+        offset += limit

ingestr/src/socrata_source/settings.py

@@ -0,0 +1,8 @@
+"""Socrata API settings and constants"""
+
+# Request timeout in seconds
+REQUEST_TIMEOUT = 30
+
+# Maximum number of records to fetch per page
+# Socrata API supports up to 50000 records per request
+DEFAULT_PAGE_SIZE = 50000