ingestr 0.13.75__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/smartsheets/__init__.py

@@ -3,6 +3,22 @@ from typing import Iterable
 import dlt
 import smartsheet  # type: ignore
 from dlt.extract import DltResource
+from smartsheet.models.enums import ColumnType  # type: ignore
+from smartsheet.models.sheet import Sheet  # type: ignore
+
+TYPE_MAPPING = {
+    ColumnType.TEXT_NUMBER: "text",
+    ColumnType.DATE: "date",
+    ColumnType.DATETIME: "timestamp",
+    ColumnType.CONTACT_LIST: "text",
+    ColumnType.CHECKBOX: "bool",
+    ColumnType.PICKLIST: "text",
+    ColumnType.DURATION: "text",
+    ColumnType.PREDECESSOR: "text",
+    ColumnType.ABSTRACT_DATETIME: "timestamp",
+    ColumnType.MULTI_CONTACT_LIST: "text",
+    ColumnType.MULTI_PICKLIST: "text",
+}
 
 
 @dlt.source
@@ -34,21 +50,33 @@ def smartsheet_source(
     )
     sheet_name = sheet_details.name
     resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
+    sheet = smartsheet_client.Sheets.get_sheet(sheet_id_int)
 
     yield dlt.resource(
-        _get_sheet_data(smartsheet_client, sheet_id_int),
+        _get_sheet_data(sheet),
         name=resource_name,
+        columns=_generate_type_hints(sheet),
         write_disposition="replace",
     )
 
 
-def _get_sheet_data(smartsheet_client: smartsheet.Smartsheet, sheet_id: int):
+def _get_sheet_data(sheet: Sheet):
     """Helper function to get all rows from a sheet."""
-    sheet = smartsheet_client.Sheets.get_sheet(sheet_id)
-    # Transform rows to a list of dictionaries
+
     column_titles = [col.title for col in sheet.columns]
     for row in sheet.rows:
-        row_data = {}
+        row_data = {"_row_id": row.id}
         for i, cell in enumerate(row.cells):
             row_data[column_titles[i]] = cell.value
         yield row_data
+
+
+def _generate_type_hints(sheet: Sheet):
+    return {
+        col.title: {
+            "data_type": TYPE_MAPPING.get(col.type.value),
+            "nullable": True,
+        }
+        for col in sheet.columns
+        if col.type.value in TYPE_MAPPING
+    }

ingestr/src/socrata_source/__init__.py

@@ -0,0 +1,83 @@
+"""A source loading data from Socrata open data platform"""
+
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+
+from .helpers import fetch_data
+
+
+@dlt.source(name="socrata", max_table_nesting=0)
+def source(
+    domain: str,
+    dataset_id: str,
+    app_token: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    incremental: Optional[Any] = None,
+    primary_key: Optional[str] = None,
+    write_disposition: Optional[str] = dlt.config.value,
+):
+    """
+    A dlt source for the Socrata open data platform.
+
+    Supports both full refresh (replace) and incremental loading (merge).
+
+    Args:
+        domain: The Socrata domain (e.g., "evergreen.data.socrata.com")
+        dataset_id: The dataset identifier (e.g., "6udu-fhnu")
+        app_token: Socrata app token for higher rate limits (recommended)
+        username: Username for authentication (if dataset is private)
+        password: Password for authentication (if dataset is private)
+        incremental: DLT incremental object for incremental loading
+        primary_key: Primary key field for merge operations (default: ":id")
+        write_disposition: Write disposition ("replace", "append", "merge").
+            If not provided, automatically determined based on incremental setting.
+
+    Returns:
+        A dlt source with a single "dataset" resource
+    """
+
+    @dlt.resource(
+        write_disposition=write_disposition or "replace",
+        primary_key=primary_key,  # type: ignore[call-overload]
+    )
+    def dataset(
+        incremental: Optional[dlt.sources.incremental] = incremental,  # type: ignore[type-arg]
+    ) -> Iterator[Dict[str, Any]]:
+        """
+        Yields records from a Socrata dataset.
+
+        Supports both full refresh (replace) and incremental loading (merge).
+        When incremental is provided, filters data using SoQL WHERE clause on the server side.
+
+        Yields:
+            Dict[str, Any]: Individual records from the dataset
+        """
+        fetch_kwargs: Dict[str, Any] = {
+            "domain": domain,
+            "dataset_id": dataset_id,
+            "app_token": app_token,
+            "username": username,
+            "password": password,
+        }
+
+        if incremental and incremental.cursor_path:
+            fetch_kwargs["incremental_key"] = incremental.cursor_path
+            fetch_kwargs["start_value"] = (
+                str(incremental.last_value)
+                if incremental.last_value is not None
+                else None
+            )
+            if getattr(incremental, "end_value", None) is not None:
+                ev = incremental.end_value  # type: ignore[attr-defined]
+                fetch_kwargs["end_value"] = (
+                    ev.isoformat()  # type: ignore[union-attr]
+                    if hasattr(ev, "isoformat")
+                    else str(ev)
+                )
+
+        # Fetch and yield records
+        yield from fetch_data(**fetch_kwargs)
+
+    return (dataset,)

ingestr/src/socrata_source/helpers.py

@@ -0,0 +1,85 @@
+"""Socrata API helpers"""
+
+from typing import Any, Dict, Iterator, Optional
+
+from dlt.sources.helpers import requests
+
+from .settings import DEFAULT_PAGE_SIZE, REQUEST_TIMEOUT
+
+
+def fetch_data(
+    domain: str,
+    dataset_id: str,
+    app_token: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    incremental_key: Optional[str] = None,
+    start_value: Optional[str] = None,
+    end_value: Optional[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch records from Socrata dataset with pagination and optional filtering.
+
+    Uses offset-based pagination to get all records, not just first 50000.
+    Supports incremental loading via SoQL WHERE clause for server-side filtering.
+
+    Args:
+        domain: Socrata domain (e.g., "data.seattle.gov")
+        dataset_id: Dataset identifier (e.g., "6udu-fhnu")
+        app_token: Socrata app token for higher rate limits
+        username: Username for authentication
+        password: Password for authentication
+        start_value: Minimum value for incremental_key (inclusive)
+        end_value: Maximum value for incremental_key (exclusive)
+
+    Yields:
+        Lists of records (one list per page)
+
+    Raises:
+        requests.HTTPError: If API request fails
+    """
+    url = f"https://{domain}/resource/{dataset_id}.json"
+
+    headers = {"Accept": "application/json"}
+    if app_token:
+        headers["X-App-Token"] = app_token
+
+    auth = (username, password) if username and password else None
+
+    limit = DEFAULT_PAGE_SIZE
+    offset = 0
+
+    while True:
+        params: Dict[str, Any] = {"$limit": limit, "$offset": offset}
+
+        if incremental_key and start_value:
+            start_value_iso = str(start_value).replace(" ", "T")
+            where_conditions = [f"{incremental_key} >= '{start_value_iso}'"]
+
+            if end_value:
+                end_value_iso = str(end_value).replace(" ", "T")
+                where_conditions.append(f"{incremental_key} < '{end_value_iso}'")
+
+            params["$where"] = " AND ".join(where_conditions)
+            params["$order"] = f"{incremental_key} ASC"
+
+        response = requests.get(
+            url,
+            headers=headers,
+            auth=auth,
+            params=params,
+            timeout=REQUEST_TIMEOUT,
+        )
+        response.raise_for_status()
+
+        data = response.json()
+
+        if not data:
+            break
+
+        yield data
+
+        if len(data) < limit:
+            break
+
+        offset += limit

ingestr/src/socrata_source/settings.py

@@ -0,0 +1,8 @@
+"""Socrata API settings and constants"""
+
+# Request timeout in seconds
+REQUEST_TIMEOUT = 30
+
+# Maximum number of records to fetch per page
+# Socrata API supports up to 50000 records per request
+DEFAULT_PAGE_SIZE = 50000