ingestr 0.14.92__py3-none-any.whl → 0.14.93__py3-none-any.whl

ingestr/src/allium/__init__.py

@@ -0,0 +1,128 @@
+"""
+Allium source for data extraction via REST API.
+
+This source provides access to Allium blockchain data via asynchronous query execution.
+"""
+
+import time
+from typing import Any, Iterator
+
+import dlt
+
+from ingestr.src.http_client import create_client
+
+
+@dlt.source(max_table_nesting=0, name="allium_source")
+def allium_source(
+    api_key: str,
+    query_id: str,
+    parameters: dict[str, Any] | None = None,
+    limit: int | None = None,
+    compute_profile: str | None = None,
+) -> Any:
+    """
+    Allium data source for blockchain data extraction.
+
+    This source connects to Allium API, runs async queries, and fetches results.
+
+    Args:
+        api_key: Allium API key for authentication
+        query_id: The query ID to execute (e.g., 'abc123')
+        parameters: Optional parameters for the query (e.g., {'start_date': '2025-02-01', 'end_date': '2025-02-02'})
+        limit: Limit the number of rows in the result (max 250,000)
+        compute_profile: Compute profile identifier
+
+    Yields:
+        DltResource: Data resources for Allium query results
+    """
+    base_url = "https://api.allium.so/api/v1/explorer"
+    session = create_client()
+    headers = {"X-API-Key": api_key}
+
+    @dlt.resource(
+        name="query_results",
+        write_disposition="replace",
+    )
+    def fetch_query_results() -> Iterator[dict[str, Any]]:
+        """
+        Fetch query results from Allium.
+
+        This function:
+        1. Starts an async query execution
+        2. Polls for completion status
+        3. Fetches and yields the results
+        """
+        # Step 1: Start async query execution
+        run_config: dict[str, Any] = {}
+        if limit is not None:
+            run_config["limit"] = limit
+        if compute_profile is not None:
+            run_config["compute_profile"] = compute_profile
+
+        run_payload = {"parameters": parameters or {}, "run_config": run_config}
+
+        run_response = session.post(
+            f"{base_url}/queries/{query_id}/run-async",
+            json=run_payload,
+            headers=headers,
+        )
+
+        run_data = run_response.json()
+
+        if "run_id" not in run_data:
+            raise ValueError(f"Failed to start query execution: {run_data}")
+
+        run_id = run_data["run_id"]
+
+        # Step 2: Poll for completion
+        max_retries = 8640  # Max 12 hours with 5-second intervals
+        retry_count = 0
+        poll_interval = 5  # seconds
+
+        while retry_count < max_retries:
+            status_response = session.get(
+                f"{base_url}/query-runs/{run_id}/status",
+                headers=headers,
+            )
+            status_response.raise_for_status()
+            status_data = status_response.json()
+
+            # Handle both string and dict responses
+            if isinstance(status_data, str):
+                status = status_data
+            else:
+                status = status_data.get("status")
+
+            if status == "success":
+                break
+            elif status == "failed":
+                error_msg = (
+                    status_data.get("error", "Unknown error")
+                    if isinstance(status_data, dict)
+                    else "Unknown error"
+                )
+                raise ValueError(f"Query execution failed: {error_msg}")
+            elif status in ["pending", "running", "queued"]:
+                time.sleep(poll_interval)
+                retry_count += 1
+            else:
+                raise ValueError(f"Unknown status: {status}")
+
+        if retry_count >= max_retries:
+            raise TimeoutError(
+                f"Query execution timed out after {max_retries * poll_interval} seconds"
+            )
+
+        # Step 3: Fetch results
+        results_response = session.get(
+            f"{base_url}/query-runs/{run_id}/results",
+            headers=headers,
+            params={"f": "json"},
+        )
+        results_response.raise_for_status()
+        query_output = results_response.json()
+
+        # Extract and yield all data
+        yield query_output.get("data", [])
+
+    return (fetch_query_results,)

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.14.92"
+version = "v0.14.93"

ingestr/src/factory.py

@@ -28,6 +28,7 @@ from ingestr.src.destinations import (
 from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
+    AlliumSource,
     AnthropicSource,
     AppleAppStoreSource,
     ApplovinMaxSource,
@@ -156,6 +157,7 @@ class SourceDestinationFactory:
     source_scheme: str
     destination_scheme: str
     sources: Dict[str, Type[SourceProtocol]] = {
+        "allium": AlliumSource,
         "anthropic": AnthropicSource,
         "csv": LocalCsvSource,
         "docebo": DoceboSource,

ingestr/src/sources.py

@@ -3997,3 +3997,72 @@ class MailchimpSource:
             ).with_resources(table)
         except ResourcesNotFoundError:
             raise UnsupportedResourceError(table, "Mailchimp")
+
+
+class AlliumSource:
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        query_params = parse_qs(parsed_uri.query)
+        api_key = query_params.get("api_key")
+
+        if api_key is None:
+            raise MissingValueError("api_key", "Allium")
+
+        # Extract query_id and custom parameters from table parameter
+        # Format: query_id or query:query_id or query:query_id:param1=value1&param2=value2
+        query_id = table
+        custom_params = {}
+        limit = None
+        compute_profile = None
+
+        if ":" in table:
+            parts = table.split(":", 2)  # Split into max 3 parts
+            if len(parts) >= 2:
+                query_id = parts[1]
+            if len(parts) == 3:
+                # Parse custom parameters from query string format
+                param_string = parts[2]
+                for param in param_string.split("&"):
+                    if "=" in param:
+                        key, value = param.split("=", 1)
+                        # Extract run_config parameters
+                        if key == "limit":
+                            limit = int(value)
+                        elif key == "compute_profile":
+                            compute_profile = value
+                        else:
+                            custom_params[key] = value
+
+        # Extract parameters from interval_start and interval_end
+        # Default: 2 days ago 00:00 to yesterday 00:00
+        now = pendulum.now()
+        default_start = now.subtract(days=2).start_of("day")
+        default_end = now.subtract(days=1).start_of("day")
+
+        parameters = {}
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        start_date = interval_start if interval_start is not None else default_start
+        end_date = interval_end if interval_end is not None else default_end
+
+        parameters["start_date"] = start_date.strftime("%Y-%m-%d")
+        parameters["end_date"] = end_date.strftime("%Y-%m-%d")
+        parameters["start_timestamp"] = str(int(start_date.timestamp()))
+        parameters["end_timestamp"] = str(int(end_date.timestamp()))
+
+        # Merge custom parameters (they override default parameters)
+        parameters.update(custom_params)
+
+        from ingestr.src.allium import allium_source
+
+        return allium_source(
+            api_key=api_key[0],
+            query_id=query_id,
+            parameters=parameters if parameters else None,
+            limit=limit,
+            compute_profile=compute_profile,
+        )

{ingestr-0.14.92.dist-info → ingestr-0.14.93.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.92
+Version: 0.14.93
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.92.dist-info → ingestr-0.14.93.dist-info}/RECORD

@@ -2,23 +2,24 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=HwY092G4VBM9UY-e_f_-cOrkOxcgUZGOsco8jwXJ7aw,21
+ingestr/src/buildinfo.py,sha256=gpczaxQtINGa_cWhMVJsfeFoUxh-gKIyba1YESpTmpk,21
 ingestr/src/destinations.py,sha256=QtjE0AGs0WkPHaI2snWPHJ8HHi4lwXUBYLJPklz8Mvk,27772
 ingestr/src/errors.py,sha256=fhJ2BxOqOsBfOxuTDKfZblvawBrPG3x_1VikIxMZBRI,874
-ingestr/src/factory.py,sha256=z6lcaGwK8o7AcaeshX7TP4WfB_jsIi0TvarcquGYnK0,7720
+ingestr/src/factory.py,sha256=k_8jgehOM2sHwCsjliYXmQhICl2B1UYoAs6vspjadv8,7770
 ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=DSJAiwfVV0czMpHX1d93gQ7xIk_RyGPubQ31KfT5N8I,139639
+ingestr/src/sources.py,sha256=D4VxA-yqilzTG0VBJBxnw9MUJ1Qeo2EpKjVGJfoMKoY,142289
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
 ingestr/src/adjust/__init__.py,sha256=-DkqpkCuwohw7BlwB9ZvtpbwQAY1Gr8J1T4KyFwsA8E,3315
 ingestr/src/adjust/adjust_helpers.py,sha256=IHSS94A7enOWkZ8cP5iW3RdYt0Xl3qZGAmDc1Xy4qkI,3802
 ingestr/src/airtable/__init__.py,sha256=XzRsS39xszUlh_s7P1_zq5v8vLfjz3m-NtTPaa8TTZU,2818
+ingestr/src/allium/__init__.py,sha256=pLNvKKy8OBVgUPK0zJQTASf6CCZIW17BfrVYXxyd5nc,4087
 ingestr/src/anthropic/__init__.py,sha256=D23oY20fE_RP9yPVkx7i6l3G1IfRLrJ2XwA8y2ot7JM,8482
 ingestr/src/anthropic/helpers.py,sha256=Co8kmWQwKMHxcUwDU9959LTU6rFxqDIIbIvVSMGatrc,16105
 ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
@@ -183,8 +184,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=zf3DXT29Y4TH2lNPBFphdjlaelUUyPJcsW2UO68RzDs,4862
-ingestr-0.14.92.dist-info/METADATA,sha256=jgxtI06WwAK2kR3fSLE7yU3iw_n686Iol31Axlf4jnU,15327
-ingestr-0.14.92.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.14.92.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.14.92.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.14.92.dist-info/RECORD,,
+ingestr-0.14.93.dist-info/METADATA,sha256=ttKTQKjoXX_xzXbQb2LisUnePWrFx5GXQf2dHCsG48g,15327
+ingestr-0.14.93.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.14.93.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.14.93.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.14.93.dist-info/RECORD,,