ingestr 0.13.13__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/blob.py

@@ -1,11 +1,15 @@
 import warnings
 from typing import Tuple, TypeAlias
-from urllib.parse import ParseResult
+from urllib.parse import ParseResult, urlparse
 
 BucketName: TypeAlias = str
 FileGlob: TypeAlias = str
 
 
+class UnsupportedEndpointError(Exception):
+    pass
+
+
 def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     """
     parse the URI of a blob storage and
@@ -14,13 +18,16 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     Supports the following Forms:
     - uri: "gs://"
       table: "bucket-name/file-glob"
+    - uri: "gs://uri-bucket-name" (uri-bucket-name is preferred)
+      table: "gs://table-bucket-name/file-glob"
+    - uri: "gs://"
+      table: "gs://bucket-name/file-glob"
     - uri: gs://bucket-name/file-glob
       table: None
     - uri: "gs://bucket-name"
       table: "file-glob"
 
-    The first form is the prefered method. Other forms are supported
-    for backward compatibility, but discouraged.
+    The first form is the prefered method. Other forms are supported but discouraged.
     """
 
     table = table.strip()
@@ -34,16 +41,36 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
         )
         return host, uri.path.lstrip("/")
 
+    table_uri = urlparse(table)
+
     if host != "":
-        warnings.warn(
-            f"Using the form '{uri.scheme}://bucket-name' is deprecated and will be removed in future versions.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return host, table.lstrip("/")
+        return host, table_uri.path.lstrip("/")
+
+    if table_uri.hostname:
+        return table_uri.hostname, table_uri.path.lstrip("/")
 
-    parts = table.lstrip("/").split("/", maxsplit=1)
+    parts = table_uri.path.lstrip("/").split("/", maxsplit=1)
     if len(parts) != 2:
         return "", parts[0]
 
     return parts[0], parts[1]
+
+
+def parse_endpoint(path: str) -> str:
+    """
+    Parse the endpoint kind from the URI.
+
+    kind is a file format. one of [csv, jsonl, parquet]
+    """
+    file_extension = path.split(".")[-1]
+    if file_extension == "gz":
+        file_extension = path.split(".")[-2]
+    if file_extension == "csv":
+        endpoint = "read_csv"
+    elif file_extension == "jsonl":
+        endpoint = "read_jsonl"
+    elif file_extension == "parquet":
+        endpoint = "read_parquet"
+    else:
+        raise UnsupportedEndpointError(f"Unsupported file format: {file_extension}")
+    return endpoint

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.13"
+version = "v0.14.104"

ingestr/src/chess/__init__.py

@@ -75,7 +75,7 @@ def players_archives(players: List[str]) -> Iterator[List[TDataItem]]:
 
 
 @dlt.resource(
-    write_disposition="append", columns={"end_time": {"data_type": "timestamp"}}
+    write_disposition="replace", columns={"end_time": {"data_type": "timestamp"}}
 )
 def players_games(
     players: List[str], start_month: str = None, end_month: str = None

ingestr/src/clickup/__init__.py

@@ -0,0 +1,85 @@
+"""Simple ClickUp source."""
+
+from datetime import datetime
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource
+
+from .helpers import ClickupClient
+
+
+@dlt.source(max_table_nesting=0)
+def clickup_source(
+    api_token: str = dlt.secrets.value,
+    start_date: datetime = None,
+    end_date: datetime = None,
+) -> Iterable[DltResource]:
+    client = ClickupClient(api_token)
+
+    @dlt.resource(
+        name="user",
+        primary_key="id",
+        write_disposition="merge",
+    )
+    def user() -> Iterable[dict]:
+        data = client.get("/user")
+        yield data["user"]
+
+    @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
+    def teams() -> Iterable[dict]:
+        for team in client.get_teams():
+            yield team
+
+    @dlt.resource(name="spaces", primary_key="id", write_disposition="merge")
+    def spaces() -> Iterable[dict]:
+        for space in client.get_spaces():
+            yield space
+
+    @dlt.resource(name="lists", write_disposition="merge", primary_key="id")
+    def lists() -> Iterable[dict]:
+        for list in client.get_lists():
+            yield list
+
+    @dlt.resource(
+        name="tasks",
+        write_disposition="merge",
+        primary_key="id",
+        columns={"date_updated": {"data_type": "timestamp"}},
+    )
+    def tasks(
+        date_updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "date_updated",
+            initial_value=ensure_pendulum_datetime(start_date).in_timezone("UTC"),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[dict]:
+        if date_updated.last_value:
+            start = ensure_pendulum_datetime(date_updated.last_value).in_timezone("UTC")
+        else:
+            start = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+
+        if date_updated.end_value is None:
+            end = pendulum.now("UTC")
+        else:
+            end = date_updated.end_value.in_timezone("UTC")
+
+        for list_obj in client.get_lists():
+            for task in client.paginated(
+                f"/list/{list_obj['id']}/task", "tasks", {"page_size": 100}
+            ):
+                task_dt = ensure_pendulum_datetime(int(task["date_updated"]) / 1000)
+                if task_dt >= start and task_dt <= end:
+                    task["date_updated"] = task_dt
+                    yield task
+
+    return (
+        user,
+        teams,
+        spaces,
+        lists,
+        tasks,
+    )

ingestr/src/clickup/helpers.py

@@ -0,0 +1,47 @@
+from typing import Iterable, Optional
+
+from ..http_client import create_client
+
+
+class ClickupClient:
+    def __init__(self, api_token: str):
+        self.session = create_client()
+        self.base_url = "https://api.clickup.com/api/v2"
+        self.headers = {"Authorization": api_token}
+
+    def get(self, endpoint: str, params: Optional[dict] = None) -> dict:
+        url = f"{self.base_url}{endpoint}"
+        resp = self.session.get(url, headers=self.headers, params=params or {})
+        resp.raise_for_status()
+        return resp.json()
+
+    def paginated(
+        self, endpoint: str, key: str, params: Optional[dict] = None
+    ) -> Iterable[dict]:
+        page = 0
+        params = params or {}
+        while True:
+            params["page"] = page
+            data = self.get(endpoint, params)
+            items = data.get(key, data)
+            if not items:
+                break
+            for item in items:
+                yield item
+            if data.get("last_page") or len(items) < params.get("page_size", 100):
+                break
+            page += 1
+
+    def get_teams(self):
+        data = self.get("/team")
+        return data.get("teams", [])
+
+    def get_spaces(self):
+        for team in self.get_teams():
+            for space in self.paginated(f"/team/{team['id']}/space", "spaces"):
+                yield space
+
+    def get_lists(self):
+        for space in self.get_spaces():
+            for lst in self.paginated(f"/space/{space['id']}/list", "lists"):
+                yield lst

ingestr/src/collector/spinner.py

@@ -0,0 +1,43 @@
+from typing import Optional
+
+from dlt.common.runtime.collector import Collector
+from rich.status import Status
+
+
+class SpinnerCollector(Collector):
+    status: Status
+    current_step: str
+    started: bool
+
+    def __init__(self) -> None:
+        self.status = Status("Ingesting data...", spinner="dots")
+        self.started = False
+
+    def update(
+        self,
+        name: str,
+        inc: int = 1,
+        total: Optional[int] = None,
+        message: Optional[str] = None,  # type: ignore
+        label: str = "",
+        **kwargs,
+    ) -> None:
+        self.status.update(self.current_step)
+
+    def _start(self, step: str) -> None:
+        self.current_step = self.__step_to_label(step)
+        self.status.start()
+
+    def __step_to_label(self, step: str) -> str:
+        verb = step.split(" ")[0].lower()
+        if verb.startswith("normalize"):
+            return "Normalizing the data"
+        elif verb.startswith("load"):
+            return "Loading the data to the destination"
+        elif verb.startswith("extract"):
+            return "Extracting the data from the source"
+
+        return f"{verb.capitalize()} the data"
+
+    def _stop(self) -> None:
+        self.status.stop()

ingestr/src/couchbase_source/__init__.py

@@ -0,0 +1,118 @@
+"""Source that loads data from Couchbase buckets, supports incremental loads."""
+
+from typing import Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import (
+    CouchbaseConfiguration,
+    client_from_credentials,
+    fetch_documents,
+)
+
+
+@dlt.source(max_table_nesting=0)
+def couchbase_source(
+    connection_string: str = dlt.secrets.value,
+    username: str = dlt.secrets.value,
+    password: str = dlt.secrets.value,
+    bucket: str = dlt.config.value,
+    scope: Optional[str] = dlt.config.value,
+    collection: Optional[str] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    limit: Optional[int] = None,
+) -> DltResource:
+    """
+    A DLT source which loads data from a Couchbase bucket using Couchbase Python SDK.
+
+    Args:
+        connection_string (str): Couchbase connection string (e.g., 'couchbase://localhost')
+        username (str): Couchbase username
+        password (str): Couchbase password
+        bucket (str): Bucket name to load data from
+        scope (Optional[str]): Scope name (defaults to '_default')
+        collection (Optional[str]): Collection name (defaults to '_default')
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading.
+            E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
+        write_disposition (str): Write disposition of the resource.
+        limit (Optional[int]): The maximum number of documents to load.
+
+    Returns:
+        DltResource: A DLT resource for the Couchbase collection.
+    """
+    # Set up Couchbase client
+    cluster = client_from_credentials(connection_string, username, password)
+
+    resource_name = f"{bucket}_{scope}_{collection}"
+
+    return dlt.resource(  # type: ignore[call-overload, arg-type]
+        fetch_documents,
+        name=resource_name,
+        primary_key="id",
+        write_disposition=write_disposition or "replace",
+        spec=CouchbaseConfiguration,
+        max_table_nesting=0,
+    )(
+        cluster=cluster,
+        bucket_name=bucket,
+        scope_name=scope,
+        collection_name=collection,
+        incremental=incremental,
+        limit=limit,
+    )
+
+
+@dlt.resource(
+    name=lambda args: f"{args['bucket']}_{args['scope']}_{args['collection']}",
+    standalone=True,
+    spec=CouchbaseConfiguration,  # type: ignore[arg-type]
+)
+def couchbase_collection(
+    connection_string: str = dlt.secrets.value,
+    username: str = dlt.secrets.value,
+    password: str = dlt.secrets.value,
+    bucket: str = dlt.config.value,
+    scope: Optional[str] = dlt.config.value,
+    collection: Optional[str] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 1000,
+) -> DltResource:
+    """
+    A DLT resource which loads a collection from Couchbase.
+
+    Args:
+        connection_string (str): Couchbase connection string (e.g., 'couchbase://localhost')
+        username (str): Couchbase username
+        password (str): Couchbase password
+        bucket (str): Bucket name to load data from
+        scope (Optional[str]): Scope name (defaults to '_default')
+        collection (Optional[str]): Collection name (defaults to '_default')
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading.
+        write_disposition (str): Write disposition of the resource.
+        limit (Optional[int]): The maximum number of documents to load.
+        chunk_size (Optional[int]): The number of documents to load in each batch.
+
+    Returns:
+        DltResource: A DLT resource for the Couchbase collection.
+    """
+    # Set up Couchbase client
+    cluster = client_from_credentials(connection_string, username, password)
+
+    return dlt.resource(  # type: ignore[call-overload]
+        fetch_documents,
+        name=f"{bucket}_{scope}_{collection}",
+        primary_key="id",
+        write_disposition=write_disposition or "replace",
+    )(
+        cluster=cluster,
+        bucket_name=bucket,
+        scope_name=scope,
+        collection_name=collection,
+        incremental=incremental,
+        limit=limit,
+        chunk_size=chunk_size,
+    )

ingestr/src/couchbase_source/helpers.py

@@ -0,0 +1,135 @@
+"""Helper functions for Couchbase source."""
+
+from datetime import datetime, timedelta
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+from couchbase.auth import PasswordAuthenticator  # type: ignore[import-untyped]
+from couchbase.cluster import Cluster  # type: ignore[import-untyped]
+from couchbase.options import (  # type: ignore[import-untyped]
+    ClusterOptions,
+    QueryOptions,
+)
+from dlt.common.configuration import configspec
+from dlt.common.time import ensure_pendulum_datetime
+
+
+@configspec
+class CouchbaseConfiguration:
+    """Configuration for Couchbase source."""
+
+    connection_string: str = dlt.secrets.value
+    username: str = dlt.secrets.value
+    password: str = dlt.secrets.value
+    bucket: str = dlt.config.value
+    scope: Optional[str] = dlt.config.value
+    collection: Optional[str] = dlt.config.value
+
+
+def client_from_credentials(
+    connection_string: str, username: str, password: str
+) -> Cluster:
+    """
+    Create a Couchbase cluster client from credentials.
+
+    Args:
+        connection_string: Couchbase connection string
+            - Local/self-hosted: 'couchbase://localhost'
+            - Capella (cloud): 'couchbases://your-instance.cloud.couchbase.com'
+        username: Couchbase username
+        password: Couchbase password
+
+    Returns:
+        Cluster: Connected Couchbase cluster instance
+    """
+    auth = PasswordAuthenticator(username, password)
+    options = ClusterOptions(auth)
+
+    # Apply wan_development profile for Capella (couchbases://) connections
+    # This helps avoid latency issues when accessing from different networks
+    if connection_string.startswith("couchbases://"):
+        options.apply_profile("wan_development")
+
+    cluster = Cluster(connection_string, options)
+    cluster.wait_until_ready(timedelta(seconds=30))
+
+    return cluster
+
+
+def fetch_documents(
+    cluster: Cluster,
+    bucket_name: str,
+    scope_name: str,
+    collection_name: str,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 1000,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch documents from a Couchbase collection using N1QL queries.
+
+    Args:
+        cluster: Couchbase cluster instance
+        bucket_name: Name of the bucket
+        scope_name: Name of the scope
+        collection_name: Name of the collection
+        incremental: Incremental loading configuration
+        limit: Maximum number of documents to fetch
+        chunk_size: Number of documents to fetch per batch
+
+    Yields:
+        Dict[str, Any]: Document data
+    """
+    # Build N1QL query with full path
+    full_collection_path = f"`{bucket_name}`.`{scope_name}`.`{collection_name}`"
+    n1ql_query = f"SELECT META().id as id, c.* FROM {full_collection_path} c"
+
+    # Add incremental filter if provided
+    if incremental and incremental.cursor_path:
+        where_clause = f" WHERE {incremental.cursor_path} >= $start_value"
+        if incremental.end_value is not None:
+            where_clause += f" AND {incremental.cursor_path} < $end_value"
+        n1ql_query += where_clause
+
+    # Add limit if provided
+    if limit:
+        n1ql_query += f" LIMIT {limit}"
+
+    # Execute query
+    try:
+        query_options = QueryOptions()
+
+        # Add parameters if incremental
+        if incremental and incremental.cursor_path:
+            named_parameters = {"start_value": incremental.last_value}
+            if incremental.end_value is not None:
+                named_parameters["end_value"] = incremental.end_value
+            query_options = QueryOptions(named_parameters=named_parameters)
+
+        result = cluster.query(n1ql_query, query_options)
+
+        # Yield documents
+        count = 0
+        for row in result:
+            doc = dict(row)
+
+            # Convert datetime fields to proper format
+            if (
+                incremental
+                and incremental.cursor_path
+                and incremental.cursor_path in doc
+            ):
+                cursor_value = doc[incremental.cursor_path]
+                if isinstance(cursor_value, (str, datetime)):
+                    doc[incremental.cursor_path] = ensure_pendulum_datetime(
+                        cursor_value
+                    )
+
+            yield doc
+
+            count += 1
+            if limit and count >= limit:
+                break
+
+    except Exception as e:
+        raise Exception(f"Error executing Couchbase N1QL query: {str(e)}")

ingestr/src/cursor/__init__.py

@@ -0,0 +1,83 @@
+"""
+This source provides data extraction from Cursor via the REST API.
+
+It fetches team member information from the Cursor API.
+"""
+
+from typing import Any, Iterable, Optional
+
+import dlt
+from dlt.common.typing import TDataItem
+
+from .helpers import CursorClient
+
+
+@dlt.source
+def cursor_source() -> Any:
+    """
+    The main function that fetches data from Cursor API.
+
+    Returns:
+        Sequence[DltResource]: A sequence of DltResource objects containing the fetched data.
+    """
+    return [
+        team_members,
+        daily_usage_data,
+        team_spend,
+        filtered_usage_events,
+    ]
+
+
+@dlt.resource(
+    write_disposition="replace",
+    max_table_nesting=0,
+)
+def team_members(
+    api_key: str = dlt.secrets.value,
+) -> Iterable[TDataItem]:
+    client = CursorClient(api_key=api_key)
+
+    members = client.get_team_members()
+    yield from members
+
+
+@dlt.resource(
+    write_disposition="replace",
+    max_table_nesting=0,
+)
+def daily_usage_data(
+    api_key: str = dlt.secrets.value,
+    start_date: Optional[int] = dlt.config.value,
+    end_date: Optional[int] = dlt.config.value,
+) -> Iterable[TDataItem]:
+    client = CursorClient(api_key=api_key)
+
+    yield from client.get_daily_usage_data(start_date=start_date, end_date=end_date)
+
+
+@dlt.resource(
+    write_disposition="replace",
+    max_table_nesting=0,
+)
+def team_spend(
+    api_key: str = dlt.secrets.value,
+) -> Iterable[TDataItem]:
+    client = CursorClient(api_key=api_key)
+
+    yield from client.get_team_spend()
+
+
+@dlt.resource(
+    write_disposition="replace",
+    max_table_nesting=0,
+)
+def filtered_usage_events(
+    api_key: str = dlt.secrets.value,
+    start_date: Optional[int] = dlt.config.value,
+    end_date: Optional[int] = dlt.config.value,
+) -> Iterable[TDataItem]:
+    client = CursorClient(api_key=api_key)
+
+    yield from client.get_filtered_usage_events(
+        start_date=start_date, end_date=end_date
+    )