ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/monday/settings.py

@@ -0,0 +1,328 @@
+"""Monday.com source settings and constants"""
+
+# GraphQL query for fetching app installs
+APP_INSTALLS_QUERY = """
+query ($app_id: ID!, $account_id: ID, $limit: Int!, $page: Int!) {
+    app_installs(
+        app_id: $app_id
+        account_id: $account_id
+        limit: $limit
+        page: $page
+    ) {
+        app_id
+        timestamp
+        app_install_account {
+            id
+        }
+        app_install_user {
+            id
+        }
+        app_version {
+            major
+            minor
+            patch
+            type
+            text
+        }
+        permissions {
+            approved_scopes
+            required_scopes
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching account information
+ACCOUNT_QUERY = """
+query {
+    account {
+        id
+        name
+        slug
+        tier
+        country_code
+        first_day_of_the_week
+        show_timeline_weekends
+        sign_up_product_kind
+        active_members_count
+        logo
+        plan {
+            max_users
+            period
+            tier
+            version
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching account roles
+ACCOUNT_ROLES_QUERY = """
+query {
+    account_roles {
+        id
+        name
+        roleType
+    }
+}
+"""
+
+# GraphQL query for fetching users
+USERS_QUERY = """
+query ($limit: Int!, $page: Int!) {
+    users(limit: $limit, page: $page) {
+        id
+        name
+        email
+        enabled
+        is_admin
+        is_guest
+        is_pending
+        is_view_only
+        created_at
+        birthday
+        country_code
+        join_date
+        location
+        mobile_phone
+        phone
+        photo_original
+        photo_thumb
+        photo_tiny
+        time_zone_identifier
+        title
+        url
+        utc_hours_diff
+        current_language
+        account {
+            id
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching boards
+BOARDS_QUERY = """
+query ($limit: Int!, $page: Int!) {
+    boards(limit: $limit, page: $page) {
+        id
+        name
+        description
+        state
+        board_kind
+        board_folder_id
+        workspace_id
+        permissions
+        item_terminology
+        items_count
+        updated_at
+        url
+        communication
+        object_type_unique_key
+        type
+        creator {
+            id
+        }
+        owners {
+            id
+        }
+        subscribers {
+            id
+        }
+        team_owners {
+            id
+        }
+        team_subscribers {
+            id
+        }
+        tags {
+            id
+            
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching custom activities
+CUSTOM_ACTIVITIES_QUERY = """
+query {
+    custom_activity {
+        id
+        name
+        type
+        color
+        icon_id
+    }
+}
+"""
+
+# GraphQL query for fetching board columns
+BOARD_COLUMNS_QUERY = """
+query ($board_ids: [ID!]) {
+    boards(ids: $board_ids) {
+        id
+        columns {
+            id
+            title
+            type
+            archived
+            description
+            settings_str
+            width
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching board views
+BOARD_VIEWS_QUERY = """
+query ($board_ids: [ID!]) {
+    boards(ids: $board_ids) {
+        id
+        views {
+            id
+            name
+            type
+            settings_str
+            view_specific_data_str
+            source_view_id
+            access_level
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching workspaces by IDs
+WORKSPACES_QUERY = """
+query ($ids: [ID!]) {
+    workspaces(ids: $ids) {
+        id
+        name
+        kind
+        description
+        created_at
+        is_default_workspace
+        state
+        account_product {
+            id
+        }
+        owners_subscribers {
+            id
+        }
+        team_owners_subscribers {
+            id
+        }
+        teams_subscribers {
+            id
+        }
+        users_subscribers {
+            id
+        }
+        settings {
+            icon
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching webhooks by board ID
+WEBHOOKS_QUERY = """
+query ($board_id: ID!) {
+    webhooks(board_id: $board_id) {
+        id
+        event
+        board_id
+        config
+    }
+}
+"""
+
+# GraphQL query for fetching updates
+UPDATES_QUERY = """
+query ($limit: Int!, $from_date: String, $to_date: String) {
+    updates(limit: $limit, from_date: $from_date, to_date: $to_date) {
+        id
+        body
+        text_body
+        created_at
+        updated_at
+        edited_at
+        creator_id
+        item_id
+        creator {
+            id
+        }
+        item {
+            id
+        }
+        assets {
+            id
+            name
+            file_extension
+            file_size
+            public_url
+            url
+            url_thumbnail
+            created_at
+            original_geometry
+            uploaded_by {
+                id
+            }
+        }
+        replies {
+            id
+            body
+            text_body
+            created_at
+            updated_at
+            creator_id
+            creator {
+                id
+            }
+        }
+        likes {
+            id
+        }
+        pinned_to_top {
+            item_id
+        }
+        viewers {
+            medium
+            user_id
+            user {
+                id
+            }
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching teams
+TEAMS_QUERY = """
+query {
+    teams {
+        id
+        name
+        picture_url
+        users {
+            id
+            created_at
+            phone
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching tags
+TAGS_QUERY = """
+query {
+    tags {
+        id
+        name
+        color
+    }
+}
+"""
+
+# Maximum number of results per page
+MAX_PAGE_SIZE = 100

ingestr/src/mongodb/__init__.py

@@ -1,8 +1,9 @@
 """Source that loads collections form any a mongo database, supports incremental loads."""
 
-from typing import Any, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Union
 
 import dlt
+from dlt.common.data_writers import TDataItemFormat
 from dlt.sources import DltResource
 
 from .helpers import (
@@ -13,7 +14,7 @@ from .helpers import (
 )
 
 
-@dlt.source
+@dlt.source(max_table_nesting=0)
 def mongodb(
     connection_url: str = dlt.secrets.value,
     database: Optional[str] = dlt.config.value,
@@ -21,6 +22,10 @@ def mongodb(
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = dlt.config.value,
+    limit: Optional[int] = None,
+    filter_: Optional[Dict[str, Any]] = None,
+    projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = None,
+    pymongoarrow_schema: Optional[Any] = None,
 ) -> Iterable[DltResource]:
     """
     A DLT source which loads data from a mongo database using PyMongo.
@@ -34,6 +39,18 @@ def mongodb(
             E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
         write_disposition (str): Write disposition of the resource.
         parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+        limit (Optional[int]):
+            The maximum number of documents to load. The limit is
+            applied to each requested collection separately.
+        filter_ (Optional[Dict[str, Any]]): The filter to apply to the collection.
+        projection: (Optional[Union[Mapping[str, Any], Iterable[str]]]): The projection to select fields of a collection
+            when loading the collection. Supported inputs:
+                include (list) - ["year", "title"]
+                include (dict) - {"year": True, "title": True}
+                exclude (dict) - {"released": False, "runtime": False}
+            Note: Can't mix include and exclude statements '{"title": True, "released": False}`
+        pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types of a collection to convert BSON to Arrow
+
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
     """
@@ -58,20 +75,39 @@ def mongodb(
             primary_key="_id",
             write_disposition=write_disposition,
             spec=MongoDbCollectionConfiguration,
-        )(client, collection, incremental=incremental, parallel=parallel)
+            max_table_nesting=0,
+        )(
+            client,
+            collection,
+            incremental=incremental,
+            parallel=parallel,
+            limit=limit,
+            filter_=filter_ or {},
+            projection=projection,
+            pymongoarrow_schema=pymongoarrow_schema,
+        )
 
 
-@dlt.common.configuration.with_config(
-    sections=("sources", "mongodb"), spec=MongoDbCollectionResourceConfiguration
+@dlt.resource(
+    name=lambda args: args["collection"],
+    standalone=True,
+    spec=MongoDbCollectionResourceConfiguration,
 )
 def mongodb_collection(
-    connection_url: str = dlt.config.value,
+    connection_url: str = dlt.secrets.value,
     database: Optional[str] = dlt.config.value,
     collection: str = dlt.config.value,
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = False,
-) -> Any:
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 1000,
+    data_item_format: Optional[TDataItemFormat] = "object",
+    filter_: Optional[Dict[str, Any]] = None,
+    projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value,
+    pymongoarrow_schema: Optional[Any] = None,
+    custom_query: Optional[List[Dict[str, Any]]] = None,
+) -> DltResource:
     """
     A DLT source which loads a collection from a mongo database using PyMongo.
 
@@ -83,6 +119,22 @@ def mongodb_collection(
             E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
         write_disposition (str): Write disposition of the resource.
         parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+        limit (Optional[int]): The number of documents load.
+        chunk_size (Optional[int]): The number of documents load in each batch.
+        data_item_format (Optional[TDataItemFormat]): The data format to use for loading.
+            Supported formats:
+                object - Python objects (dicts, lists).
+                arrow - Apache Arrow tables.
+        filter_ (Optional[Dict[str, Any]]): The filter to apply to the collection.
+        projection: (Optional[Union[Mapping[str, Any], Iterable[str]]]): The projection to select fields
+            when loading the collection. Supported inputs:
+                include (list) - ["year", "title"]
+                include (dict) - {"year": True, "title": True}
+                exclude (dict) - {"released": False, "runtime": False}
+            Note: Can't mix include and exclude statements '{"title": True, "released": False}`
+        pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types to convert BSON to Arrow
+        custom_query (Optional[List[Dict[str, Any]]]): Custom MongoDB aggregation pipeline to execute instead of find()
+
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
     """
@@ -100,4 +152,16 @@ def mongodb_collection(
         name=collection_obj.name,
         primary_key="_id",
         write_disposition=write_disposition,
-    )(client, collection_obj, incremental=incremental, parallel=parallel)
+    )(
+        client,
+        collection_obj,
+        incremental=incremental,
+        parallel=parallel,
+        limit=limit,
+        chunk_size=chunk_size,
+        data_item_format=data_item_format,
+        filter_=filter_ or {},
+        projection=projection,
+        pymongoarrow_schema=pymongoarrow_schema,
+        custom_query=custom_query,
+    )