omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/shopify/exceptions.py

@@ -0,0 +1,17 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ShopifyPartnerApiError(Exception):
+    pass

omniload/src/shopify/helpers.py

@@ -0,0 +1,202 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shopify source helpers"""
+
+from typing import Any, Iterable, Literal, Optional
+from urllib.parse import urljoin
+
+from dlt.common import jsonpath
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import Dict, DictStrAny, TDataItems
+from dlt.sources.helpers import requests
+
+from .exceptions import ShopifyPartnerApiError
+from .settings import DEFAULT_API_VERSION, DEFAULT_PARTNER_API_VERSION
+
+TOrderStatus = Literal["open", "closed", "cancelled", "any"]
+
+
+def convert_datetime_fields(item: Dict[str, Any]) -> Dict[str, Any]:
+    """Convert timestamp fields in the item to pendulum datetime objects
+
+    The item is modified in place, including nested items.
+
+    Args:
+        item: The item to convert
+
+    Returns:
+        The same data item (for convenience)
+    """
+    fields = ["created_at", "updated_at", "createdAt", "updatedAt"]
+
+    def convert_nested(obj: Any) -> Any:
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                if key in fields and isinstance(value, str):
+                    obj[key] = ensure_pendulum_datetime(value)
+                else:
+                    obj[key] = convert_nested(value)
+        elif isinstance(obj, list):
+            return [convert_nested(elem) for elem in obj]
+        return obj
+
+    return convert_nested(item)
+
+
+def remove_nodes_key(item: Any) -> Any:
+    """
+    Recursively remove the 'nodes' key from dictionaries if it's the only key and its value is an array.
+
+    Args:
+        item: The item to process (can be a dict, list, or any other type)
+
+    Returns:
+        The processed item
+    """
+    if isinstance(item, dict):
+        if len(item) == 1 and "nodes" in item and isinstance(item["nodes"], list):
+            return [remove_nodes_key(node) for node in item["nodes"]]
+        return {k: remove_nodes_key(v) for k, v in item.items()}
+    elif isinstance(item, list):
+        return [remove_nodes_key(element) for element in item]
+    else:
+        return item
+
+
+class ShopifyApi:
+    """
+    A Shopify API client that can be used to get pages of data from Shopify.
+    """
+
+    def __init__(
+        self,
+        shop_url: str,
+        private_app_password: str,
+        api_version: str = DEFAULT_API_VERSION,
+    ) -> None:
+        """
+        Args:
+            shop_url: The URL of your shop (e.g. https://my-shop.myshopify.com).
+            private_app_password: The private app password to the app on your shop.
+            api_version: The API version to use (e.g. 2023-01)
+        """
+        self.shop_url = shop_url
+        self.private_app_password = private_app_password
+        self.api_version = api_version
+
+    def get_pages(
+        self, resource: str, params: Optional[Dict[str, Any]] = None
+    ) -> Iterable[TDataItems]:
+        """Get all pages from shopify using requests.
+        Iterates through all pages and yield each page items.
+
+        Args:
+            resource: The resource to get pages for (e.g. products, orders, customers).
+            params: Query params to include in the request.
+
+        Yields:
+            List of data items from the page
+        """
+        url = urljoin(self.shop_url, f"/admin/api/{self.api_version}/{resource}.json")
+
+        resource_last = resource.split("/")[-1]
+
+        headers = {"X-Shopify-Access-Token": self.private_app_password}
+        while url:
+            response = requests.get(url, params=params, headers=headers)
+            response.raise_for_status()
+            json = response.json()
+            yield [convert_datetime_fields(item) for item in json[resource_last]]
+            url = response.links.get("next", {}).get("url")
+            # Query params are included in subsequent page URLs
+            params = None
+
+
+class ShopifyGraphQLApi:
+    """Client for Shopify GraphQL API"""
+
+    def __init__(
+        self,
+        access_token: str,
+        api_version: str = DEFAULT_PARTNER_API_VERSION,
+        base_url: str = "partners.shopify.com",
+    ) -> None:
+        self.access_token = access_token
+        self.api_version = api_version
+        self.base_url = base_url
+
+    @property
+    def graphql_url(self) -> str:
+        if self.base_url.startswith("https://"):
+            return f"{self.base_url}/admin/api/{self.api_version}/graphql.json"
+
+        return f"https://{self.base_url}/admin/api/{self.api_version}/graphql.json"
+
+    def run_graphql_query(
+        self, query: str, variables: Optional[DictStrAny] = None
+    ) -> DictStrAny:
+        """Run a graphql query against the Shopify Partner API
+
+        Args:
+            query: The query to run
+            variables: The variables to include in the query
+
+        Returns:
+            The response JSON
+        """
+        headers = {"X-Shopify-Access-Token": self.access_token}
+        response = requests.post(
+            self.graphql_url,
+            json={"query": query, "variables": variables},
+            headers=headers,
+        )
+        data = response.json()
+        if data.get("errors"):
+            raise ShopifyPartnerApiError(response.text)
+        return data  # type: ignore[no-any-return]
+
+    def get_graphql_pages(
+        self,
+        query: str,
+        data_items_path: jsonpath.TJsonPath,
+        pagination_cursor_path: jsonpath.TJsonPath,
+        pagination_variable_name: str,
+        pagination_cursor_has_next_page_path: Optional[jsonpath.TJsonPath] = None,
+        variables: Optional[DictStrAny] = None,
+    ) -> Iterable[TDataItems]:
+        variables = dict(variables or {})
+        while True:
+            data = self.run_graphql_query(query, variables)
+            data_items = jsonpath.find_values(data_items_path, data)
+
+            if not data_items:
+                break
+
+            yield [
+                remove_nodes_key(convert_datetime_fields(item)) for item in data_items
+            ]
+
+            cursors = jsonpath.find_values(pagination_cursor_path, data)
+            if not cursors:
+                break
+
+            if pagination_cursor_has_next_page_path:
+                has_next_page = jsonpath.find_values(
+                    pagination_cursor_has_next_page_path, data
+                )
+                if not has_next_page or not has_next_page[0]:
+                    break
+
+            variables[pagination_variable_name] = cursors[-1]

omniload/src/shopify/settings.py

@@ -0,0 +1,19 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FIRST_DAY_OF_MILLENNIUM = "2000-01-01"
+DEFAULT_API_VERSION = "2023-10"
+DEFAULT_ITEMS_PER_PAGE = 250
+
+DEFAULT_PARTNER_API_VERSION = "2024-01"

omniload/src/slack/__init__.py

@@ -0,0 +1,290 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Fetches Slack Conversations, History and logs."""
+
+from functools import partial
+from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple
+
+import dlt
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+from pendulum import DateTime
+
+from .helpers import SlackAPI, ensure_dt_type
+from .settings import (
+    DEFAULT_DATETIME_FIELDS,
+    DEFAULT_START_DATE,
+    MAX_PAGE_SIZE,
+    MSG_DATETIME_FIELDS,
+)
+
+
+@dlt.source(name="slack", max_table_nesting=0)
+def slack_source(
+    page_size: int = MAX_PAGE_SIZE,
+    access_token: str = dlt.secrets.value,
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+    selected_channels: Optional[List[str]] = dlt.config.value,
+    table_per_channel: bool = True,
+    replies: bool = False,
+) -> Iterable[DltResource]:
+    """
+    The source for the Slack pipeline. Available resources are conversations, conversations_history
+    and access_logs.
+
+    Args:
+        page_size: The max number of items to fetch per page. Defaults to 1000.
+        access_token: the oauth access_token used to authenticate.
+        start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+        end_date: The end time of the range for which to load data.
+        selected_channels: The list of channels to load. If None, all channels will be loaded.
+        table_per_channel: Boolean flag, True by default. If True - for each channel separate table with messages is created.
+            Otherwise, all messages are put in one table.
+        replies: Boolean flag indicating if you want a replies table to be present as well. False by default.
+
+    Returns:
+        Iterable[DltResource]: A list of DltResource objects representing the data resources.
+    """
+
+    end_dt: Optional[DateTime] = ensure_dt_type(end_date)
+    start_dt: Optional[DateTime] = ensure_dt_type(start_date)
+    write_disposition: Literal["append", "merge"] = (
+        "append" if end_date is None else "merge"
+    )
+
+    api = SlackAPI(
+        access_token=access_token,
+        page_size=page_size,
+    )
+
+    def get_channels(
+        slack_api: SlackAPI, selected_channels: Optional[List[str]]
+    ) -> Tuple[List[TDataItem], List[TDataItem]]:
+        """
+        Returns channel fetched from slack and list of selected channels.
+
+        Args:
+            slack_api: Slack API instance.
+            selected_channels: List of selected channels names or None.
+
+        Returns:
+            Tuple[List[TDataItem], List[TDataItem]]: fetched channels and selected fetched channels.
+        """
+        channels: List[TDataItem] = []
+        for page_data in slack_api.get_pages(
+            resource="conversations.list",
+            response_path="$.channels[*]",
+            datetime_fields=DEFAULT_DATETIME_FIELDS,
+        ):
+            channels.extend(page_data)
+
+        if selected_channels:
+            fetch_channels = [
+                c
+                for c in channels
+                if c["name"] in selected_channels or c["id"] in selected_channels
+            ]
+        else:
+            fetch_channels = channels
+        return channels, fetch_channels
+
+    channels, fetched_selected_channels = get_channels(api, selected_channels)
+
+    @dlt.resource(name="channels", primary_key="id", write_disposition="replace")
+    def channels_resource() -> Iterable[TDataItem]:
+        """Yield all channels as a DLT resource."""
+        yield from channels
+
+    @dlt.resource(name="users", primary_key="id", write_disposition="replace")
+    def users_resource() -> Iterable[TDataItem]:
+        """
+        Yield all users as a DLT resource.
+
+        Yields:
+            Iterable[TDataItem]: A list of users.
+        """
+
+        for page_data in api.get_pages(
+            resource="users.list",
+            response_path="$.members[*]",
+            params=dict(include_locale=True),
+            datetime_fields=DEFAULT_DATETIME_FIELDS,
+        ):
+            yield page_data
+
+    def get_messages(
+        channel_data: Dict[str, Any], start_date_ts: float, end_date_ts: float
+    ) -> Iterable[TDataItem]:
+        """
+        Generator, which gets channel messages for specific dates.
+        Args:
+            channel_data: dict with channels data.
+            start_date_ts: start timestamp.
+            end_date_ts: end timestamp.
+
+        Yields:
+            List[TDataItem]: messages.
+        """
+        params = {
+            "channel": channel_data["id"],
+            "oldest": start_date_ts,
+            "latest": end_date_ts,
+        }
+
+        for page_data in api.get_pages(
+            resource="conversations.history",
+            response_path="$.messages[*]",
+            params=params,
+            datetime_fields=MSG_DATETIME_FIELDS,
+            context={"channel": channel_data["id"]},
+        ):
+            yield page_data
+
+    def get_thread_replies(messages: List[Dict[str, Any]]) -> Iterable[TDataItem]:
+        """
+        Generator, which gets replies for each message.
+        Args:
+            messages: messages data.
+
+        Yields:
+            Li
+        """
+        for message in messages:
+            if message.get("thread_ts", None):
+                params = {
+                    "channel": message["channel"],
+                    "ts": ensure_dt_type(message["thread_ts"], to_ts=True),
+                }
+
+                for page_data in api.get_pages(
+                    resource="conversations.replies",
+                    response_path="$.messages[*]",
+                    params=params,
+                    context={"channel": message["channel"]},
+                ):
+                    yield page_data[1:]
+
+    @dlt.resource(
+        name="messages",
+        primary_key=("channel", "ts"),
+        columns={"blocks": {"data_type": "json"}},
+        write_disposition=write_disposition,
+    )
+    def messages_resource(
+        created_at: dlt.sources.incremental[DateTime] = dlt.sources.incremental(
+            "ts",
+            initial_value=start_dt,
+            end_value=end_dt,
+            allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Yield all messages for a set of selected channels as a DLT resource. Keep blocks column without normalization.
+
+        Args:
+            created_at (dlt.sources.incremental[DateTime]): The incremental created_at field.
+
+        Yields:
+            Iterable[TDataItem]: A list of messages.
+        """
+        start_date_ts = ensure_dt_type(created_at.last_value, to_ts=True)
+        end_date_ts = ensure_dt_type(created_at.end_value, to_ts=True)
+        for channel_data in fetched_selected_channels:
+            yield from get_messages(channel_data, start_date_ts, end_date_ts)
+
+    def per_table_messages_resource(
+        channel_data: Dict[str, Any],
+        created_at: dlt.sources.incremental[DateTime] = dlt.sources.incremental(
+            "ts",
+            initial_value=start_dt,
+            end_value=end_dt,
+            allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """Yield all messages for a given channel as a DLT resource. Keep blocks column without normalization.
+
+        Args:
+            channel_data (Dict[str, Any]): The channel data.
+            created_at (dlt.sources.incremental[DateTime]): The incremental created_at field.
+
+        Yields:
+            Iterable[TDataItem]: A list of messages.
+        """
+        start_date_ts = ensure_dt_type(created_at.last_value, to_ts=True)
+        end_date_ts = ensure_dt_type(created_at.end_value, to_ts=True)
+        yield from get_messages(channel_data, start_date_ts, end_date_ts)
+
+    def table_name_func(channel_name: str, payload: TDataItem) -> str:
+        """Return the table name for a given channel and payload."""
+        table_type = payload.get("subtype", payload.get("type", ""))
+        return f"{channel_name}_{table_type}"
+
+    # It will not work in the pipeline or tests because it is a paid feature,
+    # raise an error when it is not a paying account.
+    @dlt.resource(
+        name="access_logs",
+        selected=False,
+        primary_key="user_id",
+        write_disposition="append",
+    )
+    # it is not an incremental resource it just has an end_date filter
+    def logs_resource() -> Iterable[TDataItem]:
+        """The access logs resource."""
+        for page_data in api.get_pages(
+            resource="team.accessLogs",
+            response_path="$.logins[*]",
+            datetime_fields=["date_first", "date_last"],
+            params={"before": end_dt if end_dt is None else end_dt.int_timestamp},
+        ):
+            yield page_data
+
+    yield from (channels_resource, users_resource, logs_resource)
+
+    if table_per_channel:
+        for channel in fetched_selected_channels:
+            channel_name = channel["name"]
+            table_name = partial(table_name_func, channel_name)
+            messages_channel = dlt.resource(
+                per_table_messages_resource,
+                name=channel_name,
+                table_name=table_name,
+                primary_key=("channel", "ts"),
+                write_disposition=write_disposition,
+                columns={"blocks": {"data_type": "json"}},
+            )(channel)
+
+            yield messages_channel
+            if replies:
+                yield messages_channel | dlt.transformer(
+                    get_thread_replies,
+                    name=channel_name + "_replies",
+                    table_name=partial(table_name_func, channel_name + "_replies"),
+                    primary_key=("thread_ts", "ts"),
+                    write_disposition=write_disposition,
+                )
+    else:
+        yield messages_resource
+        if replies:
+            yield messages_resource | dlt.transformer(
+                get_thread_replies,
+                name="replies",
+                primary_key=("thread_ts", "ts"),
+                write_disposition=write_disposition,
+            )