omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/appstore/errors.py

@@ -0,0 +1,15 @@
+class NoReportsFoundError(Exception):
+    def __init__(self):
+        super().__init__("No Report instances found for the given date range")
+
+
+class NoOngoingReportRequestsFoundError(Exception):
+    def __init__(self):
+        super().__init__(
+            "No ONGOING report requests found (or they're stopped due to inactivity)"
+        )
+
+
+class NoSuchReportError(Exception):
+    def __init__(self, report_name):
+        super().__init__(f"No such report found: {report_name}")

omniload/src/appstore/models.py

@@ -0,0 +1,117 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from dataclasses_json import dataclass_json
+
+
+@dataclass_json
+@dataclass
+class Links:
+    self: str
+    next: Optional[str] = None
+
+
+@dataclass_json
+@dataclass
+class ReportRequestAttributes:
+    accessType: str
+    stoppedDueToInactivity: bool
+
+
+@dataclass_json
+@dataclass
+class ReportAttributes:
+    name: str
+    category: str
+
+
+@dataclass_json
+@dataclass
+class ReportInstanceAttributes:
+    granularity: str
+    processingDate: str
+
+
+@dataclass_json
+@dataclass
+class ReportSegmentAttributes:
+    checksum: str
+    url: str
+    sizeInBytes: int
+
+
+@dataclass_json
+@dataclass
+class ReportRequest:
+    type: str
+    id: str
+    attributes: ReportRequestAttributes
+
+
+@dataclass_json
+@dataclass
+class Report:
+    type: str
+    id: str
+    attributes: ReportAttributes
+
+
+@dataclass_json
+@dataclass
+class ReportInstance:
+    type: str
+    id: str
+    attributes: ReportInstanceAttributes
+
+
+@dataclass_json
+@dataclass
+class ReportSegment:
+    type: str
+    id: str
+    attributes: ReportSegmentAttributes
+
+
+@dataclass_json
+@dataclass
+class PagingMeta:
+    total: int
+    limit: int
+
+
+@dataclass_json
+@dataclass
+class Meta:
+    paging: PagingMeta
+
+
+@dataclass_json
+@dataclass
+class AnalyticsReportRequestsResponse:
+    data: List[ReportRequest]
+    meta: Meta
+    links: Links
+
+
+@dataclass_json
+@dataclass
+class AnalyticsReportResponse:
+    data: List[Report]
+    meta: Meta
+    links: Links
+
+
+@dataclass_json
+@dataclass
+class AnalyticsReportInstancesResponse:
+    data: List[ReportInstance]
+    meta: Meta
+    links: Links
+
+
+@dataclass_json
+@dataclass
+class AnalyticsReportSegmentsResponse:
+    data: List[ReportSegment]
+    meta: Meta
+    links: Links

omniload/src/appstore/resources.py

@@ -0,0 +1,179 @@
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class ResourceConfig:
+    name: str
+    primary_key: List[str]
+    columns: dict
+    report_name: str
+
+
+RESOURCES: List[ResourceConfig] = [
+    ResourceConfig(
+        name="app-downloads-detailed",
+        primary_key=[
+            "App Apple Identifier",
+            "App Name",
+            "App Version",
+            "Campaign",
+            "Date",
+            "Device",
+            "Download Type",
+            "Page Title",
+            "Page Type",
+            "Platform Version",
+            "Pre-Order",
+            "Source Info",
+            "Source Type",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Counts": {"data_type": "bigint"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Downloads Detailed",
+    ),
+    ResourceConfig(
+        name="app-store-discovery-and-engagement-detailed",
+        primary_key=[
+            "App Apple Identifier",
+            "App Name",
+            "Campaign",
+            "Date",
+            "Device",
+            "Engagement Type",
+            "Event",
+            "Page Title",
+            "Page Type",
+            "Platform Version",
+            "Source Info",
+            "Source Type",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Counts": {"data_type": "bigint"},
+            "Unique Counts": {"data_type": "bigint"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Store Discovery and Engagement Detailed",
+    ),
+    ResourceConfig(
+        name="app-sessions-detailed",
+        primary_key=[
+            "Date",
+            "App Name",
+            "App Apple Identifier",
+            "App Version",
+            "Device",
+            "Platform Version",
+            "Source Type",
+            "Source Info",
+            "Campaign",
+            "Page Type",
+            "Page Title",
+            "App Download Date",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Sessions": {"data_type": "bigint"},
+            "Total Session Duration": {"data_type": "bigint"},
+            "Unique Devices": {"data_type": "bigint"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Sessions Detailed",
+    ),
+    ResourceConfig(
+        name="app-store-installation-and-deletion-detailed",
+        primary_key=[
+            "App Apple Identifier",
+            "App Download Date",
+            "App Name",
+            "App Version",
+            "Campaign",
+            "Counts",
+            "Date",
+            "Device",
+            "Download Type",
+            "Event",
+            "Page Title",
+            "Page Type",
+            "Platform Version",
+            "Source Info",
+            "Source Type",
+            "Territory",
+            "Unique Devices",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Counts": {"data_type": "bigint"},
+            "Unique Devices": {"data_type": "bigint"},
+            "App Download Date": {"data_type": "date"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Store Installation and Deletion Detailed",
+    ),
+    ResourceConfig(
+        name="app-store-purchases-detailed",
+        primary_key=[
+            "App Apple Identifier",
+            "App Download Date",
+            "App Name",
+            "Campaign",
+            "Content Apple Identifier",
+            "Content Name",
+            "Date",
+            "Device",
+            "Page Title",
+            "Page Type",
+            "Payment Method",
+            "Platform Version",
+            "Pre-Order",
+            "Purchase Type",
+            "Source Info",
+            "Source Type",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "App Download Date": {"data_type": "date"},
+            "Content Apple Identifier": {"data_type": "bigint"},
+            "Purchases": {"data_type": "bigint"},
+            "Proceeds In USD": {"data_type": "double"},
+            "Sales In USD": {"data_type": "double"},
+            "Paying Users": {"data_type": "bigint"},
+            "processing_date": {"data_type": "date"},
+        },
+        report_name="App Store Purchases Detailed",
+    ),
+    ResourceConfig(
+        name="app-crashes-expanded",
+        primary_key=[
+            "App Name",
+            "App Version",
+            "Build",
+            "Date",
+            "Device",
+            "Platform",
+            "Release Type",
+            "Territory",
+        ],
+        columns={
+            "Date": {"data_type": "date"},
+            "processing_date": {"data_type": "date"},
+            "App Apple Identifier": {"data_type": "bigint"},
+            "Count": {"data_type": "bigint"},
+            "Unique Devices": {"data_type": "bigint"},
+        },
+        report_name="App Crashes Expanded",
+    ),
+]

omniload/src/arrow/__init__.py

@@ -0,0 +1,81 @@
+from typing import Any, Optional
+
+import dlt
+import pyarrow as pa  # type: ignore
+from dlt.common.schema.typing import TColumnNames, TTableSchemaColumns
+from dlt.extract.items import TTableHintTemplate
+
+
+def memory_mapped_arrow(
+    path: str,
+    columns: Optional[TTableSchemaColumns] = None,
+    primary_key: Optional[TTableHintTemplate[TColumnNames]] = None,
+    merge_key: Optional[TTableHintTemplate[TColumnNames]] = None,
+    incremental: Optional[dlt.sources.incremental[Any]] = None,
+):
+    @dlt.resource(
+        name="arrow_mmap",
+        columns=columns,  # type: ignore
+        primary_key=primary_key,  # type: ignore
+        merge_key=merge_key,  # type: ignore
+    )
+    def arrow_mmap(
+        incremental: Optional[dlt.sources.incremental[Any]] = incremental,
+    ):
+        import pyarrow.ipc as ipc  # type: ignore
+
+        with pa.memory_map(path, "rb") as mmap:
+            reader: ipc.RecordBatchFileReader = ipc.open_file(mmap)
+            table = reader.read_all()
+
+        last_value = None
+        end_value = None
+        if incremental:
+            if incremental.cursor_path not in table.column_names:
+                raise KeyError(
+                    f"Cursor column '{incremental.cursor_path}' does not exist in table"
+                )
+
+            last_value = incremental.last_value
+            end_value = incremental.end_value
+
+        if last_value is not None:
+            # Check if the column is a date type
+            if pa.types.is_temporal(table.schema.field(incremental.cursor_path).type):  # type: ignore
+                if not isinstance(last_value, pa.TimestampScalar):
+                    last_value = pa.scalar(last_value, type=pa.timestamp("ns"))
+
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) >= last_value  # type: ignore
+                )
+            else:
+                # For non-date types, use direct comparison
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) >= last_value  # type: ignore
+                )
+
+        if end_value is not None:
+            if pa.types.is_timestamp(table.schema.field(incremental.cursor_path).type):  # type: ignore
+                # Convert end_value to timestamp if it's not already
+                if not isinstance(end_value, pa.TimestampScalar):
+                    end_value = pa.scalar(end_value, type=pa.timestamp("ns"))
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) <= end_value  # type: ignore
+                )
+            else:
+                # For non-date types, use direct comparison
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) <= end_value  # type: ignore
+                )
+
+        yield table
+
+    return arrow_mmap
+
+
+BATCH_SIZE = 1000
+
+
+def as_list(table: pa.Table):
+    for batch in table.to_batches(BATCH_SIZE):
+        yield from batch.to_pylist()

omniload/src/asana_source/__init__.py

@@ -0,0 +1,281 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This source provides data extraction from the Asana platform via their API.
+
+It defines several functions to fetch data from different parts of Asana including
+workspaces, projects, sections, tags, tasks, stories, teams, and users. These
+functions are meant to be used as part of a data loading pipeline.
+"""
+
+import typing as t
+from typing import Any, Iterable
+
+import dlt
+from dlt.common.typing import TDataItem
+
+from .helpers import get_client
+from .settings import (
+    DEFAULT_START_DATE,
+    PROJECT_FIELDS,
+    REQUEST_TIMEOUT,
+    SECTION_FIELDS,
+    STORY_FIELDS,
+    TAG_FIELDS,
+    TASK_FIELDS,
+    TEAMS_FIELD,
+    USER_FIELDS,
+    WORKSPACE_FIELDS,
+)
+
+
+@dlt.source
+def asana_source() -> Any:  # should be Sequence[DltResource]:
+    """
+    The main function that runs all the other functions to fetch data from Asana.
+    Returns:
+        Sequence[DltResource]: A sequence of DltResource objects containing the fetched data.
+    """
+    return [
+        workspaces,
+        projects,
+        sections,
+        tags,
+        tasks,
+        stories,
+        teams,
+        users,
+    ]
+
+
+@dlt.resource(write_disposition="replace")
+def workspaces(
+    access_token: str = dlt.secrets.value, fields: Iterable[str] = WORKSPACE_FIELDS
+) -> Iterable[TDataItem]:
+    """
+    Fetches and returns a list of workspaces from Asana.
+    Args:
+        access_token (str): The access token to authenticate the Asana API client, provided in the secrets file
+        fields (Iterable[str]): The list of workspace fields to be retrieved from Asana API.
+    Yields:
+        dict: The workspace data.
+    """
+    yield from get_client(access_token).workspaces.find_all(opt_fields=",".join(fields))
+
+
+@dlt.transformer(
+    data_from=workspaces,
+    write_disposition="replace",
+)
+@dlt.defer
+def projects(
+    workspace: TDataItem,
+    access_token: str = dlt.secrets.value,
+    fields: Iterable[str] = PROJECT_FIELDS,
+) -> Iterable[TDataItem]:
+    """
+    Fetches and returns a list of projects for a given workspace from Asana.
+    Args:
+        workspace (dict): The workspace data.
+        access_token (str): The access token to authenticate the Asana API client, provided in the secrets file
+        fields (Iterable[str]): The list of workspace fields to be retrieved from Asana API.
+    Returns:
+        list[dict]: The project data for the given workspace.
+    """
+    return list(
+        get_client(access_token).projects.find_all(
+            workspace=workspace["gid"],
+            timeout=REQUEST_TIMEOUT,
+            opt_fields=",".join(fields),
+        )
+    )
+
+
+@dlt.transformer(
+    data_from=projects,
+    write_disposition="replace",
+)
+@dlt.defer
+def sections(
+    project_array: t.List[TDataItem],
+    access_token: str = dlt.secrets.value,
+    fields: Iterable[str] = SECTION_FIELDS,
+) -> Iterable[TDataItem]:
+    """
+    Fetches all sections for a given project from Asana.
+    Args:
+        project_array (list): The project data.
+        access_token (str): The access token to authenticate the Asana API client, provided in the secrets file
+        fields (Iterable[str]): The list of workspace fields to be retrieved from Asana API.
+    Returns:
+        list[dict]: The sections data for the given project.
+    """
+    return [
+        section
+        for project in project_array
+        for section in get_client(access_token).sections.get_sections_for_project(
+            project_gid=project["gid"],
+            timeout=REQUEST_TIMEOUT,
+            opt_fields=",".join(fields),
+        )
+    ]
+
+
+@dlt.transformer(data_from=workspaces, write_disposition="replace")
+@dlt.defer
+def tags(
+    workspace: TDataItem,
+    access_token: str = dlt.secrets.value,
+    fields: Iterable[str] = TAG_FIELDS,
+) -> Iterable[TDataItem]:
+    """
+    Fetches all tags for a given workspace from Asana.
+    Args:
+        workspace (dict): The workspace data.
+        access_token (str): The access token to authenticate the Asana API client, provided in the secrets file
+        fields (Iterable[str]): The list of workspace fields to be retrieved from Asana API.
+    Returns:
+        list[dict]: The tags data for the given workspace.
+    """
+    return [
+        tag
+        for tag in get_client(access_token).tags.find_all(
+            workspace=workspace["gid"],
+            timeout=REQUEST_TIMEOUT,
+            opt_fields=",".join(fields),
+        )
+    ]
+
+
+@dlt.transformer(data_from=projects, write_disposition="merge", primary_key="gid")
+def tasks(
+    project_array: t.List[TDataItem],
+    access_token: str = dlt.secrets.value,
+    modified_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "modified_at",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
+    ),
+    fields: Iterable[str] = TASK_FIELDS,
+) -> Iterable[TDataItem]:
+    """
+    Fetches all tasks for a given project from Asana.
+    Args:
+        project_array (list): The project data.
+        access_token (str): The access token to authenticate the Asana API client, provided in the secrets file
+
+        modified_at (str): The date from which to fetch modified tasks.
+        fields (Iterable[str]): The list of workspace fields to be retrieved from Asana API.
+    Yields:
+        dict: The task data for the given project.
+    """
+    yield from (
+        task
+        for project in project_array
+        for task in get_client(access_token).tasks.find_all(
+            project=project["gid"],
+            timeout=REQUEST_TIMEOUT,
+            modified_since=modified_at.start_value,
+            opt_fields=",".join(fields),
+        )
+    )
+
+
+@dlt.transformer(
+    data_from=tasks,
+    write_disposition="replace",
+)
+@dlt.defer
+def stories(
+    task: TDataItem,
+    access_token: str = dlt.secrets.value,
+    fields: Iterable[str] = STORY_FIELDS,
+) -> Iterable[TDataItem]:
+    """
+    Fetches stories for a task from Asana.
+    Args:
+        task (dict): The task data.
+        access_token (str): The access token to authenticate the Asana API client, provided in the secrets file
+        fields (Iterable[str]): The list of workspace fields to be retrieved from Asana API.
+    Returns:
+        list[dict]: The stories data for the given task.
+    """
+    return [
+        story
+        for story in get_client(access_token).stories.get_stories_for_task(
+            task_gid=task["gid"],
+            timeout=REQUEST_TIMEOUT,
+            opt_fields=",".join(fields),
+        )
+    ]
+
+
+@dlt.transformer(
+    data_from=workspaces,
+    write_disposition="replace",
+)
+@dlt.defer
+def teams(
+    workspace: TDataItem,
+    access_token: str = dlt.secrets.value,
+    fields: Iterable[str] = TEAMS_FIELD,
+) -> Iterable[TDataItem]:
+    """
+    Fetches all teams for a given workspace from Asana.
+    Args:
+        workspace (dict): The workspace data.
+        access_token (str): The access token to authenticate the Asana API client, provided in the secrets file
+        fields (Iterable[str]): The list of workspace fields to be retrieved from Asana API.
+    Returns:
+        list[dict]: The teams data for the given workspace.
+    """
+    return [
+        team
+        for team in get_client(access_token).teams.find_by_organization(
+            organization=workspace["gid"],
+            timeout=REQUEST_TIMEOUT,
+            opt_fields=",".join(fields),
+        )
+    ]
+
+
+@dlt.transformer(
+    data_from=workspaces,
+    write_disposition="replace",
+)
+@dlt.defer
+def users(
+    workspace: TDataItem,
+    access_token: str = dlt.secrets.value,
+    fields: Iterable[str] = USER_FIELDS,
+) -> Iterable[TDataItem]:
+    """
+    Fetches all users for a given workspace from Asana.
+    Args:
+        workspace (dict): The workspace data.
+        access_token (str): The access token to authenticate the Asana API client, provided in the secrets file
+        fields (Iterable[str]): The list of workspace fields to be retrieved from Asana API.
+    Returns:
+        list[dict]: The user data for the given workspace.
+    """
+    return [
+        user
+        for user in get_client(access_token).users.find_all(
+            workspace=workspace["gid"],
+            timeout=REQUEST_TIMEOUT,
+            opt_fields=",".join(fields),
+        )
+    ]

omniload/src/asana_source/helpers.py

@@ -0,0 +1,30 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Asana source helpers"""
+
+from asana import Client as AsanaClient
+
+
+def get_client(
+    access_token: str,
+) -> AsanaClient:
+    """
+    Returns an Asana API client.
+    Args:
+        access_token (str): The access token to authenticate the Asana API client.
+    Returns:
+        AsanaClient: The Asana API client.
+    """
+    return AsanaClient.access_token(access_token)