omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/freshdesk/settings.py

@@ -0,0 +1,23 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module defines default settings for the Freshdesk integration.
+
+It specifies a list of default endpoints to be used when interacting with the Freshdesk API,
+covering common entities such as agents, companies, contacts, groups, roles, and tickets.
+"""
+
+# Define default endpoints for the Freshdesk API integration.
+DEFAULT_ENDPOINTS = ["agents", "companies", "contacts", "groups", "roles", "tickets"]

omniload/src/fundraiseup/__init__.py

@@ -0,0 +1,95 @@
+"""Fundraiseup source for ingesting donations, events, fundraisers, recurring plans, and supporters."""
+
+from typing import Any, Dict, Generator, Iterable, TypedDict
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource
+
+from .client import FundraiseupClient
+
+
+class DonationCursor(TypedDict):
+    id: str
+    created_at: pendulum.DateTime
+
+
+def order_by_created(record) -> DonationCursor:
+    last_value = None
+    if len(record) == 1:
+        (record,) = record
+    else:
+        record, last_value = record
+
+    cursor: DonationCursor = {
+        "id": record["id"],
+        "created_at": ensure_pendulum_datetime(record["created_at"]),
+    }
+
+    if last_value is None:
+        return cursor
+
+    return max(cursor, last_value, key=lambda v: v["created_at"])
+
+
+@dlt.source(name="fundraiseup", max_table_nesting=0)
+def fundraiseup_source(api_key: str) -> Iterable[DltResource]:
+    """
+    Return resources for Fundraiseup API.
+
+    Args:
+        api_key: API key for authentication
+
+    Returns:
+        Iterable of DLT resources
+    """
+    client = FundraiseupClient(api_key=api_key)
+
+    # Define available resources and their configurations
+    resources = {
+        "donations": {"write_disposition": "replace", "primary_key": "id"},
+        "events": {"write_disposition": "replace", "primary_key": "id"},
+        "fundraisers": {"write_disposition": "replace", "primary_key": "id"},
+        "recurring_plans": {"write_disposition": "replace", "primary_key": "id"},
+        "supporters": {"write_disposition": "replace", "primary_key": "id"},
+    }
+
+    def create_resource(resource_name: str, config: Dict[str, Any]) -> DltResource:
+        """Create a DLT resource dynamically."""
+
+        @dlt.resource(
+            name=resource_name,
+            write_disposition=config["write_disposition"],
+            primary_key=config["primary_key"],
+        )
+        def generic_resource() -> Generator[Dict[str, Any], None, None]:
+            """Generic resource that yields batches directly."""
+            for batch in client.get_paginated_data(resource_name):
+                yield batch  # type: ignore[misc]
+
+        return generic_resource()
+
+    @dlt.resource(
+        name="donations:incremental",
+        write_disposition="merge",
+        primary_key="id",
+    )
+    def donations_incremental(
+        last_record: dlt.sources.incremental[DonationCursor] = dlt.sources.incremental(
+            "$",
+            range_start="closed",
+            range_end="closed",
+            last_value_func=order_by_created,
+        ),
+    ):
+        params = {}
+        if last_record.last_value is not None:
+            params["starting_after"] = last_record.last_value["id"]
+        for batch in client.get_paginated_data("donations", params=params):
+            yield batch  # type: ignore[misc]
+
+    # Return all resources
+    return [donations_incremental] + [
+        create_resource(name, config) for name, config in resources.items()
+    ]

omniload/src/fundraiseup/client.py

@@ -0,0 +1,81 @@
+"""Fundraiseup API Client for handling authentication and paginated requests."""
+
+from typing import Any, Dict, Iterator, Optional
+
+from omniload.src.http_client import create_client
+
+
+class FundraiseupClient:
+    """Client for interacting with Fundraiseup API v1."""
+
+    def __init__(self, api_key: str):
+        """
+        Initialize Fundraiseup API client.
+
+        Args:
+            api_key: API key for authentication
+        """
+        self.api_key = api_key
+        self.base_url = "https://api.fundraiseup.com/v1"
+        # Use shared HTTP client with retry logic for rate limiting
+        self.client = create_client(retry_status_codes=[429, 500, 502, 503, 504])
+
+    def get_paginated_data(
+        self,
+        endpoint: str,
+        params: Optional[Dict[str, Any]] = None,
+        page_size: int = 100,
+    ) -> Iterator[list[Dict[str, Any]]]:
+        """
+        Fetch paginated data from a Fundraiseup API endpoint using cursor-based pagination.
+
+        Args:
+            endpoint: API endpoint path (e.g., "donations")
+            params: Additional query parameters
+            page_size: Number of items per page (default 100)
+
+        Yields:
+            Batches of items from the API
+        """
+        url = f"{self.base_url}/{endpoint}"
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        if params is None:
+            params = {}
+
+        params["limit"] = page_size
+        starting_after = None
+
+        while True:
+            # Add cursor for pagination if not first page
+            if starting_after:
+                params["starting_after"] = starting_after
+
+            response = self.client.get(url=url, headers=headers, params=params)
+            response.raise_for_status()
+
+            data = response.json()
+
+            # Handle both list response and object with data array
+            if isinstance(data, list):
+                items = data
+                has_more = len(items) == page_size
+            else:
+                items = data.get("data", [])
+                has_more = data.get("has_more", False)
+
+            if not items:
+                break
+
+            yield items
+
+            # Set cursor for next page
+            if has_more and items:
+                starting_after = items[-1].get("id")
+                if not starting_after:
+                    break
+            else:
+                break

omniload/src/github/__init__.py

@@ -0,0 +1,202 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Source that load github issues, pull requests and reactions for a specific repository via customizable graphql query. Loads events incrementally."""
+
+import urllib.parse
+from typing import Iterator, Optional, Sequence
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItems
+from dlt.sources import DltResource
+
+from .helpers import get_reactions_data, get_rest_pages, get_stargazers
+
+
+@dlt.source(max_table_nesting=0)
+def github_reactions(
+    owner: str,
+    name: str,
+    access_token: str,
+    items_per_page: int = 100,
+    max_items: Optional[int] = None,
+) -> Sequence[DltResource]:
+    """Get reactions associated with issues, pull requests and comments in the repo `name` with owner `owner`.
+
+    This source uses graphql to retrieve all issues (`issues` resource) and pull requests (`pull requests` resource) with the associated reactions (up to 100),
+    comments (up to 100) and reactions to comments (also up to 100). Internally graphql is used to retrieve data. It is cost optimized and you are able to retrieve the
+    data for fairly large repos quickly and cheaply.
+    You can and should change the queries in `queries.py` to include for example additional fields or connections. The source can be hacked to add more resources for other
+    repository nodes easily.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic access token. Will be injected from secrets if not provided.
+        items_per_page (int, optional): How many issues/pull requests to get in single page. Defaults to 100.
+        max_items (int, optional): How many issues/pull requests to get in total. None means All.
+        max_item_age_seconds (float, optional): Do not get items older than this. Defaults to None. NOT IMPLEMENTED
+
+    Returns:
+        Sequence[DltResource]: Two DltResources: `issues` with issues and `pull_requests` with pull requests
+    """
+    return (
+        dlt.resource(
+            get_reactions_data(
+                "issues",
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="issues",
+            write_disposition="replace",
+        ),
+        dlt.resource(
+            get_reactions_data(
+                "pullRequests",
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="pull_requests",
+            write_disposition="replace",
+        ),
+    )
+
+
+@dlt.source(max_table_nesting=0)
+def github_repo_events(
+    owner: str,
+    name: str,
+    access_token: str,
+    start_date: pendulum.DateTime,
+    end_date: Optional[pendulum.DateTime] = None,
+) -> DltResource:
+    """Gets events for repository `name` with owner `owner` incrementally.
+
+    This source contains a single resource `repo_events` that gets given repository's events and dispatches them to separate tables with names based on event type.
+    The data is loaded incrementally. Subsequent runs will get only new events and append them to tables.
+    Please note that Github allows only for 300 events to be retrieved for public repositories. You should get the events frequently for the active repos.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic or fine-grained access token. If not provided, calls are made anonymously
+
+    Returns:
+        DltSource: source with the `repo_events` resource
+
+    """
+
+    # use naming function in table name to generate separate tables for each event
+    @dlt.resource(
+        primary_key="id", table_name=lambda i: i["type"], write_disposition="merge"
+    )
+    def repo_events(
+        last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            last_value_func=max,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterator[TDataItems]:
+        repos_path = (
+            f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
+        )
+
+        # Get the date range from the incremental state
+        start_filter = pendulum.parse(
+            last_created_at.last_value or last_created_at.initial_value
+        )
+        end_filter = (
+            pendulum.parse(last_created_at.end_value)
+            if last_created_at.end_value
+            else pendulum.now()
+        )
+
+        for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
+            # Filter events by date range
+            filtered_events = []
+            for event in page:
+                event_date = pendulum.parse(event["created_at"])
+
+                # Check if event is within the date range
+                if event_date >= start_filter:
+                    if end_filter is None or event_date <= end_filter:
+                        filtered_events.append(event)
+                    elif event_date > end_filter:
+                        # Skip events that are newer than our end date
+                        continue
+                else:
+                    # Events are ordered by date desc, so if we hit an older event, we can stop
+                    break
+
+            if filtered_events:
+                yield filtered_events
+
+            # stop requesting pages if the last element was already older than initial value
+            # note: incremental will skip those items anyway, we just do not want to use the api limits
+            if last_created_at.start_out_of_range:
+                print(
+                    f"Overlap with previous run created at {last_created_at.initial_value}"
+                )
+                break
+
+    return repo_events
+
+
+@dlt.source(max_table_nesting=0)
+def github_stargazers(
+    owner: str,
+    name: str,
+    access_token: str,
+    items_per_page: int = 100,
+    max_items: Optional[int] = None,
+) -> Sequence[DltResource]:
+    """Get stargazers in the repo `name` with owner `owner`.
+
+    This source uses graphql to retrieve all stargazers with the associated starred date,
+    Internally graphql is used to retrieve data. It is cost optimized and you are able to retrieve the
+    data for fairly large repos quickly and cheaply.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic access token. Will be injected from secrets if not provided.
+        items_per_page (int, optional): How many issues/pull requests to get in single page. Defaults to 100.
+        max_items (int, optional): How many issues/pull requests to get in total. None means All.
+
+    Returns:
+        Sequence[DltResource]: One DltResource: `stargazers`
+    """
+    return (
+        dlt.resource(
+            get_stargazers(
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="stargazers",
+            write_disposition="replace",
+        ),
+    )

omniload/src/github/helpers.py

@@ -0,0 +1,207 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Iterator, List, Optional, Tuple
+
+from dlt.common.typing import DictStrAny, StrAny
+from dlt.common.utils import chunks
+from dlt.sources.helpers import requests
+
+from .queries import COMMENT_REACTIONS_QUERY, ISSUES_QUERY, RATE_LIMIT, STARGAZERS_QUERY
+from .settings import GRAPHQL_API_BASE_URL, REST_API_BASE_URL
+
+
+#
+# Shared
+#
+def _get_auth_header(access_token: Optional[str]) -> StrAny:
+    if access_token:
+        return {"Authorization": f"Bearer {access_token}"}
+    else:
+        # REST API works without access token (with high rate limits)
+        return {}
+
+
+#
+# Rest API helpers
+#
+def get_rest_pages(access_token: Optional[str], query: str) -> Iterator[List[StrAny]]:
+    def _request(page_url: str) -> requests.Response:
+        r = requests.get(page_url, headers=_get_auth_header(access_token))
+        print(
+            f"got page {page_url}, requests left: " + r.headers["x-ratelimit-remaining"]
+        )
+        return r
+
+    next_page_url = REST_API_BASE_URL + query
+    while True:
+        r: requests.Response = _request(next_page_url)
+        page_items = r.json()
+        if len(page_items) == 0:
+            break
+        yield page_items
+        if "next" not in r.links:
+            break
+        next_page_url = r.links["next"]["url"]
+
+
+#
+# GraphQL API helpers
+#
+def get_stargazers(
+    owner: str,
+    name: str,
+    access_token: str,
+    items_per_page: int,
+    max_items: Optional[int],
+) -> Iterator[Iterator[StrAny]]:
+    variables = {"owner": owner, "name": name, "items_per_page": items_per_page}
+    for page_items in _get_graphql_pages(
+        access_token, STARGAZERS_QUERY, variables, "stargazers", max_items
+    ):
+        yield map(
+            lambda item: {"starredAt": item["starredAt"], "user": item["node"]},
+            page_items,
+        )
+
+
+def get_reactions_data(
+    node_type: str,
+    owner: str,
+    name: str,
+    access_token: str,
+    items_per_page: int,
+    max_items: Optional[int],
+) -> Iterator[Iterator[StrAny]]:
+    variables = {
+        "owner": owner,
+        "name": name,
+        "issues_per_page": items_per_page,
+        "first_reactions": 100,
+        "first_comments": 100,
+        "node_type": node_type,
+    }
+    for page_items in _get_graphql_pages(
+        access_token, ISSUES_QUERY % node_type, variables, node_type, max_items
+    ):
+        # use reactionGroups to query for reactions to comments that have any reactions. reduces cost by 10-50x
+        reacted_comment_ids = {}
+        for item in page_items:
+            for comment in item["comments"]["nodes"]:
+                if any(group["createdAt"] for group in comment["reactionGroups"]):
+                    # print(f"for comment {comment['id']}: has reaction")
+                    reacted_comment_ids[comment["id"]] = comment
+                # if "reactionGroups" in comment:
+                comment.pop("reactionGroups", None)
+
+        # get comment reactions by querying comment nodes separately
+        comment_reactions = _get_comment_reaction(
+            list(reacted_comment_ids.keys()), access_token
+        )
+        # attach the reaction nodes where they should be
+        for comment in comment_reactions.values():
+            comment_id = comment["id"]
+            reacted_comment_ids[comment_id]["reactions"] = comment["reactions"]
+        yield map(_extract_nested_nodes, page_items)
+
+
+def _extract_top_connection(data: StrAny, node_type: str) -> StrAny:
+    assert isinstance(data, dict) and len(data) == 1, (
+        f"The data with list of {node_type} must be a dictionary and contain only one element"
+    )
+    data = next(iter(data.values()))
+    return data[node_type]  # type: ignore
+
+
+def _extract_nested_nodes(item: DictStrAny) -> DictStrAny:
+    """Recursively moves `nodes` and `totalCount` to reduce nesting."""
+    item["reactions_totalCount"] = item["reactions"].get("totalCount", 0)
+    item["reactions"] = item["reactions"]["nodes"]
+    comments = item["comments"]
+    item["comments_totalCount"] = item["comments"].get("totalCount", 0)
+    for comment in comments["nodes"]:
+        if "reactions" in comment:
+            comment["reactions_totalCount"] = comment["reactions"].get("totalCount", 0)
+            comment["reactions"] = comment["reactions"]["nodes"]
+    item["comments"] = comments["nodes"]
+    return item
+
+
+def _run_graphql_query(
+    access_token: str, query: str, variables: DictStrAny
+) -> Tuple[StrAny, StrAny]:
+    def _request() -> requests.Response:
+        r = requests.post(
+            GRAPHQL_API_BASE_URL,
+            json={"query": query, "variables": variables},
+            headers=_get_auth_header(access_token),
+        )
+        return r
+
+    data = _request().json()
+    if "errors" in data:
+        raise ValueError(data)
+    data = data["data"]
+    # pop rate limits
+    rate_limit = data.pop("rateLimit", {"cost": 0, "remaining": 0})
+    return data, rate_limit
+
+
+def _get_graphql_pages(
+    access_token: str, query: str, variables: DictStrAny, node_type: str, max_items: int
+) -> Iterator[List[DictStrAny]]:
+    items_count = 0
+    while True:
+        data, rate_limit = _run_graphql_query(access_token, query, variables)
+        top_connection = _extract_top_connection(data, node_type)
+        data_items = (
+            top_connection["nodes"]
+            if "nodes" in top_connection
+            else top_connection["edges"]
+        )
+        items_count += len(data_items)
+        print(
+            f"Got {len(data_items)}/{items_count} {node_type}s, query cost {rate_limit['cost']}, remaining credits: {rate_limit['remaining']}"
+        )
+        if data_items:
+            yield data_items
+        else:
+            return
+        # print(data["repository"][node_type]["pageInfo"]["endCursor"])
+        variables["page_after"] = _extract_top_connection(data, node_type)["pageInfo"][
+            "endCursor"
+        ]
+        if max_items and items_count >= max_items:
+            print(f"Max items limit reached: {items_count} >= {max_items}")
+            return
+
+
+def _get_comment_reaction(comment_ids: List[str], access_token: str) -> StrAny:
+    """Builds a query from a list of comment nodes and returns associated reactions."""
+    idx = 0
+    data: DictStrAny = {}
+    for page_chunk in chunks(comment_ids, 50):
+        subs = []
+        for comment_id in page_chunk:
+            subs.append(COMMENT_REACTIONS_QUERY % (idx, comment_id))
+            idx += 1
+        subs.append(RATE_LIMIT)
+        query = "{" + ",\n".join(subs) + "}"
+        # print(query)
+        page, rate_limit = _run_graphql_query(access_token, query, {})
+        print(
+            f"Got {len(page)} comments, query cost {rate_limit['cost']}, remaining credits: {rate_limit['remaining']}"
+        )
+        data.update(page)
+    return data