ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/partition.py

@@ -0,0 +1,32 @@
+from typing import Dict
+
+from dlt.common.schema.typing import TColumnSchema
+from dlt.sources import DltResource, DltSource
+
+import ingestr.src.resource as resource
+
+
+def apply_athena_hints(
+    source: DltSource | DltResource,
+    partition_column: str,
+    additional_hints: Dict[str, TColumnSchema] = {},
+) -> None:
+    from dlt.destinations.adapters import athena_adapter, athena_partition
+
+    def _apply_partition_hint(resource: DltResource) -> None:
+        columns = resource.columns if resource.columns else {}
+
+        partition_hint = (
+            columns.get(partition_column)  # type: ignore
+            or additional_hints.get(partition_column)
+        )
+
+        athena_adapter(
+            resource,
+            athena_partition.day(partition_column)
+            if partition_hint
+            and partition_hint.get("data_type") in ("timestamp", "date")
+            else partition_column,
+        )
+
+    resource.for_each(source, _apply_partition_hint)

ingestr/src/personio/__init__.py

@@ -0,0 +1,331 @@
+"""Fetches Personio Employees, Absences, Attendances."""
+
+from typing import Iterable, Optional
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+
+from .helpers import PersonioAPI
+
+
+@dlt.source(name="personio", max_table_nesting=0)
+def personio_source(
+    start_date: TAnyDateTime,
+    end_date: Optional[TAnyDateTime] = None,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+    items_per_page: int = 200,
+) -> Iterable[DltResource]:
+    """
+    The source for the Personio pipeline. Available resources are employees, absences, and attendances.
+
+    Args:
+        client_id: The client ID of your app.
+        client_secret: The client secret of your app.
+        items_per_page: The max number of items to fetch per page. Defaults to 200.
+    Returns:
+        Iterable: A list of DltResource objects representing the data resources.
+    """
+
+    client = PersonioAPI(client_id, client_secret)
+
+    @dlt.resource(primary_key="id", write_disposition="merge", max_table_nesting=0)
+    def employees(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "last_modified_at", initial_value=None, allow_external_schedulers=True
+        ),
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for employees, supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'last_modified_at' value.
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of employees.
+        """
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            """Converts an employee item."""
+            attributes = item.get("attributes", {})
+            output = {}
+            for value in attributes.values():
+                name = value["universal_id"]
+                if not name:
+                    label: str = value["label"].replace(" ", "_")
+                    name = label.lower()
+
+                if value["type"] == "date" and value["value"]:
+                    output[name] = ensure_pendulum_datetime(value["value"])
+                else:
+                    output[name] = value["value"]
+            return output
+
+        if updated_at.last_value:
+            last_value = updated_at.last_value.format("YYYY-MM-DDTHH:mm:ss")
+        else:
+            last_value = None
+
+        params = {"limit": items_per_page, "updated_since": last_value}
+
+        pages = client.get_pages("company/employees", params=params)
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="replace", max_table_nesting=0)
+    def absence_types(items_per_page: int = items_per_page) -> Iterable[TDataItem]:
+        """
+        The resource for absence types (time-off-types), supports pagination.
+
+        Args:
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of absences.
+        """
+
+        pages = client.get_pages(
+            "company/time-off-types", params={"limit": items_per_page}
+        )
+
+        for page in pages:
+            yield [item.get("attributes", {}) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="merge", max_table_nesting=0)
+    def absences(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at", initial_value=None, allow_external_schedulers=True
+        ),
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for absence (time-offs), supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'updated_at' value.
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of absences.
+        """
+        if updated_at.last_value:
+            updated_iso = updated_at.last_value.format("YYYY-MM-DDTHH:mm:ss")
+        else:
+            updated_iso = None
+
+        params = {
+            "limit": items_per_page,
+            "updated_since": updated_iso,
+        }
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            output = item.get("attributes", {})
+            output["created_at"] = ensure_pendulum_datetime(output["created_at"])
+            output["updated_at"] = ensure_pendulum_datetime(output["updated_at"])
+            return output
+
+        pages = client.get_pages(
+            "company/time-offs",
+            params=params,
+            offset_by_page=True,
+        )
+
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="merge", max_table_nesting=0)
+    def attendances(
+        start_date: TAnyDateTime = start_date,
+        end_date: Optional[TAnyDateTime] = end_date,
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at", initial_value=None, allow_external_schedulers=True
+        ),
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for attendances, supports incremental loading and pagination.
+
+        Args:
+            start_date: The start date to fetch attendances from.
+            end_date: The end date to fetch attendances from. Defaults to now.
+            updated_at: The saved state of the last 'updated_at' value.
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of attendances.
+        """
+
+        end_date = end_date or pendulum.now()
+        if updated_at.last_value:
+            updated_iso = updated_at.last_value.format("YYYY-MM-DDTHH:mm:ss")
+        else:
+            updated_iso = None
+
+        params = {
+            "limit": items_per_page,
+            "start_date": ensure_pendulum_datetime(start_date).to_date_string(),
+            "end_date": ensure_pendulum_datetime(end_date).to_date_string(),
+            "updated_from": updated_iso,
+            "includePending": True,
+        }
+        pages = client.get_pages(
+            "company/attendances",
+            params=params,
+        )
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            """Converts an attendance item."""
+            output = dict(id=item["id"], **item.get("attributes"))
+            output["date"] = ensure_pendulum_datetime(output["date"]).date()
+            output["updated_at"] = ensure_pendulum_datetime(output["updated_at"])
+            return output
+
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="replace", max_table_nesting=0)
+    def projects() -> Iterable[TDataItem]:
+        """
+        The resource for projects.
+
+        Returns:
+            Iterable: A generator of projects.
+        """
+
+        pages = client.get_pages("company/attendances/projects")
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            """Converts an attendance item."""
+            output = dict(id=item["id"], **item.get("attributes"))
+            output["created_at"] = ensure_pendulum_datetime(output["created_at"])
+            output["updated_at"] = ensure_pendulum_datetime(output["updated_at"])
+            return output
+
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="replace", max_table_nesting=0)
+    def document_categories() -> Iterable[TDataItem]:
+        """
+        The resource for document_categories.
+
+        Returns:
+            Iterable: A generator of document_categories.
+        """
+
+        pages = client.get_pages("company/document-categories")
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            """Converts an document_categories item."""
+            output = dict(id=item["id"], **item.get("attributes"))
+            return output
+
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="replace", max_table_nesting=0)
+    def custom_reports_list() -> Iterable[TDataItem]:
+        """
+        The resource for custom_reports.
+
+        Returns:
+            Iterable: A generator of custom_reports.
+        """
+
+        pages = client.get_pages("company/custom-reports/reports")
+
+        for page in pages:
+            yield [item.get("attributes", {}) for item in page]
+
+    @dlt.transformer(
+        data_from=employees,
+        write_disposition="merge",
+        primary_key=["employee_id", "id"],
+    )
+    @dlt.defer
+    def employees_absences_balance(employees_item: TDataItem) -> Iterable[TDataItem]:
+        """
+        The transformer for employees_absences_balance.
+
+        Args:
+            employees_item: The employee data.
+
+        Returns:
+            Iterable: A generator of employees_absences_balance for each employee.
+        """
+        for employee in employees_item:
+            employee_id = employee["id"]
+            pages = client.get_pages(
+                f"company/employees/{employee_id}/absences/balance",
+            )
+
+            for page in pages:
+                yield [dict(employee_id=employee_id, **i) for i in page]
+
+    @dlt.transformer(
+        data_from=custom_reports_list,
+        write_disposition="merge",
+        primary_key=["report_id", "item_id"],
+    )
+    @dlt.defer
+    def custom_reports(
+        custom_reports_item: TDataItem, items_per_page: int = items_per_page
+    ) -> Iterable[TDataItem]:
+        """
+        The transformer for custom reports, supports pagination.
+
+        Args:
+            custom_reports_item: The custom_report data.
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of employees_absences_balance for each employee.
+        """
+
+        def convert_item(item: TDataItem, report_id: str) -> TDataItem:
+            """Converts an employee item."""
+            attributes = item.pop("attributes")
+            output = dict(report_id=report_id, item_id=list(item.values())[0])
+            for value in attributes:
+                name = value["attribute_id"]
+                if value["data_type"] == "date" and value["value"]:
+                    output[name] = ensure_pendulum_datetime(value["value"])
+                else:
+                    output[name] = value["value"]
+            return output
+
+        for custom_report in custom_reports_item:
+            report_id = custom_report["id"]
+            pages = client.get_pages(
+                f"company/custom-reports/reports/{report_id}",
+                params={"limit": items_per_page},
+                offset_by_page=True,
+            )
+
+            for page in pages:
+                for report in page:
+                    report_items = report.get("attributes", {}).get("items", [])
+                    yield [convert_item(item, report_id) for item in report_items]
+
+    return (
+        employees,
+        absence_types,
+        absences,
+        attendances,
+        projects,
+        document_categories,
+        employees_absences_balance,
+        custom_reports_list,
+        custom_reports,
+    )

ingestr/src/personio/helpers.py

@@ -0,0 +1,86 @@
+"""Personio source helpers"""
+
+from typing import Any, Iterable, Optional
+from urllib.parse import urljoin
+
+from dlt.common.typing import Dict, TDataItems
+from dlt.sources.helpers import requests
+
+
+class PersonioAPI:
+    """A Personio API client."""
+
+    base_url = "https://api.personio.de/v1/"
+
+    def __init__(self, client_id: str, client_secret: str) -> None:
+        """
+        Args:
+            client_id: The client ID of your app.
+            client_secret: The client secret of your app.
+        """
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.access_token = self.get_token()
+
+    def get_token(self) -> str:
+        """Get an access token from Personio.
+
+        Returns:
+            The access token.
+        """
+        headers = {"Content-Type": "application/json", "Accept": "application/json"}
+        data = {"client_id": self.client_id, "client_secret": self.client_secret}
+        url = urljoin(self.base_url, "auth")
+        response = requests.request("POST", url, headers=headers, json=data)
+        json_response = response.json()
+        token: str = json_response["data"]["token"]
+        return token
+
+    def get_pages(
+        self,
+        resource: str,
+        params: Optional[Dict[str, Any]] = None,
+        offset_by_page: bool = False,
+    ) -> Iterable[TDataItems]:
+        """Get all pages from Personio using requests.
+
+        Args:
+            resource: The resource to get pages for (e.g. employees, absences, attendances).
+            params: The parameters for the resource.
+            offset_by_page (bool): If True, offset increases by 1 per page; else, increases by page_size.
+
+        Yields:
+            List of data items from the page
+        """
+        params = params or {}
+        headers = {"Authorization": f"Bearer {self.access_token}"}
+        params.update({"offset": int(offset_by_page), "page": int(offset_by_page)})
+        url = urljoin(self.base_url, resource)
+        starts_from_zero = False
+        while True:
+            response = requests.get(url, headers=headers, params=params)
+            json_response = response.json()
+            # Get an item list from the page
+            yield json_response["data"]
+
+            metadata = json_response.get("metadata")
+            if not metadata:
+                break
+
+            total_pages = metadata.get("total_pages")
+            current_page = metadata.get("current_page")
+            if current_page == 0:
+                starts_from_zero = True
+
+            if (
+                current_page >= (total_pages - int(starts_from_zero))
+                or not json_response["data"]
+            ):
+                break
+
+            if offset_by_page:
+                params["offset"] += 1
+                params["page"] += 1
+            else:
+                params["offset"] += params["limit"]
+                params["page"] += 1

ingestr/src/phantombuster/__init__.py

@@ -0,0 +1,65 @@
+from typing import Iterable, Optional
+
+import dlt
+import pendulum
+import requests
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+
+from ingestr.src.phantombuster.client import PhantombusterClient
+
+
+def retry_on_limit(
+    response: Optional[requests.Response], exception: Optional[BaseException]
+) -> bool:
+    if response is not None and response.status_code == 429:
+        return True
+    return False
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+
+
+@dlt.source(max_table_nesting=0)
+def phantombuster_source(
+    api_key: str, agent_id: str, start_date: TAnyDateTime, end_date: TAnyDateTime | None
+) -> Iterable[DltResource]:
+    client = PhantombusterClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key="container_id",
+        columns={
+            "partition_dt": {"data_type": "date", "partition": True},
+        },
+    )
+    def completed_phantoms(
+        dateTime=(
+            dlt.sources.incremental(
+                "ended_at",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterable[TDataItem]:
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+
+        yield client.fetch_containers_result(
+            create_client(), agent_id, start_date=start_dt, end_date=end_dt
+        )
+
+    return completed_phantoms

ingestr/src/phantombuster/client.py

@@ -0,0 +1,87 @@
+from typing import Union
+
+import pendulum
+import requests
+
+
+class PhantombusterClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+
+    def _get_headers(self):
+        return {
+            "X-Phantombuster-Key-1": self.api_key,
+            "accept": "application/json",
+        }
+
+    def fetch_containers_result(
+        self,
+        session: requests.Session,
+        agent_id: str,
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
+    ):
+        url = "https://api.phantombuster.com/api/v2/containers/fetch-all/"
+        before_ended_at = None
+        limit = 100
+
+        started_at = start_date.int_timestamp * 1000 + int(
+            start_date.microsecond / 1000
+        )
+        ended_at = end_date.int_timestamp * 1000 + int(end_date.microsecond / 1000)
+
+        while True:
+            params: dict[str, Union[str, int, float, bytes, None]] = {
+                "agentId": agent_id,
+                "limit": limit,
+                "mode": "finalized",
+            }
+
+            if before_ended_at:
+                params["beforeEndedAt"] = before_ended_at
+
+            response = session.get(url=url, headers=self._get_headers(), params=params)
+            data = response.json()
+            containers = data.get("containers", [])
+
+            for container in containers:
+                container_ended_at = container.get("endedAt")
+
+                if before_ended_at is None or before_ended_at > container_ended_at:
+                    before_ended_at = container_ended_at
+
+                if container_ended_at < started_at or container_ended_at > ended_at:
+                    continue
+
+                try:
+                    result = self.fetch_result_object(session, container["id"])
+                    partition_dt = pendulum.from_timestamp(
+                        container_ended_at / 1000, tz="UTC"
+                    ).date()
+                    container_ended_at_datetime = pendulum.from_timestamp(
+                        container_ended_at / 1000, tz="UTC"
+                    )
+                    row = {
+                        "container_id": container["id"],
+                        "container": container,
+                        "result": result,
+                        "partition_dt": partition_dt,
+                        "ended_at": container_ended_at_datetime,
+                    }
+                    yield row
+
+                except requests.RequestException as e:
+                    print(f"Error fetching result for container {container['id']}: {e}")
+
+            if data["maxLimitReached"] is False:
+                break
+
+    def fetch_result_object(self, session: requests.Session, container_id: str):
+        result_url = (
+            "https://api.phantombuster.com/api/v2/containers/fetch-result-object"
+        )
+        params = {"id": container_id}
+        response = session.get(result_url, headers=self._get_headers(), params=params)
+        response.raise_for_status()
+
+        return response.json()

ingestr/src/pinterest/__init__.py

@@ -0,0 +1,82 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers import requests
+
+
+@dlt.source(name="pinterest", max_table_nesting=0)
+def pinterest_source(
+    start_date: pendulum.DateTime,
+    access_token: str,
+    page_size: int = 200,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    session = requests.Session()
+    session.headers.update({"Authorization": f"Bearer {access_token}"})
+    base_url = "https://api.pinterest.com/v5"
+
+    def fetch_data(
+        endpoint: str,
+        start_dt: pendulum.DateTime,
+        end_dt: pendulum.DateTime,
+    ) -> Iterable[TDataItem]:
+        url = f"{base_url}/{endpoint}"
+        params = {"page_size": page_size}
+        bookmark = None
+        while True:
+            if bookmark:
+                params["bookmark"] = bookmark
+
+            resp = session.get(url, params=params)
+            resp.raise_for_status()
+            data = resp.json()
+            items = data.get("items") or []
+
+            for item in items:
+                item_created = ensure_pendulum_datetime(item["created_at"])
+                if item_created <= start_dt:
+                    continue
+                if item_created > end_dt:
+                    continue
+                item["created_at"] = item_created
+                yield item
+
+            bookmark = data.get("bookmark")
+            if not bookmark:
+                break
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def pins(
+        datetime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+        ),
+    ) -> Iterable[TDataItem]:
+        _start_date = datetime.last_value or start_date
+        if end_date is None:
+            _end_date = pendulum.now("UTC")
+        else:
+            _end_date = datetime.end_value
+        yield from fetch_data("pins", _start_date, _end_date)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def boards(
+        datetime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+        ),
+    ) -> Iterable[TDataItem]:
+        _start_date = datetime.last_value or start_date
+        if end_date is None:
+            _end_date = pendulum.now("UTC")
+        else:
+            _end_date = datetime.end_value
+        yield from fetch_data("boards", _start_date, _end_date)
+
+    return pins, boards