ingestr 0.13.9__tar.gz → 0.13.10__tar.gz

{ingestr-0.13.9 → ingestr-0.13.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.9
+Version: 0.13.10
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -161,6 +161,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>DynamoDB</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -247,11 +252,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
-    <tr>
-        <td>DynamoDB</td>
-        <td>✅</td>
-        <td>-</td>
-    </tr>
     <tr>
         <td>Facebook Ads</td>
         <td>✅</td>
@@ -301,6 +301,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Notion</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+    <tr>
+        <td>Personio</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>

{ingestr-0.13.9 → ingestr-0.13.10}/README.md

@@ -97,6 +97,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>DynamoDB</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -183,11 +188,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
-    <tr>
-        <td>DynamoDB</td>
-        <td>✅</td>
-        <td>-</td>
-    </tr>
     <tr>
         <td>Facebook Ads</td>
         <td>✅</td>
@@ -237,6 +237,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Notion</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+    <tr>
+        <td>Personio</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>

{ingestr-0.13.9 → ingestr-0.13.10}/docs/.vitepress/config.mjs

@@ -62,6 +62,7 @@ export default defineConfig({
               { text: "ClickHouse", link: "/supported-sources/clickhouse.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },
+              { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },
               {
                 text: "Google BigQuery",
                 link: "/supported-sources/bigquery.md",
@@ -100,7 +101,6 @@ export default defineConfig({
               { text: "Applovin Max", link: "/supported-sources/applovin_max.md"},
               { text: "Asana", link: "/supported-sources/asana.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
-              { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },
               {
                 text: "Facebook Ads",
                 link: "/supported-sources/facebook-ads.md",
@@ -115,6 +115,7 @@ export default defineConfig({
               { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
+              { text: "Personio", link: "/supported-sources/personio.md" },
               { text: "S3", link: "/supported-sources/s3.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },
               { text: "Slack", link: "/supported-sources/slack.md" },

ingestr-0.13.10/docs/supported-sources/personio.md

@@ -0,0 +1,51 @@
+# Personio
+[Personio](https://personio.de/) is a human resources management software that helps businesses
+streamline HR processes, including recruitment, employee data management, and payroll, in one
+platform.
+
+ingestr supports Personio as a source.
+
+## URI format
+
+The URI format for Personio is as follows:
+
+```plaintext
+personio://?client_id=<client-id>&client_secret=<client-secret>
+```
+
+URI parameters:
+
+- `client_id`: the client ID used for authentication with the Personio API
+- `client_secret`: the client secret used for authentication with the Personio API
+
+## Setting up a Personio Integration
+
+To grab personio credentials, please follow the guide [here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/personio#grab-credentials).
+
+Once you complete the guide, you should have a client ID and client secret. Let's say your `client_id` is id_123 and your `client_secret` is secret_123, here's a sample command that will copy the data from Personio into a DuckDB database:
+
+```bash
+ingestr ingest --source-uri 'personio://?client_id=id_123&client_secret=secret_123' \
+ --source-table 'employees' \
+ --dest-uri duckdb:///personio.duckdb \
+ --dest-table 'dest.employees'
+```
+
+<img alt="personio_img" src="../media/personio.png"/>
+
+
+Personio source allows ingesting the following resources into separate tables:
+- [employees](https://developer.personio.de/reference/get_company-employees) : Retrieves company employees details
+- [absences](https://developer.personio.de/reference/get_company-time-offs) : Retrieves absence periods for absences tracked in days
+- [absence_types](https://developer.personio.de/reference/get_company-time-off-types) : Retrieves list of various types of employee absences
+- [attendances](https://developer.personio.de/reference/get_company-attendances) : Retrieves attendance records for each employee
+- [projects](https://developer.personio.de/reference/get_company-attendances-projects) : Retrieves a list of all company projects
+- [document_categories](https://developer.personio.de/reference/get_company-document-categories) : Retrieves all document categories of the company
+- [employees_absences_balance](https://developer.personio.de/reference/get_company-employees-employee-id-absences-balance) : Retrieves the absence balance for a specific employee
+- [custom_reports_list](https://developer.personio.de/reference/listreports) : Retrieves metadata about existing custom reports in your Personio account, such as report name, report type, report date / timeframe.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.\
+
+
+> [!WARNING]
+> Personio does not support incremental loading for many endpoints, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.

ingestr-0.13.10/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.10"

{ingestr-0.13.9 → ingestr-0.13.10}/ingestr/src/factory.py

@@ -48,6 +48,7 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     ZendeskSource,
+    PersonioSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -136,6 +137,7 @@ class SourceDestinationFactory:
         "linkedinads": LinkedInAdsSource,
         "applovin": AppLovinSource,
         "applovinmax": ApplovinMaxSource,
+        "personio": PersonioSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr-0.13.10/ingestr/src/personio/__init__.py

@@ -0,0 +1,331 @@
+"""Fetches Personio Employees, Absences, Attendances."""
+
+from typing import Iterable, Optional
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+
+from .helpers import PersonioAPI
+
+
+@dlt.source(name="personio", max_table_nesting=0)
+def personio_source(
+    start_date: TAnyDateTime,
+    end_date: Optional[TAnyDateTime] = None,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+    items_per_page: int = 200,
+) -> Iterable[DltResource]:
+    """
+    The source for the Personio pipeline. Available resources are employees, absences, and attendances.
+
+    Args:
+        client_id: The client ID of your app.
+        client_secret: The client secret of your app.
+        items_per_page: The max number of items to fetch per page. Defaults to 200.
+    Returns:
+        Iterable: A list of DltResource objects representing the data resources.
+    """
+
+    client = PersonioAPI(client_id, client_secret)
+
+    @dlt.resource(primary_key="id", write_disposition="merge", max_table_nesting=0)
+    def employees(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "last_modified_at", initial_value=None, allow_external_schedulers=True
+        ),
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for employees, supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'last_modified_at' value.
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of employees.
+        """
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            """Converts an employee item."""
+            attributes = item.get("attributes", {})
+            output = {}
+            for value in attributes.values():
+                name = value["universal_id"]
+                if not name:
+                    label: str = value["label"].replace(" ", "_")
+                    name = label.lower()
+
+                if value["type"] == "date" and value["value"]:
+                    output[name] = ensure_pendulum_datetime(value["value"])
+                else:
+                    output[name] = value["value"]
+            return output
+
+        if updated_at.last_value:
+            last_value = updated_at.last_value.format("YYYY-MM-DDTHH:mm:ss")
+        else:
+            last_value = None
+
+        params = {"limit": items_per_page, "updated_since": last_value}
+
+        pages = client.get_pages("company/employees", params=params)
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="replace", max_table_nesting=0)
+    def absence_types(items_per_page: int = items_per_page) -> Iterable[TDataItem]:
+        """
+        The resource for absence types (time-off-types), supports pagination.
+
+        Args:
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of absences.
+        """
+
+        pages = client.get_pages(
+            "company/time-off-types", params={"limit": items_per_page}
+        )
+
+        for page in pages:
+            yield [item.get("attributes", {}) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="merge", max_table_nesting=0)
+    def absences(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at", initial_value=None, allow_external_schedulers=True
+        ),
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for absence (time-offs), supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'updated_at' value.
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of absences.
+        """
+        if updated_at.last_value:
+            updated_iso = updated_at.last_value.format("YYYY-MM-DDTHH:mm:ss")
+        else:
+            updated_iso = None
+
+        params = {
+            "limit": items_per_page,
+            "updated_since": updated_iso,
+        }
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            output = item.get("attributes", {})
+            output["created_at"] = ensure_pendulum_datetime(output["created_at"])
+            output["updated_at"] = ensure_pendulum_datetime(output["updated_at"])
+            return output
+
+        pages = client.get_pages(
+            "company/time-offs",
+            params=params,
+            offset_by_page=True,
+        )
+
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="merge", max_table_nesting=0)
+    def attendances(
+        start_date: TAnyDateTime = start_date,
+        end_date: Optional[TAnyDateTime] = end_date,
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at", initial_value=None, allow_external_schedulers=True
+        ),
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for attendances, supports incremental loading and pagination.
+
+        Args:
+            start_date: The start date to fetch attendances from.
+            end_date: The end date to fetch attendances from. Defaults to now.
+            updated_at: The saved state of the last 'updated_at' value.
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of attendances.
+        """
+        
+        end_date = end_date or pendulum.now()
+        if updated_at.last_value:
+            updated_iso = updated_at.last_value.format("YYYY-MM-DDTHH:mm:ss")
+        else:
+            updated_iso = None
+
+        params = {
+            "limit": items_per_page,
+            "start_date": ensure_pendulum_datetime(start_date).to_date_string(),
+            "end_date": ensure_pendulum_datetime(end_date).to_date_string(),
+            "updated_from": updated_iso,
+            "includePending": True,
+        }
+        pages = client.get_pages(
+            "company/attendances",
+            params=params,
+        )
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            """Converts an attendance item."""
+            output = dict(id=item["id"], **item.get("attributes"))
+            output["date"] = ensure_pendulum_datetime(output["date"]).date()
+            output["updated_at"] = ensure_pendulum_datetime(output["updated_at"])
+            return output
+
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="replace", max_table_nesting=0)
+    def projects() -> Iterable[TDataItem]:
+        """
+        The resource for projects.
+
+        Returns:
+            Iterable: A generator of projects.
+        """
+
+        pages = client.get_pages("company/attendances/projects")
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            """Converts an attendance item."""
+            output = dict(id=item["id"], **item.get("attributes"))
+            output["created_at"] = ensure_pendulum_datetime(output["created_at"])
+            output["updated_at"] = ensure_pendulum_datetime(output["updated_at"])
+            return output
+
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="replace", max_table_nesting=0)
+    def document_categories() -> Iterable[TDataItem]:
+        """
+        The resource for document_categories.
+
+        Returns:
+            Iterable: A generator of document_categories.
+        """
+
+        pages = client.get_pages("company/document-categories")
+
+        def convert_item(item: TDataItem) -> TDataItem:
+            """Converts an document_categories item."""
+            output = dict(id=item["id"], **item.get("attributes"))
+            return output
+
+        for page in pages:
+            yield [convert_item(item) for item in page]
+
+    @dlt.resource(primary_key="id", write_disposition="replace", max_table_nesting=0)
+    def custom_reports_list() -> Iterable[TDataItem]:
+        """
+        The resource for custom_reports.
+
+        Returns:
+            Iterable: A generator of custom_reports.
+        """
+
+        pages = client.get_pages("company/custom-reports/reports")
+
+        for page in pages:
+            yield [item.get("attributes", {}) for item in page]
+
+    @dlt.transformer(
+        data_from=employees,
+        write_disposition="merge",
+        primary_key=["employee_id", "id"],
+    )
+    @dlt.defer
+    def employees_absences_balance(employees_item: TDataItem) -> Iterable[TDataItem]:
+        """
+        The transformer for employees_absences_balance.
+
+        Args:
+            employees_item: The employee data.
+
+        Returns:
+            Iterable: A generator of employees_absences_balance for each employee.
+        """
+        for employee in employees_item:
+            employee_id = employee["id"]
+            pages = client.get_pages(
+                f"company/employees/{employee_id}/absences/balance",
+            )
+
+            for page in pages:
+                yield [dict(employee_id=employee_id, **i) for i in page]
+
+    @dlt.transformer(
+        data_from=custom_reports_list,
+        write_disposition="merge",
+        primary_key=["report_id", "item_id"],
+    )
+    @dlt.defer
+    def custom_reports(
+        custom_reports_item: TDataItem, items_per_page: int = items_per_page
+    ) -> Iterable[TDataItem]:
+        """
+        The transformer for custom reports, supports pagination.
+
+        Args:
+            custom_reports_item: The custom_report data.
+            items_per_page: The max number of items to fetch per page. Defaults to 200.
+
+        Returns:
+            Iterable: A generator of employees_absences_balance for each employee.
+        """
+
+        def convert_item(item: TDataItem, report_id: str) -> TDataItem:
+            """Converts an employee item."""
+            attributes = item.pop("attributes")
+            output = dict(report_id=report_id, item_id=list(item.values())[0])
+            for value in attributes:
+                name = value["attribute_id"]
+                if value["data_type"] == "date" and value["value"]:
+                    output[name] = ensure_pendulum_datetime(value["value"])
+                else:
+                    output[name] = value["value"]
+            return output
+
+        for custom_report in custom_reports_item:
+            report_id = custom_report["id"]
+            pages = client.get_pages(
+                f"company/custom-reports/reports/{report_id}",
+                params={"limit": items_per_page},
+                offset_by_page=True,
+            )
+
+            for page in pages:
+                for report in page:
+                    report_items = report.get("attributes", {}).get("items", [])
+                    yield [convert_item(item, report_id) for item in report_items]
+
+    return (
+        employees,
+        absence_types,
+        absences,
+        attendances,
+        projects,
+        document_categories,
+        employees_absences_balance,
+        custom_reports_list,
+        custom_reports,
+    )

ingestr-0.13.10/ingestr/src/personio/helpers.py

@@ -0,0 +1,85 @@
+"""Personio source helpers"""
+from typing import Any, Iterable, Optional
+from urllib.parse import urljoin
+
+from dlt.common.typing import Dict, TDataItems
+from dlt.sources.helpers import requests
+
+
+class PersonioAPI:
+    """A Personio API client."""
+
+    base_url = "https://api.personio.de/v1/"
+
+    def __init__(self, client_id: str, client_secret: str) -> None:
+        """
+        Args:
+            client_id: The client ID of your app.
+            client_secret: The client secret of your app.
+        """
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.access_token = self.get_token()
+
+    def get_token(self) -> str:
+        """Get an access token from Personio.
+
+        Returns:
+            The access token.
+        """
+        headers = {"Content-Type": "application/json", "Accept": "application/json"}
+        data = {"client_id": self.client_id, "client_secret": self.client_secret}
+        url = urljoin(self.base_url, "auth")
+        response = requests.request("POST", url, headers=headers, json=data)
+        json_response = response.json()
+        token: str = json_response["data"]["token"]
+        return token
+
+    def get_pages(
+        self,
+        resource: str,
+        params: Optional[Dict[str, Any]] = None,
+        offset_by_page: bool = False,
+    ) -> Iterable[TDataItems]:
+        """Get all pages from Personio using requests.
+
+        Args:
+            resource: The resource to get pages for (e.g. employees, absences, attendances).
+            params: The parameters for the resource.
+            offset_by_page (bool): If True, offset increases by 1 per page; else, increases by page_size.
+
+        Yields:
+            List of data items from the page
+        """
+        params = params or {}
+        headers = {"Authorization": f"Bearer {self.access_token}"}
+        params.update({"offset": int(offset_by_page), "page": int(offset_by_page)})
+        url = urljoin(self.base_url, resource)
+        starts_from_zero = False
+        while True:
+            response = requests.get(url, headers=headers, params=params)
+            json_response = response.json()
+            # Get an item list from the page
+            yield json_response["data"]
+
+            metadata = json_response.get("metadata")
+            if not metadata:
+                break
+
+            total_pages = metadata.get("total_pages")
+            current_page = metadata.get("current_page")
+            if current_page == 0:
+                starts_from_zero = True
+
+            if (
+                current_page >= (total_pages - int(starts_from_zero))
+                or not json_response["data"]
+            ):
+                break
+
+            if offset_by_page:
+                params["offset"] += 1
+                params["page"] += 1
+            else:
+                params["offset"] += params["limit"]
+                params["page"] += 1

{ingestr-0.13.9 → ingestr-0.13.10}/ingestr/src/sources.py

@@ -83,6 +83,7 @@ from ingestr.src.linkedin_ads.dimension_time_enum import (
 )
 from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
+from ingestr.src.personio import personio_source
 from ingestr.src.shopify import shopify_source
 from ingestr.src.slack import slack_source
 from ingestr.src.sql_database.callbacks import (
@@ -1833,3 +1834,50 @@ class ApplovinMaxSource:
             api_key=api_key[0],
             application=application[0],
         ).with_resources(table)
+
+
+class PersonioSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    # applovin://?client_id=123&client_secret=123
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        client_id = params.get("client_id")
+        client_secret = params.get("client_secret")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        interval_start_date = (
+            interval_start if interval_start is not None else "2018-01-01"
+        )
+
+        interval_end_date = (
+            interval_end.strftime("%Y-%m-%d") if interval_end is not None else None
+        )
+
+        if client_id is None:
+            raise MissingValueError("client_id", "Personio")
+        if client_secret is None:
+            raise MissingValueError("client_secret", "Personio")
+        if table not in [
+            "employees",
+            "absences",
+            "absence_types",
+            "attendances",
+            "projects",
+            "document_categories",
+            "employees_absences_balance",
+            "custom_reports_list",
+        ]:
+            raise UnsupportedResourceError(table, "Personio")
+        
+        return personio_source(
+            client_id=client_id[0],
+            client_secret=client_secret[0],
+            start_date=interval_start_date,
+            end_date=interval_end_date,
+        ).with_resources(table)

ingestr-0.13.9/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.9"