ingestr 0.13.9__tar.gz → 0.13.11__tar.gz

{ingestr-0.13.9 → ingestr-0.13.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.9
+Version: 0.13.11
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -18,42 +18,43 @@ Requires-Dist: asana==3.2.3
 Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
-Requires-Dist: confluent-kafka>=2.6.1
+Requires-Dist: confluent-kafka>=2.8.0
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dataclasses-json==0.6.7
-Requires-Dist: dlt==1.5.0
-Requires-Dist: duckdb-engine==0.13.5
-Requires-Dist: duckdb==1.1.3
+Requires-Dist: dlt==1.6.1
+Requires-Dist: duckdb-engine==0.15.0
+Requires-Dist: duckdb==1.2.0
 Requires-Dist: facebook-business==20.0.0
 Requires-Dist: flatten-json==0.1.14
 Requires-Dist: gcsfs==2024.10.0
 Requires-Dist: google-ads==25.1.0
-Requires-Dist: google-analytics-data==0.18.16
+Requires-Dist: google-analytics-data==0.18.17
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
-Requires-Dist: mysql-connector-python==9.1.0
+Requires-Dist: mysql-connector-python==9.2.0
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: psutil==6.1.1
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3
 Requires-Dist: pyarrow==18.1.0
-Requires-Dist: pyathena==3.9.0
-Requires-Dist: pymongo==4.10.1
+Requires-Dist: pyathena==3.12.2
+Requires-Dist: pymongo==4.11.1
 Requires-Dist: pymysql==1.1.1
 Requires-Dist: pyrate-limiter==3.7.0
 Requires-Dist: redshift-connector==2.1.5
 Requires-Dist: rich==13.9.4
 Requires-Dist: rudder-sdk-python==2.1.4
 Requires-Dist: s3fs==2024.10.0
+Requires-Dist: simple-salesforce==1.12.6
 Requires-Dist: snowflake-sqlalchemy==1.6.1
-Requires-Dist: sqlalchemy-bigquery==1.12.0
+Requires-Dist: sqlalchemy-bigquery==1.12.1
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: stripe==10.7.0
-Requires-Dist: tqdm==4.67.0
+Requires-Dist: tqdm==4.67.1
 Requires-Dist: typer==0.13.1
 Requires-Dist: types-requests==2.32.0.20240907
 Provides-Extra: odbc
@@ -161,6 +162,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>DynamoDB</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -247,11 +253,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
-    <tr>
-        <td>DynamoDB</td>
-        <td>✅</td>
-        <td>-</td>
-    </tr>
     <tr>
         <td>Facebook Ads</td>
         <td>✅</td>
@@ -301,12 +302,22 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Notion</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+    <tr>
+        <td>Personio</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Salesforce</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Shopify</td>
         <td>✅</td>

{ingestr-0.13.9 → ingestr-0.13.11}/README.md

@@ -97,6 +97,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>DynamoDB</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -183,11 +188,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
-    <tr>
-        <td>DynamoDB</td>
-        <td>✅</td>
-        <td>-</td>
-    </tr>
     <tr>
         <td>Facebook Ads</td>
         <td>✅</td>
@@ -237,12 +237,22 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Notion</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+    <tr>
+        <td>Personio</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Salesforce</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Shopify</td>
         <td>✅</td>

{ingestr-0.13.9 → ingestr-0.13.11}/docs/.vitepress/config.mjs

@@ -62,6 +62,7 @@ export default defineConfig({
               { text: "ClickHouse", link: "/supported-sources/clickhouse.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },
+              { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },
               {
                 text: "Google BigQuery",
                 link: "/supported-sources/bigquery.md",
@@ -100,7 +101,6 @@ export default defineConfig({
               { text: "Applovin Max", link: "/supported-sources/applovin_max.md"},
               { text: "Asana", link: "/supported-sources/asana.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
-              { text: "DynamoDB", link: "/supported-sources/dynamodb.md" },
               {
                 text: "Facebook Ads",
                 link: "/supported-sources/facebook-ads.md",
@@ -115,7 +115,9 @@ export default defineConfig({
               { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
+              { text: "Personio", link: "/supported-sources/personio.md" },
               { text: "S3", link: "/supported-sources/s3.md" },
+              { text: "Salesforce", link: "/supported-sources/salesforce.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },
               { text: "Slack", link: "/supported-sources/slack.md" },
               { text: "Stripe", link: "/supported-sources/stripe.md" },

ingestr-0.13.11/docs/supported-sources/personio.md

@@ -0,0 +1,51 @@
+# Personio
+[Personio](https://personio.de/) is a human resources management software that helps businesses
+streamline HR processes, including recruitment, employee data management, and payroll, in one
+platform.
+
+ingestr supports Personio as a source.
+
+## URI format
+
+The URI format for Personio is as follows:
+
+```plaintext
+personio://?client_id=<client-id>&client_secret=<client-secret>
+```
+
+URI parameters:
+
+- `client_id`: the client ID used for authentication with the Personio API
+- `client_secret`: the client secret used for authentication with the Personio API
+
+## Setting up a Personio Integration
+
+To grab personio credentials, please follow the guide [here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/personio#grab-credentials).
+
+Once you complete the guide, you should have a client ID and client secret. Let's say your `client_id` is id_123 and your `client_secret` is secret_123, here's a sample command that will copy the data from Personio into a DuckDB database:
+
+```bash
+ingestr ingest --source-uri 'personio://?client_id=id_123&client_secret=secret_123' \
+ --source-table 'employees' \
+ --dest-uri duckdb:///personio.duckdb \
+ --dest-table 'dest.employees'
+```
+
+<img alt="personio_img" src="../media/personio.png"/>
+
+
+Personio source allows ingesting the following resources into separate tables:
+- [employees](https://developer.personio.de/reference/get_company-employees) : Retrieves company employees details
+- [absences](https://developer.personio.de/reference/get_company-time-offs) : Retrieves absence periods for absences tracked in days
+- [absence_types](https://developer.personio.de/reference/get_company-time-off-types) : Retrieves list of various types of employee absences
+- [attendances](https://developer.personio.de/reference/get_company-attendances) : Retrieves attendance records for each employee
+- [projects](https://developer.personio.de/reference/get_company-attendances-projects) : Retrieves a list of all company projects
+- [document_categories](https://developer.personio.de/reference/get_company-document-categories) : Retrieves all document categories of the company
+- [employees_absences_balance](https://developer.personio.de/reference/get_company-employees-employee-id-absences-balance) : Retrieves the absence balance for a specific employee
+- [custom_reports_list](https://developer.personio.de/reference/listreports) : Retrieves metadata about existing custom reports in your Personio account, such as report name, report type, report date / timeframe.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.\
+
+
+> [!WARNING]
+> Personio does not support incremental loading for many endpoints, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.

ingestr-0.13.11/docs/supported-sources/salesforce.md

@@ -0,0 +1,68 @@
+# Salesforce
+[Salesforce](https://www.salesforce.com/) is a cloud-based customer relationship management (CRM) platform that helps businesses manage sales, customer interactions, and business processes. It provides tools for sales automation, customer service, marketing, analytics, and application development.
+
+Ingestr supports Salesforce as a source.
+
+## URI format
+
+The URI format for Salesforce is as follows:
+```
+salesforce://?username=<username>&password=<password>&token=<token>
+```
+
+URI parameters:
+- `username` is your Salesforce account username.
+- `password` is your Salesforce account password.
+- `token` is your Salesforce security token.
+
+You can obtain your security token by logging into your Salesforce account and navigating to the user settings under "Reset My Security Token."
+
+## Setting up a Salesforce Integration
+
+You can obtain an OAuth access token by setting up a connected app in Salesforce and using OAuth 2.0 authentication. For more information, see [Salesforce API Authentication](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm).
+
+## Example
+
+Let's say:
+* Your Salesforce username is `user`.
+* Your password is `password123`.
+* Your security token is `fake_token`.
+* You want to ingest `account` data from your salesforce account
+* You want to save this data in a duckdb database `sf.db` under the table `public.account`
+
+You can run the following command to achieve this:
+```sh
+ingestr ingest \
+  --source-uri "salesforce://?username=user&password=password123&token=fake_token" \
+  --source-table "account" \
+  --dest-uri "duckdb:///sf.db" \
+  --dest-table "public.account"
+```
+
+## Tables
+
+Salesforce source allows ingesting the following objects into separate tables:
+
+| **Table**                 | **Mode**   | **Description** |
+|---------------------------|-----------|----------------|
+| `user`                  | replace   | Refers to an individual who has access to a Salesforce org or instance. |
+| `user_role`             | replace   | A standard object that represents a role within the organization's hierarchy. |
+| `opportunity`          | merge     | Represents a sales opportunity for a specific account or contact. |
+| `opportunity_line_item` | merge     | Represents individual line items or products associated with an Opportunity. |
+| `opportunity_contact_role` | merge  | Represents the association between an Opportunity and a Contact. |
+| `account`               | merge     | Individual or organization that interacts with your business. |
+| `contact`               | replace   | An individual person associated with an account or organization. |
+| `lead`                  | replace   | Prospective customer/individual/org. that has shown interest in a company's products/services. |
+| `campaign`              | replace   | Marketing initiative or project designed to achieve specific goals, such as generating leads. |
+| `campaign_member`       | merge     | Association between a Contact or Lead and a Campaign. |
+| `product`               | replace   | For managing and organizing your product-related data within the Salesforce ecosystem. |
+| `pricebook`             | replace   | Used to manage product pricing and create price books. |
+| `pricebook_entry`      | replace   | Represents a specific price for a product in a price book. |
+| `task`                 | merge     | Used to track and manage various activities and tasks within the Salesforce platform. |
+| `event`                | merge     | Used to track and manage calendar-based events, such as meetings, appointments, or calls. |
+
+Use these as `--source-table` parameters in the `ingestr ingest` command.
+
+> [!WARNING]
+> Salesforce API limits may affect the frequency and volume of data ingestion. Incremental loading is supported for objects with the `SystemModstamp` field, but some objects may require full-refresh loads. This is indicated by `mode` in the tables above. Tables with mode `replace` don't support incremental loads, while the ones with `merge` do.
+

{ingestr-0.13.9 → ingestr-0.13.11}/ingestr/src/applovin/__init__.py

@@ -1,10 +1,10 @@
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta, timezone
 from enum import Enum
 from typing import Dict, List, Optional
-from requests import Response
 
 import dlt
 from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
+from requests import Response
 
 
 class InvalidCustomReportError(Exception):
@@ -13,9 +13,11 @@ class InvalidCustomReportError(Exception):
             "Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
         )
 
+
 class ClientError(Exception):
     pass
 
+
 TYPE_HINTS = {
     "application_is_hidden": {"data_type": "bool"},
     "average_cpa": {"data_type": "double"},
@@ -119,7 +121,6 @@ def applovin_source(
     end_date: Optional[str],
     custom: Optional[str],
 ):
-
     backfill = False
     if end_date is None:
         backfill = True
@@ -127,7 +128,7 @@ def applovin_source(
         # use the greatest of yesterday and start_date
         end_date = max(
             datetime.now(timezone.utc) - timedelta(days=1),
-            datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc)
+            datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc),
         ).strftime("%Y-%m-%d")
 
     config: RESTAPIConfig = {
@@ -157,7 +158,7 @@ def applovin_source(
                 "paginator": "single_page",
                 "response_actions": [
                     http_error_handler,
-                ]
+                ],
             },
         },
         "resources": [
@@ -177,8 +178,7 @@ def applovin_source(
                 "advertiser-probabilistic-report",
                 "probabilisticReport",
                 exclude(
-                    REPORT_SCHEMA[ReportType.ADVERTISER],
-                    PROBABILISTIC_REPORT_EXCLUDE
+                    REPORT_SCHEMA[ReportType.ADVERTISER], PROBABILISTIC_REPORT_EXCLUDE
                 ),
                 ReportType.ADVERTISER,
             ),
@@ -256,6 +256,7 @@ def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
 def build_type_hints(cols: List[str]) -> dict:
     return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}
 
+
 def http_error_handler(resp: Response):
     if not resp.ok:
         raise ClientError(f"HTTP Status {resp.status_code}: {resp.text}")

ingestr-0.13.11/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.11"

{ingestr-0.13.9 → ingestr-0.13.11}/ingestr/src/factory.py

@@ -42,12 +42,14 @@ from ingestr.src.sources import (
     MongoDbSource,
     NotionSource,
     S3Source,
+    SalesforceSource,
     ShopifySource,
     SlackSource,
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
     ZendeskSource,
+    PersonioSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -136,6 +138,8 @@ class SourceDestinationFactory:
         "linkedinads": LinkedInAdsSource,
         "applovin": AppLovinSource,
         "applovinmax": ApplovinMaxSource,
+        "salesforce": SalesforceSource,
+        "personio": PersonioSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,