omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/salesforce/__init__.py

@@ -0,0 +1,170 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Iterable
+
+import dlt
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource, incremental
+from simple_salesforce import Salesforce
+
+from .helpers import get_records
+
+
+@dlt.source(name="salesforce")
+def salesforce_source(
+    username: str,
+    password: str,
+    token: str,
+    domain: str,
+    custom_object: str = None,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from Salesforce using the Salesforce API.
+
+    Args:
+        username (str): The username for authentication.
+        password (str): The password for authentication.
+        token (str): The security token for authentication.
+
+    Yields:
+        DltResource: Data resources from Salesforce.
+    """
+
+    client = Salesforce(username, password, token, domain=domain)
+
+    # define resources
+    @dlt.resource(write_disposition="replace")
+    def user() -> Iterable[TDataItem]:
+        yield get_records(client, "User")
+
+    @dlt.resource(write_disposition="replace")
+    def user_role() -> Iterable[TDataItem]:
+        yield get_records(client, "UserRole")
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def opportunity(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client, "Opportunity", last_timestamp.last_value, "SystemModstamp"
+        )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def opportunity_line_item(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client, "OpportunityLineItem", last_timestamp.last_value, "SystemModstamp"
+        )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def opportunity_contact_role(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client,
+            "OpportunityContactRole",
+            last_timestamp.last_value,
+            "SystemModstamp",
+        )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def account(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "LastModifiedDate", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client, "Account", last_timestamp.last_value, "LastModifiedDate"
+        )
+
+    @dlt.resource(write_disposition="replace")
+    def contact() -> Iterable[TDataItem]:
+        yield get_records(client, "Contact")
+
+    @dlt.resource(write_disposition="replace")
+    def lead() -> Iterable[TDataItem]:
+        yield get_records(client, "Lead")
+
+    @dlt.resource(write_disposition="replace")
+    def campaign() -> Iterable[TDataItem]:
+        yield get_records(client, "Campaign")
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def campaign_member(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client, "CampaignMember", last_timestamp.last_value, "SystemModstamp"
+        )
+
+    @dlt.resource(write_disposition="replace")
+    def product() -> Iterable[TDataItem]:
+        yield get_records(client, "Product2")
+
+    @dlt.resource(write_disposition="replace")
+    def pricebook() -> Iterable[TDataItem]:
+        yield get_records(client, "Pricebook2")
+
+    @dlt.resource(write_disposition="replace")
+    def pricebook_entry() -> Iterable[TDataItem]:
+        yield get_records(client, "PricebookEntry")
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def task(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(client, "Task", last_timestamp.last_value, "SystemModstamp")
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def event(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(client, "Event", last_timestamp.last_value, "SystemModstamp")
+
+    @dlt.resource(write_disposition="replace")
+    def custom() -> Iterable[TDataItem]:
+        yield get_records(client, custom_object)
+
+    return (
+        user,
+        user_role,
+        opportunity,
+        opportunity_line_item,
+        opportunity_contact_role,
+        account,
+        contact,
+        lead,
+        campaign,
+        campaign_member,
+        product,
+        pricebook,
+        pricebook_entry,
+        task,
+        event,
+        custom,
+    )

omniload/src/salesforce/helpers.py

@@ -0,0 +1,78 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Salesforce source helpers"""
+
+from typing import Iterable, Optional
+
+import pendulum
+from dlt.common.typing import TDataItem
+from simple_salesforce import Salesforce
+
+
+def get_records(
+    sf: Salesforce,
+    sobject: str,
+    last_state: Optional[str] = None,
+    replication_key: Optional[str] = None,
+) -> Iterable[TDataItem]:
+    """
+    Retrieves records from Salesforce for a specified sObject.
+
+    Args:
+        sf (Salesforce): An instance of the Salesforce API client.
+        sobject (str): The name of the sObject to retrieve records from.
+        last_state (str, optional): The last known state for incremental loading. Defaults to None.
+        replication_key (str, optional): The replication key for incremental loading. Defaults to None.
+
+    Yields:
+        Dict[TDataItem]: A dictionary representing a record from the Salesforce sObject.
+    """
+
+    # Get all fields for the sobject
+    desc = getattr(sf, sobject).describe()
+    # Salesforce returns compound fields as separate fields, so we need to filter them out
+    compound_fields = {
+        f["compoundFieldName"]
+        for f in desc["fields"]
+        if f["compoundFieldName"] is not None
+    } - {"Name"}
+    # Salesforce returns datetime fields as timestamps, so we need to convert them
+    date_fields = {
+        f["name"] for f in desc["fields"] if f["type"] in ("datetime",) and f["name"]
+    }
+    # If no fields are specified, use all fields except compound fields
+    fields = [f["name"] for f in desc["fields"] if f["name"] not in compound_fields]
+
+    # Generate a predicate to filter records by the replication key
+    predicate, order_by, n_records = "", "", 0
+    if replication_key:
+        if last_state:
+            predicate = f"WHERE {replication_key} > {last_state}"
+        order_by = f"ORDER BY {replication_key} ASC"
+    query = f"SELECT {', '.join(fields)} FROM {sobject} {predicate} {order_by}"
+
+    # Query all records in batches
+    for page in getattr(sf.bulk, sobject).query_all(query, lazy_operation=True):
+        for record in page:
+            # Strip out the attributes field
+            record.pop("attributes", None)
+            for field in date_fields:
+                # Convert Salesforce timestamps to ISO 8601
+                if record.get(field):
+                    record[field] = pendulum.from_timestamp(
+                        record[field] / 1000,
+                    ).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        yield from page
+        n_records += len(page)