omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/hubspot/settings.py

@@ -0,0 +1,329 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Hubspot source settings and constants"""
+
+from dlt.common import pendulum
+
+STARTDATE = pendulum.datetime(year=2000, month=1, day=1)
+
+CRM_CONTACTS_ENDPOINT = (
+    "/crm/v3/objects/contacts?associations=companies,deals,products,tickets,quotes"
+)
+CRM_COMPANIES_ENDPOINT = "/crm/v3/objects/companies?associations=products"
+CRM_DEALS_ENDPOINT = (
+    "/crm/v3/objects/deals?associations=companies,contacts,products,tickets,quotes"
+)
+CRM_PRODUCTS_ENDPOINT = (
+    "/crm/v3/objects/products?associations=companies,contacts,deals,tickets,quotes"
+)
+CRM_TICKETS_ENDPOINT = (
+    "/crm/v3/objects/tickets?associations=companies,contacts,deals,products,quotes"
+)
+CRM_QUOTES_ENDPOINT = (
+    "/crm/v3/objects/quotes?associations=companies,contacts,deals,products,tickets"
+)
+CRM_CALLS_ENDPOINT = (
+    "/crm/v3/objects/calls?associations=contacts,companies,deals,products,quotes"
+)
+CRM_EMAILS_ENDPOINT = (
+    "/crm/v3/objects/emails?associations=contacts,companies,deals,products,quotes"
+)
+CRM_FEEDBACK_SUBMISSIONS_ENDPOINT = "/crm/v3/objects/feedback_submissions?associations=contacts,companies,deals,products,quotes"
+CRM_LINE_ITEMS_ENDPOINT = (
+    "/crm/v3/objects/line_items?associations=contacts,companies,deals,products,quotes"
+)
+CRM_MEETINGS_ENDPOINT = (
+    "/crm/v3/objects/meetings?associations=contacts,companies,deals,products,quotes"
+)
+CRM_NOTES_ENDPOINT = (
+    "/crm/v3/objects/notes?associations=contacts,companies,deals,products,quotes"
+)
+CRM_TASKS_ENDPOINT = (
+    "/crm/v3/objects/tasks?associations=contacts,companies,deals,products,quotes"
+)
+CRM_CARTS_ENDPOINT = (
+    "/crm/v3/objects/carts?associations=contacts,companies,deals,products,quotes"
+)
+CRM_DISCOUNTS_ENDPOINT = "/crm/v3/objects/discounts?associations=contacts,line_items,companies,deals,products,quotes"
+CRM_FEES_ENDPOINT = "/crm/v3/objects/fees?associations=contacts,line_items,companies,deals,products,quotes"
+CRM_INVOICES_ENDPOINT = "/crm/v3/objects/invoices?associations=contacts,line_items,companies,fees,products,quotes"
+CRM_COMMERCE_PAYMENTS_ENDPOINT = "/crm/v3/objects/commerce_payments?associations=contacts,companies,deals,quotes,invoices,products,fees"
+CRM_TAXES_ENDPOINT = (
+    "/crm/v3/objects/taxes?associations=line_items,companies,deals,products,quotes,fees"
+)
+CRM_OWNERS_ENDPOINT = "/crm/v3/owners"
+CRM_SCHEMAS_ENDPOINT = "/crm/v3/schemas"
+
+CRM_OBJECT_ENDPOINTS = {
+    "contact": CRM_CONTACTS_ENDPOINT,
+    "company": CRM_COMPANIES_ENDPOINT,
+    "deal": CRM_DEALS_ENDPOINT,
+    "product": CRM_PRODUCTS_ENDPOINT,
+    "ticket": CRM_TICKETS_ENDPOINT,
+    "quote": CRM_QUOTES_ENDPOINT,
+    "call": CRM_CALLS_ENDPOINT,
+    "email": CRM_EMAILS_ENDPOINT,
+    "feedback_submission": CRM_FEEDBACK_SUBMISSIONS_ENDPOINT,
+    "line_item": CRM_LINE_ITEMS_ENDPOINT,
+    "meeting": CRM_MEETINGS_ENDPOINT,
+    "note": CRM_NOTES_ENDPOINT,
+    "task": CRM_TASKS_ENDPOINT,
+    "cart": CRM_CARTS_ENDPOINT,
+    "discount": CRM_DISCOUNTS_ENDPOINT,
+    "fee": CRM_FEES_ENDPOINT,
+    "invoice": CRM_INVOICES_ENDPOINT,
+    "commerce_payment": CRM_COMMERCE_PAYMENTS_ENDPOINT,
+    "tax": CRM_TAXES_ENDPOINT,
+}
+
+WEB_ANALYTICS_EVENTS_ENDPOINT = "/events/v3/events?objectType={objectType}&objectId={objectId}&occurredAfter={occurredAfter}&occurredBefore={occurredBefore}&sort=-occurredAt"
+
+OBJECT_TYPE_SINGULAR = {
+    "companies": "company",
+    "contacts": "contact",
+    "deals": "deal",
+    "tickets": "ticket",
+    "products": "product",
+    "quotes": "quote",
+    "calls": "call",
+    "emails": "email",
+    "feedback_submissions": "feedback_submission",
+    "line_items": "line_item",
+    "meetings": "meeting",
+    "notes": "note",
+    "tasks": "task",
+    "carts": "cart",
+    "discounts": "discount",
+    "fees": "fee",
+    "invoices": "invoice",
+    "commerce_payments": "commerce_payment",
+    "taxes": "tax",
+}
+
+OBJECT_TYPE_PLURAL = {v: k for k, v in OBJECT_TYPE_SINGULAR.items()}
+
+# Contacts use "lastmodifieddate"; all other CRM objects use "hs_lastmodifieddate"
+LAST_MODIFIED_PROPERTY = {
+    "contact": "lastmodifieddate",
+}
+DEFAULT_LAST_MODIFIED_PROPERTY = "hs_lastmodifieddate"
+
+DEFAULT_DEAL_PROPS = [
+    "amount",
+    "closedate",
+    "createdate",
+    "dealname",
+    "dealstage",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "pipeline",
+]
+
+DEFAULT_COMPANY_PROPS = [
+    "createdate",
+    "domain",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "name",
+]
+
+DEFAULT_CONTACT_PROPS = [
+    "createdate",
+    "email",
+    "firstname",
+    "hs_object_id",
+    "lastmodifieddate",
+    "lastname",
+]
+
+DEFAULT_TICKET_PROPS = [
+    "createdate",
+    "content",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "hs_pipeline",
+    "hs_pipeline_stage",
+    "hs_ticket_category",
+    "hs_ticket_priority",
+    "subject",
+]
+
+DEFAULT_PRODUCT_PROPS = [
+    "createdate",
+    "description",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "name",
+    "price",
+]
+
+DEFAULT_QUOTE_PROPS = [
+    "hs_createdate",
+    "hs_expiration_date",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "hs_public_url_key",
+    "hs_status",
+    "hs_title",
+]
+
+DEFAULT_CALL_PROPS = [
+    "hs_call_body",
+    "hs_call_direction",
+    "hs_call_disposition",
+    "hs_call_duration",
+    "hs_call_from_number",
+    "hs_call_status",
+    "hs_call_title",
+    "hs_call_to_number",
+    "hs_lastmodifieddate",
+    "hs_timestamp",
+]
+
+DEFAULT_EMAIL_PROPS = [
+    "hs_attachment_ids",
+    "hs_email_direction",
+    "hs_email_headers",
+    "hs_email_html",
+    "hs_email_status",
+    "hs_email_subject",
+    "hs_email_text",
+    "hs_timestamp",
+    "hs_lastmodifieddate",
+    "hubspot_owner_id",
+]
+
+DEFAULT_FEEDBACK_SUBMISSION_PROPS = [
+    "hs_createdate",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "hs_sentiment",
+    "hs_survey_channel",
+]
+
+DEFAULT_LINE_ITEM_PROPS = [
+    "amount",
+    "description",
+    "hs_line_item_currency_code",
+    "hs_recurring_billing_end_date",
+    "hs_recurring_billing_start_date",
+    "hs_lastmodifieddate",
+    "hs_sku",
+    "name",
+    "price",
+    "quantity",
+    "recurringbillingfrequency",
+]
+
+DEFAULT_MEETING_PROPS = [
+    "hs_internal_meeting_notes",
+    "hs_meeting_body",
+    "hs_meeting_end_time",
+    "hs_meeting_external_url",
+    "hs_meeting_location",
+    "hs_meeting_outcome",
+    "hs_meeting_start_time",
+    "hs_meeting_title",
+    "hs_timestamp",
+    "hs_lastmodifieddate",
+    "hubspot_owner_id",
+]
+
+DEFAULT_NOTE_PROPS = [
+    "hs_attachment_ids",
+    "hs_note_body",
+    "hs_timestamp",
+    "hs_lastmodifieddate",
+    "hubspot_owner_id",
+]
+
+DEFAULT_TASK_PROPS = [
+    "hs_task_body",
+    "hs_task_priority",
+    "hs_task_status",
+    "hs_task_subject",
+    "hs_task_type",
+    "hs_timestamp",
+    "hs_lastmodifieddate",
+    "hubspot_owner_id",
+]
+
+DEFAULT_CART_PROPS = [
+    "hs_cart_discount",
+    "hs_cart_name",
+    "hs_cart_url",
+    "hs_createdate",
+    "hs_currency_code",
+    "hs_external_cart_id",
+    "hs_external_status",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "hs_shipping_cost",
+    "hs_source_store",
+    "hs_tags",
+    "hs_tax",
+    "hs_total_price",
+]
+
+DEFAULT_DISCOUNT_PROPS = [
+    "hs_duration",
+    "hs_label",
+    "hs_lastmodifieddate",
+    "hs_sort_order",
+    "hs_type",
+    "hs_value",
+]
+
+DEFAULT_FEE_PROPS = [
+    "hs_label",
+    "hs_lastmodifieddate",
+    "hs_type",
+    "hs_value",
+]
+
+DEFAULT_INVOICE_PROPS = [
+    "hs_currency",
+    "hs_due_date",
+    "hs_invoice_date",
+    "hs_lastmodifieddate",
+    "hs_tax_id",
+]
+
+DEFAULT_COMMERCE_PAYMENT_PROPS = [
+    "hs_currency_code",
+    "hs_customer_email",
+    "hs_fees_amount",
+    "hs_initial_amount",
+    "hs_initiated_date",
+    "hs_internal_comment",
+    "hs_lastmodifieddate",
+    "hs_latest_status",
+    "hs_payment_method_type",
+    "hs_payout_date",
+    "hs_processor_type",
+    "hs_reference_number",
+    "hs_refunds_amount",
+    "hs_billing_address_city",
+    "hs_billing_address_country",
+]
+
+DEFAULT_TAX_PROPS = [
+    "hs_label",
+    "hs_lastmodifieddate",
+    "hs_type",
+    "hs_value",
+]
+
+ALL = ("ALL",)

omniload/src/indeed/__init__.py

@@ -0,0 +1,153 @@
+from typing import Any, Dict, Iterable, Iterator
+
+import dlt
+import pendulum
+
+from .helpers import (
+    _get_account,
+    _get_campaign_budget,
+    _get_campaign_details,
+    _get_campaign_jobs,
+    _get_campaign_properties,
+    _get_campaign_stats,
+    _get_oauth_token,
+    _get_traffic_report,
+    _paginate_campaigns,
+)
+
+
+@dlt.source(name="indeed", max_table_nesting=0)
+def indeed_source(
+    client_id: str,
+    client_secret: str,
+    employer_id: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[dlt.sources.DltResource]:
+    token = _get_oauth_token(client_id, client_secret, employer_id)
+
+    @dlt.resource(name="campaigns", write_disposition="merge", primary_key="Id")
+    def campaigns() -> Iterator[Dict[str, Any]]:
+        for campaign in _paginate_campaigns(token):
+            yield campaign
+
+    @dlt.transformer(
+        name="campaign_details",
+        write_disposition="merge",
+        primary_key="campaignId",
+        data_from=campaigns,
+        parallelized=True,
+    )
+    def campaign_details(campaign: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
+        details = _get_campaign_details(token, campaign["Id"])
+        yield details
+
+    @dlt.transformer(
+        name="campaign_budget",
+        write_disposition="merge",
+        primary_key="campaignId",
+        data_from=campaigns,
+        parallelized=True,
+    )
+    def campaign_budget(campaign: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
+        budget = _get_campaign_budget(token, campaign["Id"])
+        if budget:
+            yield budget
+
+    @dlt.transformer(
+        name="campaign_jobs",
+        write_disposition="merge",
+        primary_key=["campaignId", "jobKey"],
+        data_from=campaigns,
+        parallelized=True,
+    )
+    def campaign_jobs(campaign: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
+        for job in _get_campaign_jobs(token, campaign["Id"]):
+            yield job
+
+    @dlt.transformer(
+        name="campaign_properties",
+        write_disposition="merge",
+        primary_key="campaignId",
+        data_from=campaigns,
+        parallelized=True,
+    )
+    def campaign_properties(campaign: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
+        props = _get_campaign_properties(token, campaign["Id"])
+        if props:
+            yield props
+
+    @dlt.transformer(
+        name="campaign_stats",
+        write_disposition="merge",
+        primary_key=["campaignId", "Date"],
+        merge_key="Date",
+        data_from=campaigns,
+        parallelized=True,
+    )
+    def campaign_stats(
+        campaign: Dict[str, Any],
+        date: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "Date",
+            initial_value=start_date.to_date_string(),
+            end_value=end_date.to_date_string() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        current_start = date.last_value or start_date.to_date_string()
+        current_end = date.end_value or pendulum.now("UTC").to_date_string()
+
+        for stat in _get_campaign_stats(
+            token, campaign["Id"], current_start, current_end
+        ):
+            yield stat
+
+    @dlt.resource(
+        name="account",
+        write_disposition="merge",
+        primary_key=["employerId", "jobSourceId"],
+    )
+    def account() -> Iterator[Dict[str, Any]]:
+        data = _get_account(token)
+        employer_id = data.get("employerId")
+        contact = data.get("contact")
+        company = data.get("company")
+        email = data.get("email")
+
+        for job_source in data.get("jobSourceList", []):
+            yield {
+                "employerId": employer_id,
+                "contact": contact,
+                "company": company,
+                "email": email,
+                "jobSourceId": job_source.get("id"),
+                "jobSourceSiteName": job_source.get("siteName"),
+            }
+
+    @dlt.resource(name="traffic_stats", write_disposition="merge", merge_key="date")
+    def traffic_stats(
+        date: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "date",
+            initial_value=start_date.to_date_string(),
+            end_value=end_date.to_date_string() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        current_start = date.last_value or start_date.to_date_string()
+        current_end = date.end_value or pendulum.now("UTC").to_date_string()
+
+        for row in _get_traffic_report(token, current_start, current_end):
+            yield row
+
+    return [
+        campaigns,
+        campaign_details,
+        campaign_budget,
+        campaign_jobs,
+        campaign_properties,
+        campaign_stats,
+        account,
+        traffic_stats,
+    ]

omniload/src/indeed/helpers.py

@@ -0,0 +1,228 @@
+import csv
+import io
+import time
+from datetime import datetime, timedelta
+from typing import Any, Dict, Iterator, Optional
+
+import requests
+
+INDEED_TOKEN_URL = "https://apis.indeed.com/oauth/v2/tokens"
+INDEED_API_BASE_URL = "https://apis.indeed.com/ads/v1"
+
+DEFAULT_SCOPES = [
+    "employer.advertising.campaign.read",
+    "employer.advertising.campaign_report.read",
+    "employer.advertising.account.read",
+]
+
+
+def _get_oauth_token(
+    client_id: str,
+    client_secret: str,
+    employer_id: str,
+) -> str:
+    response = requests.post(
+        INDEED_TOKEN_URL,
+        data={
+            "client_id": client_id,
+            "client_secret": client_secret,
+            "grant_type": "client_credentials",
+            "scope": " ".join(DEFAULT_SCOPES),
+            "employer": employer_id,
+        },
+        headers={"Content-Type": "application/x-www-form-urlencoded"},
+    )
+    response.raise_for_status()
+    payload = response.json()
+
+    if "access_token" not in payload:
+        raise ValueError(f"Token response missing access_token: {payload}")
+
+    return payload["access_token"]
+
+
+def _api_request(
+    token: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+) -> requests.Response:
+    url = f"{INDEED_API_BASE_URL}{endpoint}"
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/json",
+    }
+    response = requests.get(url, headers=headers, params=params)
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        raise RuntimeError(f"{e} - Response body: {response.text}") from e
+    return response
+
+
+def _paginate_campaigns(token: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        params: Dict[str, Any] = {"perPage": 500, "status": "ACTIVE"}
+        if cursor:
+            params["start"] = cursor
+        response = _api_request(token, "/campaigns", params=params)
+        data = response.json()
+
+        campaigns = data.get("data", {}).get("Campaigns", [])
+        for campaign in campaigns:
+            yield campaign
+
+        links = data.get("meta", {}).get("links", [])
+        next_link = next((link for link in links if link.get("rel") == "next"), None)
+
+        if not next_link:
+            break
+
+        href = next_link.get("href", "")
+        if "start=" in href:
+            cursor = href.split("start=")[1].split("&")[0]
+        else:
+            break
+
+
+def _get_campaign_details(token: str, campaign_id: str) -> Dict[str, Any]:
+    response = _api_request(token, f"/campaigns/{campaign_id}")
+    data = response.json().get("data", {})
+    data["campaignId"] = campaign_id
+    return data
+
+
+def _get_campaign_budget(token: str, campaign_id: str) -> Optional[Dict[str, Any]]:
+    try:
+        response = _api_request(token, f"/campaigns/{campaign_id}/budget")
+        data = response.json().get("data", {})
+        data["campaignId"] = campaign_id
+        return data
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 404:
+            return None
+        raise
+
+
+def _get_campaign_jobs(token: str, campaign_id: str) -> Iterator[Dict[str, Any]]:
+    try:
+        response = _api_request(
+            token,
+            f"/campaigns/{campaign_id}/jobDetails",
+        )
+        data = response.json()
+
+        jobs = data.get("data", {}).get("entries", [])
+        for job in jobs:
+            job["campaignId"] = campaign_id
+            yield job
+    except requests.exceptions.HTTPError as e:
+        if e.response is not None and e.response.status_code == 404:
+            return
+        if e.response is not None and e.response.status_code == 429:
+            time.sleep(5)
+            yield from _get_campaign_jobs(token, campaign_id)
+            return
+        raise
+
+
+def _get_campaign_properties(token: str, campaign_id: str) -> Optional[Dict[str, Any]]:
+    try:
+        response = _api_request(token, f"/campaigns/{campaign_id}/properties")
+        data = response.json().get("data", {})
+        data["campaignId"] = campaign_id
+        return data
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 404:
+            return None
+        raise
+
+
+def _get_campaign_stats(
+    token: str, campaign_id: str, start_date: str, end_date: str
+) -> Iterator[Dict[str, Any]]:
+    try:
+        response = _api_request(
+            token,
+            f"/campaigns/{campaign_id}/stats",
+            params={"startDate": start_date, "endDate": end_date},
+        )
+        data = response.json().get("data", {})
+
+        stats = data.get("Stats", []) or data.get("entries", [])
+        for stat in stats:
+            stat["campaignId"] = campaign_id
+            yield stat
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 404:
+            return
+        raise
+
+
+def _get_account(token: str) -> Dict[str, Any]:
+    response = _api_request(token, "/account")
+    return response.json().get("data", {})
+
+
+def _get_traffic_report_for_day(
+    token: str, date: str, max_retries: int = 10, retry_delay: int = 5
+) -> Iterator[Dict[str, Any]]:
+    try:
+        start_dt = datetime.strptime(date, "%Y-%m-%d")
+        end_dt = start_dt + timedelta(days=1)
+        end_date_str = end_dt.strftime("%Y-%m-%d")
+
+        response = _api_request(
+            token,
+            "/stats",
+            params={"startDate": date, "endDate": end_date_str, "v": "8"},
+        )
+
+        if response.status_code == 202:
+            location = response.json().get("data", {}).get("location", "")
+            if not location:
+                raise ValueError(f"API returned 202 but no location for date {date}")
+
+            if location.startswith("/v1"):
+                location = location[3:]
+
+            for attempt in range(max_retries):
+                time.sleep(retry_delay)
+                report_response = _api_request(token, location)
+
+                if report_response.status_code == 200:
+                    content_type = report_response.headers.get("Content-Type", "")
+                    if "text/csv" in content_type or "application/csv" in content_type:
+                        for row in _parse_csv(report_response.text):
+                            row["date"] = date
+                            yield row
+                        return
+            return
+    except requests.exceptions.HTTPError as e:
+        body = e.response.text if e.response is not None else "N/A"
+        raise RuntimeError(f"{e} - Response body: {body}") from e
+
+
+def _get_traffic_report(
+    token: str,
+    start_date: str,
+    end_date: str,
+    max_retries: int = 10,
+    retry_delay: int = 5,
+) -> Iterator[Dict[str, Any]]:
+    start = datetime.strptime(start_date, "%Y-%m-%d")
+    end = datetime.strptime(end_date, "%Y-%m-%d") + timedelta(days=1)
+
+    current = start
+    while current < end:
+        date_str = current.strftime("%Y-%m-%d")
+        yield from _get_traffic_report_for_day(
+            token, date_str, max_retries, retry_delay
+        )
+        current += timedelta(days=1)
+
+
+def _parse_csv(csv_content: str) -> Iterator[Dict[str, Any]]:
+    reader = csv.DictReader(io.StringIO(csv_content))
+    for row in reader:
+        yield dict(row)