ingestr 0.13.25__tar.gz → 0.13.26__tar.gz

{ingestr-0.13.25 → ingestr-0.13.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.25
+Version: 0.13.26
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

ingestr-0.13.26/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.26"

{ingestr-0.13.25 → ingestr-0.13.26}/ingestr/src/hubspot/__init__.py

@@ -36,6 +36,7 @@ from .helpers import _get_property_names, fetch_data, fetch_property_history
 from .settings import (
     ALL,
     CRM_OBJECT_ENDPOINTS,
+    CRM_SCHEMAS_ENDPOINT,
     DEFAULT_COMPANY_PROPS,
     DEFAULT_CONTACT_PROPS,
     DEFAULT_DEAL_PROPS,
@@ -161,6 +162,13 @@ def hubspot(
             props,
             include_custom_props,
         )
+      
+    @dlt.resource(name="schemas", write_disposition="merge", primary_key="id")
+    def schemas(
+        api_key: str = api_key,
+    ) -> Iterator[TDataItems]:
+        """Hubspot schemas resource"""
+        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key)
 
     @dlt.resource(name="quotes", write_disposition="replace")
     def quotes(
@@ -178,7 +186,7 @@ def hubspot(
             include_custom_props,
         )
 
-    return companies, contacts, deals, tickets, products, quotes
+    return companies, contacts, deals, tickets, products, quotes, schemas
 
 
 def crm_objects(

{ingestr-0.13.25 → ingestr-0.13.26}/ingestr/src/hubspot/helpers.py

@@ -132,16 +132,27 @@ def fetch_data(
         if "results" in _data:
             _objects: List[Dict[str, Any]] = []
             for _result in _data["results"]:
-                _obj = _result.get("properties", _result)
-                if "id" not in _obj and "id" in _result:
-                    # Move id from properties to top level
-                    _obj["id"] = _result["id"]
-                if "associations" in _result:
-                    for association in _result["associations"]:
-                        __values = [
-                            {
-                                "value": _obj["hs_object_id"],
-                                f"{association}_id": __r["id"],
+                if endpoint == "/crm/v3/schemas":
+                    _objects.append({
+                        "name": _result["labels"].get("singular", ""),
+                        "objectTypeId": _result.get("objectTypeId", ""),
+                        "id": _result.get("id", ""),
+                        "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
+                        "properties": _result.get("properties", ""),
+                        "createdAt": _result.get("createdAt", ""),
+                        "updatedAt": _result.get("updatedAt", "")
+                    })
+                else:
+                    _obj = _result.get("properties", _result)
+                    if "id" not in _obj and "id" in _result:
+                        # Move id from properties to top level
+                        _obj["id"] = _result["id"]
+                    if "associations" in _result:
+                        for association in _result["associations"]:
+                            __values = [
+                                {
+                                    "value": _obj["hs_object_id"],
+                                    f"{association}_id": __r["id"],
                             }
                             for __r in _result["associations"][association]["results"]
                         ]
@@ -152,7 +163,7 @@ def fetch_data(
                         ]
 
                         _obj[association] = __values
-                _objects.append(_obj)
+                    _objects.append(_obj)
             yield _objects
 
         # Follow pagination links if they exist

{ingestr-0.13.25 → ingestr-0.13.26}/ingestr/src/hubspot/settings.py

@@ -14,6 +14,7 @@ CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals"
 CRM_PRODUCTS_ENDPOINT = "/crm/v3/objects/products"
 CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets"
 CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes"
+CRM_SCHEMAS_ENDPOINT = "/crm/v3/schemas"
 
 CRM_OBJECT_ENDPOINTS = {
     "contact": CRM_CONTACTS_ENDPOINT,

{ingestr-0.13.25 → ingestr-0.13.26}/ingestr/src/sources.py

@@ -803,7 +803,7 @@ class HubspotSource:
             raise ValueError("api_key in the URI is required to connect to Hubspot")
 
         endpoint = None
-        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes"]:
+        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
             endpoint = table
         else:
             raise ValueError(
@@ -2064,6 +2064,11 @@ class FrankfurterSource:
                 end_date = pendulum.now()
             validate_dates(start_date=start_date, end_date=end_date)
 
+        # For currencies and latest tables, set start and end dates to current date
+        else:
+            start_date = pendulum.now()
+            end_date = pendulum.now()
+
         # Validate table
         if table not in ["currencies", "latest", "exchange_rates"]:
             raise ValueError(

ingestr-0.13.25/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.25"