ingestr 0.13.27__py3-none-any.whl → 0.13.29__py3-none-any.whl

ingestr/main.py

@@ -302,6 +302,13 @@ def ingest(
             envvar=["COLUMNS", "INGESTR_COLUMNS"],
         ),
     ] = None,  # type: ignore
+    yield_limit: Annotated[
+        Optional[int],
+        typer.Option(
+            help="Limit the number of pages yielded from the source",
+            envvar=["YIELD_LIMIT", "INGESTR_YIELD_LIMIT"],
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -556,6 +563,9 @@ def ingest(
         if factory.source_scheme.startswith("mysql"):
             resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
 
+        if yield_limit:
+            resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
+
         def col_h(x):
             if column_hints:
                 x.apply_hints(columns=column_hints)

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.27"
+version = "v0.13.29"

ingestr/src/hubspot/__init__.py

@@ -56,6 +56,7 @@ def hubspot(
     api_key: str = dlt.secrets.value,
     include_history: bool = False,
     include_custom_props: bool = True,
+    custom_object: str = None,
 ) -> Sequence[DltResource]:
     """
     A DLT source that retrieves data from the HubSpot API using the
@@ -162,13 +163,13 @@ def hubspot(
             props,
             include_custom_props,
         )
-      
+
     @dlt.resource(name="schemas", write_disposition="merge", primary_key="id")
     def schemas(
         api_key: str = api_key,
     ) -> Iterator[TDataItems]:
         """Hubspot schemas resource"""
-        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key)
+        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key, resource_name="schemas")
 
     @dlt.resource(name="quotes", write_disposition="replace")
     def quotes(
@@ -186,7 +187,33 @@ def hubspot(
             include_custom_props,
         )
 
-    return companies, contacts, deals, tickets, products, quotes, schemas
+    @dlt.resource(write_disposition="merge", primary_key="hs_object_id")
+    def custom(
+        api_key: str = api_key,
+        custom_object_name: str = custom_object,
+    ) -> Iterator[TDataItems]:
+        get_custom_object = schemas(api_key)
+        object_type_id = None
+        for custom_object in get_custom_object:
+            if custom_object["name"] == custom_object_name.capitalize():
+                object_type_id = custom_object["objectTypeId"]
+                break
+        if object_type_id is None:
+            raise ValueError(f"There is no such custom object as {custom_object_name}")
+        custom_object_properties = f"crm/v3/properties/{object_type_id}"
+
+        props_pages = fetch_data(custom_object_properties, api_key)
+        props = []
+        for page in props_pages:
+            props.extend([prop["name"] for prop in page])
+        props = ",".join(sorted(list(set(props))))
+
+        custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
+
+        """Hubspot custom object details resource"""
+        yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")
+
+    return companies, contacts, deals, tickets, products, quotes, schemas, custom
 
 
 def crm_objects(

ingestr/src/hubspot/helpers.py

@@ -90,7 +90,10 @@ def fetch_property_history(
 
 
 def fetch_data(
-    endpoint: str, api_key: str, params: Optional[Dict[str, Any]] = None
+    endpoint: str,
+    api_key: str,
+    params: Optional[Dict[str, Any]] = None,
+    resource_name: str = None,
 ) -> Iterator[List[Dict[str, Any]]]:
     """
     Fetch data from HUBSPOT endpoint using a specified API key and yield the properties of each result.
@@ -132,16 +135,22 @@ def fetch_data(
         if "results" in _data:
             _objects: List[Dict[str, Any]] = []
             for _result in _data["results"]:
-                if endpoint == "/crm/v3/schemas":
-                    _objects.append({
-                        "name": _result["labels"].get("singular", ""),
-                        "objectTypeId": _result.get("objectTypeId", ""),
-                        "id": _result.get("id", ""),
-                        "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
-                        "properties": _result.get("properties", ""),
-                        "createdAt": _result.get("createdAt", ""),
-                        "updatedAt": _result.get("updatedAt", "")
-                    })
+                if resource_name == "schemas":
+                    _objects.append(
+                        {
+                            "name": _result["labels"].get("singular", ""),
+                            "objectTypeId": _result.get("objectTypeId", ""),
+                            "id": _result.get("id", ""),
+                            "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
+                            "properties": _result.get("properties", ""),
+                            "createdAt": _result.get("createdAt", ""),
+                            "updatedAt": _result.get("updatedAt", ""),
+                        }
+                    )
+                elif resource_name == "custom":
+                    _objects.append(
+                        _result.get("properties", ""),
+                    )
                 else:
                     _obj = _result.get("properties", _result)
                     if "id" not in _obj and "id" in _result:
@@ -153,9 +162,11 @@ def fetch_data(
                                 {
                                     "value": _obj["hs_object_id"],
                                     f"{association}_id": __r["id"],
-                            }
-                            for __r in _result["associations"][association]["results"]
-                        ]
+                                }
+                                for __r in _result["associations"][association][
+                                    "results"
+                                ]
+                            ]
 
                         # remove duplicates from list of dicts
                         __values = [

ingestr/src/hubspot/settings.py

@@ -5,15 +5,21 @@ from dlt.common import pendulum
 STARTDATE = pendulum.datetime(year=2000, month=1, day=1)
 
 CRM_CONTACTS_ENDPOINT = (
-    "/crm/v3/objects/contacts?associations=deals,products,tickets,quotes"
+    "/crm/v3/objects/contacts?associations=companies,deals,products,tickets,quotes"
 )
-CRM_COMPANIES_ENDPOINT = (
-    "/crm/v3/objects/companies?associations=contacts,deals,products,tickets,quotes"
+CRM_COMPANIES_ENDPOINT = "/crm/v3/objects/companies?associations=products"
+CRM_DEALS_ENDPOINT = (
+    "/crm/v3/objects/deals?associations=companies,contacts,products,tickets,quotes"
+)
+CRM_PRODUCTS_ENDPOINT = (
+    "/crm/v3/objects/products?associations=companies,contacts,deals,tickets,quotes"
+)
+CRM_TICKETS_ENDPOINT = (
+    "/crm/v3/objects/tickets?associations=companies,contacts,deals,products,quotes"
+)
+CRM_QUOTES_ENDPOINT = (
+    "/crm/v3/objects/quotes?associations=companies,contacts,deals,products,tickets"
 )
-CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals"
-CRM_PRODUCTS_ENDPOINT = "/crm/v3/objects/products"
-CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets"
-CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes"
 CRM_SCHEMAS_ENDPOINT = "/crm/v3/schemas"
 
 CRM_OBJECT_ENDPOINTS = {

ingestr/src/sources.py

@@ -803,7 +803,20 @@ class HubspotSource:
             raise ValueError("api_key in the URI is required to connect to Hubspot")
 
         endpoint = None
-        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
+
+        if table.startswith("custom:"):
+            fields = table.split(":", 2)
+            if len(fields) != 2:
+                raise ValueError(
+                    "Invalid Hubspot custom table format. Expected format: custom:<custom_object_type>"
+                )
+            endpoint = fields[1]
+            return hubspot(
+                api_key=api_key[0],
+                custom_object=endpoint,
+            ).with_resources("custom")
+
+        elif table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
             endpoint = table
         else:
             raise ValueError(

{ingestr-0.13.27.dist-info → ingestr-0.13.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.27
+Version: 0.13.29
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.27.dist-info → ingestr-0.13.29.dist-info}/RECORD

@@ -1,8 +1,8 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=wvbRCJ2--M0Zw2cYtSH874TxTtlD0wadHREeLG3anOY,25618
+ingestr/main.py,sha256=Vt5NFN59BUlkYrOwiF9xF5MCFYp9r-aVSnQ99f1ypRE,25964
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=KAR_TXhR11JwQNb01TN64yrPzC6Xv6G-HVq71tG_n90,21
+ingestr/src/buildinfo.py,sha256=fraAO4VHAeBDwo-MJW7EhZKMHMio-7qqLE2lULqrTHA,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
@@ -10,7 +10,7 @@ ingestr/src/filters.py,sha256=5LNpBgm8FJXdrFHGyM7dLVyphKykSpPk7yuQAZ8GML4,1133
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=Zmjg3pDsCjSl8IG_bwH1eGiBxsrUMVesPLlYtz7YU-U,73859
+ingestr/src/sources.py,sha256=GNX3dLFFqNxe6-humD46PZI9nh84tyuiwh9Cl0RwUg8,74309
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -61,9 +61,9 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=nHjMvqUAUNVy9dX-1YiaY6PeplDnzc_LjQl25DdVrYg,9765
-ingestr/src/hubspot/helpers.py,sha256=vJPaF9qU71-w-LUe4NaRowioIJG3GfX4MObyitHJC6U,7388
-ingestr/src/hubspot/settings.py,sha256=hXnIhE4_FlhtgUiCjMLGGZJ_lBlBQJ0YWi6lHJ6Q3WU,2234
+ingestr/src/hubspot/__init__.py,sha256=rSYmN8h6qqxhWCW6elD-pC7iqHXlIofb1F9wvTzziUE,10962
+ingestr/src/hubspot/helpers.py,sha256=fscilfO_K7HS2XQxzf7MeZwVXLTP0WdqnV-NhdeqQAA,7748
+ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=u5ThH1y8uObZKXgIo71em1UnX6MsVHWOjcf1jKqKbE8,6205
@@ -121,8 +121,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.27.dist-info/METADATA,sha256=cq7IIe2m-ecKzlBDJVATmzn-t82fASfuQEgwHRf4v10,13659
-ingestr-0.13.27.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.27.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.27.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.27.dist-info/RECORD,,
+ingestr-0.13.29.dist-info/METADATA,sha256=bHCOtgvVHv64YRJ9hylunKHeARE2caXv7SuBfD9x1fc,13659
+ingestr-0.13.29.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.29.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.29.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.29.dist-info/RECORD,,