ingestr 0.13.30__py3-none-any.whl → 0.13.31__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.30"
+version = "v0.13.31"

ingestr/src/hubspot/__init__.py

@@ -32,7 +32,12 @@ from dlt.common import pendulum
 from dlt.common.typing import TDataItems
 from dlt.sources import DltResource
 
-from .helpers import _get_property_names, fetch_data, fetch_property_history
+from .helpers import (
+    _get_property_names,
+    fetch_data,
+    fetch_data_raw,
+    fetch_property_history,
+)
 from .settings import (
     ALL,
     CRM_OBJECT_ENDPOINTS,
@@ -192,12 +197,21 @@ def hubspot(
         api_key: str = api_key,
         custom_object_name: str = custom_object,
     ) -> Iterator[TDataItems]:
-        get_custom_object = schemas(api_key)
+        get_custom_object = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
         object_type_id = None
-        for custom_object in get_custom_object:
-            if custom_object["name"] == custom_object_name.capitalize():
+
+        custom_object_lowercase = custom_object_name.lower()
+        for custom_object in get_custom_object["results"]:
+            if custom_object["name"].lower() == custom_object_lowercase:
                 object_type_id = custom_object["objectTypeId"]
                 break
+
+            # sometimes people use the plural name of the object type by accident, we should try to match that if we can
+            if "labels" in custom_object:
+                if custom_object_lowercase == custom_object["labels"]["plural"].lower():
+                    object_type_id = custom_object["objectTypeId"]
+                    break
+
         if object_type_id is None:
             raise ValueError(f"There is no such custom object as {custom_object_name}")
         custom_object_properties = f"crm/v3/properties/{object_type_id}"

ingestr/src/hubspot/helpers.py

@@ -130,6 +130,7 @@ def fetch_data(
     # Parse the API response and yield the properties of each result
     # Parse the response JSON data
     _data = r.json()
+
     # Yield the properties of each result in the API response
     while _data is not None:
         if "results" in _data:
@@ -156,6 +157,7 @@ def fetch_data(
                     if "id" not in _obj and "id" in _result:
                         # Move id from properties to top level
                         _obj["id"] = _result["id"]
+
                     if "associations" in _result:
                         for association in _result["associations"]:
                             __values = [
@@ -168,12 +170,13 @@ def fetch_data(
                                 ]
                             ]
 
-                        # remove duplicates from list of dicts
-                        __values = [
-                            dict(t) for t in {tuple(d.items()) for d in __values}
-                        ]
+                            # remove duplicates from list of dicts
+                            __values = [
+                                dict(t) for t in {tuple(d.items()) for d in __values}
+                            ]
+
+                            _obj[association] = __values
 
-                        _obj[association] = __values
                     _objects.append(_obj)
             yield _objects
 
@@ -208,3 +211,12 @@ def _get_property_names(api_key: str, object_type: str) -> List[str]:
         properties.extend([prop["name"] for prop in page])
 
     return properties
+
+
+def fetch_data_raw(
+    endpoint: str, api_key: str, params: Optional[Dict[str, Any]] = None
+) -> Iterator[List[Dict[str, Any]]]:
+    url = get_url(endpoint)
+    headers = _get_headers(api_key)
+    r = requests.get(url, headers=headers, params=params)
+    return r.json()

{ingestr-0.13.30.dist-info → ingestr-0.13.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.30
+Version: 0.13.31
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -74,7 +74,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.0
+Requires-Dist: greenlet==3.2.1
 Requires-Dist: grpcio-status==1.62.3
 Requires-Dist: grpcio==1.70.0
 Requires-Dist: hdbcli==2.23.27

{ingestr-0.13.30.dist-info → ingestr-0.13.31.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=Vt5NFN59BUlkYrOwiF9xF5MCFYp9r-aVSnQ99f1ypRE,25964
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=4uN8Z1CmXZ2T6X2YHEC3ZWFuS5uL_sTMUZiwgVyotHo,21
+ingestr/src/buildinfo.py,sha256=b5zsXj7M33l1DC6hlhdWhnmeLEBJhFFtKbG2j0gA0ZM,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
@@ -61,8 +61,8 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=rSYmN8h6qqxhWCW6elD-pC7iqHXlIofb1F9wvTzziUE,10962
-ingestr/src/hubspot/helpers.py,sha256=fscilfO_K7HS2XQxzf7MeZwVXLTP0WdqnV-NhdeqQAA,7748
+ingestr/src/hubspot/__init__.py,sha256=QheZb_F2TEBdzd29SEgkU3AMdIIF7Gpj-t27EXnSIZ4,11448
+ingestr/src/hubspot/helpers.py,sha256=4aVOSzIsQV3RemqRJEJLu7BWMdcOuubwNvrjuMu87rg,8045
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
@@ -121,8 +121,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.30.dist-info/METADATA,sha256=h7uzURTF300XZEFjInqhpEjO6b-RFeBHWgS5jf4qCM0,13659
-ingestr-0.13.30.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.30.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.30.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.30.dist-info/RECORD,,
+ingestr-0.13.31.dist-info/METADATA,sha256=tYd7c8Yc5Zqv0vDeKd4haCgBnl8PfvpLKaHToxHF5vs,13659
+ingestr-0.13.31.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.31.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.31.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.31.dist-info/RECORD,,