ingestr 0.13.25__py3-none-any.whl → 0.13.27__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.25"
+version = "v0.13.27"

ingestr/src/hubspot/__init__.py

@@ -36,6 +36,7 @@ from .helpers import _get_property_names, fetch_data, fetch_property_history
 from .settings import (
     ALL,
     CRM_OBJECT_ENDPOINTS,
+    CRM_SCHEMAS_ENDPOINT,
     DEFAULT_COMPANY_PROPS,
     DEFAULT_CONTACT_PROPS,
     DEFAULT_DEAL_PROPS,
@@ -161,6 +162,13 @@ def hubspot(
             props,
             include_custom_props,
         )
+      
+    @dlt.resource(name="schemas", write_disposition="merge", primary_key="id")
+    def schemas(
+        api_key: str = api_key,
+    ) -> Iterator[TDataItems]:
+        """Hubspot schemas resource"""
+        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key)
 
     @dlt.resource(name="quotes", write_disposition="replace")
     def quotes(
@@ -178,7 +186,7 @@ def hubspot(
             include_custom_props,
         )
 
-    return companies, contacts, deals, tickets, products, quotes
+    return companies, contacts, deals, tickets, products, quotes, schemas
 
 
 def crm_objects(

ingestr/src/hubspot/helpers.py

@@ -132,16 +132,27 @@ def fetch_data(
         if "results" in _data:
             _objects: List[Dict[str, Any]] = []
             for _result in _data["results"]:
-                _obj = _result.get("properties", _result)
-                if "id" not in _obj and "id" in _result:
-                    # Move id from properties to top level
-                    _obj["id"] = _result["id"]
-                if "associations" in _result:
-                    for association in _result["associations"]:
-                        __values = [
-                            {
-                                "value": _obj["hs_object_id"],
-                                f"{association}_id": __r["id"],
+                if endpoint == "/crm/v3/schemas":
+                    _objects.append({
+                        "name": _result["labels"].get("singular", ""),
+                        "objectTypeId": _result.get("objectTypeId", ""),
+                        "id": _result.get("id", ""),
+                        "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
+                        "properties": _result.get("properties", ""),
+                        "createdAt": _result.get("createdAt", ""),
+                        "updatedAt": _result.get("updatedAt", "")
+                    })
+                else:
+                    _obj = _result.get("properties", _result)
+                    if "id" not in _obj and "id" in _result:
+                        # Move id from properties to top level
+                        _obj["id"] = _result["id"]
+                    if "associations" in _result:
+                        for association in _result["associations"]:
+                            __values = [
+                                {
+                                    "value": _obj["hs_object_id"],
+                                    f"{association}_id": __r["id"],
                             }
                             for __r in _result["associations"][association]["results"]
                         ]
@@ -152,7 +163,7 @@ def fetch_data(
                         ]
 
                         _obj[association] = __values
-                _objects.append(_obj)
+                    _objects.append(_obj)
             yield _objects
 
         # Follow pagination links if they exist

ingestr/src/hubspot/settings.py

@@ -14,6 +14,7 @@ CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals"
 CRM_PRODUCTS_ENDPOINT = "/crm/v3/objects/products"
 CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets"
 CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes"
+CRM_SCHEMAS_ENDPOINT = "/crm/v3/schemas"
 
 CRM_OBJECT_ENDPOINTS = {
     "contact": CRM_CONTACTS_ENDPOINT,

ingestr/src/sources.py

@@ -803,7 +803,7 @@ class HubspotSource:
             raise ValueError("api_key in the URI is required to connect to Hubspot")
 
         endpoint = None
-        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes"]:
+        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
             endpoint = table
         else:
             raise ValueError(
@@ -1398,13 +1398,19 @@ class GoogleAnalyticsSource:
         parse_uri = urlparse(uri)
         source_fields = parse_qs(parse_uri.query)
         cred_path = source_fields.get("credentials_path")
+        cred_base64 = source_fields.get("credentials_base64")
 
-        if not cred_path:
-            raise ValueError("credentials_path is required to connect Google Analytics")
-        credentials = {}
+        if not cred_path and not cred_base64:
+            raise ValueError("credentials_path or credentials_base64 is required to connect Google Analytics")
 
-        with open(cred_path[0], "r") as f:
-            credentials = json.load(f)
+        credentials = {}
+        if cred_path:
+            with open(cred_path[0], "r") as f:
+                credentials = json.load(f)
+        elif cred_base64:
+            credentials = json.loads(
+                base64.b64decode(cred_base64[0]).decode("utf-8")
+            )
 
         property_id = source_fields.get("property_id")
         if not property_id:
@@ -2064,6 +2070,11 @@ class FrankfurterSource:
                 end_date = pendulum.now()
             validate_dates(start_date=start_date, end_date=end_date)
 
+        # For currencies and latest tables, set start and end dates to current date
+        else:
+            start_date = pendulum.now()
+            end_date = pendulum.now()
+
         # Validate table
         if table not in ["currencies", "latest", "exchange_rates"]:
             raise ValueError(

{ingestr-0.13.25.dist-info → ingestr-0.13.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.25
+Version: 0.13.27
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -74,7 +74,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.1.1
+Requires-Dist: greenlet==3.2.0
 Requires-Dist: grpcio-status==1.62.3
 Requires-Dist: grpcio==1.70.0
 Requires-Dist: hdbcli==2.23.27

{ingestr-0.13.25.dist-info → ingestr-0.13.27.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=wvbRCJ2--M0Zw2cYtSH874TxTtlD0wadHREeLG3anOY,25618
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=lkdWOsV5VTf0j0DW9a0BdkPbOZMZEYAUvs8aGZ2V-uE,21
+ingestr/src/buildinfo.py,sha256=KAR_TXhR11JwQNb01TN64yrPzC6Xv6G-HVq71tG_n90,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
@@ -10,7 +10,7 @@ ingestr/src/filters.py,sha256=5LNpBgm8FJXdrFHGyM7dLVyphKykSpPk7yuQAZ8GML4,1133
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=u2Kh2K9v1YhrKXss9yGGxFXr-fMp1E6pvbcwjZqYiGM,73394
+ingestr/src/sources.py,sha256=Zmjg3pDsCjSl8IG_bwH1eGiBxsrUMVesPLlYtz7YU-U,73859
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -61,9 +61,9 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=NYgSIAPXQh2Qp1eKun7TgcerKogq6pWtNkr-_f0FXbI,9464
-ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
-ingestr/src/hubspot/settings.py,sha256=9P1OKiRL88kl_m8n1HhuG-Qpq9VGbqPLn5Q0QYneToU,2193
+ingestr/src/hubspot/__init__.py,sha256=nHjMvqUAUNVy9dX-1YiaY6PeplDnzc_LjQl25DdVrYg,9765
+ingestr/src/hubspot/helpers.py,sha256=vJPaF9qU71-w-LUe4NaRowioIJG3GfX4MObyitHJC6U,7388
+ingestr/src/hubspot/settings.py,sha256=hXnIhE4_FlhtgUiCjMLGGZJ_lBlBQJ0YWi6lHJ6Q3WU,2234
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=u5ThH1y8uObZKXgIo71em1UnX6MsVHWOjcf1jKqKbE8,6205
@@ -121,8 +121,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.25.dist-info/METADATA,sha256=IiZGRzg-cRXN10snt3Y_97yODJ9KkDF-x9XjP_cx9vo,13659
-ingestr-0.13.25.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.25.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.25.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.25.dist-info/RECORD,,
+ingestr-0.13.27.dist-info/METADATA,sha256=cq7IIe2m-ecKzlBDJVATmzn-t82fASfuQEgwHRf4v10,13659
+ingestr-0.13.27.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.27.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.27.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.27.dist-info/RECORD,,