ingestr 0.13.26__tar.gz → 0.13.28__tar.gz

{ingestr-0.13.26 → ingestr-0.13.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.26
+Version: 0.13.28
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -74,7 +74,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.1.1
+Requires-Dist: greenlet==3.2.0
 Requires-Dist: grpcio-status==1.62.3
 Requires-Dist: grpcio==1.70.0
 Requires-Dist: hdbcli==2.23.27

{ingestr-0.13.26 → ingestr-0.13.28}/docs/supported-sources/google_analytics.md

@@ -9,10 +9,15 @@ The URI format for Google Analytics is as follows:
 ```plaintext
 googleanalytics://?credentials_path=/path/to/service/account.json&property_id=<property_id>
 ```
+Alternatively, you can use base64 encoded credentials:
+
+```
+googleanalytics://?credentials_base64=<base64_encoded_credentials>&property_id=<property_id>
+```
 
 URI parameters:
 - `credentials_path`: The path to the service account JSON file.
-- `property_id`: It is a unique number that identifies a particular property on Google Analytics. [Follow this guide](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) if you don't know your property ID.
+- `property_id`: It is a unique number that identifies a particular property on Google Analytics. [Follow this guide](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) to know more about property ID.
 
 ## Setting up an Google Analytics Integration
 Google Analytics requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/google_analytics#grab-google-service-account-credentials). Once you complete the guide, you should have an `.json` file  and `project_id`. 

ingestr-0.13.28/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.28"

{ingestr-0.13.26 → ingestr-0.13.28}/ingestr/src/hubspot/__init__.py

@@ -56,6 +56,7 @@ def hubspot(
     api_key: str = dlt.secrets.value,
     include_history: bool = False,
     include_custom_props: bool = True,
+    custom_object: str = None,
 ) -> Sequence[DltResource]:
     """
     A DLT source that retrieves data from the HubSpot API using the
@@ -162,13 +163,13 @@ def hubspot(
             props,
             include_custom_props,
         )
-      
+
     @dlt.resource(name="schemas", write_disposition="merge", primary_key="id")
     def schemas(
         api_key: str = api_key,
     ) -> Iterator[TDataItems]:
         """Hubspot schemas resource"""
-        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key)
+        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key,resource_name="schemas")
 
     @dlt.resource(name="quotes", write_disposition="replace")
     def quotes(
@@ -186,7 +187,34 @@ def hubspot(
             include_custom_props,
         )
 
-    return companies, contacts, deals, tickets, products, quotes, schemas
+    @dlt.resource(write_disposition="merge", primary_key="hs_object_id")
+    def custom(
+        api_key: str = api_key,
+        custom_object_name: str = custom_object,
+    ) -> Iterator[TDataItems]:
+
+        get_custom_object =  schemas(api_key)
+        object_type_id = None
+        for custom_object in get_custom_object:
+            if custom_object["name"] == custom_object_name.capitalize():
+                object_type_id = custom_object["objectTypeId"]
+                break
+        if object_type_id is None:
+            raise ValueError(f"There is no such custom object as {custom_object_name}")
+        custom_object_properties= f"crm/v3/properties/{object_type_id}"
+        
+        props_pages = fetch_data(custom_object_properties, api_key)
+        props = []
+        for page in props_pages:
+            props.extend([prop["name"] for prop in page])
+        props = ",".join(sorted(list(set(props))))
+       
+        custom_object_endpoint= f"crm/v3/objects/{object_type_id}/?properties={props}"
+        
+        """Hubspot custom object details resource"""
+        yield from fetch_data(custom_object_endpoint, api_key,resource_name="custom")
+
+    return companies, contacts, deals, tickets, products, quotes, schemas, custom
 
 
 def crm_objects(

{ingestr-0.13.26 → ingestr-0.13.28}/ingestr/src/hubspot/helpers.py

@@ -90,7 +90,7 @@ def fetch_property_history(
 
 
 def fetch_data(
-    endpoint: str, api_key: str, params: Optional[Dict[str, Any]] = None
+    endpoint: str, api_key: str, params: Optional[Dict[str, Any]] = None,resource_name: str = None
 ) -> Iterator[List[Dict[str, Any]]]:
     """
     Fetch data from HUBSPOT endpoint using a specified API key and yield the properties of each result.
@@ -132,7 +132,7 @@ def fetch_data(
         if "results" in _data:
             _objects: List[Dict[str, Any]] = []
             for _result in _data["results"]:
-                if endpoint == "/crm/v3/schemas":
+                if resource_name == "schemas":
                     _objects.append({
                         "name": _result["labels"].get("singular", ""),
                         "objectTypeId": _result.get("objectTypeId", ""),
@@ -142,6 +142,10 @@ def fetch_data(
                         "createdAt": _result.get("createdAt", ""),
                         "updatedAt": _result.get("updatedAt", "")
                     })
+                elif resource_name == "custom":
+                    _objects.append(
+                        _result.get("properties", ""),
+                    )
                 else:
                     _obj = _result.get("properties", _result)
                     if "id" not in _obj and "id" in _result:

{ingestr-0.13.26 → ingestr-0.13.28}/ingestr/src/sources.py

@@ -803,7 +803,20 @@ class HubspotSource:
             raise ValueError("api_key in the URI is required to connect to Hubspot")
 
         endpoint = None
-        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
+
+        if table.startswith("custom:"):
+            fields = table.split(":", 2)
+            if len(fields) != 2:
+                raise ValueError(
+                    "Invalid Hubspot custom table format. Expected format: custom:<custom_object_type>"
+                )
+            endpoint = fields[1]
+            return hubspot(
+                api_key=api_key[0],
+                custom_object=endpoint,
+            ).with_resources("custom")
+
+        elif table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
             endpoint = table
         else:
             raise ValueError(
@@ -1398,13 +1411,19 @@ class GoogleAnalyticsSource:
         parse_uri = urlparse(uri)
         source_fields = parse_qs(parse_uri.query)
         cred_path = source_fields.get("credentials_path")
+        cred_base64 = source_fields.get("credentials_base64")
 
-        if not cred_path:
-            raise ValueError("credentials_path is required to connect Google Analytics")
-        credentials = {}
+        if not cred_path and not cred_base64:
+            raise ValueError("credentials_path or credentials_base64 is required to connect Google Analytics")
 
-        with open(cred_path[0], "r") as f:
-            credentials = json.load(f)
+        credentials = {}
+        if cred_path:
+            with open(cred_path[0], "r") as f:
+                credentials = json.load(f)
+        elif cred_base64:
+            credentials = json.loads(
+                base64.b64decode(cred_base64[0]).decode("utf-8")
+            )
 
         property_id = source_fields.get("property_id")
         if not property_id:

{ingestr-0.13.26 → ingestr-0.13.28}/requirements.txt

@@ -187,7 +187,7 @@ googleapis-common-protos==1.69.0
     #   google-ads
     #   google-api-core
     #   grpcio-status
-greenlet==3.1.1
+greenlet==3.2.0
     # via sqlalchemy
 grpcio==1.70.0
     # via

ingestr-0.13.26/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.26"