ingestr 0.13.28__tar.gz → 0.13.29__tar.gz

{ingestr-0.13.28 → ingestr-0.13.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.28
+Version: 0.13.29
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.28 → ingestr-0.13.29}/ingestr/main.py

@@ -302,6 +302,13 @@ def ingest(
             envvar=["COLUMNS", "INGESTR_COLUMNS"],
         ),
     ] = None,  # type: ignore
+    yield_limit: Annotated[
+        Optional[int],
+        typer.Option(
+            help="Limit the number of pages yielded from the source",
+            envvar=["YIELD_LIMIT", "INGESTR_YIELD_LIMIT"],
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -556,6 +563,9 @@ def ingest(
         if factory.source_scheme.startswith("mysql"):
             resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
 
+        if yield_limit:
+            resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
+
         def col_h(x):
             if column_hints:
                 x.apply_hints(columns=column_hints)

ingestr-0.13.29/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.29"

{ingestr-0.13.28 → ingestr-0.13.29}/ingestr/src/hubspot/__init__.py

@@ -169,7 +169,7 @@ def hubspot(
         api_key: str = api_key,
     ) -> Iterator[TDataItems]:
         """Hubspot schemas resource"""
-        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key,resource_name="schemas")
+        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key, resource_name="schemas")
 
     @dlt.resource(name="quotes", write_disposition="replace")
     def quotes(
@@ -192,8 +192,7 @@ def hubspot(
         api_key: str = api_key,
         custom_object_name: str = custom_object,
     ) -> Iterator[TDataItems]:
-
-        get_custom_object =  schemas(api_key)
+        get_custom_object = schemas(api_key)
         object_type_id = None
         for custom_object in get_custom_object:
             if custom_object["name"] == custom_object_name.capitalize():
@@ -201,18 +200,18 @@ def hubspot(
                 break
         if object_type_id is None:
             raise ValueError(f"There is no such custom object as {custom_object_name}")
-        custom_object_properties= f"crm/v3/properties/{object_type_id}"
-        
+        custom_object_properties = f"crm/v3/properties/{object_type_id}"
+
         props_pages = fetch_data(custom_object_properties, api_key)
         props = []
         for page in props_pages:
             props.extend([prop["name"] for prop in page])
         props = ",".join(sorted(list(set(props))))
-       
-        custom_object_endpoint= f"crm/v3/objects/{object_type_id}/?properties={props}"
-        
+
+        custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
+
         """Hubspot custom object details resource"""
-        yield from fetch_data(custom_object_endpoint, api_key,resource_name="custom")
+        yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")
 
     return companies, contacts, deals, tickets, products, quotes, schemas, custom
 

{ingestr-0.13.28 → ingestr-0.13.29}/ingestr/src/hubspot/helpers.py

@@ -90,7 +90,10 @@ def fetch_property_history(
 
 
 def fetch_data(
-    endpoint: str, api_key: str, params: Optional[Dict[str, Any]] = None,resource_name: str = None
+    endpoint: str,
+    api_key: str,
+    params: Optional[Dict[str, Any]] = None,
+    resource_name: str = None,
 ) -> Iterator[List[Dict[str, Any]]]:
     """
     Fetch data from HUBSPOT endpoint using a specified API key and yield the properties of each result.
@@ -133,15 +136,17 @@ def fetch_data(
             _objects: List[Dict[str, Any]] = []
             for _result in _data["results"]:
                 if resource_name == "schemas":
-                    _objects.append({
-                        "name": _result["labels"].get("singular", ""),
-                        "objectTypeId": _result.get("objectTypeId", ""),
-                        "id": _result.get("id", ""),
-                        "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
-                        "properties": _result.get("properties", ""),
-                        "createdAt": _result.get("createdAt", ""),
-                        "updatedAt": _result.get("updatedAt", "")
-                    })
+                    _objects.append(
+                        {
+                            "name": _result["labels"].get("singular", ""),
+                            "objectTypeId": _result.get("objectTypeId", ""),
+                            "id": _result.get("id", ""),
+                            "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
+                            "properties": _result.get("properties", ""),
+                            "createdAt": _result.get("createdAt", ""),
+                            "updatedAt": _result.get("updatedAt", ""),
+                        }
+                    )
                 elif resource_name == "custom":
                     _objects.append(
                         _result.get("properties", ""),
@@ -157,9 +162,11 @@ def fetch_data(
                                 {
                                     "value": _obj["hs_object_id"],
                                     f"{association}_id": __r["id"],
-                            }
-                            for __r in _result["associations"][association]["results"]
-                        ]
+                                }
+                                for __r in _result["associations"][association][
+                                    "results"
+                                ]
+                            ]
 
                         # remove duplicates from list of dicts
                         __values = [

{ingestr-0.13.28 → ingestr-0.13.29}/ingestr/src/hubspot/settings.py

@@ -5,15 +5,21 @@ from dlt.common import pendulum
 STARTDATE = pendulum.datetime(year=2000, month=1, day=1)
 
 CRM_CONTACTS_ENDPOINT = (
-    "/crm/v3/objects/contacts?associations=deals,products,tickets,quotes"
+    "/crm/v3/objects/contacts?associations=companies,deals,products,tickets,quotes"
 )
-CRM_COMPANIES_ENDPOINT = (
-    "/crm/v3/objects/companies?associations=contacts,deals,products,tickets,quotes"
+CRM_COMPANIES_ENDPOINT = "/crm/v3/objects/companies?associations=products"
+CRM_DEALS_ENDPOINT = (
+    "/crm/v3/objects/deals?associations=companies,contacts,products,tickets,quotes"
+)
+CRM_PRODUCTS_ENDPOINT = (
+    "/crm/v3/objects/products?associations=companies,contacts,deals,tickets,quotes"
+)
+CRM_TICKETS_ENDPOINT = (
+    "/crm/v3/objects/tickets?associations=companies,contacts,deals,products,quotes"
+)
+CRM_QUOTES_ENDPOINT = (
+    "/crm/v3/objects/quotes?associations=companies,contacts,deals,products,tickets"
 )
-CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals"
-CRM_PRODUCTS_ENDPOINT = "/crm/v3/objects/products"
-CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets"
-CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes"
 CRM_SCHEMAS_ENDPOINT = "/crm/v3/schemas"
 
 CRM_OBJECT_ENDPOINTS = {

ingestr-0.13.28/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.28"