ingestr 0.7.8__py3-none-any.whl → 0.8.1__py3-none-any.whl

ingestr/src/shopify/helpers.py

@@ -14,6 +14,53 @@ from .settings import DEFAULT_API_VERSION, DEFAULT_PARTNER_API_VERSION
 TOrderStatus = Literal["open", "closed", "cancelled", "any"]
 
 
+def convert_datetime_fields(item: Dict[str, Any]) -> Dict[str, Any]:
+    """Convert timestamp fields in the item to pendulum datetime objects
+
+    The item is modified in place, including nested items.
+
+    Args:
+        item: The item to convert
+
+    Returns:
+        The same data item (for convenience)
+    """
+    fields = ["created_at", "updated_at", "createdAt", "updatedAt"]
+
+    def convert_nested(obj: Any) -> Any:
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                if key in fields and isinstance(value, str):
+                    obj[key] = ensure_pendulum_datetime(value)
+                else:
+                    obj[key] = convert_nested(value)
+        elif isinstance(obj, list):
+            return [convert_nested(elem) for elem in obj]
+        return obj
+
+    return convert_nested(item)
+
+
+def remove_nodes_key(item: Any) -> Any:
+    """
+    Recursively remove the 'nodes' key from dictionaries if it's the only key and its value is an array.
+
+    Args:
+        item: The item to process (can be a dict, list, or any other type)
+
+    Returns:
+        The processed item
+    """
+    if isinstance(item, dict):
+        if len(item) == 1 and "nodes" in item and isinstance(item["nodes"], list):
+            return [remove_nodes_key(node) for node in item["nodes"]]
+        return {k: remove_nodes_key(v) for k, v in item.items()}
+    elif isinstance(item, list):
+        return [remove_nodes_key(element) for element in item]
+    else:
+        return item
+
+
 class ShopifyApi:
     """
     A Shopify API client that can be used to get pages of data from Shopify.
@@ -50,57 +97,38 @@ class ShopifyApi:
         """
         url = urljoin(self.shop_url, f"/admin/api/{self.api_version}/{resource}.json")
 
+        resource_last = resource.split("/")[-1]
+
         headers = {"X-Shopify-Access-Token": self.private_app_password}
         while url:
             response = requests.get(url, params=params, headers=headers)
             response.raise_for_status()
             json = response.json()
-            # Get item list from the page
-            yield [self._convert_datetime_fields(item) for item in json[resource]]
+            yield [convert_datetime_fields(item) for item in json[resource_last]]
             url = response.links.get("next", {}).get("url")
             # Query params are included in subsequent page URLs
             params = None
 
-    def _convert_datetime_fields(self, item: Dict[str, Any]) -> Dict[str, Any]:
-        """Convert timestamp fields in the item to pendulum datetime objects
-
-        The item is modified in place.
 
-        Args:
-            item: The item to convert
-
-        Returns:
-            The same data item (for convenience)
-        """
-        fields = ["created_at", "updated_at"]
-        for field in fields:
-            if field in item:
-                item[field] = ensure_pendulum_datetime(item[field])
-        return item
-
-
-class ShopifyPartnerApi:
-    """Client for Shopify Partner grapql API"""
+class ShopifyGraphQLApi:
+    """Client for Shopify GraphQL API"""
 
     def __init__(
         self,
         access_token: str,
-        organization_id: str,
         api_version: str = DEFAULT_PARTNER_API_VERSION,
+        base_url: str = "partners.shopify.com",
     ) -> None:
-        """
-        Args:
-            access_token: The access token to use
-            organization_id: The organization id to query
-            api_version: The API version to use (e.g. 2023-01)
-        """
         self.access_token = access_token
-        self.organization_id = organization_id
         self.api_version = api_version
+        self.base_url = base_url
 
     @property
     def graphql_url(self) -> str:
-        return f"https://partners.shopify.com/{self.organization_id}/api/{self.api_version}/graphql.json"
+        if self.base_url.startswith("https://"):
+            return f"{self.base_url}/admin/api/{self.api_version}/graphql.json"
+
+        return f"https://{self.base_url}/admin/api/{self.api_version}/graphql.json"
 
     def run_graphql_query(
         self, query: str, variables: Optional[DictStrAny] = None
@@ -130,18 +158,31 @@ class ShopifyPartnerApi:
         query: str,
         data_items_path: jsonpath.TJsonPath,
         pagination_cursor_path: jsonpath.TJsonPath,
+        pagination_cursor_has_next_page_path: jsonpath.TJsonPath,
         pagination_variable_name: str,
         variables: Optional[DictStrAny] = None,
     ) -> Iterable[TDataItems]:
         variables = dict(variables or {})
         while True:
             data = self.run_graphql_query(query, variables)
-            print(data)
             data_items = jsonpath.find_values(data_items_path, data)
+
             if not data_items:
                 break
-            yield data_items
+
+            yield [
+                remove_nodes_key(convert_datetime_fields(item)) for item in data_items
+            ]
+
             cursors = jsonpath.find_values(pagination_cursor_path, data)
             if not cursors:
                 break
+
+            if pagination_cursor_has_next_page_path:
+                has_next_page = jsonpath.find_values(
+                    pagination_cursor_has_next_page_path, data
+                )
+                if not has_next_page or not has_next_page[0]:
+                    break
+
             variables[pagination_variable_name] = cursors[-1]

ingestr/src/sources.py

@@ -1,13 +1,15 @@
 import base64
 import csv
 import json
-from datetime import date
+from datetime import date, datetime
 from typing import Any, Callable, Optional
 from urllib.parse import parse_qs, urlparse
 
 import dlt
 
+from ingestr.src.adjust._init_ import adjust_source
 from ingestr.src.airtable import airtable_source
+from ingestr.src.appsflyer._init_ import appsflyer_source
 from ingestr.src.chess import source
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.google_sheets import google_spreadsheet
@@ -188,11 +190,6 @@ class ShopifySource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
-        if kwargs.get("incremental_key"):
-            raise ValueError(
-                "Shopify takes care of incrementality on its own, you should not provide incremental_key"
-            )
-
         source_fields = urlparse(uri)
         source_params = parse_qs(source_fields.query)
         api_key = source_params.get("api_key")
@@ -207,7 +204,19 @@ class ShopifySource:
             date_args["end_date"] = kwargs.get("interval_end")
 
         resource = None
-        if table in ["products", "orders", "customers"]:
+        if table in [
+            "products",
+            "products_legacy",
+            "orders",
+            "customers",
+            "inventory_items",
+            "transactions",
+            "balance",
+            "events",
+            "price_rules",
+            "discounts",
+            "taxonomy",
+        ]:
             resource = table
         else:
             raise ValueError(
@@ -652,3 +661,76 @@ class KafkaSource:
             batch_size=int(batch_size[0]),
             batch_timeout=int(batch_timeout[0]),
         )
+
+
+class AdjustSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Adjust takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        source_part = urlparse(uri)
+        source_params = parse_qs(source_part.query)
+        api_key = source_params.get("api_key")
+
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Adjust")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        start_date = (
+            interval_start.strftime("%Y-%m-%d") if interval_start else "2000-01-01"
+        )
+        end_date = (
+            interval_end.strftime("%Y-%m-%d")
+            if interval_end
+            else datetime.now().strftime("%Y-%m-%d")
+        )
+
+        Endpoint = None
+        if table in ["campaigns", "creatives"]:
+            Endpoint = table
+
+        return adjust_source(
+            start_date=start_date, end_date=end_date, api_key=api_key[0]
+        ).with_resources(Endpoint)
+
+
+class AppsflyerSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Appsflyer_Source takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        source_fields = urlparse(uri)
+        source_params = parse_qs(source_fields.query)
+        api_key = source_params.get("api_key")
+
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Appsflyer")
+
+        resource = None
+        if table in ["campaigns", "creatives"]:
+            resource = table
+        else:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Appsflyer source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        start_date = kwargs.get("interval_start") or "2024-01-02"
+        end_date = kwargs.get("interval_end") or "2024-01-29"
+
+        return appsflyer_source(
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(resource)

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.7.8"
+__version__ = "0.8.1"

{ingestr-0.7.8.dist-info → ingestr-0.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.8
+Version: 0.8.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -44,6 +44,7 @@ Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: stripe==10.7.0
 Requires-Dist: tqdm==4.66.2
 Requires-Dist: typer==0.12.3
+Requires-Dist: types-requests==2.32.0.20240907
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -178,10 +179,18 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <td>Adjust</td>
+        <td>✅</td>
+        <td>-</td>
     <tr>
         <td>Airtable</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>AppsFlyer</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Chess.com</td>

{ingestr-0.7.8.dist-info → ingestr-0.8.1.dist-info}/RECORD

@@ -1,11 +1,15 @@
-ingestr/main.py,sha256=Hlcb8mUAWoGZr4ZKtQnoEhjLkjroiwx2-J86C6fN37E,17596
+ingestr/main.py,sha256=U66TM57ePv-RdoAftQ0pFZx8woZUQnLepKa50C-bA5I,17655
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/factory.py,sha256=CTVaFeMVgZO1fC9AKOqx-Wu89l5_YL6GlmvDF-FkAew,4442
-ingestr/src/sources.py,sha256=BlMsajIMcu_oqmU38uqlasXz2vtN_J8yXa24NHFcwJA,22696
+ingestr/src/factory.py,sha256=-_KwBpbNAegv_oXIB57klyjUb3K6e0lw_xdi5bwmarI,4645
+ingestr/src/sources.py,sha256=0IGguMm85E3Rahu6zVLawoe2d4lqRY31uHuxlqCsiQc,25324
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=uC8wB9mRblQ0jUBAOUyCQLUQJ39MC2xybVLB_8ZsevU,22
+ingestr/src/version.py,sha256=Ocl79hbbH8_jdr5dGC90VR1cAvZc05Rc0tkZttUnMjo,22
+ingestr/src/adjust/_init_.py,sha256=_jJE3Ywvv-YyJ7ywICdht_X2Gnd1cKm6F1wAfnpXuWM,890
+ingestr/src/adjust/helpers.py,sha256=kkYC3MqMHLNucuQ50klZWrvd3o8VfUysNtZTQSsKZ_c,2588
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
+ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
+ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
 ingestr/src/chess/__init__.py,sha256=PaxT2DObudOGlhyoENE5LjR6rTdsxiqKKpAZeyzVLCA,6791
 ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,549
 ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
@@ -35,9 +39,9 @@ ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMO
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
-ingestr/src/shopify/__init__.py,sha256=EWjpvZz7K6Pms7uUoqqkM4Wj0XeE2NrDvVp4BNM8dPk,9163
+ingestr/src/shopify/__init__.py,sha256=uqYoodwLHLuZHwvQx_lwOI23TzxnzHbGuZUUNzTA2Os,62031
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
-ingestr/src/shopify/helpers.py,sha256=OO_Tw-HwVLnRhwT3vqUWEQEEcWIS9KWE6VDDe8BCC2w,4972
+ingestr/src/shopify/helpers.py,sha256=2MlqyCc7VJxpYlGAxw-bYpSuCZNfat0E5zDe1jQYVP4,6279
 ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
 ingestr/src/slack/__init__.py,sha256=UfUhkS6FnCKJeXkkJ5QrmdT5nZm5czjtomsQu_x9WUM,9987
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
@@ -60,8 +64,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.7.8.dist-info/METADATA,sha256=JGJ_76vC0icT_tJSYDkbtRXuc_63sgHXJYYIksTSyOE,6561
-ingestr-0.7.8.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.7.8.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.7.8.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.7.8.dist-info/RECORD,,
+ingestr-0.8.1.dist-info/METADATA,sha256=r3eBIXMFdnmLPzlK7mh0BZWRJoHvXyZjls7UxkPb93Q,6755
+ingestr-0.8.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.8.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.8.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.8.1.dist-info/RECORD,,