ingestr 0.9.5__py3-none-any.whl → 0.10.0rc0__py3-none-any.whl

ingestr/main.py

@@ -1,19 +1,13 @@
-import hashlib
-import tempfile
 from datetime import datetime
 from enum import Enum
 from typing import Optional
 
-import dlt
-import humanize
 import typer
-from dlt.common.pipeline import LoadInfo
-from dlt.common.runtime.collector import Collector, LogCollector
+from dlt.common.runtime.collector import Collector
 from rich.console import Console
 from rich.status import Status
 from typing_extensions import Annotated
 
-from ingestr.src.factory import SourceDestinationFactory
 from ingestr.src.telemetry.event import track
 
 app = typer.Typer(
@@ -181,6 +175,20 @@ def ingest(
             envvar="PRIMARY_KEY",
         ),
     ] = None,  # type: ignore
+    partition_by: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The partition key to be used for partitioning the destination table",
+            envvar="PARTITION_BY",
+        ),
+    ] = None,  # type: ignore
+    cluster_by: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The clustering key to be used for clustering the destination table, not every destination supports clustering.",
+            envvar="CLUSTER_BY",
+        ),
+    ] = None,  # type: ignore
     yes: Annotated[
         Optional[bool],
         typer.Option(
@@ -252,6 +260,66 @@ def ingest(
         ),
     ] = 5,  # type: ignore
 ):
+    import hashlib
+    import tempfile
+    from datetime import datetime
+
+    import dlt
+    import humanize
+    import typer
+    from dlt.common.destination import Destination
+    from dlt.common.pipeline import LoadInfo
+    from dlt.common.runtime.collector import Collector, LogCollector
+    from dlt.common.schema.typing import TColumnSchema
+
+    from ingestr.src.factory import SourceDestinationFactory
+    from ingestr.src.telemetry.event import track
+
+    def report_errors(run_info: LoadInfo):
+        for load_package in run_info.load_packages:
+            failed_jobs = load_package.jobs["failed_jobs"]
+            if len(failed_jobs) == 0:
+                continue
+
+            print()
+            print("[bold red]Failed jobs:[/bold red]")
+            print()
+            for job in failed_jobs:
+                print(f"[bold red]  {job.job_file_info.job_id()}[/bold red]")
+                print(f"    [bold yellow]Error:[/bold yellow] {job.failed_message}")
+
+            raise typer.Exit(1)
+
+    def validate_source_dest_tables(
+        source_table: str, dest_table: str
+    ) -> tuple[str, str]:
+        if not dest_table:
+            if len(source_table.split(".")) != 2:
+                print(
+                    "[red]Table name must be in the format schema.table for source table when dest-table is not given.[/red]"
+                )
+                raise typer.Abort()
+
+            print()
+            print(
+                "[yellow]Destination table is not given, defaulting to the source table.[/yellow]"
+            )
+            dest_table = source_table
+        return (source_table, dest_table)
+
+    def validate_loader_file_format(
+        dlt_dest: Destination, loader_file_format: Optional[LoaderFileFormat]
+    ):
+        if (
+            loader_file_format
+            and loader_file_format.value
+            not in dlt_dest.capabilities().supported_loader_file_formats
+        ):
+            print(
+                f"[red]Loader file format {loader_file_format.value} is not supported by the destination.[/red]"
+            )
+            raise typer.Abort()
+
     track(
         "command_triggered",
         {
@@ -267,29 +335,23 @@ def ingest(
         dlt.config["schema.naming"] = schema_naming.value
 
     try:
-        if not dest_table:
-            if len(source_table.split(".")) != 2:
-                print(
-                    "[red]Table name must be in the format schema.table for source table when dest-table is not given.[/red]"
-                )
-                raise typer.Abort()
-
-            print()
-            print(
-                "[yellow]Destination table is not given, defaulting to the source table.[/yellow]"
-            )
-            dest_table = source_table
+        (source_table, dest_table) = validate_source_dest_tables(
+            source_table, dest_table
+        )
 
         factory = SourceDestinationFactory(source_uri, dest_uri)
         source = factory.get_source()
         destination = factory.get_destination()
 
+        column_hints: dict[str, TColumnSchema] = {}
         original_incremental_strategy = incremental_strategy
 
         merge_key = None
         if incremental_strategy == IncrementalStrategy.delete_insert:
             merge_key = incremental_key
             incremental_strategy = IncrementalStrategy.merge
+            if incremental_key:
+                column_hints[incremental_key] = {"merge_key": True}
 
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
@@ -303,11 +365,31 @@ def ingest(
             pipelines_dir = tempfile.mkdtemp()
             is_pipelines_dir_temp = True
 
+        dlt_dest = destination.dlt_dest(uri=dest_uri)
+        validate_loader_file_format(dlt_dest, loader_file_format)
+
+        if partition_by:
+            if partition_by not in column_hints:
+                column_hints[partition_by] = {}
+
+            column_hints[partition_by]["partition"] = True
+
+        if cluster_by:
+            if cluster_by not in column_hints:
+                column_hints[cluster_by] = {}
+
+            column_hints[cluster_by]["cluster"] = True
+
+        if primary_key:
+            for key in primary_key:
+                if key not in column_hints:
+                    column_hints[key] = {}
+
+                column_hints[key]["primary_key"] = True
+
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),
-            destination=destination.dlt_dest(
-                uri=dest_uri,
-            ),
+            destination=dlt_dest,
             progress=progressInstance,
             pipelines_dir=pipelines_dir,
             refresh="drop_resources" if full_refresh else None,
@@ -400,27 +482,17 @@ def ingest(
             loader_file_format=loader_file_format.value
             if loader_file_format is not None
             else None,  # type: ignore
+            columns=column_hints,
         )
 
-        for load_package in run_info.load_packages:
-            failed_jobs = load_package.jobs["failed_jobs"]
-            if len(failed_jobs) > 0:
-                print()
-                print("[bold red]Failed jobs:[/bold red]")
-                print()
-                for job in failed_jobs:
-                    print(f"[bold red]  {job.job_file_info.job_id()}[/bold red]")
-                    print(f"    [bold yellow]Error:[/bold yellow] {job.failed_message}")
-
-                raise typer.Exit(1)
+        report_errors(run_info)
 
         destination.post_load()
 
         end_time = datetime.now()
         elapsedHuman = ""
-        if run_info.started_at:
-            elapsed = end_time - start_time
-            elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
+        elapsed = end_time - start_time
+        elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
 
         # remove the pipelines_dir folder if it was created by ingestr
         if is_pipelines_dir_temp:

ingestr/src/gorgias/__init__.py

@@ -99,12 +99,12 @@ def gorgias_source(
                 "description": "When the user was last updated.",
             },
             "meta": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Meta information associated with the user.",
             },
             "data": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Additional data associated with the user.",
             },
@@ -185,17 +185,17 @@ def gorgias_source(
                 "description": "Indicates if the ticket was created by an agent",
             },
             "customer": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": False,
                 "description": "The customer linked to the ticket.",
             },
             "assignee_user": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "User assigned to the ticket",
             },
             "assignee_team": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Team assigned to the ticket",
             },
@@ -210,17 +210,17 @@ def gorgias_source(
                 "description": "Excerpt of the ticket",
             },
             "integrations": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": False,
                 "description": "Integration information related to the ticket",
             },
             "meta": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Meta information related to the ticket",
             },
             "tags": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": False,
                 "description": "Tags associated with the ticket",
             },
@@ -354,7 +354,7 @@ def gorgias_source(
                 "description": "How the message has been received, or sent from Gorgias.",
             },
             "sender": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": False,
                 "description": "The person who sent the message. It can be a user or a customer.",
             },
@@ -364,7 +364,7 @@ def gorgias_source(
                 "description": "ID of the integration that either received or sent the message.",
             },
             "intents": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "",
             },
@@ -379,7 +379,7 @@ def gorgias_source(
                 "description": "Whether the message was sent by your company to a customer, or the opposite.",
             },
             "receiver": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The primary receiver of the message. It can be a user or a customer. Optional when the source type is 'internal-note'.",
             },
@@ -414,27 +414,27 @@ def gorgias_source(
                 "description": "",
             },
             "headers": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Headers of the message",
             },
             "attachments": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of files attached to the message.",
             },
             "actions": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of actions performed on the message.",
             },
             "macros": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of macros",
             },
             "meta": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Message metadata",
             },
@@ -526,7 +526,7 @@ def gorgias_source(
                 "description": "ID of the customer linked to the survey.",
             },
             "meta": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Meta information associated with the survey.",
             },

ingestr/src/shopify/__init__.py

@@ -89,12 +89,12 @@ def shopify_source(
                 "description": "An unsigned 64-bit integer that's used as a unique identifier for the product.",
             },
             "images": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of product image objects, each one representing an image associated with the product.",
             },
             "options": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The custom product properties. For example, Size, Color, and Material.",
             },
@@ -139,7 +139,7 @@ def shopify_source(
                 "description": "The date and time (ISO 8601 format) when the product was last modified.",
             },
             "variants": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "An array of product variants, each representing a different version of the product.",
             },
@@ -191,7 +191,7 @@ def shopify_source(
                 "description": "The ID of the app that created the order.",
             },
             "billing_address": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The mailing address associated with the payment method.",
             },
@@ -226,7 +226,7 @@ def shopify_source(
                 "description": "A unique value referencing the checkout associated with the order.",
             },
             "client_details": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Information about the browser the customer used when placing the order.",
             },
@@ -236,7 +236,7 @@ def shopify_source(
                 "description": "The date and time when the order was closed.",
             },
             "company": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Information about the purchasing company for the order.",
             },
@@ -261,7 +261,7 @@ def shopify_source(
                 "description": "The three-letter code (ISO 4217 format) for the shop currency.",
             },
             "current_total_additional_fees_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The current total additional fees on the order in shop and presentment currencies.",
             },
@@ -271,12 +271,12 @@ def shopify_source(
                 "description": "The current total discounts on the order in the shop currency.",
             },
             "current_total_discounts_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The current total discounts on the order in shop and presentment currencies.",
             },
             "current_total_duties_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The current total duties charged on the order in shop and presentment currencies.",
             },
@@ -286,7 +286,7 @@ def shopify_source(
                 "description": "The current total price of the order in the shop currency.",
             },
             "current_total_price_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The current total price of the order in shop and presentment currencies.",
             },
@@ -296,7 +296,7 @@ def shopify_source(
                 "description": "The sum of prices for all line items after discounts and returns in the shop currency.",
             },
             "current_subtotal_price_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The sum of the prices for all line items after discounts and returns in shop and presentment currencies.",
             },
@@ -306,12 +306,12 @@ def shopify_source(
                 "description": "The sum of the prices for all tax lines applied to the order in the shop currency.",
             },
             "current_total_tax_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The sum of the prices for all tax lines applied to the order in shop and presentment currencies.",
             },
             "customer": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Information about the customer.",
             },
@@ -321,12 +321,12 @@ def shopify_source(
                 "description": "The two or three-letter language code, optionally followed by a region modifier.",
             },
             "discount_applications": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "An ordered list of stacked discount applications.",
             },
             "discount_codes": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of discounts applied to the order.",
             },
@@ -346,7 +346,7 @@ def shopify_source(
                 "description": "The status of payments associated with the order.",
             },
             "fulfillments": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "An array of fulfillments associated with the order.",
             },
@@ -372,7 +372,7 @@ def shopify_source(
                 "description": "The URL for the page where the buyer landed when they entered the shop.",
             },
             "line_items": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of line item objects containing information about an item in the order.",
             },
@@ -397,7 +397,7 @@ def shopify_source(
                 "description": "An optional note that a shop owner can attach to the order.",
             },
             "note_attributes": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "Extra information added to the order as key-value pairs.",
             },
@@ -412,22 +412,22 @@ def shopify_source(
                 "description": "The order's position in the shop's count of orders, starting at 1001.",
             },
             "original_total_additional_fees_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The original total additional fees on the order in shop and presentment currencies.",
             },
             "original_total_duties_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The original total duties charged on the order in shop and presentment currencies.",
             },
             "payment_terms": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The terms and conditions under which a payment should be processed.",
             },
             "payment_gateway_names": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The list of payment gateways used for the order.",
             },
@@ -457,17 +457,17 @@ def shopify_source(
                 "description": "The website where the customer clicked a link to the shop.",
             },
             "refunds": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of refunds applied to the order.",
             },
             "shipping_address": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The mailing address where the order will be shipped.",
             },
             "shipping_lines": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "An array detailing the shipping methods used.",
             },
@@ -492,7 +492,7 @@ def shopify_source(
                 "description": "The price of the order in the shop currency after discounts but before shipping, duties, taxes, and tips.",
             },
             "subtotal_price_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The subtotal of the order in shop and presentment currencies after discounts but before shipping, duties, taxes, and tips.",
             },
@@ -502,7 +502,7 @@ def shopify_source(
                 "description": "Tags attached to the order, formatted as a string of comma-separated values.",
             },
             "tax_lines": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "An array of tax line objects detailing taxes applied to the order.",
             },
@@ -527,7 +527,7 @@ def shopify_source(
                 "description": "The total discounts applied to the price of the order in the shop currency.",
             },
             "total_discounts_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The total discounts applied to the price of the order in shop and presentment currencies.",
             },
@@ -537,7 +537,7 @@ def shopify_source(
                 "description": "The sum of all line item prices in the shop currency.",
             },
             "total_line_items_price_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The total of all line item prices in shop and presentment currencies.",
             },
@@ -552,12 +552,12 @@ def shopify_source(
                 "description": "The sum of all line item prices, discounts, shipping, taxes, and tips in the shop currency.",
             },
             "total_price_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The total price of the order in shop and presentment currencies.",
             },
             "total_shipping_price_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The total shipping price of the order in shop and presentment currencies.",
             },
@@ -567,7 +567,7 @@ def shopify_source(
                 "description": "The sum of the prices for all tax lines applied to the order in the shop currency.",
             },
             "total_tax_set": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The sum of the prices for all tax lines applied to the order in shop and presentment currencies.",
             },
@@ -1669,27 +1669,27 @@ query discountNodes($after: String, $query: String, $first: Int)  {
                 "description": "A globally unique ID for the product.",
             },
             "availablePublicationsCount": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": False,
                 "description": "The number of publications that a resource is published to",
             },
             "category": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The category of the product from Shopify's Standard Product Taxonomy.",
             },
             "combinedListing": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A special product type that combines separate products into a single product listing.",
             },
             "combinedListingRole": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The role of the product in a combined listing.",
             },
             "compareAtPriceRange": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "The compare-at price range of the product in the shop's default currency.",
             },
@@ -1719,17 +1719,17 @@ query discountNodes($after: String, $query: String, $first: Int)  {
                 "description": "A unique, human-readable string of the product's title.",
             },
             "metafields": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of custom fields associated with the product.",
             },
             "options": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": True,
                 "description": "A list of product options, e.g., size, color.",
             },
             "priceRangeV2": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": False,
                 "description": "The minimum and maximum prices of a product.",
             },
@@ -1784,12 +1784,12 @@ query discountNodes($after: String, $query: String, $first: Int)  {
                 "description": "The date and time when the product was last modified.",
             },
             "variantsFirst250": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": False,
                 "description": "A list of variants associated with the product, first 250.",
             },
             "variantsCount": {
-                "data_type": "complex",
+                "data_type": "json",
                 "nullable": False,
                 "description": "The number of variants associated with the product.",
             },

ingestr/src/slack/__init__.py

@@ -166,7 +166,7 @@ def slack_source(
     @dlt.resource(
         name="messages",
         primary_key=("channel", "ts"),
-        columns={"blocks": {"data_type": "complex"}},
+        columns={"blocks": {"data_type": "json"}},
         write_disposition=write_disposition,
     )
     def messages_resource(
@@ -249,7 +249,7 @@ def slack_source(
                 table_name=table_name,
                 primary_key=("channel", "ts"),
                 write_disposition=write_disposition,
-                columns={"blocks": {"data_type": "complex"}},
+                columns={"blocks": {"data_type": "json"}},
             )(channel)
 
             yield messages_channel