ingestr 0.13.22__py3-none-any.whl → 0.13.24__py3-none-any.whl

ingestr/conftest.py

@@ -0,0 +1,63 @@
+import os
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+
+from main_test import DESTINATIONS, SOURCES  # type: ignore
+
+
+def pytest_configure(config):
+    if is_master(config):
+        config.shared_directory = tempfile.mkdtemp()
+
+
+def pytest_configure_node(node):
+    """xdist hook"""
+    node.workerinput["shared_directory"] = node.config.shared_directory
+
+
+def is_master(config):
+    """True if the code running the given pytest.config object is running in a xdist master
+    node or not running xdist at all.
+    """
+    return not hasattr(config, "workerinput")
+
+
+def start_containers(config):
+    if hasattr(config, "workerinput"):
+        return
+
+    unique_containers = set(SOURCES.values()) | set(DESTINATIONS.values())
+    for container in unique_containers:
+        container.container_lock_dir = config.shared_directory
+
+    with ThreadPoolExecutor() as executor:
+        for container in unique_containers:
+            executor.submit(container.start_fully)
+        # futures = [
+        #     executor.submit(container.start_fully) for container in unique_containers
+        # ]
+        # # Wait for all futures to complete
+        # for future in futures:
+        #     future.result()
+
+
+def stop_containers(config):
+    if hasattr(config, "workerinput"):
+        return
+
+    should_manage_containers = os.environ.get("PYTEST_XDIST_WORKER", "gw0") == "gw0"
+    if not should_manage_containers:
+        return
+
+    unique_containers = set(SOURCES.values()) | set(DESTINATIONS.values())
+
+    for container in unique_containers:
+        container.stop_fully()
+
+
+def pytest_sessionstart(session):
+    start_containers(session.config)
+
+
+def pytest_sessionfinish(session, exitstatus):
+    stop_containers(session.config)

ingestr/main.py

@@ -11,7 +11,7 @@ from typing_extensions import Annotated
 import ingestr.src.partition as partition
 import ingestr.src.resource as resource
 from ingestr.src.destinations import AthenaDestination
-from ingestr.src.filters import cast_set_to_list
+from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
 from ingestr.src.telemetry.event import track
 
 app = typer.Typer(
@@ -35,7 +35,7 @@ DATE_FORMATS = [
 
 # https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
 PARQUET_SUPPORTED_DESTINATIONS = [
-    "athena" "bigquery",
+    "athenabigquery",
     "duckdb",
     "snowflake",
     "databricks",
@@ -553,6 +553,8 @@ def ingest(
         )
 
         resource.for_each(dlt_source, lambda x: x.add_map(cast_set_to_list))
+        if factory.source_scheme.startswith("mysql"):
+            resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
 
         def col_h(x):
             if column_hints:

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.22"
+version = "v0.13.24"

ingestr/src/filters.py

@@ -10,6 +10,27 @@ def cast_set_to_list(row):
     return row
 
 
+def handle_mysql_empty_dates(row):
+    # MySQL returns empty dates as 0000-00-00, which is not a valid date, we handle them here.
+    if not isinstance(row, dict):
+        return row
+
+    for key in row.keys():
+        if not isinstance(row[key], str):
+            continue
+
+        if row[key] == "0000-00-00":
+            from datetime import date
+
+            row[key] = date(1970, 1, 1)
+
+        elif row[key] == "0000-00-00 00:00:00":
+            from datetime import datetime
+
+            row[key] = datetime(1970, 1, 1, 0, 0, 0)
+    return row
+
+
 def table_adapter_exclude_columns(cols: list[str]):
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore

ingestr/src/github/helpers.py

@@ -103,9 +103,9 @@ def get_reactions_data(
 
 
 def _extract_top_connection(data: StrAny, node_type: str) -> StrAny:
-    assert (
-        isinstance(data, dict) and len(data) == 1
-    ), f"The data with list of {node_type} must be a dictionary and contain only one element"
+    assert isinstance(data, dict) and len(data) == 1, (
+        f"The data with list of {node_type} must be a dictionary and contain only one element"
+    )
     data = next(iter(data.values()))
     return data[node_type]  # type: ignore
 
@@ -158,7 +158,7 @@ def _get_graphql_pages(
         )
         items_count += len(data_items)
         print(
-            f'Got {len(data_items)}/{items_count} {node_type}s, query cost {rate_limit["cost"]}, remaining credits: {rate_limit["remaining"]}'
+            f"Got {len(data_items)}/{items_count} {node_type}s, query cost {rate_limit['cost']}, remaining credits: {rate_limit['remaining']}"
         )
         if data_items:
             yield data_items
@@ -187,7 +187,7 @@ def _get_comment_reaction(comment_ids: List[str], access_token: str) -> StrAny:
         # print(query)
         page, rate_limit = _run_graphql_query(access_token, query, {})
         print(
-            f'Got {len(page)} comments, query cost {rate_limit["cost"]}, remaining credits: {rate_limit["remaining"]}'
+            f"Got {len(page)} comments, query cost {rate_limit['cost']}, remaining credits: {rate_limit['remaining']}"
         )
         data.update(page)
     return data

ingestr/src/google_sheets/__init__.py

@@ -70,9 +70,9 @@ def google_spreadsheet(
         spreadsheet_id=spreadsheet_id,
         range_names=list(all_range_names),
     )
-    assert len(all_range_names) == len(
-        all_range_data
-    ), "Google Sheets API must return values for all requested ranges"
+    assert len(all_range_names) == len(all_range_data), (
+        "Google Sheets API must return values for all requested ranges"
+    )
 
     # get metadata for two first rows of each range
     # first should contain headers
@@ -126,7 +126,7 @@ def google_spreadsheet(
         headers = get_range_headers(headers_metadata, name)
         if headers is None:
             # generate automatic headers and treat the first row as data
-            headers = [f"col_{idx+1}" for idx in range(len(headers_metadata))]
+            headers = [f"col_{idx + 1}" for idx in range(len(headers_metadata))]
             data_row_metadata = headers_metadata
             rows_data = values[0:]
             logger.warning(

ingestr/src/google_sheets/helpers/data_processing.py

@@ -149,12 +149,12 @@ def get_range_headers(headers_metadata: List[DictStrAny], range_name: str) -> Li
                     header_val = str(f"col_{idx + 1}")
                 else:
                     logger.warning(
-                        f"In range {range_name}, header value: {header_val} at position {idx+1} is not a string!"
+                        f"In range {range_name}, header value: {header_val} at position {idx + 1} is not a string!"
                     )
                     return None
         else:
             logger.warning(
-                f"In range {range_name}, header at position {idx+1} is not missing!"
+                f"In range {range_name}, header at position {idx + 1} is not missing!"
             )
             return None
         headers.append(header_val)

{ingestr-0.13.22.dist-info → ingestr-0.13.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.22
+Version: 0.13.24
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -46,10 +46,10 @@ Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
-Requires-Dist: dlt==1.6.1
+Requires-Dist: dlt==1.9.0
 Requires-Dist: dnspython==2.7.0
-Requires-Dist: duckdb-engine==0.15.0
-Requires-Dist: duckdb==1.2.0
+Requires-Dist: duckdb-engine==0.17.0
+Requires-Dist: duckdb==1.2.1
 Requires-Dist: et-xmlfile==2.0.0
 Requires-Dist: facebook-business==20.0.0
 Requires-Dist: filelock==3.17.0
@@ -168,6 +168,7 @@ Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
+Requires-Dist: sqlglot==26.12.1
 Requires-Dist: stripe==10.7.0
 Requires-Dist: tenacity==9.0.0
 Requires-Dist: thrift==0.16.0

{ingestr-0.13.22.dist-info → ingestr-0.13.24.dist-info}/RECORD

@@ -1,11 +1,12 @@
-ingestr/main.py,sha256=74lbiWEa27MUKFPbyUNGIlrwD5fRxej5cKFwe_LX1pE,25452
+ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
+ingestr/main.py,sha256=wvbRCJ2--M0Zw2cYtSH874TxTtlD0wadHREeLG3anOY,25618
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=ExEPLDyz3-FkQx0OHsblNsR-B9G1fUx77cQtxlv6CXA,21
+ingestr/src/buildinfo.py,sha256=x-bxDOuFDQ1rgDJf03eHD1bXb9Yfo3wX39XyaBE0LkU,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=1jqcLv_QUUGeyg1OYN3ywrRdcDZyDRtMOongwyjDapU,5268
-ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
+ingestr/src/filters.py,sha256=5LNpBgm8FJXdrFHGyM7dLVyphKykSpPk7yuQAZ8GML4,1133
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
@@ -41,7 +42,7 @@ ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351Aj
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
 ingestr/src/github/__init__.py,sha256=xVijF-Wi4p88hkVJnKH-oTixismjD3aUcGqGa6Wr4e4,5889
-ingestr/src/github/helpers.py,sha256=Tmnik9811zBWNO6cJwV9PFQxEx2j32LHAQCvNbubsEI,6759
+ingestr/src/github/helpers.py,sha256=rpv_3HzuOl4PQ-FUeA66pev-pgze9SaE8RUHIPYfZ_A,6759
 ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
 ingestr/src/github/settings.py,sha256=N5ahWrDIQ_4IWV9i-hTXxyYduqY9Ym2BTwqsWxcDdJ8,258
 ingestr/src/google_ads/__init__.py,sha256=bH0TtnRWcOUESezpvoA7VEUHAq_0ITGQeX4GGVBfl1I,3725
@@ -52,10 +53,10 @@ ingestr/src/google_ads/reports.py,sha256=AVY1pPt5yaIFskQe1k5VW2Dhlux3bzewsHlDrdG
 ingestr/src/google_analytics/__init__.py,sha256=8Evpmoy464YpNbCI_NmvFHIzWCu7J7SjJw-RrPZ6AL8,3674
 ingestr/src/google_analytics/helpers.py,sha256=vLmFyQ_IEJEK5LlxBJQeJw0VHaE5gRRZdBa54U72CaQ,5965
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
-ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
+ingestr/src/google_sheets/__init__.py,sha256=CL0HfY74uxX8-ge0ucI0VhWMYZVAfoX7WRPBitRi-CI,6647
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
 ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
-ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
+ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
 ingestr/src/hubspot/__init__.py,sha256=NYgSIAPXQh2Qp1eKun7TgcerKogq6pWtNkr-_f0FXbI,9464
@@ -118,8 +119,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.22.dist-info/METADATA,sha256=SC89LgkVuV22LAaSCETkDoT6bFYCgIkHjLgs2UP4q4c,13627
-ingestr-0.13.22.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.22.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.22.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.22.dist-info/RECORD,,
+ingestr-0.13.24.dist-info/METADATA,sha256=qzKyEOTPIb6cTD49Q6zC-bqSn7ax45JmCKcGh0jtJRw,13659
+ingestr-0.13.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.24.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.24.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.24.dist-info/RECORD,,