ingestr 0.13.22__tar.gz → 0.13.24__tar.gz

{ingestr-0.13.22 → ingestr-0.13.24}/Dockerfile

@@ -3,6 +3,10 @@ FROM python:3.11-slim
 WORKDIR /app
 
 COPY ./requirements.txt /app/requirements.txt
+COPY ./requirements_arm64.txt /app/requirements_arm64.txt
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+    cp /app/requirements_arm64.txt /app/requirements.txt; \
+fi
 
 # Setup dependencies for pyodbc
 RUN \
@@ -31,7 +35,10 @@ RUN /install.sh && rm /install.sh
 RUN $HOME/.local/bin/uv pip install --system --no-cache -r requirements.txt
 
 COPY . /app
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+    cp /app/requirements_arm64.txt /app/requirements.txt; \
+fi
 
-RUN pip3 install -e . && pip3 install pyodbc
+RUN $HOME/.local/bin/uv pip install --system . && $HOME/.local/bin/uv pip install --system pyodbc
 
 ENTRYPOINT ["ingestr"]

{ingestr-0.13.22 → ingestr-0.13.24}/Makefile

@@ -13,6 +13,7 @@ venv/touchfile: requirements-dev.txt requirements.txt
 
 lock-deps:
 	@uv pip compile requirements.in --quiet -o requirements.txt 
+	@uv pip compile requirements.in --quiet -o requirements_arm64.txt --python-platform aarch64-unknown-linux-gnu
 
 deps: lock-deps
 	uv pip install -r requirements-dev.txt
@@ -27,7 +28,7 @@ test : venv lock-deps
 	. venv/bin/activate; $(MAKE) test-ci
 
 test-specific: venv lock-deps
-	. venv/bin/activate; pytest -rP -vv --tb=short --capture=no -k $(test)
+	. venv/bin/activate; TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto  -rP -vv --tb=short --capture=no -k $(test)
 
 lint-ci:
 	ruff format ingestr && ruff check ingestr --fix

{ingestr-0.13.22 → ingestr-0.13.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.22
+Version: 0.13.24
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -46,10 +46,10 @@ Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
-Requires-Dist: dlt==1.6.1
+Requires-Dist: dlt==1.9.0
 Requires-Dist: dnspython==2.7.0
-Requires-Dist: duckdb-engine==0.15.0
-Requires-Dist: duckdb==1.2.0
+Requires-Dist: duckdb-engine==0.17.0
+Requires-Dist: duckdb==1.2.1
 Requires-Dist: et-xmlfile==2.0.0
 Requires-Dist: facebook-business==20.0.0
 Requires-Dist: filelock==3.17.0
@@ -168,6 +168,7 @@ Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
+Requires-Dist: sqlglot==26.12.1
 Requires-Dist: stripe==10.7.0
 Requires-Dist: tenacity==9.0.0
 Requires-Dist: thrift==0.16.0

ingestr-0.13.24/ingestr/conftest.py

@@ -0,0 +1,63 @@
+import os
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+
+from main_test import DESTINATIONS, SOURCES  # type: ignore
+
+
+def pytest_configure(config):
+    if is_master(config):
+        config.shared_directory = tempfile.mkdtemp()
+
+
+def pytest_configure_node(node):
+    """xdist hook"""
+    node.workerinput["shared_directory"] = node.config.shared_directory
+
+
+def is_master(config):
+    """True if the code running the given pytest.config object is running in a xdist master
+    node or not running xdist at all.
+    """
+    return not hasattr(config, "workerinput")
+
+
+def start_containers(config):
+    if hasattr(config, "workerinput"):
+        return
+
+    unique_containers = set(SOURCES.values()) | set(DESTINATIONS.values())
+    for container in unique_containers:
+        container.container_lock_dir = config.shared_directory
+
+    with ThreadPoolExecutor() as executor:
+        for container in unique_containers:
+            executor.submit(container.start_fully)
+        # futures = [
+        #     executor.submit(container.start_fully) for container in unique_containers
+        # ]
+        # # Wait for all futures to complete
+        # for future in futures:
+        #     future.result()
+
+
+def stop_containers(config):
+    if hasattr(config, "workerinput"):
+        return
+
+    should_manage_containers = os.environ.get("PYTEST_XDIST_WORKER", "gw0") == "gw0"
+    if not should_manage_containers:
+        return
+
+    unique_containers = set(SOURCES.values()) | set(DESTINATIONS.values())
+
+    for container in unique_containers:
+        container.stop_fully()
+
+
+def pytest_sessionstart(session):
+    start_containers(session.config)
+
+
+def pytest_sessionfinish(session, exitstatus):
+    stop_containers(session.config)

{ingestr-0.13.22 → ingestr-0.13.24}/ingestr/main.py

@@ -11,7 +11,7 @@ from typing_extensions import Annotated
 import ingestr.src.partition as partition
 import ingestr.src.resource as resource
 from ingestr.src.destinations import AthenaDestination
-from ingestr.src.filters import cast_set_to_list
+from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
 from ingestr.src.telemetry.event import track
 
 app = typer.Typer(
@@ -35,7 +35,7 @@ DATE_FORMATS = [
 
 # https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
 PARQUET_SUPPORTED_DESTINATIONS = [
-    "athena" "bigquery",
+    "athenabigquery",
     "duckdb",
     "snowflake",
     "databricks",
@@ -553,6 +553,8 @@ def ingest(
         )
 
         resource.for_each(dlt_source, lambda x: x.add_map(cast_set_to_list))
+        if factory.source_scheme.startswith("mysql"):
+            resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
 
         def col_h(x):
             if column_hints:

ingestr-0.13.24/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.24"

{ingestr-0.13.22 → ingestr-0.13.24}/ingestr/src/filters.py

@@ -10,6 +10,27 @@ def cast_set_to_list(row):
     return row
 
 
+def handle_mysql_empty_dates(row):
+    # MySQL returns empty dates as 0000-00-00, which is not a valid date, we handle them here.
+    if not isinstance(row, dict):
+        return row
+
+    for key in row.keys():
+        if not isinstance(row[key], str):
+            continue
+
+        if row[key] == "0000-00-00":
+            from datetime import date
+
+            row[key] = date(1970, 1, 1)
+
+        elif row[key] == "0000-00-00 00:00:00":
+            from datetime import datetime
+
+            row[key] = datetime(1970, 1, 1, 0, 0, 0)
+    return row
+
+
 def table_adapter_exclude_columns(cols: list[str]):
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore

{ingestr-0.13.22 → ingestr-0.13.24}/ingestr/src/github/helpers.py

@@ -103,9 +103,9 @@ def get_reactions_data(
 
 
 def _extract_top_connection(data: StrAny, node_type: str) -> StrAny:
-    assert (
-        isinstance(data, dict) and len(data) == 1
-    ), f"The data with list of {node_type} must be a dictionary and contain only one element"
+    assert isinstance(data, dict) and len(data) == 1, (
+        f"The data with list of {node_type} must be a dictionary and contain only one element"
+    )
     data = next(iter(data.values()))
     return data[node_type]  # type: ignore
 
@@ -158,7 +158,7 @@ def _get_graphql_pages(
         )
         items_count += len(data_items)
         print(
-            f'Got {len(data_items)}/{items_count} {node_type}s, query cost {rate_limit["cost"]}, remaining credits: {rate_limit["remaining"]}'
+            f"Got {len(data_items)}/{items_count} {node_type}s, query cost {rate_limit['cost']}, remaining credits: {rate_limit['remaining']}"
         )
         if data_items:
             yield data_items
@@ -187,7 +187,7 @@ def _get_comment_reaction(comment_ids: List[str], access_token: str) -> StrAny:
         # print(query)
         page, rate_limit = _run_graphql_query(access_token, query, {})
         print(
-            f'Got {len(page)} comments, query cost {rate_limit["cost"]}, remaining credits: {rate_limit["remaining"]}'
+            f"Got {len(page)} comments, query cost {rate_limit['cost']}, remaining credits: {rate_limit['remaining']}"
         )
         data.update(page)
     return data

{ingestr-0.13.22 → ingestr-0.13.24}/ingestr/src/google_sheets/__init__.py

@@ -70,9 +70,9 @@ def google_spreadsheet(
         spreadsheet_id=spreadsheet_id,
         range_names=list(all_range_names),
     )
-    assert len(all_range_names) == len(
-        all_range_data
-    ), "Google Sheets API must return values for all requested ranges"
+    assert len(all_range_names) == len(all_range_data), (
+        "Google Sheets API must return values for all requested ranges"
+    )
 
     # get metadata for two first rows of each range
     # first should contain headers
@@ -126,7 +126,7 @@ def google_spreadsheet(
         headers = get_range_headers(headers_metadata, name)
         if headers is None:
             # generate automatic headers and treat the first row as data
-            headers = [f"col_{idx+1}" for idx in range(len(headers_metadata))]
+            headers = [f"col_{idx + 1}" for idx in range(len(headers_metadata))]
             data_row_metadata = headers_metadata
             rows_data = values[0:]
             logger.warning(

{ingestr-0.13.22 → ingestr-0.13.24}/ingestr/src/google_sheets/helpers/data_processing.py

@@ -149,12 +149,12 @@ def get_range_headers(headers_metadata: List[DictStrAny], range_name: str) -> Li
                     header_val = str(f"col_{idx + 1}")
                 else:
                     logger.warning(
-                        f"In range {range_name}, header value: {header_val} at position {idx+1} is not a string!"
+                        f"In range {range_name}, header value: {header_val} at position {idx + 1} is not a string!"
                     )
                     return None
         else:
             logger.warning(
-                f"In range {range_name}, header at position {idx+1} is not missing!"
+                f"In range {range_name}, header at position {idx + 1} is not missing!"
             )
             return None
         headers.append(header_val)

{ingestr-0.13.22 → ingestr-0.13.24}/requirements-dev.txt

@@ -1,13 +1,14 @@
 -r requirements.txt
 
-mypy==1.13.0
+mypy==1.15.0
 pytest-cov==4.1.0
 pytest==8.3.3
-ruff==0.8.6
+ruff==0.11.4
 hatchling==1.27.0
 build==1.2.1
 pyodbc==5.2.0
 twine==6.0.1
 testcontainers[postgres,mysql]==4.8.2
 pytest-xdist[psutil]==3.6.1
-pkginfo==1.12.0
+pkginfo==1.12.0
+pytest-repeat==0.9.3

{ingestr-0.13.22 → ingestr-0.13.24}/requirements.in

@@ -2,9 +2,9 @@ asana==3.2.3
 confluent-kafka==2.8.0
 databricks-sql-connector==2.9.3
 dataclasses-json==0.6.7
-dlt==1.6.1
-duckdb_engine==0.15.0
-duckdb==1.2.0
+dlt==1.9.0
+duckdb_engine==0.17.0
+duckdb==1.2.1
 google-ads==25.1.0
 facebook-business==20.0.0
 google-api-python-client==2.130.0
@@ -46,6 +46,6 @@ clickhouse-connect==0.8.14
 clickhouse-driver==0.2.9
 clickhouse-sqlalchemy==0.2.7
 zstd==1.5.6.5
-ibm-db==3.2.6
-ibm-db-sa==0.4.1
+ibm-db==3.2.6; platform_machine != "aarch64"
+ibm-db-sa==0.4.1; platform_machine != "aarch64"
 aiohttp>=3.11.15

{ingestr-0.13.22 → ingestr-0.13.24}/requirements.txt

@@ -94,15 +94,15 @@ decorator==5.2.1
     # via gcsfs
 deprecation==2.1.0
     # via rudder-sdk-python
-dlt==1.6.1
+dlt==1.9.0
     # via -r requirements.in
 dnspython==2.7.0
     # via pymongo
-duckdb==1.2.0
+duckdb==1.2.1
     # via
     #   -r requirements.in
     #   duckdb-engine
-duckdb-engine==0.15.0
+duckdb-engine==0.17.0
     # via -r requirements.in
 et-xmlfile==2.0.0
     # via openpyxl
@@ -496,6 +496,8 @@ sqlalchemy-redshift==0.8.14
     # via -r requirements.in
 sqlalchemy2-stubs==0.0.2a38
     # via -r requirements.in
+sqlglot==26.12.1
+    # via dlt
 stripe==10.7.0
     # via -r requirements.in
 tenacity==9.0.0