ingestr 0.9.1__tar.gz → 0.9.2__tar.gz

{ingestr-0.9.1 → ingestr-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.9.1
+Version: 0.9.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -28,7 +28,6 @@ Requires-Dist: py-machineid==0.5.1
 Requires-Dist: pyairtable==2.3.3
 Requires-Dist: pymongo==4.6.3
 Requires-Dist: pymysql==1.1.0
-Requires-Dist: pyodbc==5.1.0
 Requires-Dist: pyrate-limiter==3.6.1
 Requires-Dist: redshift-connector==2.1.0
 Requires-Dist: rich==13.7.1
@@ -44,6 +43,8 @@ Requires-Dist: stripe==10.7.0
 Requires-Dist: tqdm==4.66.2
 Requires-Dist: typer==0.12.3
 Requires-Dist: types-requests==2.32.0.20240907
+Provides-Extra: odbc
+Requires-Dist: pyodbc==5.1.0; extra == 'odbc'
 Provides-Extra: oracle
 Requires-Dist: cx-oracle==8.3.0; extra == 'oracle'
 Description-Content-Type: text/markdown

{ingestr-0.9.1 → ingestr-0.9.2}/docs/supported-sources/mssql.md

@@ -3,6 +3,14 @@ Microsoft SQL Server is a relational database management system developed by Mic
 
 ingestr supports Microsoft SQL Server as both a source and destination.
 
+## Installation
+
+To use Microsoft SQL Server with ingestr, you need to install the `pyodbc` add-on as well. You can do this by running:
+
+```bash
+pip install ingestr[odbc]
+```
+
 ## URI Format
 The URI format for Microsoft SQL Server is as follows:
 

ingestr-0.9.2/ingestr/src/arrow/__init__.py

@@ -0,0 +1,77 @@
+"""Source that loads tables form Airtable.
+Supports whitelisting of tables or loading of all tables from a specified base.
+"""
+
+from typing import Any, Optional
+
+import dlt
+from dlt.common.schema.typing import TColumnNames, TTableSchemaColumns
+from dlt.extract.items import TTableHintTemplate
+
+
+def memory_mapped_arrow(
+    path: str,
+    columns: Optional[TTableSchemaColumns] = None,
+    primary_key: Optional[TTableHintTemplate[TColumnNames]] = None,
+    merge_key: Optional[TTableHintTemplate[TColumnNames]] = None,
+    incremental: Optional[dlt.sources.incremental[Any]] = None,
+):
+    @dlt.resource(
+        name="arrow_mmap",
+        columns=columns,  # type: ignore
+        primary_key=primary_key,  # type: ignore
+        merge_key=merge_key,  # type: ignore
+    )
+    def arrow_mmap(
+        incremental: Optional[dlt.sources.incremental[Any]] = incremental,
+    ):
+        import pyarrow as pa  # type: ignore
+        import pyarrow.ipc as ipc  # type: ignore
+
+        with pa.memory_map(path, "rb") as mmap:
+            reader: ipc.RecordBatchFileReader = ipc.open_file(mmap)
+            table = reader.read_all()
+
+        last_value = None
+        end_value = None
+        if incremental:
+            if incremental.cursor_path not in table.column_names:
+                raise KeyError(
+                    f"Cursor column '{incremental.cursor_path}' does not exist in table"
+                )
+
+            last_value = incremental.last_value
+            end_value = incremental.end_value
+
+        if last_value is not None:
+            # Check if the column is a date type
+            if pa.types.is_temporal(table.schema.field(incremental.cursor_path).type):  # type: ignore
+                if not isinstance(last_value, pa.TimestampScalar):
+                    last_value = pa.scalar(last_value, type=pa.timestamp("ns"))
+
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) > last_value  # type: ignore
+                )
+            else:
+                # For non-date types, use direct comparison
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) > last_value  # type: ignore
+                )
+
+        if end_value is not None:
+            if pa.types.is_timestamp(table.schema.field(incremental.cursor_path).type):  # type: ignore
+                # Convert end_value to timestamp if it's not already
+                if not isinstance(end_value, pa.TimestampScalar):
+                    end_value = pa.scalar(end_value, type=pa.timestamp("ns"))
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) < end_value  # type: ignore
+                )
+            else:
+                # For non-date types, use direct comparison
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) < end_value  # type: ignore
+                )
+
+        yield table
+
+    return arrow_mmap

{ingestr-0.9.1 → ingestr-0.9.2}/ingestr/src/factory.py

@@ -18,6 +18,7 @@ from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
     AppsflyerSource,
+    ArrowMemoryMappedSource,
     ChessSource,
     FacebookAdsSource,
     GoogleSheetsSource,
@@ -136,6 +137,8 @@ class SourceDestinationFactory:
             return AdjustSource()
         elif self.source_scheme == "zendesk":
             return ZendeskSource()
+        elif self.source_scheme == "mmap":
+            return ArrowMemoryMappedSource()
         elif self.source_scheme == "s3":
             return S3Source()
         else:

{ingestr-0.9.1 → ingestr-0.9.2}/ingestr/src/mongodb/__init__.py

@@ -65,7 +65,7 @@ def mongodb(
     sections=("sources", "mongodb"), spec=MongoDbCollectionResourceConfiguration
 )
 def mongodb_collection(
-    connection_url: str = dlt.secrets.value,
+    connection_url: str = dlt.config.value,
     database: Optional[str] = dlt.config.value,
     collection: str = dlt.config.value,
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]

{ingestr-0.9.1 → ingestr-0.9.2}/ingestr/src/mongodb/helpers.py

@@ -155,7 +155,7 @@ class MongoDbCollectionConfiguration(BaseConfiguration):
 
 @configspec
 class MongoDbCollectionResourceConfiguration(BaseConfiguration):
-    connection_url: str = dlt.secrets.value
+    connection_url: str = dlt.config.value
     database: Optional[str] = dlt.config.value
     collection: str = dlt.config.value
     incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]

{ingestr-0.9.1 → ingestr-0.9.2}/ingestr/src/sources.py

@@ -12,6 +12,7 @@ from dlt.common.typing import TSecretStrValue
 from ingestr.src.adjust._init_ import adjust_source
 from ingestr.src.airtable import airtable_source
 from ingestr.src.appsflyer._init_ import appsflyer_source
+from ingestr.src.arrow import memory_mapped_arrow
 from ingestr.src.chess import source
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
@@ -75,6 +76,51 @@ class SqlSource:
         return table_instance
 
 
+class ArrowMemoryMappedSource:
+    table_builder: Callable
+
+    def __init__(self, table_builder=memory_mapped_arrow) -> None:
+        self.table_builder = table_builder
+
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        import os
+
+        incremental = None
+        if kwargs.get("incremental_key"):
+            start_value = kwargs.get("interval_start")
+            end_value = kwargs.get("interval_end")
+
+            incremental = dlt.sources.incremental(
+                kwargs.get("incremental_key", ""),
+                initial_value=start_value,
+                end_value=end_value,
+            )
+
+        file_path = uri.split("://")[1]
+        if not os.path.exists(file_path):
+            raise ValueError(f"File at path {file_path} does not exist")
+
+        if os.path.isdir(file_path):
+            raise ValueError(
+                f"Path {file_path} is a directory, it should be an Arrow memory mapped file"
+            )
+
+        primary_key = kwargs.get("primary_key")
+        merge_key = kwargs.get("merge_key")
+
+        table_instance = self.table_builder(
+            path=file_path,
+            incremental=incremental,
+            merge_key=merge_key,
+            primary_key=primary_key,
+        )
+
+        return table_instance
+
+
 class MongoDbSource:
     table_builder: Callable
 
@@ -656,12 +702,12 @@ class KafkaSource:
             credentials=KafkaCredentials(
                 bootstrap_servers=bootstrap_servers[0],
                 group_id=group_id[0],
-                security_protocol=security_protocol[0]
-                if len(security_protocol) > 0
-                else None,  # type: ignore
-                sasl_mechanisms=sasl_mechanisms[0]
-                if len(sasl_mechanisms) > 0
-                else None,  # type: ignore
+                security_protocol=(
+                    security_protocol[0] if len(security_protocol) > 0 else None
+                ),  # type: ignore
+                sasl_mechanisms=(
+                    sasl_mechanisms[0] if len(sasl_mechanisms) > 0 else None
+                ),  # type: ignore
                 sasl_username=sasl_username[0] if len(sasl_username) > 0 else None,  # type: ignore
                 sasl_password=sasl_password[0] if len(sasl_password) > 0 else None,  # type: ignore
             ),

ingestr-0.9.2/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.9.2"

{ingestr-0.9.1 → ingestr-0.9.2}/pyproject.toml

@@ -132,6 +132,9 @@ classifiers = [
 oracle = [
     "cx_Oracle==8.3.0",
 ]
+odbc = [
+    "pyodbc==5.1.0",
+]
 
 [project.urls]
 Homepage = "https://github.com/bruin-data/ingestr"

{ingestr-0.9.1 → ingestr-0.9.2}/requirements.txt

@@ -11,7 +11,6 @@ psycopg2-binary==2.9.9
 py-machineid==0.5.1
 pymongo==4.6.3
 pymysql==1.1.0
-pyodbc==5.1.0
 pyrate-limiter==3.6.1
 redshift-connector==2.1.0
 rich==13.7.1

ingestr-0.9.1/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.9.1"