ingestr 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

ingestr/src/arrow/__init__.py

@@ -0,0 +1,77 @@
+"""Source that loads tables form Airtable.
+Supports whitelisting of tables or loading of all tables from a specified base.
+"""
+
+from typing import Any, Optional
+
+import dlt
+from dlt.common.schema.typing import TColumnNames, TTableSchemaColumns
+from dlt.extract.items import TTableHintTemplate
+
+
+def memory_mapped_arrow(
+    path: str,
+    columns: Optional[TTableSchemaColumns] = None,
+    primary_key: Optional[TTableHintTemplate[TColumnNames]] = None,
+    merge_key: Optional[TTableHintTemplate[TColumnNames]] = None,
+    incremental: Optional[dlt.sources.incremental[Any]] = None,
+):
+    @dlt.resource(
+        name="arrow_mmap",
+        columns=columns,  # type: ignore
+        primary_key=primary_key,  # type: ignore
+        merge_key=merge_key,  # type: ignore
+    )
+    def arrow_mmap(
+        incremental: Optional[dlt.sources.incremental[Any]] = incremental,
+    ):
+        import pyarrow as pa  # type: ignore
+        import pyarrow.ipc as ipc  # type: ignore
+
+        with pa.memory_map(path, "rb") as mmap:
+            reader: ipc.RecordBatchFileReader = ipc.open_file(mmap)
+            table = reader.read_all()
+
+        last_value = None
+        end_value = None
+        if incremental:
+            if incremental.cursor_path not in table.column_names:
+                raise KeyError(
+                    f"Cursor column '{incremental.cursor_path}' does not exist in table"
+                )
+
+            last_value = incremental.last_value
+            end_value = incremental.end_value
+
+        if last_value is not None:
+            # Check if the column is a date type
+            if pa.types.is_temporal(table.schema.field(incremental.cursor_path).type):  # type: ignore
+                if not isinstance(last_value, pa.TimestampScalar):
+                    last_value = pa.scalar(last_value, type=pa.timestamp("ns"))
+
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) > last_value  # type: ignore
+                )
+            else:
+                # For non-date types, use direct comparison
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) > last_value  # type: ignore
+                )
+
+        if end_value is not None:
+            if pa.types.is_timestamp(table.schema.field(incremental.cursor_path).type):  # type: ignore
+                # Convert end_value to timestamp if it's not already
+                if not isinstance(end_value, pa.TimestampScalar):
+                    end_value = pa.scalar(end_value, type=pa.timestamp("ns"))
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) < end_value  # type: ignore
+                )
+            else:
+                # For non-date types, use direct comparison
+                table = table.filter(
+                    pa.compute.field(incremental.cursor_path) < end_value  # type: ignore
+                )
+
+        yield table
+
+    return arrow_mmap

ingestr/src/factory.py

@@ -18,6 +18,7 @@ from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
     AppsflyerSource,
+    ArrowMemoryMappedSource,
     ChessSource,
     FacebookAdsSource,
     GoogleSheetsSource,
@@ -136,6 +137,8 @@ class SourceDestinationFactory:
             return AdjustSource()
         elif self.source_scheme == "zendesk":
             return ZendeskSource()
+        elif self.source_scheme == "mmap":
+            return ArrowMemoryMappedSource()
         elif self.source_scheme == "s3":
             return S3Source()
         else:

ingestr/src/mongodb/__init__.py

@@ -65,7 +65,7 @@ def mongodb(
     sections=("sources", "mongodb"), spec=MongoDbCollectionResourceConfiguration
 )
 def mongodb_collection(
-    connection_url: str = dlt.secrets.value,
+    connection_url: str = dlt.config.value,
     database: Optional[str] = dlt.config.value,
     collection: str = dlt.config.value,
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]

ingestr/src/mongodb/helpers.py

@@ -155,7 +155,7 @@ class MongoDbCollectionConfiguration(BaseConfiguration):
 
 @configspec
 class MongoDbCollectionResourceConfiguration(BaseConfiguration):
-    connection_url: str = dlt.secrets.value
+    connection_url: str = dlt.config.value
     database: Optional[str] = dlt.config.value
     collection: str = dlt.config.value
     incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]

ingestr/src/sources.py

@@ -12,6 +12,7 @@ from dlt.common.typing import TSecretStrValue
 from ingestr.src.adjust._init_ import adjust_source
 from ingestr.src.airtable import airtable_source
 from ingestr.src.appsflyer._init_ import appsflyer_source
+from ingestr.src.arrow import memory_mapped_arrow
 from ingestr.src.chess import source
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
@@ -75,6 +76,51 @@ class SqlSource:
         return table_instance
 
 
+class ArrowMemoryMappedSource:
+    table_builder: Callable
+
+    def __init__(self, table_builder=memory_mapped_arrow) -> None:
+        self.table_builder = table_builder
+
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        import os
+
+        incremental = None
+        if kwargs.get("incremental_key"):
+            start_value = kwargs.get("interval_start")
+            end_value = kwargs.get("interval_end")
+
+            incremental = dlt.sources.incremental(
+                kwargs.get("incremental_key", ""),
+                initial_value=start_value,
+                end_value=end_value,
+            )
+
+        file_path = uri.split("://")[1]
+        if not os.path.exists(file_path):
+            raise ValueError(f"File at path {file_path} does not exist")
+
+        if os.path.isdir(file_path):
+            raise ValueError(
+                f"Path {file_path} is a directory, it should be an Arrow memory mapped file"
+            )
+
+        primary_key = kwargs.get("primary_key")
+        merge_key = kwargs.get("merge_key")
+
+        table_instance = self.table_builder(
+            path=file_path,
+            incremental=incremental,
+            merge_key=merge_key,
+            primary_key=primary_key,
+        )
+
+        return table_instance
+
+
 class MongoDbSource:
     table_builder: Callable
 
@@ -656,12 +702,12 @@ class KafkaSource:
             credentials=KafkaCredentials(
                 bootstrap_servers=bootstrap_servers[0],
                 group_id=group_id[0],
-                security_protocol=security_protocol[0]
-                if len(security_protocol) > 0
-                else None,  # type: ignore
-                sasl_mechanisms=sasl_mechanisms[0]
-                if len(sasl_mechanisms) > 0
-                else None,  # type: ignore
+                security_protocol=(
+                    security_protocol[0] if len(security_protocol) > 0 else None
+                ),  # type: ignore
+                sasl_mechanisms=(
+                    sasl_mechanisms[0] if len(sasl_mechanisms) > 0 else None
+                ),  # type: ignore
                 sasl_username=sasl_username[0] if len(sasl_username) > 0 else None,  # type: ignore
                 sasl_password=sasl_password[0] if len(sasl_password) > 0 else None,  # type: ignore
             ),

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.9.0"
+__version__ = "0.9.2"

{ingestr-0.9.0.dist-info → ingestr-0.9.2.dist-info}/METADATA

@@ -1,11 +1,10 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.9.0
+Version: 0.9.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
 Author-email: Burak Karakan <burak.karakan@getbruin.com>
-License-File: LICENSE.md
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -15,7 +14,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: confluent-kafka>=2.3.0
-Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dlt==0.5.1
 Requires-Dist: duckdb-engine==0.11.5
@@ -30,7 +28,6 @@ Requires-Dist: py-machineid==0.5.1
 Requires-Dist: pyairtable==2.3.3
 Requires-Dist: pymongo==4.6.3
 Requires-Dist: pymysql==1.1.0
-Requires-Dist: pyodbc==5.1.0
 Requires-Dist: pyrate-limiter==3.6.1
 Requires-Dist: redshift-connector==2.1.0
 Requires-Dist: rich==13.7.1
@@ -46,6 +43,10 @@ Requires-Dist: stripe==10.7.0
 Requires-Dist: tqdm==4.66.2
 Requires-Dist: typer==0.12.3
 Requires-Dist: types-requests==2.32.0.20240907
+Provides-Extra: odbc
+Requires-Dist: pyodbc==5.1.0; extra == 'odbc'
+Provides-Extra: oracle
+Requires-Dist: cx-oracle==8.3.0; extra == 'oracle'
 Description-Content-Type: text/markdown
 
 <div align="center">

{ingestr-0.9.0.dist-info → ingestr-0.9.2.dist-info}/RECORD

@@ -1,15 +1,16 @@
 ingestr/main.py,sha256=U66TM57ePv-RdoAftQ0pFZx8woZUQnLepKa50C-bA5I,17655
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/factory.py,sha256=NbSLbF2ClHRcptpmL0n9p7bGVz8Uj6xyG0Y93OZTfKM,4830
-ingestr/src/sources.py,sha256=Ny7qYulZUmwrCsG8UZ_SeR6FIqVR3E1FWdvj_yQD41M,30786
+ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
+ingestr/src/sources.py,sha256=iZbCY-pzv6jbgdHOh0Vdsl3cBoC71eiFZgu_a5RoaDE,32188
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=H9NWRZb7NbeRRPLP_V1fARmLNXranorVM-OOY-8_2ug,22
+ingestr/src/version.py,sha256=gqT-BGoeEItda9fICQDvLbxEjWRIBhFJxPxxKvmHLUo,22
 ingestr/src/adjust/_init_.py,sha256=_jJE3Ywvv-YyJ7ywICdht_X2Gnd1cKm6F1wAfnpXuWM,890
 ingestr/src/adjust/helpers.py,sha256=kkYC3MqMHLNucuQ50klZWrvd3o8VfUysNtZTQSsKZ_c,2588
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
+ingestr/src/arrow/__init__.py,sha256=AgU7S9Ra3ZeeG00Mf32zxO5sgMFfRnTdOSirUJ1Pu10,2976
 ingestr/src/chess/__init__.py,sha256=PaxT2DObudOGlhyoENE5LjR6rTdsxiqKKpAZeyzVLCA,6791
 ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,549
 ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
@@ -35,8 +36,8 @@ ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,
 ingestr/src/klaviyo/_init_.py,sha256=nq2T1p3Xc7yiwGabsZBp2Jy2fa8_n5oxqxBnUGhKOgg,6592
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
-ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
-ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
+ingestr/src/mongodb/__init__.py,sha256=aMr1PFIDUMRv--ne61lR17HudsN-fsrzMeyxe9PqK2s,4335
+ingestr/src/mongodb/helpers.py,sha256=y9rYKR8eyIqam_eNsZmwSYevgi8mghh7Zp8qhTHl65s,5652
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
 ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -73,8 +74,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.9.0.dist-info/METADATA,sha256=yJZ6Tdum4TirrssuFG40UZ_zm4pzhzSSweJRIignFck,6949
-ingestr-0.9.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.9.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.9.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.9.0.dist-info/RECORD,,
+ingestr-0.9.2.dist-info/METADATA,sha256=ZnA32SeV-3jKU3g13UsczdNiKtvu1lXvF-Gb2mZUlzw,7004
+ingestr-0.9.2.dist-info/WHEEL,sha256=wukiCwsxxsuzcQTdnC_ZWHZECE4wwOh3xCCrap6i6Ts,87
+ingestr-0.9.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.9.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.9.2.dist-info/RECORD,,

{ingestr-0.9.0.dist-info → ingestr-0.9.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.26.0
 Root-Is-Purelib: true
 Tag: py3-none-any