ingestr 0.13.41__tar.gz → 0.13.42__tar.gz

{ingestr-0.13.41 → ingestr-0.13.42}/Makefile

@@ -8,7 +8,7 @@ venv: venv/touchfile
 
 venv/touchfile: requirements-dev.txt requirements.txt
 	test -d venv || python3 -m venv venv
-	. venv/bin/activate; pip install uv; $(MAKE) deps
+	. venv/bin/activate; pip install --disable-pip-version-check uv; $(MAKE) deps
 	touch venv/touchfile
 
 lock-deps:

{ingestr-0.13.41 → ingestr-0.13.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.41
+Version: 0.13.42
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

ingestr-0.13.42/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.42"

{ingestr-0.13.41 → ingestr-0.13.42}/ingestr/src/factory.py

@@ -120,6 +120,7 @@ class SourceDestinationFactory:
     sources: Dict[str, Type[SourceProtocol]] = {
         "csv": LocalCsvSource,
         "mongodb": MongoDbSource,
+        "mongodb+srv": MongoDbSource,
         "notion": NotionSource,
         "gsheets": GoogleSheetsSource,
         "shopify": ShopifySource,

{ingestr-0.13.41 → ingestr-0.13.42}/ingestr/src/mongodb/__init__.py

@@ -1,8 +1,9 @@
 """Source that loads collections form any a mongo database, supports incremental loads."""
 
-from typing import Any, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Union
 
 import dlt
+from dlt.common.data_writers import TDataItemFormat
 from dlt.sources import DltResource
 
 from .helpers import (
@@ -21,6 +22,10 @@ def mongodb(
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = dlt.config.value,
+    limit: Optional[int] = None,
+    filter_: Optional[Dict[str, Any]] = None,
+    projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = None,
+    pymongoarrow_schema: Optional[Any] = None,
 ) -> Iterable[DltResource]:
     """
     A DLT source which loads data from a mongo database using PyMongo.
@@ -34,6 +39,18 @@ def mongodb(
             E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
         write_disposition (str): Write disposition of the resource.
         parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+        limit (Optional[int]):
+            The maximum number of documents to load. The limit is
+            applied to each requested collection separately.
+        filter_ (Optional[Dict[str, Any]]): The filter to apply to the collection.
+        projection: (Optional[Union[Mapping[str, Any], Iterable[str]]]): The projection to select fields of a collection
+            when loading the collection. Supported inputs:
+                include (list) - ["year", "title"]
+                include (dict) - {"year": True, "title": True}
+                exclude (dict) - {"released": False, "runtime": False}
+            Note: Can't mix include and exclude statements '{"title": True, "released": False}`
+        pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types of a collection to convert BSON to Arrow
+
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
     """
@@ -58,19 +75,36 @@ def mongodb(
             primary_key="_id",
             write_disposition=write_disposition,
             spec=MongoDbCollectionConfiguration,
-        )(client, collection, incremental=incremental, parallel=parallel)
+        )(
+            client,
+            collection,
+            incremental=incremental,
+            parallel=parallel,
+            limit=limit,
+            filter_=filter_ or {},
+            projection=projection,
+            pymongoarrow_schema=pymongoarrow_schema,
+        )
 
 
-@dlt.common.configuration.with_config(
-    sections=("sources", "mongodb"), spec=MongoDbCollectionResourceConfiguration
+@dlt.resource(
+    name=lambda args: args["collection"],
+    standalone=True,
+    spec=MongoDbCollectionResourceConfiguration,
 )
 def mongodb_collection(
-    connection_url: str = dlt.config.value,
+    connection_url: str = dlt.secrets.value,
     database: Optional[str] = dlt.config.value,
     collection: str = dlt.config.value,
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = False,
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 10000,
+    data_item_format: Optional[TDataItemFormat] = "object",
+    filter_: Optional[Dict[str, Any]] = None,
+    projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value,
+    pymongoarrow_schema: Optional[Any] = None,
 ) -> Any:
     """
     A DLT source which loads a collection from a mongo database using PyMongo.
@@ -83,6 +117,21 @@ def mongodb_collection(
             E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
         write_disposition (str): Write disposition of the resource.
         parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+        limit (Optional[int]): The number of documents load.
+        chunk_size (Optional[int]): The number of documents load in each batch.
+        data_item_format (Optional[TDataItemFormat]): The data format to use for loading.
+            Supported formats:
+                object - Python objects (dicts, lists).
+                arrow - Apache Arrow tables.
+        filter_ (Optional[Dict[str, Any]]): The filter to apply to the collection.
+        projection: (Optional[Union[Mapping[str, Any], Iterable[str]]]): The projection to select fields
+            when loading the collection. Supported inputs:
+                include (list) - ["year", "title"]
+                include (dict) - {"year": True, "title": True}
+                exclude (dict) - {"released": False, "runtime": False}
+            Note: Can't mix include and exclude statements '{"title": True, "released": False}`
+        pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types to convert BSON to Arrow
+
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
     """
@@ -100,4 +149,15 @@ def mongodb_collection(
         name=collection_obj.name,
         primary_key="_id",
         write_disposition=write_disposition,
-    )(client, collection_obj, incremental=incremental, parallel=parallel)
+    )(
+        client,
+        collection_obj,
+        incremental=incremental,
+        parallel=parallel,
+        limit=limit,
+        chunk_size=chunk_size,
+        data_item_format=data_item_format,
+        filter_=filter_ or {},
+        projection=projection,
+        pymongoarrow_schema=pymongoarrow_schema,
+    )