ingestr 0.13.40__tar.gz → 0.13.42__tar.gz

{ingestr-0.13.40 → ingestr-0.13.42}/Makefile

@@ -8,7 +8,7 @@ venv: venv/touchfile
 
 venv/touchfile: requirements-dev.txt requirements.txt
 	test -d venv || python3 -m venv venv
-	. venv/bin/activate; pip install uv; $(MAKE) deps
+	. venv/bin/activate; pip install --disable-pip-version-check uv; $(MAKE) deps
 	touch venv/touchfile
 
 lock-deps:

{ingestr-0.13.40 → ingestr-0.13.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.40
+Version: 0.13.42
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -72,11 +72,14 @@ Requires-Dist: google-auth==2.38.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: google-cloud-bigquery==3.30.0
 Requires-Dist: google-cloud-core==2.4.2
+Requires-Dist: google-cloud-spanner==3.54.0
 Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
 Requires-Dist: greenlet==3.2.2
+Requires-Dist: grpc-google-iam-v1==0.14.2
+Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3
 Requires-Dist: grpcio==1.70.0
 Requires-Dist: hdbcli==2.23.27
@@ -168,9 +171,11 @@ Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
+Requires-Dist: sqlalchemy-spanner==1.11.0
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: sqlglot==26.12.1
+Requires-Dist: sqlparse==0.5.3
 Requires-Dist: stripe==10.7.0
 Requires-Dist: tenacity==9.0.0
 Requires-Dist: thrift==0.16.0

{ingestr-0.13.40 → ingestr-0.13.42}/docs/.vitepress/config.mjs

@@ -138,6 +138,7 @@ export default defineConfig({
               { text: "Salesforce", link: "/supported-sources/salesforce.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },
               { text: "Slack", link: "/supported-sources/slack.md" },
+              { text: "Spanner", link: "/supported-sources/spanner.md" },
               { text: "Stripe", link: "/supported-sources/stripe.md" },
               { text: "TikTok Ads", link: "/supported-sources/tiktok-ads.md" },
               { text: "Zendesk", link: "/supported-sources/zendesk.md" },

ingestr-0.13.42/docs/supported-sources/spanner.md

@@ -0,0 +1,41 @@
+# Spanner
+[Spanner](https://cloud.google.com/spanner) is a fully managed, scalable, and strongly consistent database service.
+
+ingestr supports Spanner as a source.
+
+## URI format
+The URI format for Spanner is as follows:
+
+
+```plaintext
+spanner://?project_id=<project_id>&instance_id=<instance_id>&database=<database>&credentials_path=</path/to/service/account.json>
+```
+or
+```plaintext
+spanner://?project_id=<project_id>&instance_id=<instance_id>&database=<database>&credentials_base64=<base64_encoded_credentials>
+```
+
+URI parameters:
+- `project_id`: Your Google Cloud project ID (e.g., "my-project-123")
+- `instance_id`: The Spanner instance ID (e.g., "my-instance")
+- `database`: The database name (e.g., "my-database")
+- `credentials_path`: Path to your Google Cloud service account JSON file
+- `credentials_base64`: Base64-encoded service account credentials (alternative to credentials_path)
+
+Let's say we have the above credentials. Here's a sample command that will copy the data from Spanner (GoogleSQL) to DuckDB.
+
+```sh
+ingestr ingest \
+  --source-uri 'spanner://?project_id=id_123&instance_id=instance_123&database=db_123&credentials_path=./service_account.json' \
+  --source-table 'users' \
+  --dest-uri duckdb:///spanner.duckdb \
+  --dest-table 'dest.output'
+
+```
+
+<img alt="pipedrive_img" src="../media/spanner_ingestion.png"/>
+
+> [!NOTE]
+> Currently, only GoogleSQL databases are supported.
+
+

ingestr-0.13.42/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.42"

{ingestr-0.13.40 → ingestr-0.13.42}/ingestr/src/factory.py

@@ -79,6 +79,7 @@ SQL_SOURCE_SCHEMES = [
     "clickhouse",
     "databricks",
     "db2",
+    "spanner",
 ]
 
 
@@ -119,6 +120,7 @@ class SourceDestinationFactory:
     sources: Dict[str, Type[SourceProtocol]] = {
         "csv": LocalCsvSource,
         "mongodb": MongoDbSource,
+        "mongodb+srv": MongoDbSource,
         "notion": NotionSource,
         "gsheets": GoogleSheetsSource,
         "shopify": ShopifySource,

{ingestr-0.13.40 → ingestr-0.13.42}/ingestr/src/mongodb/__init__.py

@@ -1,8 +1,9 @@
 """Source that loads collections form any a mongo database, supports incremental loads."""
 
-from typing import Any, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Union
 
 import dlt
+from dlt.common.data_writers import TDataItemFormat
 from dlt.sources import DltResource
 
 from .helpers import (
@@ -21,6 +22,10 @@ def mongodb(
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = dlt.config.value,
+    limit: Optional[int] = None,
+    filter_: Optional[Dict[str, Any]] = None,
+    projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = None,
+    pymongoarrow_schema: Optional[Any] = None,
 ) -> Iterable[DltResource]:
     """
     A DLT source which loads data from a mongo database using PyMongo.
@@ -34,6 +39,18 @@ def mongodb(
             E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
         write_disposition (str): Write disposition of the resource.
         parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+        limit (Optional[int]):
+            The maximum number of documents to load. The limit is
+            applied to each requested collection separately.
+        filter_ (Optional[Dict[str, Any]]): The filter to apply to the collection.
+        projection: (Optional[Union[Mapping[str, Any], Iterable[str]]]): The projection to select fields of a collection
+            when loading the collection. Supported inputs:
+                include (list) - ["year", "title"]
+                include (dict) - {"year": True, "title": True}
+                exclude (dict) - {"released": False, "runtime": False}
+            Note: Can't mix include and exclude statements '{"title": True, "released": False}`
+        pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types of a collection to convert BSON to Arrow
+
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
     """
@@ -58,19 +75,36 @@ def mongodb(
             primary_key="_id",
             write_disposition=write_disposition,
             spec=MongoDbCollectionConfiguration,
-        )(client, collection, incremental=incremental, parallel=parallel)
+        )(
+            client,
+            collection,
+            incremental=incremental,
+            parallel=parallel,
+            limit=limit,
+            filter_=filter_ or {},
+            projection=projection,
+            pymongoarrow_schema=pymongoarrow_schema,
+        )
 
 
-@dlt.common.configuration.with_config(
-    sections=("sources", "mongodb"), spec=MongoDbCollectionResourceConfiguration
+@dlt.resource(
+    name=lambda args: args["collection"],
+    standalone=True,
+    spec=MongoDbCollectionResourceConfiguration,
 )
 def mongodb_collection(
-    connection_url: str = dlt.config.value,
+    connection_url: str = dlt.secrets.value,
     database: Optional[str] = dlt.config.value,
     collection: str = dlt.config.value,
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = False,
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 10000,
+    data_item_format: Optional[TDataItemFormat] = "object",
+    filter_: Optional[Dict[str, Any]] = None,
+    projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value,
+    pymongoarrow_schema: Optional[Any] = None,
 ) -> Any:
     """
     A DLT source which loads a collection from a mongo database using PyMongo.
@@ -83,6 +117,21 @@ def mongodb_collection(
             E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
         write_disposition (str): Write disposition of the resource.
         parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+        limit (Optional[int]): The number of documents load.
+        chunk_size (Optional[int]): The number of documents load in each batch.
+        data_item_format (Optional[TDataItemFormat]): The data format to use for loading.
+            Supported formats:
+                object - Python objects (dicts, lists).
+                arrow - Apache Arrow tables.
+        filter_ (Optional[Dict[str, Any]]): The filter to apply to the collection.
+        projection: (Optional[Union[Mapping[str, Any], Iterable[str]]]): The projection to select fields
+            when loading the collection. Supported inputs:
+                include (list) - ["year", "title"]
+                include (dict) - {"year": True, "title": True}
+                exclude (dict) - {"released": False, "runtime": False}
+            Note: Can't mix include and exclude statements '{"title": True, "released": False}`
+        pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types to convert BSON to Arrow
+
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
     """
@@ -100,4 +149,15 @@ def mongodb_collection(
         name=collection_obj.name,
         primary_key="_id",
         write_disposition=write_disposition,
-    )(client, collection_obj, incremental=incremental, parallel=parallel)
+    )(
+        client,
+        collection_obj,
+        incremental=incremental,
+        parallel=parallel,
+        limit=limit,
+        chunk_size=chunk_size,
+        data_item_format=data_item_format,
+        filter_=filter_ or {},
+        projection=projection,
+        pymongoarrow_schema=pymongoarrow_schema,
+    )