ingestr 0.13.40__tar.gz → 0.13.41__tar.gz

{ingestr-0.13.40 → ingestr-0.13.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.40
+Version: 0.13.41
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -72,11 +72,14 @@ Requires-Dist: google-auth==2.38.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: google-cloud-bigquery==3.30.0
 Requires-Dist: google-cloud-core==2.4.2
+Requires-Dist: google-cloud-spanner==3.54.0
 Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
 Requires-Dist: greenlet==3.2.2
+Requires-Dist: grpc-google-iam-v1==0.14.2
+Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3
 Requires-Dist: grpcio==1.70.0
 Requires-Dist: hdbcli==2.23.27
@@ -168,9 +171,11 @@ Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
+Requires-Dist: sqlalchemy-spanner==1.11.0
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: sqlglot==26.12.1
+Requires-Dist: sqlparse==0.5.3
 Requires-Dist: stripe==10.7.0
 Requires-Dist: tenacity==9.0.0
 Requires-Dist: thrift==0.16.0

{ingestr-0.13.40 → ingestr-0.13.41}/docs/.vitepress/config.mjs

@@ -138,6 +138,7 @@ export default defineConfig({
               { text: "Salesforce", link: "/supported-sources/salesforce.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },
               { text: "Slack", link: "/supported-sources/slack.md" },
+              { text: "Spanner", link: "/supported-sources/spanner.md" },
               { text: "Stripe", link: "/supported-sources/stripe.md" },
               { text: "TikTok Ads", link: "/supported-sources/tiktok-ads.md" },
               { text: "Zendesk", link: "/supported-sources/zendesk.md" },

ingestr-0.13.41/docs/supported-sources/spanner.md

@@ -0,0 +1,41 @@
+# Spanner
+[Spanner](https://cloud.google.com/spanner) is a fully managed, scalable, and strongly consistent database service.
+
+ingestr supports Spanner as a source.
+
+## URI format
+The URI format for Spanner is as follows:
+
+
+```plaintext
+spanner://?project_id=<project_id>&instance_id=<instance_id>&database=<database>&credentials_path=</path/to/service/account.json>
+```
+or
+```plaintext
+spanner://?project_id=<project_id>&instance_id=<instance_id>&database=<database>&credentials_base64=<base64_encoded_credentials>
+```
+
+URI parameters:
+- `project_id`: Your Google Cloud project ID (e.g., "my-project-123")
+- `instance_id`: The Spanner instance ID (e.g., "my-instance")
+- `database`: The database name (e.g., "my-database")
+- `credentials_path`: Path to your Google Cloud service account JSON file
+- `credentials_base64`: Base64-encoded service account credentials (alternative to credentials_path)
+
+Let's say we have the above credentials. Here's a sample command that will copy the data from Spanner (GoogleSQL) to DuckDB.
+
+```sh
+ingestr ingest \
+  --source-uri 'spanner://?project_id=id_123&instance_id=instance_123&database=db_123&credentials_path=./service_account.json' \
+  --source-table 'users' \
+  --dest-uri duckdb:///spanner.duckdb \
+  --dest-table 'dest.output'
+
+```
+
+<img alt="pipedrive_img" src="../media/spanner_ingestion.png"/>
+
+> [!NOTE]
+> Currently, only GoogleSQL databases are supported.
+
+

ingestr-0.13.41/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.41"

{ingestr-0.13.40 → ingestr-0.13.41}/ingestr/src/factory.py

@@ -79,6 +79,7 @@ SQL_SOURCE_SCHEMES = [
     "clickhouse",
     "databricks",
     "db2",
+    "spanner",
 ]
 
 

{ingestr-0.13.40 → ingestr-0.13.41}/ingestr/src/sources.py

@@ -52,7 +52,10 @@ class SqlSource:
     def dlt_source(self, uri: str, table: str, **kwargs):
         table_fields = TableDefinition(dataset="custom", table="custom")
         if not table.startswith("query:"):
-            table_fields = table_string_to_dataclass(table)
+            if uri.startswith("spanner://"):
+                table_fields = TableDefinition(dataset="", table=table)
+            else:
+                table_fields = table_string_to_dataclass(table)
 
         incremental = None
         if kwargs.get("incremental_key"):
@@ -113,6 +116,45 @@ class SqlSource:
         if uri.startswith("db2://"):
             uri = uri.replace("db2://", "db2+ibm_db://")
 
+        if uri.startswith("spanner://"):
+            parsed_uri = urlparse(uri)
+            query_params = parse_qs(parsed_uri.query)
+
+            project_id_param = query_params.get("project_id")
+            instance_id_param = query_params.get("instance_id")
+            database_param = query_params.get("database")
+
+            cred_path = query_params.get("credentials_path")
+            cred_base64 = query_params.get("credentials_base64")
+
+            if not project_id_param or not instance_id_param or not database_param:
+                raise ValueError(
+                    "project_id, instance_id and database are required in the URI to get data from Google Spanner"
+                )
+
+            project_id = project_id_param[0]
+            instance_id = instance_id_param[0]
+            database = database_param[0]
+
+            if not cred_path and not cred_base64:
+                raise ValueError(
+                    "credentials_path or credentials_base64 is required in the URI to get data from Google Sheets"
+                )
+            if cred_path:
+                os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cred_path[0]
+            elif cred_base64:
+                credentials = json.loads(
+                    base64.b64decode(cred_base64[0]).decode("utf-8")
+                )
+                temp = tempfile.NamedTemporaryFile(
+                    mode="w", delete=False, suffix=".json"
+                )
+                json.dump(credentials, temp)
+                temp.close()
+                os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = temp.name
+
+            uri = f"spanner+spanner:///projects/{project_id}/instances/{instance_id}/databases/{database}"
+
         from dlt.common.libs.sql_alchemy import (
             Engine,
             MetaData,
@@ -2325,7 +2367,9 @@ class ElasticsearchSource:
 
         index = table
         if not index:
-            raise ValueError("Table name must be provided which is the index name in elasticsearch")
+            raise ValueError(
+                "Table name must be provided which is the index name in elasticsearch"
+            )
 
         query_params = parsed.query
         params = parse_qs(query_params)

{ingestr-0.13.40 → ingestr-0.13.41}/requirements.in

@@ -50,3 +50,5 @@ ibm-db==3.2.6; platform_machine != "aarch64"
 ibm-db-sa==0.4.1; platform_machine != "aarch64"
 aiohttp>=3.11.15
 elasticsearch==8.10.1
+sqlalchemy-spanner==1.11.0
+google-cloud-spanner==3.54.0

{ingestr-0.13.40 → ingestr-0.13.41}/requirements.txt

@@ -16,7 +16,9 @@ aioitertools==0.12.0
 aiosignal==1.3.2
     # via aiohttp
 alembic==1.15.1
-    # via databricks-sql-connector
+    # via
+    #   databricks-sql-connector
+    #   sqlalchemy-spanner
 annotated-types==0.7.0
     # via pydantic
 asana==3.2.3
@@ -147,6 +149,7 @@ google-api-core==2.24.1
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
     #   google-cloud-core
+    #   google-cloud-spanner
     #   google-cloud-storage
     #   sqlalchemy-bigquery
 google-api-python-client==2.130.0
@@ -176,7 +179,12 @@ google-cloud-bigquery-storage==2.24.0
 google-cloud-core==2.4.2
     # via
     #   google-cloud-bigquery
+    #   google-cloud-spanner
     #   google-cloud-storage
+google-cloud-spanner==3.54.0
+    # via
+    #   -r requirements.in
+    #   sqlalchemy-spanner
 google-cloud-storage==3.1.0
     # via gcsfs
 google-crc32c==1.6.0
@@ -191,13 +199,21 @@ googleapis-common-protos==1.69.0
     # via
     #   google-ads
     #   google-api-core
+    #   grpc-google-iam-v1
     #   grpcio-status
 greenlet==3.2.2
     # via sqlalchemy
+grpc-google-iam-v1==0.14.2
+    # via google-cloud-spanner
+grpc-interceptor==0.15.4
+    # via google-cloud-spanner
 grpcio==1.70.0
     # via
     #   google-ads
     #   google-api-core
+    #   googleapis-common-protos
+    #   grpc-google-iam-v1
+    #   grpc-interceptor
     #   grpcio-status
 grpcio-status==1.62.3
     # via
@@ -321,13 +337,16 @@ proto-plus==1.26.0
     #   google-analytics-data
     #   google-api-core
     #   google-cloud-bigquery-storage
+    #   google-cloud-spanner
 protobuf==4.25.6
     # via
     #   google-ads
     #   google-analytics-data
     #   google-api-core
     #   google-cloud-bigquery-storage
+    #   google-cloud-spanner
     #   googleapis-common-protos
+    #   grpc-google-iam-v1
     #   grpcio-status
     #   proto-plus
 psutil==6.1.1
@@ -493,16 +512,21 @@ sqlalchemy==1.4.52
     #   sqlalchemy-bigquery
     #   sqlalchemy-hana
     #   sqlalchemy-redshift
+    #   sqlalchemy-spanner
 sqlalchemy-bigquery==1.12.1
     # via -r requirements.in
 sqlalchemy-hana==2.0.0
     # via -r requirements.in
 sqlalchemy-redshift==0.8.14
     # via -r requirements.in
+sqlalchemy-spanner==1.11.0
+    # via -r requirements.in
 sqlalchemy2-stubs==0.0.2a38
     # via -r requirements.in
 sqlglot==26.12.1
     # via dlt
+sqlparse==0.5.3
+    # via google-cloud-spanner
 stripe==10.7.0
     # via -r requirements.in
 tenacity==9.0.0

{ingestr-0.13.40 → ingestr-0.13.41}/requirements_arm64.txt

@@ -16,7 +16,9 @@ aioitertools==0.12.0
 aiosignal==1.3.2
     # via aiohttp
 alembic==1.15.2
-    # via databricks-sql-connector
+    # via
+    #   databricks-sql-connector
+    #   sqlalchemy-spanner
 annotated-types==0.7.0
     # via pydantic
 asana==3.2.3
@@ -147,6 +149,7 @@ google-api-core==2.24.2
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
     #   google-cloud-core
+    #   google-cloud-spanner
     #   google-cloud-storage
     #   sqlalchemy-bigquery
 google-api-python-client==2.130.0
@@ -176,7 +179,12 @@ google-cloud-bigquery-storage==2.24.0
 google-cloud-core==2.4.3
     # via
     #   google-cloud-bigquery
+    #   google-cloud-spanner
     #   google-cloud-storage
+google-cloud-spanner==3.54.0
+    # via
+    #   -r requirements.in
+    #   sqlalchemy-spanner
 google-cloud-storage==3.1.0
     # via gcsfs
 google-crc32c==1.7.1
@@ -191,13 +199,21 @@ googleapis-common-protos==1.69.2
     # via
     #   google-ads
     #   google-api-core
+    #   grpc-google-iam-v1
     #   grpcio-status
 greenlet==3.1.1
     # via sqlalchemy
+grpc-google-iam-v1==0.14.2
+    # via google-cloud-spanner
+grpc-interceptor==0.15.4
+    # via google-cloud-spanner
 grpcio==1.71.0
     # via
     #   google-ads
     #   google-api-core
+    #   googleapis-common-protos
+    #   grpc-google-iam-v1
+    #   grpc-interceptor
     #   grpcio-status
 grpcio-status==1.62.3
     # via
@@ -315,13 +331,16 @@ proto-plus==1.26.1
     #   google-analytics-data
     #   google-api-core
     #   google-cloud-bigquery-storage
+    #   google-cloud-spanner
 protobuf==4.25.6
     # via
     #   google-ads
     #   google-analytics-data
     #   google-api-core
     #   google-cloud-bigquery-storage
+    #   google-cloud-spanner
     #   googleapis-common-protos
+    #   grpc-google-iam-v1
     #   grpcio-status
     #   proto-plus
 psutil==6.1.1
@@ -486,16 +505,21 @@ sqlalchemy==1.4.52
     #   sqlalchemy-bigquery
     #   sqlalchemy-hana
     #   sqlalchemy-redshift
+    #   sqlalchemy-spanner
 sqlalchemy-bigquery==1.12.1
     # via -r requirements.in
 sqlalchemy-hana==2.0.0
     # via -r requirements.in
 sqlalchemy-redshift==0.8.14
     # via -r requirements.in
+sqlalchemy-spanner==1.11.0
+    # via -r requirements.in
 sqlalchemy2-stubs==0.0.2a38
     # via -r requirements.in
 sqlglot==26.12.1
     # via dlt
+sqlparse==0.5.3
+    # via google-cloud-spanner
 stripe==10.7.0
     # via -r requirements.in
 tenacity==9.0.0

ingestr-0.13.40/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.40"