ingestr 0.13.40__py3-none-any.whl → 0.13.41__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.40"
+version = "v0.13.41"

ingestr/src/factory.py

@@ -79,6 +79,7 @@ SQL_SOURCE_SCHEMES = [
     "clickhouse",
     "databricks",
     "db2",
+    "spanner",
 ]
 
 

ingestr/src/sources.py

@@ -52,7 +52,10 @@ class SqlSource:
     def dlt_source(self, uri: str, table: str, **kwargs):
         table_fields = TableDefinition(dataset="custom", table="custom")
         if not table.startswith("query:"):
-            table_fields = table_string_to_dataclass(table)
+            if uri.startswith("spanner://"):
+                table_fields = TableDefinition(dataset="", table=table)
+            else:
+                table_fields = table_string_to_dataclass(table)
 
         incremental = None
         if kwargs.get("incremental_key"):
@@ -113,6 +116,45 @@ class SqlSource:
         if uri.startswith("db2://"):
             uri = uri.replace("db2://", "db2+ibm_db://")
 
+        if uri.startswith("spanner://"):
+            parsed_uri = urlparse(uri)
+            query_params = parse_qs(parsed_uri.query)
+
+            project_id_param = query_params.get("project_id")
+            instance_id_param = query_params.get("instance_id")
+            database_param = query_params.get("database")
+
+            cred_path = query_params.get("credentials_path")
+            cred_base64 = query_params.get("credentials_base64")
+
+            if not project_id_param or not instance_id_param or not database_param:
+                raise ValueError(
+                    "project_id, instance_id and database are required in the URI to get data from Google Spanner"
+                )
+
+            project_id = project_id_param[0]
+            instance_id = instance_id_param[0]
+            database = database_param[0]
+
+            if not cred_path and not cred_base64:
+                raise ValueError(
+                    "credentials_path or credentials_base64 is required in the URI to get data from Google Sheets"
+                )
+            if cred_path:
+                os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cred_path[0]
+            elif cred_base64:
+                credentials = json.loads(
+                    base64.b64decode(cred_base64[0]).decode("utf-8")
+                )
+                temp = tempfile.NamedTemporaryFile(
+                    mode="w", delete=False, suffix=".json"
+                )
+                json.dump(credentials, temp)
+                temp.close()
+                os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = temp.name
+
+            uri = f"spanner+spanner:///projects/{project_id}/instances/{instance_id}/databases/{database}"
+
         from dlt.common.libs.sql_alchemy import (
             Engine,
             MetaData,
@@ -2325,7 +2367,9 @@ class ElasticsearchSource:
 
         index = table
         if not index:
-            raise ValueError("Table name must be provided which is the index name in elasticsearch")
+            raise ValueError(
+                "Table name must be provided which is the index name in elasticsearch"
+            )
 
         query_params = parsed.query
         params = parse_qs(query_params)

{ingestr-0.13.40.dist-info → ingestr-0.13.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.40
+Version: 0.13.41
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -72,11 +72,14 @@ Requires-Dist: google-auth==2.38.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: google-cloud-bigquery==3.30.0
 Requires-Dist: google-cloud-core==2.4.2
+Requires-Dist: google-cloud-spanner==3.54.0
 Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
 Requires-Dist: greenlet==3.2.2
+Requires-Dist: grpc-google-iam-v1==0.14.2
+Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3
 Requires-Dist: grpcio==1.70.0
 Requires-Dist: hdbcli==2.23.27
@@ -168,9 +171,11 @@ Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
+Requires-Dist: sqlalchemy-spanner==1.11.0
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: sqlglot==26.12.1
+Requires-Dist: sqlparse==0.5.3
 Requires-Dist: stripe==10.7.0
 Requires-Dist: tenacity==9.0.0
 Requires-Dist: thrift==0.16.0

{ingestr-0.13.40.dist-info → ingestr-0.13.41.dist-info}/RECORD

@@ -2,15 +2,15 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=Pe_rzwcDRKIYa7baEVUAAPOHyqQbX29RUexMl0F_S1k,25273
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=kx4THAPAkQ2P32re7w8VrOWSESl3Fz3mnN83MSygHeE,21
+ingestr/src/buildinfo.py,sha256=AK7sGUNx6CPDKJOXeMexFRun9bjpyv2c9t4DII73Pes,21
 ingestr/src/destinations.py,sha256=MctbeJUyNr0DRB0XYt2xAbEKkHZ40-nXXEOYCs4KuoE,15420
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=x-Ym3uHMgzj_svUk7Lopn3Jj-IhcQLCuDqA_eUPFLAI,5582
+ingestr/src/factory.py,sha256=KJKIL9q7kU4oAVXy5o0wDwLAU0nG9y0xC8D7HzksYak,5597
 ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=RitbAjFVnq1I7MsjbD7hrn6Akd_92P6OCEg--YHivDw,80770
+ingestr/src/sources.py,sha256=SWZAa6bokLurQRPtH7rxi8K-GSVLp_p9Ig1ArGRsxCo,82635
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -128,8 +128,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.40.dist-info/METADATA,sha256=DV_PkyMFlK4isa37puXrTKAfFPb4oQ4_cKv-b1lojI4,13653
-ingestr-0.13.40.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.40.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.40.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.40.dist-info/RECORD,,
+ingestr-0.13.41.dist-info/METADATA,sha256=UzGBs9s0Kr6R1xji_ULG5Tuc383Klx2AIzfyZdXLBp4,13852
+ingestr-0.13.41.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.41.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.41.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.41.dist-info/RECORD,,