ingestr 0.13.60__py3-none-any.whl → 0.13.61__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.60"
+version = "v0.13.61"

ingestr/src/destinations.py

@@ -1,3 +1,4 @@
+import abc
 import base64
 import csv
 import json
@@ -9,6 +10,7 @@ from urllib.parse import parse_qs, quote, urlparse
 import dlt
 import dlt.destinations.impl.filesystem.filesystem
 from dlt.common.configuration.specs import AwsCredentials
+from dlt.common.storages.configuration import FileSystemCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
 )
@@ -111,6 +113,14 @@ class BigQueryDestination:
         pass
 
 
+class CrateDBDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        uri = uri.replace("cratedb://", "postgres://")
+        import dlt_cratedb.impl.cratedb.factory
+
+        return dlt_cratedb.impl.cratedb.factory.cratedb(credentials=uri, **kwargs)
+
+
 class PostgresDestination(GenericSqlDestination):
     def dlt_dest(self, uri: str, **kwargs):
         return dlt.destinations.postgres(credentials=uri, **kwargs)
@@ -386,43 +396,62 @@ class ClickhouseDestination:
         pass
 
 
-class S3FSClient(dlt.destinations.impl.filesystem.filesystem.FilesystemClient):
+class BlobFSClient(dlt.destinations.impl.filesystem.filesystem.FilesystemClient):
     @property
     def dataset_path(self):
         # override to remove dataset path
         return self.bucket_path
 
 
-class S3FS(dlt.destinations.filesystem):
+class BlobFS(dlt.destinations.filesystem):
     @property
     def client_class(self):
-        return S3FSClient
+        return BlobFSClient
 
 
-class S3Destination:
+class SqliteDestination(GenericSqlDestination):
     def dlt_dest(self, uri: str, **kwargs):
-        parsed_uri = urlparse(uri)
-        params = parse_qs(parsed_uri.query)
+        return dlt.destinations.sqlalchemy(credentials=uri)
 
-        access_key_id = params.get("access_key_id", [None])[0]
-        if access_key_id is None:
-            raise MissingValueError("access_key_id", "S3")
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        return {
+            # https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
+            "dataset_name": "main",
+            "table_name": table,
+        }
 
-        secret_access_key = params.get("secret_access_key", [None])[0]
-        if secret_access_key is None:
-            raise MissingValueError("secret_access_key", "S3")
 
-        endpoint_url = params.get("endpoint_url", [None])[0]
-        if endpoint_url is not None:
-            parsed_endpoint = urlparse(endpoint_url)
-            if not parsed_endpoint.scheme or not parsed_endpoint.netloc:
-                raise ValueError("Invalid endpoint_url. Must be a valid URL.")
+class MySqlDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
 
-        creds = AwsCredentials(
-            aws_access_key_id=access_key_id,
-            aws_secret_access_key=secret_access_key,
-            endpoint_url=endpoint_url,
-        )
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        database = parsed.path.lstrip("/")
+        if not database:
+            raise ValueError("You need to specify a database")
+        return {
+            "dataset_name": database,
+            "table_name": table,
+        }
+
+
+class BlobStorageDestination(abc.ABC):
+    @abc.abstractmethod
+    def credentials(self, params: dict) -> FileSystemCredentials:
+        """Build credentials for the blob storage destination."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def protocol(self) -> str:
+        """The protocol used for the blob storage destination."""
+        pass
+
+    def dlt_dest(self, uri: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        creds = self.credentials(params)
 
         dest_table = kwargs["dest_table"]
 
@@ -442,7 +471,7 @@ class S3Destination:
         base_path = "/".join(table_parts[:-1])
 
         opts = {
-            "bucket_url": f"s3://{base_path}",
+            "bucket_url": f"{self.protocol}://{base_path}",
             "credentials": creds,
             # supresses dlt warnings about dataset name normalization.
             # we don't use dataset names in S3 so it's fine to disable this.
@@ -452,7 +481,7 @@ class S3Destination:
         if layout is not None:
             opts["layout"] = layout
 
-        return S3FS(**opts)  # type: ignore
+        return BlobFS(**opts)  # type: ignore
 
     def validate_table(self, table: str):
         table = table.strip("/ ")
@@ -470,28 +499,56 @@ class S3Destination:
         pass
 
 
-class SqliteDestination(GenericSqlDestination):
-    def dlt_dest(self, uri: str, **kwargs):
-        return dlt.destinations.sqlalchemy(credentials=uri)
+class S3Destination(BlobStorageDestination):
+    @property
+    def protocol(self) -> str:
+        return "s3"
 
-    def dlt_run_params(self, uri: str, table: str, **kwargs):
-        return {
-            # https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
-            "dataset_name": "main",
-            "table_name": table,
-        }
+    def credentials(self, params: dict) -> FileSystemCredentials:
+        access_key_id = params.get("access_key_id", [None])[0]
+        if access_key_id is None:
+            raise MissingValueError("access_key_id", "S3")
 
+        secret_access_key = params.get("secret_access_key", [None])[0]
+        if secret_access_key is None:
+            raise MissingValueError("secret_access_key", "S3")
 
-class MySqlDestination(GenericSqlDestination):
-    def dlt_dest(self, uri: str, **kwargs):
-        return dlt.destinations.sqlalchemy(credentials=uri)
+        endpoint_url = params.get("endpoint_url", [None])[0]
+        if endpoint_url is not None:
+            parsed_endpoint = urlparse(endpoint_url)
+            if not parsed_endpoint.scheme or not parsed_endpoint.netloc:
+                raise ValueError("Invalid endpoint_url. Must be a valid URL.")
 
-    def dlt_run_params(self, uri: str, table: str, **kwargs):
-        parsed = urlparse(uri)
-        database = parsed.path.lstrip("/")
-        if not database:
-            raise ValueError("You need to specify a database")
-        return {
-            "dataset_name": database,
-            "table_name": table,
-        }
+        return AwsCredentials(
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            endpoint_url=endpoint_url,
+        )
+
+
+class GCSDestination(BlobStorageDestination):
+    @property
+    def protocol(self) -> str:
+        return "gs"
+
+    def credentials(self, params: dict) -> FileSystemCredentials:
+        """Builds GCS credentials from the provided parameters."""
+        credentials_path = params.get("credentials_path")
+        credentials_base64 = params.get("credentials_base64")
+        credentials_available = any(
+            map(
+                lambda x: x is not None,
+                [credentials_path, credentials_base64],
+            )
+        )
+        if credentials_available is False:
+            raise MissingValueError("credentials_path or credentials_base64", "GCS")
+
+        credentials = None
+        if credentials_path:
+            with open(credentials_path[0], "r") as f:
+                credentials = json.load(f)
+        else:
+            credentials = json.loads(base64.b64decode(credentials_base64[0]).decode())  # type: ignore
+
+        return credentials

ingestr/src/factory.py

@@ -7,9 +7,11 @@ from ingestr.src.destinations import (
     AthenaDestination,
     BigQueryDestination,
     ClickhouseDestination,
+    CrateDBDestination,
     CsvDestination,
     DatabricksDestination,
     DuckDBDestination,
+    GCSDestination,
     MsSQLDestination,
     MySqlDestination,
     PostgresDestination,
@@ -181,6 +183,7 @@ class SourceDestinationFactory:
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,
+        "cratedb": CrateDBDestination,
         "databricks": DatabricksDestination,
         "duckdb": DuckDBDestination,
         "mssql": MsSQLDestination,
@@ -197,6 +200,7 @@ class SourceDestinationFactory:
         "clickhouse+native": ClickhouseDestination,
         "clickhouse": ClickhouseDestination,
         "s3": S3Destination,
+        "gs": GCSDestination,
         "sqlite": SqliteDestination,
         "mysql": MySqlDestination,
         "mysql+pymysql": MySqlDestination,

ingestr/src/isoc_pulse/__init__.py

@@ -73,7 +73,7 @@ def pulse_source(
             "write_disposition": "merge",
             "primary_key": "date",
         },
-        "resources": resources, # type:ignore
+        "resources": resources,  # type:ignore
     }
     res = rest_api_resources(config)
     if metric == "net_loss":

ingestr/src/sources.py

@@ -1885,7 +1885,7 @@ class GCSSource:
             endpoint = blob.parse_endpoint(path_to_file)
         except blob.UnsupportedEndpointError:
             raise ValueError(
-                "S3 Source only supports specific formats files: csv, jsonl, parquet"
+                "GCS Source only supports specific formats files: csv, jsonl, parquet"
             )
         except Exception as e:
             raise ValueError(

{ingestr-0.13.60.dist-info → ingestr-0.13.61.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.60
+Version: 0.13.61
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -47,6 +47,7 @@ Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
+Requires-Dist: dlt-cratedb==0.0.1
 Requires-Dist: dlt==1.10.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
@@ -305,7 +306,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td>CrateDB</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>✅</td>
     </tr>
     <tr>
         <td>Databricks</td>

{ingestr-0.13.60.dist-info → ingestr-0.13.61.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=taDyHyaVSpB17iNLl8zA0gmr4CqDO-MSTQX1CaRBB9U,26364
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=1sTup4WLO36DuLnh5cnxtmEDBjKKYxAOSisEvjELy1w,21
-ingestr/src/destinations.py,sha256=TcxM2rcwHfgMMP6U0yRNcfWKlEzkBbZbqCIDww7lkTY,16882
+ingestr/src/buildinfo.py,sha256=slTtbF5rCzvz-7Z_L3XF0kUk-fOkQu3INJIPtywQb5Y,21
+ingestr/src/destinations.py,sha256=YU7c5cNqViCIJ9NDhgGuorh5jMvLi7yViPWpEJ57Xx0,18788
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=OKqjYqvHhgaOF48-eSNSabcfXt4Gmr1yZ8cFGizXh0g,6319
+ingestr/src/factory.py,sha256=j_ANB1X5BUa_iUOLqreHMGWnDPFV4bGgd0lL35lZ6XE,6432
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=sJmiiInFb-KCPsaIy4qus6lx59MDCOobWgxJ7lfKH08,99047
+ingestr/src/sources.py,sha256=SZo44oi7DBTEJrAvVhbj0lpXWd4aGSQTMgwZ17lgO9c,99048
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -73,7 +73,7 @@ ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOIN
 ingestr/src/hubspot/__init__.py,sha256=wqHefhc_YRI5dNFCcpvH-UUilNThE49sbGouSBiHYsw,11776
 ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
-ingestr/src/isoc_pulse/__init__.py,sha256=WDgKBn15gyQheXE6oJ_2OuMUQwKPbAjflKAsnucu7u8,4647
+ingestr/src/isoc_pulse/__init__.py,sha256=9b4eN4faatpiwTuRNPuYcEt1hEFDEjua9XhfakUigBk,4648
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=YretSz4F28tbkcPhd55mBp2Xk7XE9unyWx0nmvl8iEc,6235
@@ -143,8 +143,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.60.dist-info/METADATA,sha256=FwdcfGIPPRKlSV8wJX1HAqHriGUZBl_XXi0Yco8O874,14993
-ingestr-0.13.60.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.60.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.60.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.60.dist-info/RECORD,,
+ingestr-0.13.61.dist-info/METADATA,sha256=0VM6DuyeROIX3TUb8FBXRMiUg4KktwK8Gr95ZespXHE,15027
+ingestr-0.13.61.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.61.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.61.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.61.dist-info/RECORD,,