ingestr 0.13.51__py3-none-any.whl → 0.13.53__py3-none-any.whl

ingestr/main.py

@@ -58,6 +58,7 @@ class LoaderFileFormat(str, Enum):
 
 
 class SqlBackend(str, Enum):
+    default = "default"
     sqlalchemy = "sqlalchemy"
     pyarrow = "pyarrow"
     connectorx = "connectorx"
@@ -187,7 +188,7 @@ def ingest(
             help="The SQL backend to use",
             envvar=["SQL_BACKEND", "INGESTR_SQL_BACKEND"],
         ),
-    ] = SqlBackend.pyarrow,  # type: ignore
+    ] = SqlBackend.default,  # type: ignore
     loader_file_format: Annotated[
         Optional[LoaderFileFormat],
         typer.Option(
@@ -289,7 +290,11 @@ def ingest(
     from ingestr.src.collector.spinner import SpinnerCollector
     from ingestr.src.destinations import AthenaDestination
     from ingestr.src.factory import SourceDestinationFactory
-    from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
+    from ingestr.src.filters import (
+        cast_set_to_list,
+        cast_spanner_types,
+        handle_mysql_empty_dates,
+    )
     from ingestr.src.sources import MongoDbSource
 
     def report_errors(run_info: LoadInfo):
@@ -517,6 +522,15 @@ def ingest(
             if interval_end:
                 interval_end = interval_end.date()  # type: ignore
 
+        if factory.source_scheme.startswith("spanner"):
+            # we tend to use the 'pyarrow' backend in general, however, it has issues with JSON objects, so we override it to 'sqlalchemy' for Spanner.
+            if sql_backend.value == SqlBackend.default:
+                sql_backend = SqlBackend.sqlalchemy
+
+        # this allows us to identify the cases where the user does not have a preference, so that for some sources we can override it.
+        if sql_backend == SqlBackend.default:
+            sql_backend = SqlBackend.pyarrow
+
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,
@@ -535,6 +549,9 @@ def ingest(
         if factory.source_scheme.startswith("mysql"):
             resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
 
+        if factory.source_scheme.startswith("spanner"):
+            resource.for_each(dlt_source, lambda x: x.add_map(cast_spanner_types))
+
         if yield_limit:
             resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
 

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.51"
+version = "v0.13.53"

ingestr/src/destinations.py

@@ -468,3 +468,33 @@ class S3Destination:
 
     def post_load(self) -> None:
         pass
+
+
+class SqliteDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        return {
+            #https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
+            "dataset_name": "main",
+            "table_name": table,
+        }
+
+
+class MySqlDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        database = parsed.path.lstrip("/")
+        if not database:
+            raise ValueError("You need to specify a database")
+        return {
+            "dataset_name": database,
+            "table_name": table,
+        }
+
+
+

ingestr/src/factory.py

@@ -11,10 +11,12 @@ from ingestr.src.destinations import (
     DatabricksDestination,
     DuckDBDestination,
     MsSQLDestination,
+    MySqlDestination,
     PostgresDestination,
     RedshiftDestination,
     S3Destination,
     SnowflakeDestination,
+    SqliteDestination,
     SynapseDestination,
 )
 from ingestr.src.sources import (
@@ -52,18 +54,20 @@ from ingestr.src.sources import (
     PipedriveSource,
     S3Source,
     SalesforceSource,
+    SFTPSource,
     ShopifySource,
     SlackSource,
+    SmartsheetSource,
     SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
     ZendeskSource,
-    SmartsheetSource,
 )
 
 SQL_SOURCE_SCHEMES = [
     "bigquery",
+    "crate",
     "duckdb",
     "mssql",
     "mysql",
@@ -163,6 +167,7 @@ class SourceDestinationFactory:
         "attio": AttioSource,
         "solidgate": SolidgateSource,
         "smartsheet": SmartsheetSource,
+        "sftp": SFTPSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,
@@ -182,6 +187,9 @@ class SourceDestinationFactory:
         "clickhouse+native": ClickhouseDestination,
         "clickhouse": ClickhouseDestination,
         "s3": S3Destination,
+        "sqlite": SqliteDestination,
+        "mysql": MySqlDestination,
+        "mysql+pymysql": MySqlDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

ingestr/src/filters.py

@@ -7,6 +7,20 @@ def cast_set_to_list(row):
     return row
 
 
+def cast_spanner_types(row):
+    if not isinstance(row, dict):
+        return row
+
+    from google.cloud.spanner_v1.data_types import JsonObject
+
+    for key in row.keys():
+        if isinstance(row[key], JsonObject):
+            import json
+
+            row[key] = json.loads(row[key].serialize())
+    return row
+
+
 def handle_mysql_empty_dates(row):
     # MySQL returns empty dates as 0000-00-00, which is not a valid date, we handle them here.
     if not isinstance(row, dict):

ingestr/src/solidgate/__init__.py

@@ -43,7 +43,7 @@ def solidgate_source(
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
     @dlt.resource(
-        name="apm-orders",
+        name="apm_orders",
         write_disposition="merge",
         primary_key="order_id",
         columns={
@@ -69,7 +69,7 @@ def solidgate_source(
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
     @dlt.resource(
-        name="card-orders",
+        name="card_orders",
         write_disposition="merge",
         primary_key="order_id",
         columns={
@@ -95,7 +95,7 @@ def solidgate_source(
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
     @dlt.resource(
-        name="financial-entries",
+        name="financial_entries",
         write_disposition="merge",
         primary_key="id",
         columns={

ingestr/src/sources.py

@@ -18,6 +18,7 @@ from typing import (
 )
 from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
 
+import fsspec  # type: ignore
 import pendulum
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.extract import Incremental
@@ -78,31 +79,7 @@ class SqlSource:
         # clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
         if uri.startswith("clickhouse://"):
             parsed_uri = urlparse(uri)
-
-            username = parsed_uri.username
-            if not username:
-                raise ValueError(
-                    "A username is required to connect to the ClickHouse database."
-                )
-
-            password = parsed_uri.password
-            if not password:
-                raise ValueError(
-                    "A password is required to authenticate with the ClickHouse database."
-                )
-
-            host = parsed_uri.hostname
-            if not host:
-                raise ValueError(
-                    "The hostname or IP address of the ClickHouse server is required to establish a connection."
-                )
-
-            port = parsed_uri.port
-            if not port:
-                raise ValueError(
-                    "The TCP port of the ClickHouse server is required to establish a connection."
-                )
-
+            
             query_params = parse_qs(parsed_uri.query)
 
             if "http_port" in query_params:
@@ -2506,3 +2483,56 @@ class SolidgateSource:
             ).with_resources(table_name)
         except ResourcesNotFoundError:
             raise UnsupportedResourceError(table_name, "Solidgate")
+
+
+class SFTPSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        host = parsed_uri.hostname
+        if not host:
+            raise MissingValueError("host", "SFTP URI")
+        port = parsed_uri.port or 22
+        username = parsed_uri.username
+        password = parsed_uri.password
+
+        params: Dict[str, Any] = {
+            "host": host,
+            "port": port,
+            "username": username,
+            "password": password,
+            "look_for_keys": False,
+            "allow_agent": False,
+        }
+
+        try:
+            fs = fsspec.filesystem("sftp", **params)
+        except Exception as e:
+            raise ConnectionError(
+                f"Failed to connect or authenticate to sftp server {host}:{port}. Error: {e}"
+            )
+        bucket_url = f"sftp://{host}:{port}"
+
+        if table.startswith("/"):
+            file_glob = table
+        else:
+            file_glob = f"/{table}"
+
+        file_extension = table.split(".")[-1].lower()
+        endpoint: str
+        if file_extension == "csv":
+            endpoint = "read_csv"
+        elif file_extension == "jsonl":
+            endpoint = "read_jsonl"
+        elif file_extension == "parquet":
+            endpoint = "read_parquet"
+        else:
+            raise ValueError(
+                "FTPServer Source only supports specific file formats: csv, jsonl, parquet."
+            )
+        from ingestr.src.filesystem import readers
+
+        dlt_source_resource = readers(bucket_url, fs, file_glob)
+        return dlt_source_resource.with_resources(endpoint)

{ingestr-0.13.51.dist-info → ingestr-0.13.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.51
+Version: 0.13.53
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,6 +26,7 @@ Requires-Dist: asn1crypto==1.5.1
 Requires-Dist: asynch==0.2.4
 Requires-Dist: attrs==25.1.0
 Requires-Dist: backoff==2.2.1
+Requires-Dist: bcrypt==4.3.0
 Requires-Dist: beautifulsoup4==4.13.3
 Requires-Dist: boto3==1.37.1
 Requires-Dist: botocore==1.37.1
@@ -39,6 +40,7 @@ Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: crate==2.0.0
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: curlify==2.2.1
 Requires-Dist: databricks-sql-connector==2.9.3
@@ -59,6 +61,7 @@ Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0
 Requires-Dist: fsspec==2025.3.2
 Requires-Dist: gcsfs==2025.3.2
+Requires-Dist: geojson==3.2.0
 Requires-Dist: gitdb==4.0.12
 Requires-Dist: gitpython==3.1.44
 Requires-Dist: giturlparse==0.12.0
@@ -77,7 +80,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.2
+Requires-Dist: greenlet==3.2.3
 Requires-Dist: grpc-google-iam-v1==0.14.2
 Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3
@@ -113,6 +116,7 @@ Requires-Dist: openpyxl==3.1.5
 Requires-Dist: orjson==3.10.15
 Requires-Dist: packaging==24.2
 Requires-Dist: pandas==2.2.3
+Requires-Dist: paramiko==3.5.1
 Requires-Dist: pathvalidate==3.2.3
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: platformdirs==4.3.6
@@ -137,6 +141,7 @@ Requires-Dist: pygments==2.19.1
 Requires-Dist: pyjwt==2.10.1
 Requires-Dist: pymongo==4.11.1
 Requires-Dist: pymysql==1.1.1
+Requires-Dist: pynacl==1.5.0
 Requires-Dist: pyopenssl==25.0.0
 Requires-Dist: pyparsing==3.2.1
 Requires-Dist: pyrate-limiter==3.7.0
@@ -170,6 +175,7 @@ Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sortedcontainers==2.4.0
 Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
+Requires-Dist: sqlalchemy-cratedb==0.42.0.dev2
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy-spanner==1.11.0
@@ -192,6 +198,7 @@ Requires-Dist: tzdata==2025.1
 Requires-Dist: tzlocal==5.3
 Requires-Dist: uritemplate==4.1.1
 Requires-Dist: urllib3==2.3.0
+Requires-Dist: verlib2==0.3.1
 Requires-Dist: wrapt==1.17.2
 Requires-Dist: yarl==1.18.3
 Requires-Dist: zeep==4.3.1
@@ -292,11 +299,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>CrateDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>IBM Db2</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>DuckDB</td>
         <td>✅</td>
@@ -307,6 +324,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Elasticsearch</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+     <tr>
+        <td>GCP Spanner</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -393,6 +420,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Attio</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Chess.com</td>
         <td>✅</td>
@@ -404,7 +436,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Github</td>
+        <td>GitHub</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -447,6 +479,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Personio</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Phantombuster</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Pipedrive</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>
@@ -469,7 +511,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Smartsheet</td>
+        <td>Smartsheets</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Solidgate</td>
         <td>✅</td>
         <td>-</td>
     </tr>

{ingestr-0.13.51.dist-info → ingestr-0.13.53.dist-info}/RECORD

@@ -1,17 +1,17 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=rHxHQAbd0ccW2e2kQSWlv7-5qcc2ZB6Eh3vyjm4Nzns,25550
+ingestr/main.py,sha256=GkC1hdq8AVGrvolc95zMfjmibI95p2pmFkbgCOVf-Og,26311
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=ZgugQZPIlr-_7kRH5qIupQ5e48JTOHSlRWEsmNx4wXw,21
-ingestr/src/destinations.py,sha256=41Bj1UgxR8a2KcZWqtGw74AKZKnSBrueQRnBdrf3c-A,16003
+ingestr/src/buildinfo.py,sha256=TRZbcB7mFeJZcDA_BqTjZvRtxs0161pLvM9EmkNt90U,21
+ingestr/src/destinations.py,sha256=wYE6p_DC9HrQ5KehhgbLxgnkS1P9wE9L21Hw_lgAZ70,16884
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=JeK0M8C05-_KxxIhtevs-MJGLlngXUQupd-Pm4Fg904,5803
-ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
+ingestr/src/factory.py,sha256=wDa3F7xbnQcQwfnQIbxWypuJ1G8GGhRpdssWRPtEO_Q,6020
+ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=SnFxil6gU5RIYiSUfE6Upq2se0YeRSuTIKK3Jxn-YKI,86587
+ingestr/src/sources.py,sha256=sEi-09LXySuhHqUpscHlfQTmQ_7Bgq1GDY_y5mma-sg,87437
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -109,7 +109,7 @@ ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
 ingestr/src/smartsheets/__init__.py,sha256=pdzSV7rA0XYD5Xa1u4zb6vziy5iFXIQNROkpJ9oYas0,1623
-ingestr/src/solidgate/__init__.py,sha256=DZYQ4M3Cc7AIbdQcNQm_6yX2whnFhE-iM10-ACJ3W3A,3626
+ingestr/src/solidgate/__init__.py,sha256=JdaXvAu5QGuf9-FY294vwCQCEmfrqIld9oqbzqCJS3g,3626
 ingestr/src/solidgate/helpers.py,sha256=oePEc9nnvmN3IaKrfJCvyKL79xdGM0-gRTN3-8tY4Fc,4952
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
@@ -135,8 +135,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.51.dist-info/METADATA,sha256=danPRyBhqlvyOwtw8kBmBh5FEY_xH6upsy5GrsFaMg4,13983
-ingestr-0.13.51.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.51.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.51.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.51.dist-info/RECORD,,
+ingestr-0.13.53.dist-info/METADATA,sha256=vVMGPiZ4snksSxZGUKgNl0a3bfSxUBeTrWpUG0lHFhw,14902
+ingestr-0.13.53.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.53.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.53.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.53.dist-info/RECORD,,