ingestr 0.13.52__py3-none-any.whl → 0.13.53__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.52"
+version = "v0.13.53"

ingestr/src/destinations.py

@@ -480,3 +480,21 @@ class SqliteDestination(GenericSqlDestination):
             "dataset_name": "main",
             "table_name": table,
         }
+
+
+class MySqlDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        database = parsed.path.lstrip("/")
+        if not database:
+            raise ValueError("You need to specify a database")
+        return {
+            "dataset_name": database,
+            "table_name": table,
+        }
+
+
+

ingestr/src/factory.py

@@ -11,6 +11,7 @@ from ingestr.src.destinations import (
     DatabricksDestination,
     DuckDBDestination,
     MsSQLDestination,
+    MySqlDestination,
     PostgresDestination,
     RedshiftDestination,
     S3Destination,
@@ -53,6 +54,7 @@ from ingestr.src.sources import (
     PipedriveSource,
     S3Source,
     SalesforceSource,
+    SFTPSource,
     ShopifySource,
     SlackSource,
     SmartsheetSource,
@@ -65,6 +67,7 @@ from ingestr.src.sources import (
 
 SQL_SOURCE_SCHEMES = [
     "bigquery",
+    "crate",
     "duckdb",
     "mssql",
     "mysql",
@@ -164,6 +167,7 @@ class SourceDestinationFactory:
         "attio": AttioSource,
         "solidgate": SolidgateSource,
         "smartsheet": SmartsheetSource,
+        "sftp": SFTPSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,
@@ -184,6 +188,8 @@ class SourceDestinationFactory:
         "clickhouse": ClickhouseDestination,
         "s3": S3Destination,
         "sqlite": SqliteDestination,
+        "mysql": MySqlDestination,
+        "mysql+pymysql": MySqlDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

ingestr/src/sources.py

@@ -18,6 +18,7 @@ from typing import (
 )
 from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
 
+import fsspec  # type: ignore
 import pendulum
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.extract import Incremental
@@ -78,31 +79,7 @@ class SqlSource:
         # clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
         if uri.startswith("clickhouse://"):
             parsed_uri = urlparse(uri)
-
-            username = parsed_uri.username
-            if not username:
-                raise ValueError(
-                    "A username is required to connect to the ClickHouse database."
-                )
-
-            password = parsed_uri.password
-            if not password:
-                raise ValueError(
-                    "A password is required to authenticate with the ClickHouse database."
-                )
-
-            host = parsed_uri.hostname
-            if not host:
-                raise ValueError(
-                    "The hostname or IP address of the ClickHouse server is required to establish a connection."
-                )
-
-            port = parsed_uri.port
-            if not port:
-                raise ValueError(
-                    "The TCP port of the ClickHouse server is required to establish a connection."
-                )
-
+            
             query_params = parse_qs(parsed_uri.query)
 
             if "http_port" in query_params:
@@ -2506,3 +2483,56 @@ class SolidgateSource:
             ).with_resources(table_name)
         except ResourcesNotFoundError:
             raise UnsupportedResourceError(table_name, "Solidgate")
+
+
+class SFTPSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        host = parsed_uri.hostname
+        if not host:
+            raise MissingValueError("host", "SFTP URI")
+        port = parsed_uri.port or 22
+        username = parsed_uri.username
+        password = parsed_uri.password
+
+        params: Dict[str, Any] = {
+            "host": host,
+            "port": port,
+            "username": username,
+            "password": password,
+            "look_for_keys": False,
+            "allow_agent": False,
+        }
+
+        try:
+            fs = fsspec.filesystem("sftp", **params)
+        except Exception as e:
+            raise ConnectionError(
+                f"Failed to connect or authenticate to sftp server {host}:{port}. Error: {e}"
+            )
+        bucket_url = f"sftp://{host}:{port}"
+
+        if table.startswith("/"):
+            file_glob = table
+        else:
+            file_glob = f"/{table}"
+
+        file_extension = table.split(".")[-1].lower()
+        endpoint: str
+        if file_extension == "csv":
+            endpoint = "read_csv"
+        elif file_extension == "jsonl":
+            endpoint = "read_jsonl"
+        elif file_extension == "parquet":
+            endpoint = "read_parquet"
+        else:
+            raise ValueError(
+                "FTPServer Source only supports specific file formats: csv, jsonl, parquet."
+            )
+        from ingestr.src.filesystem import readers
+
+        dlt_source_resource = readers(bucket_url, fs, file_glob)
+        return dlt_source_resource.with_resources(endpoint)

{ingestr-0.13.52.dist-info → ingestr-0.13.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.52
+Version: 0.13.53
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,6 +26,7 @@ Requires-Dist: asn1crypto==1.5.1
 Requires-Dist: asynch==0.2.4
 Requires-Dist: attrs==25.1.0
 Requires-Dist: backoff==2.2.1
+Requires-Dist: bcrypt==4.3.0
 Requires-Dist: beautifulsoup4==4.13.3
 Requires-Dist: boto3==1.37.1
 Requires-Dist: botocore==1.37.1
@@ -39,6 +40,7 @@ Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: crate==2.0.0
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: curlify==2.2.1
 Requires-Dist: databricks-sql-connector==2.9.3
@@ -59,6 +61,7 @@ Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0
 Requires-Dist: fsspec==2025.3.2
 Requires-Dist: gcsfs==2025.3.2
+Requires-Dist: geojson==3.2.0
 Requires-Dist: gitdb==4.0.12
 Requires-Dist: gitpython==3.1.44
 Requires-Dist: giturlparse==0.12.0
@@ -77,7 +80,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.2
+Requires-Dist: greenlet==3.2.3
 Requires-Dist: grpc-google-iam-v1==0.14.2
 Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3
@@ -113,6 +116,7 @@ Requires-Dist: openpyxl==3.1.5
 Requires-Dist: orjson==3.10.15
 Requires-Dist: packaging==24.2
 Requires-Dist: pandas==2.2.3
+Requires-Dist: paramiko==3.5.1
 Requires-Dist: pathvalidate==3.2.3
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: platformdirs==4.3.6
@@ -137,6 +141,7 @@ Requires-Dist: pygments==2.19.1
 Requires-Dist: pyjwt==2.10.1
 Requires-Dist: pymongo==4.11.1
 Requires-Dist: pymysql==1.1.1
+Requires-Dist: pynacl==1.5.0
 Requires-Dist: pyopenssl==25.0.0
 Requires-Dist: pyparsing==3.2.1
 Requires-Dist: pyrate-limiter==3.7.0
@@ -170,6 +175,7 @@ Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sortedcontainers==2.4.0
 Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
+Requires-Dist: sqlalchemy-cratedb==0.42.0.dev2
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy-spanner==1.11.0
@@ -192,6 +198,7 @@ Requires-Dist: tzdata==2025.1
 Requires-Dist: tzlocal==5.3
 Requires-Dist: uritemplate==4.1.1
 Requires-Dist: urllib3==2.3.0
+Requires-Dist: verlib2==0.3.1
 Requires-Dist: wrapt==1.17.2
 Requires-Dist: yarl==1.18.3
 Requires-Dist: zeep==4.3.1
@@ -292,11 +299,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>CrateDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>IBM Db2</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>DuckDB</td>
         <td>✅</td>
@@ -307,6 +324,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Elasticsearch</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+     <tr>
+        <td>GCP Spanner</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -393,6 +420,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Attio</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Chess.com</td>
         <td>✅</td>
@@ -404,7 +436,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Github</td>
+        <td>GitHub</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -447,6 +479,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Personio</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Phantombuster</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Pipedrive</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>
@@ -469,7 +511,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Smartsheet</td>
+        <td>Smartsheets</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Solidgate</td>
         <td>✅</td>
         <td>-</td>
     </tr>

{ingestr-0.13.52.dist-info → ingestr-0.13.53.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=GkC1hdq8AVGrvolc95zMfjmibI95p2pmFkbgCOVf-Og,26311
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=qcKqSPI861BBapk7rbcYkc9M_18QQnT44YM1Zr_Sg80,21
-ingestr/src/destinations.py,sha256=axSm0GLM_bKQmWYb74fnHebNktWgtfAyyqdHV-zBOsA,16405
+ingestr/src/buildinfo.py,sha256=TRZbcB7mFeJZcDA_BqTjZvRtxs0161pLvM9EmkNt90U,21
+ingestr/src/destinations.py,sha256=wYE6p_DC9HrQ5KehhgbLxgnkS1P9wE9L21Hw_lgAZ70,16884
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=5yrg-XkaixuCkiTz3B7mraE8LaANXtzItenbx8TdPrE,5863
+ingestr/src/factory.py,sha256=wDa3F7xbnQcQwfnQIbxWypuJ1G8GGhRpdssWRPtEO_Q,6020
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=SnFxil6gU5RIYiSUfE6Upq2se0YeRSuTIKK3Jxn-YKI,86587
+ingestr/src/sources.py,sha256=sEi-09LXySuhHqUpscHlfQTmQ_7Bgq1GDY_y5mma-sg,87437
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -135,8 +135,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.52.dist-info/METADATA,sha256=jfSWJ6mE6vPz-5XsIIjqdLxd45-OrOUKbhbwJqiZlSc,13983
-ingestr-0.13.52.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.52.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.52.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.52.dist-info/RECORD,,
+ingestr-0.13.53.dist-info/METADATA,sha256=vVMGPiZ4snksSxZGUKgNl0a3bfSxUBeTrWpUG0lHFhw,14902
+ingestr-0.13.53.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.53.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.53.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.53.dist-info/RECORD,,