ingestr 0.13.52__tar.gz → 0.13.53__tar.gz

{ingestr-0.13.52 → ingestr-0.13.53}/.gitignore

@@ -3,6 +3,7 @@ __pycache__
 .env
 venv
 !.gitkeep
+.idea
 .vscode/*
 .metals/*
 .coverage.*

{ingestr-0.13.52 → ingestr-0.13.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.52
+Version: 0.13.53
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,6 +26,7 @@ Requires-Dist: asn1crypto==1.5.1
 Requires-Dist: asynch==0.2.4
 Requires-Dist: attrs==25.1.0
 Requires-Dist: backoff==2.2.1
+Requires-Dist: bcrypt==4.3.0
 Requires-Dist: beautifulsoup4==4.13.3
 Requires-Dist: boto3==1.37.1
 Requires-Dist: botocore==1.37.1
@@ -39,6 +40,7 @@ Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: crate==2.0.0
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: curlify==2.2.1
 Requires-Dist: databricks-sql-connector==2.9.3
@@ -59,6 +61,7 @@ Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0
 Requires-Dist: fsspec==2025.3.2
 Requires-Dist: gcsfs==2025.3.2
+Requires-Dist: geojson==3.2.0
 Requires-Dist: gitdb==4.0.12
 Requires-Dist: gitpython==3.1.44
 Requires-Dist: giturlparse==0.12.0
@@ -77,7 +80,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.2
+Requires-Dist: greenlet==3.2.3
 Requires-Dist: grpc-google-iam-v1==0.14.2
 Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3
@@ -113,6 +116,7 @@ Requires-Dist: openpyxl==3.1.5
 Requires-Dist: orjson==3.10.15
 Requires-Dist: packaging==24.2
 Requires-Dist: pandas==2.2.3
+Requires-Dist: paramiko==3.5.1
 Requires-Dist: pathvalidate==3.2.3
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: platformdirs==4.3.6
@@ -137,6 +141,7 @@ Requires-Dist: pygments==2.19.1
 Requires-Dist: pyjwt==2.10.1
 Requires-Dist: pymongo==4.11.1
 Requires-Dist: pymysql==1.1.1
+Requires-Dist: pynacl==1.5.0
 Requires-Dist: pyopenssl==25.0.0
 Requires-Dist: pyparsing==3.2.1
 Requires-Dist: pyrate-limiter==3.7.0
@@ -170,6 +175,7 @@ Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sortedcontainers==2.4.0
 Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
+Requires-Dist: sqlalchemy-cratedb==0.42.0.dev2
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy-spanner==1.11.0
@@ -192,6 +198,7 @@ Requires-Dist: tzdata==2025.1
 Requires-Dist: tzlocal==5.3
 Requires-Dist: uritemplate==4.1.1
 Requires-Dist: urllib3==2.3.0
+Requires-Dist: verlib2==0.3.1
 Requires-Dist: wrapt==1.17.2
 Requires-Dist: yarl==1.18.3
 Requires-Dist: zeep==4.3.1
@@ -292,11 +299,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>CrateDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>IBM Db2</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>DuckDB</td>
         <td>✅</td>
@@ -307,6 +324,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Elasticsearch</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+     <tr>
+        <td>GCP Spanner</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -393,6 +420,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Attio</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Chess.com</td>
         <td>✅</td>
@@ -404,7 +436,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Github</td>
+        <td>GitHub</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -447,6 +479,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Personio</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Phantombuster</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Pipedrive</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>
@@ -469,7 +511,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Smartsheet</td>
+        <td>Smartsheets</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Solidgate</td>
         <td>✅</td>
         <td>-</td>
     </tr>

{ingestr-0.13.52 → ingestr-0.13.53}/README.md

@@ -87,11 +87,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>CrateDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
     <tr>
         <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>IBM Db2</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>DuckDB</td>
         <td>✅</td>
@@ -102,6 +112,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Elasticsearch</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+     <tr>
+        <td>GCP Spanner</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -188,6 +208,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Attio</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Chess.com</td>
         <td>✅</td>
@@ -199,7 +224,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Github</td>
+        <td>GitHub</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -242,6 +267,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Personio</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Phantombuster</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Pipedrive</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>
@@ -264,7 +299,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Smartsheet</td>
+        <td>Smartsheets</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Solidgate</td>
         <td>✅</td>
         <td>-</td>
     </tr>

{ingestr-0.13.52 → ingestr-0.13.53}/docs/.vitepress/config.mjs

@@ -136,8 +136,9 @@ export default defineConfig({
               { text: "Personio", link: "/supported-sources/personio.md" },
               { text: "PhantomBuster", link: "/supported-sources/phantombuster.md" },
               { text: "Pipedrive", link: "/supported-sources/pipedrive.md" },
-              { text: "S3", link: "/supported-sources/s3.md" },
+              { text: "S3", link: "/supported-sources/s3.md"},
               { text: "Salesforce", link: "/supported-sources/salesforce.md" },
+              { text: "SFTP", link: "/supported-sources/sftp.md"},
               { text: "Shopify", link: "/supported-sources/shopify.md" },
               { text: "Slack", link: "/supported-sources/slack.md" },
               { text: "Smartsheet", link: "/supported-sources/smartsheets.md" },

ingestr-0.13.53/docs/supported-sources/cratedb.md

@@ -0,0 +1,80 @@
+# CrateDB
+
+[CrateDB] is a distributed and scalable SQL database for storing and analyzing
+massive amounts of data in near real-time, even with complex queries. It is
+PostgreSQL-compatible, and based on Lucene.
+
+ingestr supports CrateDB as a source database.
+
+## Source
+
+For connecting to CrateDB as a database source, ingestr uses its SQLAlchemy
+dialect package [sqlalchemy-cratedb].
+
+### URI format
+
+The URI format for CrateDB as a source is as follows:
+```plaintext
+crate://<username>:<password>@<host>:<port>?ssl=<ssl>
+```
+> [!INFO]
+> The driver does not require any option, and will default to
+> `crate://crate@localhost:4200?ssl=false` when just using `crate://`.
+>
+> When connecting to CrateDB on localhost, use:
+> ```plaintext
+> crate://crate@localhost:4200?ssl=false
+> ```
+>
+> When connecting to [CrateDB Cloud], the URI looks like this:
+> ```plaintext
+> crate://admin:<PASSWORD>@<CLUSTERNAME>.eks1.eu-west-1.aws.cratedb.net:4200?ssl=true
+> ```
+
+### URI parameters
+- `username` (required): The username is required to authenticate with the CrateDB server.
+- `password` (required): The password is required to authenticate the provided username.
+- `host` (required): The hostname or IP address of the CrateDB server where the database is hosted.
+- `port` (required): The TCP port number used by the CrateDB server. Mostly `4200`.
+- `ssl` (optional): Set to `true` for a secure HTTPS connection. By default, SSL is disabled (`false`).
+
+### Example
+
+This is an example command that will copy the `sys.summits` table from CrateDB
+to DuckDB, then display the content from DuckDB.
+
+```shell
+ingestr ingest \
+    --source-uri 'crate://crate@localhost:4200/' \
+    --source-table 'sys.summits' \
+    --dest-uri 'duckdb:///cratedb.duckdb' \
+    --dest-table 'dest.summits'
+```
+```shell
+duckdb cratedb.duckdb 'SELECT * FROM dest.summits LIMIT 5'
+```
+
+<img alt="CrateDB_img" src="../media/cratedb-source.png" />
+
+
+## Appendix
+
+To start a single-node instance of CrateDB for evaluation purposes,
+use Docker or Podman.
+```shell
+docker run --rm -it --name=cratedb \
+  --publish=4200:4200 --publish=5432:5432 \
+  --env=CRATE_HEAP_SIZE=2g crate/crate:nightly \
+  -Cdiscovery.type=single-node
+```
+
+We are tracking development progress and incompatibilities at 
+[Support for ingestr/CrateDB] and ["tool: dlt/ingestr"]. Please join the discussion
+or share relevant issue reports that help us improve interoperability. Thanks!
+
+
+[CrateDB]: https://github.com/crate/crate
+[CrateDB Cloud]: https://console.cratedb.cloud/
+[sqlalchemy-cratedb]: https://pypi.org/project/sqlalchemy-cratedb/
+[Support for ingestr/CrateDB]: https://github.com/crate/crate-clients-tools/issues/86
+["tool: dlt/ingestr"]: https://github.com/crate/crate/issues?q=state%3Aopen%20label%3A%22tool%3A%20dlt%2Fingestr%22

{ingestr-0.13.52 → ingestr-0.13.53}/docs/supported-sources/mysql.md

@@ -1,13 +1,13 @@
 # MySQL
 MySQL is an open source relational database management system, known for its speed and reliability.
 
-ingestr supports MySQL as a source.
+ingestr supports MySQL as a source and a destination.
 
 ## URI format
 The URI format for MySQL is as follows:
 
 ```plaintext
-mysql://user:password@host:port/dbname 
+mysql://user:password@host:port/dbname
 ```
 
 URI parameters:
@@ -17,4 +17,4 @@ URI parameters:
 - `port`: the port number the database server is listening on, the default is 3306
 - `dbname`: the name of the database to connect to
 
-The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's MySQL dialect [here](https://docs.sqlalchemy.org/en/20/core/engines.html#mysql).
+The same URI structure and table can be used both for sources and destinations. You can read more about SQLAlchemy's MySQL dialect [here](https://docs.sqlalchemy.org/en/20/core/engines.html#mysql).

ingestr-0.13.53/docs/supported-sources/sftp.md

@@ -0,0 +1,43 @@
+# SFTP
+
+SFTP (SSH File Transfer Protocol) is a secure file transfer protocol that runs over the SSH protocol. It provides a secure way to transfer files between a local and a remote computer.
+
+`ingestr` supports SFTP as a data source.
+
+## URI Format
+
+The URI for connecting to an SFTP server is structured as follows:
+
+```plaintext
+sftp://<username>:<password>@<host>:<port>
+```
+
+## URI Components:
+- `username`: The username for the SFTP server.
+- `password`: The password for the SFTP server.
+- `host`: The hostname or IP address of the SFTP server.
+- `port`: The port number of the SFTP server (defaults to 22 if not specified).
+
+## Setting up an SFTP Integration
+
+To integrate `ingestr` with an SFTP server, you need the server's hostname, port, a valid username, and a password.
+
+Once you have your credentials, you can load data to desired destaination.
+
+### Example: Loading data from SFTP TO DUCKDB
+
+```sh
+ingestr ingest \
+    --source-uri 'sftp://myuser:MySecretPassword123@sftp.example.com' \
+    --source-table 'user.csv' \
+    --dest-uri duckdb:///sftp_data.duckdb \
+    --dest-table 'dest.users_deatils'
+```
+
+<img alt="sftp" src="../media/sftp.png"/>
+
+
+The `--source-table` specifies `/path/to/directory` The base directory on the server where `ingestr` should start looking for files.
+
+
+

ingestr-0.13.53/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.53"

{ingestr-0.13.52 → ingestr-0.13.53}/ingestr/src/destinations.py

@@ -480,3 +480,21 @@ class SqliteDestination(GenericSqlDestination):
             "dataset_name": "main",
             "table_name": table,
         }
+
+
+class MySqlDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        database = parsed.path.lstrip("/")
+        if not database:
+            raise ValueError("You need to specify a database")
+        return {
+            "dataset_name": database,
+            "table_name": table,
+        }
+
+
+

{ingestr-0.13.52 → ingestr-0.13.53}/ingestr/src/factory.py

@@ -11,6 +11,7 @@ from ingestr.src.destinations import (
     DatabricksDestination,
     DuckDBDestination,
     MsSQLDestination,
+    MySqlDestination,
     PostgresDestination,
     RedshiftDestination,
     S3Destination,
@@ -53,6 +54,7 @@ from ingestr.src.sources import (
     PipedriveSource,
     S3Source,
     SalesforceSource,
+    SFTPSource,
     ShopifySource,
     SlackSource,
     SmartsheetSource,
@@ -65,6 +67,7 @@ from ingestr.src.sources import (
 
 SQL_SOURCE_SCHEMES = [
     "bigquery",
+    "crate",
     "duckdb",
     "mssql",
     "mysql",
@@ -164,6 +167,7 @@ class SourceDestinationFactory:
         "attio": AttioSource,
         "solidgate": SolidgateSource,
         "smartsheet": SmartsheetSource,
+        "sftp": SFTPSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,
@@ -184,6 +188,8 @@ class SourceDestinationFactory:
         "clickhouse": ClickhouseDestination,
         "s3": S3Destination,
         "sqlite": SqliteDestination,
+        "mysql": MySqlDestination,
+        "mysql+pymysql": MySqlDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

{ingestr-0.13.52 → ingestr-0.13.53}/ingestr/src/sources.py

@@ -18,6 +18,7 @@ from typing import (
 )
 from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
 
+import fsspec  # type: ignore
 import pendulum
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.extract import Incremental
@@ -78,31 +79,7 @@ class SqlSource:
         # clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
         if uri.startswith("clickhouse://"):
             parsed_uri = urlparse(uri)
-
-            username = parsed_uri.username
-            if not username:
-                raise ValueError(
-                    "A username is required to connect to the ClickHouse database."
-                )
-
-            password = parsed_uri.password
-            if not password:
-                raise ValueError(
-                    "A password is required to authenticate with the ClickHouse database."
-                )
-
-            host = parsed_uri.hostname
-            if not host:
-                raise ValueError(
-                    "The hostname or IP address of the ClickHouse server is required to establish a connection."
-                )
-
-            port = parsed_uri.port
-            if not port:
-                raise ValueError(
-                    "The TCP port of the ClickHouse server is required to establish a connection."
-                )
-
+            
             query_params = parse_qs(parsed_uri.query)
 
             if "http_port" in query_params:
@@ -2506,3 +2483,56 @@ class SolidgateSource:
             ).with_resources(table_name)
         except ResourcesNotFoundError:
             raise UnsupportedResourceError(table_name, "Solidgate")
+
+
+class SFTPSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        host = parsed_uri.hostname
+        if not host:
+            raise MissingValueError("host", "SFTP URI")
+        port = parsed_uri.port or 22
+        username = parsed_uri.username
+        password = parsed_uri.password
+
+        params: Dict[str, Any] = {
+            "host": host,
+            "port": port,
+            "username": username,
+            "password": password,
+            "look_for_keys": False,
+            "allow_agent": False,
+        }
+
+        try:
+            fs = fsspec.filesystem("sftp", **params)
+        except Exception as e:
+            raise ConnectionError(
+                f"Failed to connect or authenticate to sftp server {host}:{port}. Error: {e}"
+            )
+        bucket_url = f"sftp://{host}:{port}"
+
+        if table.startswith("/"):
+            file_glob = table
+        else:
+            file_glob = f"/{table}"
+
+        file_extension = table.split(".")[-1].lower()
+        endpoint: str
+        if file_extension == "csv":
+            endpoint = "read_csv"
+        elif file_extension == "jsonl":
+            endpoint = "read_jsonl"
+        elif file_extension == "parquet":
+            endpoint = "read_parquet"
+        else:
+            raise ValueError(
+                "FTPServer Source only supports specific file formats: csv, jsonl, parquet."
+            )
+        from ingestr.src.filesystem import readers
+
+        dlt_source_resource = readers(bucket_url, fs, file_glob)
+        return dlt_source_resource.with_resources(endpoint)

{ingestr-0.13.52 → ingestr-0.13.53}/requirements.in

@@ -25,6 +25,7 @@ rich==13.9.4
 rudder-sdk-python==2.1.4
 snowflake-sqlalchemy==1.6.1
 sqlalchemy-bigquery==1.12.1
+sqlalchemy-cratedb==0.42.0.dev2
 sqlalchemy-hana==2.0.0
 sqlalchemy-redshift==0.8.14
 SQLAlchemy==1.4.52
@@ -53,3 +54,4 @@ elasticsearch==8.10.1
 sqlalchemy-spanner==1.11.0
 google-cloud-spanner==3.54.0
 smartsheet-python-sdk==3.0.5
+paramiko==3.5.1