ingestr 0.2.6__tar.gz → 0.3.0__tar.gz

{ingestr-0.2.6 → ingestr-0.3.0}/Makefile

@@ -22,7 +22,7 @@ test-specific: venv
 
 lint-ci:
 	ruff check ingestr --fix && ruff format ingestr
-	mypy  --explicit-package-bases ingestr --config-file pyproject.toml
+	mypy --config-file pyproject.toml --explicit-package-bases ingestr
 
 lint: venv
 	. venv/bin/activate; $(MAKE) lint-ci

{ingestr-0.2.6 → ingestr-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.2.6
+Version: 0.3.0
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,26 +16,26 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==0.4.6
-Requires-Dist: duckdb-engine==0.11.2
-Requires-Dist: duckdb==0.10.1
+Requires-Dist: dlt==0.4.8
+Requires-Dist: duckdb-engine==0.11.5
+Requires-Dist: duckdb==0.10.2
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.9
 Requires-Dist: py-machineid==0.5.1
-Requires-Dist: pymongo==4.6.2
+Requires-Dist: pymongo==4.6.3
 Requires-Dist: pymysql==1.1.0
 Requires-Dist: pyodbc==5.1.0
 Requires-Dist: redshift-connector==2.1.0
-Requires-Dist: rich==13.7.0
+Requires-Dist: rich==13.7.1
 Requires-Dist: rudder-sdk-python==2.1.0
-Requires-Dist: snowflake-sqlalchemy==1.5.1
-Requires-Dist: sqlalchemy-bigquery==1.10.0
+Requires-Dist: snowflake-sqlalchemy==1.5.3
+Requires-Dist: sqlalchemy-bigquery==1.11.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: tqdm==4.66.2
-Requires-Dist: typer==0.9.0
+Requires-Dist: typer==0.12.3
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -91,20 +91,84 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
 
 ## Supported Sources & Destinations
 
-| Database             | Source | Destination |
-|----------------------|--------|-------------|
-| Postgres             | ✅      | ✅         |
-| BigQuery             | ✅      | ✅         |
-| Snowflake            | ✅      | ✅         |
-| Redshift             | ✅      | ✅         |
-| Databricks           | ✅      | ✅         |
-| DuckDB               | ✅      | ✅         |
-| Microsoft SQL Server | ✅      | ✅         |
-| Local CSV file       | ✅      | ✅         |
-| MongoDB              | ✅      | ❌         |
-| Oracle               | ✅      | ❌         |
-| SQLite               | ✅      | ❌         |
-| MySQL                | ✅      | ❌         |
+<table>
+    <tr>
+        <th></th>
+        <th>Source</th>
+        <th>Destination</th>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
+    </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>BigQuery</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Snowflake</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Databricks</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>DuckDB</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Microsoft SQL Server</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Local CSV file</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>MongoDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>Oracle</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>SQLite</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>MySQL</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
+    </tr>
+    <tr>
+        <td>Notion</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+</table>
 
 More to come soon!
 

{ingestr-0.2.6 → ingestr-0.3.0}/README.md

@@ -51,20 +51,84 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
 
 ## Supported Sources & Destinations
 
-| Database             | Source | Destination |
-|----------------------|--------|-------------|
-| Postgres             | ✅      | ✅         |
-| BigQuery             | ✅      | ✅         |
-| Snowflake            | ✅      | ✅         |
-| Redshift             | ✅      | ✅         |
-| Databricks           | ✅      | ✅         |
-| DuckDB               | ✅      | ✅         |
-| Microsoft SQL Server | ✅      | ✅         |
-| Local CSV file       | ✅      | ✅         |
-| MongoDB              | ✅      | ❌         |
-| Oracle               | ✅      | ❌         |
-| SQLite               | ✅      | ❌         |
-| MySQL                | ✅      | ❌         |
+<table>
+    <tr>
+        <th></th>
+        <th>Source</th>
+        <th>Destination</th>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
+    </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>BigQuery</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Snowflake</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Databricks</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>DuckDB</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Microsoft SQL Server</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Local CSV file</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>MongoDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>Oracle</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>SQLite</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>MySQL</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
+    </tr>
+    <tr>
+        <td>Notion</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+</table>
 
 More to come soon!
 

{ingestr-0.2.6 → ingestr-0.3.0}/docs/.vitepress/config.mjs

@@ -4,20 +4,17 @@ import { defineConfig } from "vitepress";
 export default defineConfig({
   title: "ingestr",
   description: "Ingest & copy data between any source and any destination",
-  base: '/ingestr/',
+  base: "/ingestr/",
   head: [
+    ["script", { async: "", src: "https://www.googletagmanager.com/gtag/js?id=G-MZJ20PP4MJ" }],
     [
-      'script',
-      { async: '', src: 'https://www.googletagmanager.com/gtag/js?id=G-MZJ20PP4MJ' }
-    ],
-    [
-      'script',
+      "script",
       {},
       `window.dataLayer = window.dataLayer || [];
       function gtag(){dataLayer.push(arguments);}
       gtag('js', new Date());
-      gtag('config', 'G-MZJ20PP4MJ');`
-    ]
+      gtag('config', 'G-MZJ20PP4MJ');`,
+    ],
   ],
   themeConfig: {
     // https://vitepress.dev/reference/default-theme-config
@@ -46,19 +43,31 @@ export default defineConfig({
       {
         text: "Sources & Destinations",
         items: [
-          { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
-          { text: "Databricks", link: "/supported-sources/databricks.md" },
-          { text: "DuckDB", link: "/supported-sources/duckdb.md" },
-          { text: "Google BigQuery", link: "/supported-sources/bigquery.md" },
-          { text: "Local CSV Files", link: "/supported-sources/csv.md" },
-          { text: "Microsoft SQL Server", link: "/supported-sources/mssql.md" },
-          { text: "MongoDB", link: "/supported-sources/mongodb.md" },
-          { text: "MySQL", link: "/supported-sources/mysql.md" },
-          { text: "Oracle", link: "/supported-sources/oracle.md" },
           { text: "Overview", link: "/supported-sources/overview.md" },
-          { text: "Postgres", link: "/supported-sources/postgres.md" },
-          { text: "Snowflake", link: "/supported-sources/snowflake.md" },
-          { text: "SQLite", link: "/supported-sources/sqlite.md" },
+          {
+            text: "Databases",
+            collapsed: false,
+            items: [
+              { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
+              { text: "Databricks", link: "/supported-sources/databricks.md" },
+              { text: "DuckDB", link: "/supported-sources/duckdb.md" },
+              { text: "Google BigQuery", link: "/supported-sources/bigquery.md" },
+              { text: "Local CSV Files", link: "/supported-sources/csv.md" },
+              { text: "Microsoft SQL Server", link: "/supported-sources/mssql.md" },
+              { text: "MongoDB", link: "/supported-sources/mongodb.md" },
+              { text: "MySQL", link: "/supported-sources/mysql.md" },
+              { text: "Oracle", link: "/supported-sources/oracle.md" },
+              { text: "Postgres", link: "/supported-sources/postgres.md" },
+              { text: "Snowflake", link: "/supported-sources/snowflake.md" },
+              { text: "SQLite", link: "/supported-sources/sqlite.md" },
+            ],
+          },
+
+          {
+            text: "Platforms",
+            collapsed: false,
+            items: [{ text: "Notion", link: "/supported-sources/notion.md" }],
+          },
         ],
       },
     ],

ingestr-0.3.0/docs/supported-sources/notion.md

@@ -0,0 +1,49 @@
+# Notion
+[Notion](https://www.notion.so/) is an all-in-one workspace for note-taking, project management, and database management.
+
+ingestr supports Notion as a source.
+
+## URI Format
+The URI format for Notion is as follows:
+
+```plaintext
+notion://?api_key=token
+```
+
+URI parameters:
+- `api_key`: the integration token used for authentication with the Notion API
+
+The URI is used to connect to the Notion API for extracting data. More details on setting up Notion integrations can be found [here](https://developers.notion.com/docs/getting-started).
+
+## Setting up a Notion Integration
+
+Notion requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/notion#setup-guide).
+
+Once you complete the guide, you should have an API key, and the table ID to connect to. Let's say your API token is `secret_12345` and the database you'd like to connect to is `bfeaafc0c25f40a9asdasd672a9456f3`, here's a sample command that will copy the data from the Notion table into a duckdb database:
+
+```sh
+ingestr ingest --source-uri 'notion://?api_key=secret_12345' --source-table 'bfeaafc0c25f40a9asdasd672a9456f3' --dest-uri duckdb:///notion.duckdb --dest-table 'notion.output'
+```
+
+The result of this command will be a bunch of tables in the `notion.duckdb` database. The Notion integration creates a bunch of extra tables in the schema to keep track of additional information about every field in a database. You should take some time to play around with the data and understand how it's structured, and take a good look at `_dlt_parent_id` column in the tables to understand the relationships between tables.
+
+Take a look at the following Notion table:
+![an example Notion database](./images/notion_example.png)
+
+Ingesting this table using ingestr will create a bunch of new tables with quite a lot of details in them. The following query is a reconstruction of the table as it looks on Notion:
+
+```sql
+select n.text__content, s.text__content, o.properties__numerical_value__number, r.text__content
+from notion.output o
+         join notion.output__properties__name__title n on n._dlt_parent_id = o._dlt_id
+         join notion.output__properties__another_col__rich_text r on r._dlt_parent_id = o._dlt_id
+         join notion.output__properties__second_value__rich_text s on s._dlt_parent_id = o._dlt_id
+order by 1;
+```
+
+Take this as a starting point and play around with the data.
+
+
+> [!CAUTION]
+> Notion does not support incremental loading, which means every time you run the command, it will copy the entire table from Notion to the destination. This can be slow for large tables.
+

ingestr-0.3.0/docs/supported-sources/overview.md

@@ -0,0 +1,84 @@
+# Supported Sources & Destinations
+ingestr supports the following sources and destinations:
+
+
+<table>
+    <tr>
+        <th></th>
+        <th>Source</th>
+        <th>Destination</th>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
+    </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>BigQuery</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Snowflake</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Databricks</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>DuckDB</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Microsoft SQL Server</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>Local CSV file</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>MongoDB</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>Oracle</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>SQLite</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td>MySQL</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+    <tr>
+        <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
+    </tr>
+    <tr>
+        <td>Notion</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
+</table>
+
+More to come soon!

{ingestr-0.2.6 → ingestr-0.3.0}/docs/supported-sources/snowflake.md

@@ -7,7 +7,7 @@ ingestr supports Snowflake as both a source and destination.
 The URI format for Snowflake is as follows:
 
 ```plaintext
-snowflake://user:password@account/dbname?warehouse=COMPUTE_WH
+snowflake://user:password@account/dbname?warehouse=COMPUTE_WH&role=data_scientist
 ```
 
 URI parameters:
@@ -15,6 +15,7 @@ URI parameters:
 - `password`: the password for the user
 - `account`: your Snowflake account identifier
 - `dbname`: the name of the database to connect to
-- `warehouse`: the name of the warehouse to use
+- `warehouse`: the name of the warehouse to use (optional)
+- `role`: the name of the role to use (optional)
 
 The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's Snowflake dialect [here](https://docs.snowflake.com/en/developer-guide/python-connector/sqlalchemy#connection-parameters).

{ingestr-0.2.6 → ingestr-0.3.0}/ingestr/main.py

@@ -5,7 +5,7 @@ from typing import Optional
 import dlt
 import humanize
 import typer
-from dlt.common.runtime.collector import Collector
+from dlt.common.runtime.collector import Collector, LogCollector
 from rich.console import Console
 from rich.status import Status
 from typing_extensions import Annotated
@@ -34,8 +34,6 @@ DATE_FORMATS = [
 
 
 class SpinnerCollector(Collector):
-    """A Collector that shows progress with `tqdm` progress bars"""
-
     status: Status
     current_step: str
     started: bool
@@ -150,6 +148,13 @@ def ingest(
             envvar="FULL_REFRESH",
         ),
     ] = False,  # type: ignore
+    progress: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The progress display type, must be one of 'interactive', 'log'",
+            envvar="PROGRESS",
+        ),
+    ] = "interactive",  # type: ignore
 ):
     track(
         "command_triggered",
@@ -186,12 +191,16 @@ def ingest(
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
 
+        progressInstance: Collector = SpinnerCollector()
+        if progress == "log":
+            progressInstance = LogCollector()
+
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),
             destination=destination.dlt_dest(
                 uri=dest_uri,
             ),
-            progress=SpinnerCollector(),
+            progress=progressInstance,
             pipelines_dir="pipeline_data",
             full_refresh=full_refresh,
         )

{ingestr-0.2.6 → ingestr-0.3.0}/ingestr/main_test.py

@@ -93,6 +93,7 @@ def test_create_replace():
         "testschema.output",
     )
 
+    print(result.stdout)
     assert result.exit_code == 0
 
     res = conn.sql(
@@ -333,10 +334,10 @@ def test_delete_insert_without_primary_key():
         "CREATE TABLE testschema_delete_insert.input (id INTEGER, val VARCHAR, updated_at TIMESTAMP WITH TIME ZONE)"
     )
     conn.execute(
-        "INSERT INTO testschema_delete_insert.input VALUES (1, 'val1', '2022-01-01')"
+        "INSERT INTO testschema_delete_insert.input VALUES (1, 'val1', '2022-01-01 00:00:00+00:00')"
     )
     conn.execute(
-        "INSERT INTO testschema_delete_insert.input VALUES (2, 'val2', '2022-02-01')"
+        "INSERT INTO testschema_delete_insert.input VALUES (2, 'val2', '2022-02-01 00:00:00+00:00')"
     )
 
     res = conn.sql("select count(*) from testschema_delete_insert.input").fetchall()
@@ -357,7 +358,7 @@ def test_delete_insert_without_primary_key():
     def get_output_rows():
         conn.execute("CHECKPOINT")
         return conn.sql(
-            "select id, val, strftime(updated_at, '%Y-%m-%d') as updated_at from testschema_delete_insert.output order by id asc"
+            "select id, val, strftime(CAST(updated_at AT TIME ZONE 'UTC' AS TIMESTAMP), '%Y-%m-%d %H:%M:%S') from testschema_delete_insert.output order by id asc"
         ).fetchall()
 
     def assert_output_equals(expected):
@@ -367,7 +368,9 @@ def test_delete_insert_without_primary_key():
             assert res[i] == row
 
     run()
-    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-02-01")])
+    assert_output_equals(
+        [(1, "val1", "2022-01-01 00:00:00"), (2, "val2", "2022-02-01 00:00:00")]
+    )
 
     first_run_id = conn.sql(
         "select _dlt_load_id from testschema_delete_insert.output limit 1"
@@ -375,8 +378,10 @@ def test_delete_insert_without_primary_key():
 
     ##############################
     # we'll run again, since this is a delete+insert, we expect the run ID to change for the last one
-    run()
-    assert_output_equals([(1, "val1", "2022-01-01"), (2, "val2", "2022-02-01")])
+    res = run()
+    assert_output_equals(
+        [(1, "val1", "2022-01-01 00:00:00"), (2, "val2", "2022-02-01 00:00:00")]
+    )
 
     # we ensure that one of the rows is updated with a new run
     count_by_run_id = conn.sql(
@@ -392,17 +397,17 @@ def test_delete_insert_without_primary_key():
     ##############################
     # now we'll insert a few more lines for the same day, the new rows should show up
     conn.execute(
-        "INSERT INTO testschema_delete_insert.input VALUES (3, 'val3', '2022-02-01'), (4, 'val4', '2022-02-01')"
+        "INSERT INTO testschema_delete_insert.input VALUES (3, 'val3', '2022-02-01 00:00:00+00:00'), (4, 'val4', '2022-02-01 00:00:00+00:00')"
     )
     conn.execute("CHECKPOINT")
 
     run()
     assert_output_equals(
         [
-            (1, "val1", "2022-01-01"),
-            (2, "val2", "2022-02-01"),
-            (3, "val3", "2022-02-01"),
-            (4, "val4", "2022-02-01"),
+            (1, "val1", "2022-01-01 00:00:00"),
+            (2, "val2", "2022-02-01 00:00:00"),
+            (3, "val3", "2022-02-01 00:00:00"),
+            (4, "val4", "2022-02-01 00:00:00"),
         ]
     )
 

{ingestr-0.2.6 → ingestr-0.3.0}/ingestr/src/destinations.py

@@ -177,7 +177,8 @@ class CsvDestination(GenericSqlDestination):
         )
 
         output_path = self.uri.split("://")[1]
-        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        if output_path.count("/") > 1:
+            os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
         with gzip.open(first_file_path, "rt", encoding="utf-8") as jsonl_file:  # type: ignore
             with open(output_path, "w", newline="") as csv_file:

{ingestr-0.2.6 → ingestr-0.3.0}/ingestr/src/factory.py

@@ -14,7 +14,7 @@ from ingestr.src.destinations import (
     SnowflakeDestination,
     SynapseDestination,
 )
-from ingestr.src.sources import LocalCsvSource, MongoDbSource, SqlSource
+from ingestr.src.sources import LocalCsvSource, MongoDbSource, NotionSource, SqlSource
 
 SQL_SOURCE_SCHEMES = [
     "bigquery",
@@ -80,6 +80,8 @@ class SourceDestinationFactory:
             return LocalCsvSource()
         elif self.source_scheme == "mongodb":
             return MongoDbSource()
+        elif self.source_scheme == "notion":
+            return NotionSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")
 

{ingestr-0.2.6 → ingestr-0.3.0}/ingestr/src/mongodb/__init__.py

@@ -70,7 +70,7 @@ def mongodb_collection(
     collection: str = dlt.config.value,
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
-    parallel: Optional[bool] = dlt.config.value,
+    parallel: Optional[bool] = False,
 ) -> Any:
     """
     A DLT source which loads a collection from a mongo database using PyMongo.

{ingestr-0.2.6 → ingestr-0.3.0}/ingestr/src/mongodb/helpers.py

@@ -83,7 +83,7 @@ class CollectionLoaderParallell(CollectionLoader):
     def _get_cursor(self) -> TCursor:
         cursor = self.collection.find(filter=self._filter_op)
         if self._sort_op:
-            cursor = cursor.sort(self._sort_op)  # type: ignore
+            cursor = cursor.sort(self._sort_op)
         return cursor
 
     @dlt.defer
@@ -155,11 +155,11 @@ class MongoDbCollectionConfiguration(BaseConfiguration):
 
 @configspec
 class MongoDbCollectionResourceConfiguration(BaseConfiguration):
-    connection_url: str
-    database: Optional[str]
-    collection: str
+    connection_url: str = dlt.secrets.value
+    database: Optional[str] = dlt.config.value
+    collection: str = dlt.config.value
     incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]
-    write_disposition: Optional[str] = None
+    write_disposition: Optional[str] = dlt.config.value
     parallel: Optional[bool] = False