ingestr 0.1.4__tar.gz → 0.2.1__tar.gz

{ingestr-0.1.4 → ingestr-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestr
-Version: 0.1.4
+Version: 0.2.1
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -23,6 +23,7 @@ Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.9
 Requires-Dist: py-machineid==0.5.1
+Requires-Dist: pymongo==4.6.2
 Requires-Dist: pymysql==1.1.0
 Requires-Dist: pyodbc==5.1.0
 Requires-Dist: redshift-connector==2.1.0
@@ -100,6 +101,7 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
 | DuckDB               | ✅      | ✅         |
 | Microsoft SQL Server | ✅      | ✅         |
 | Local CSV file       | ✅      | ✅         |
+| MongoDB              | ✅      | ❌         |
 | Oracle               | ✅      | ❌         |
 | SQLite               | ✅      | ❌         |
 | MySQL                | ✅      | ❌         |

{ingestr-0.1.4 → ingestr-0.2.1}/README.md

@@ -61,6 +61,7 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
 | DuckDB               | ✅      | ✅         |
 | Microsoft SQL Server | ✅      | ✅         |
 | Local CSV file       | ✅      | ✅         |
+| MongoDB              | ✅      | ❌         |
 | Oracle               | ✅      | ❌         |
 | SQLite               | ✅      | ❌         |
 | MySQL                | ✅      | ❌         |

{ingestr-0.1.4 → ingestr-0.2.1}/docs/.vitepress/config.mjs

@@ -46,15 +46,16 @@ export default defineConfig({
       {
         text: "Sources & Destinations",
         items: [
-          { text: "Overview", link: "/supported-sources/overview.md" },
           { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
           { text: "Databricks", link: "/supported-sources/databricks.md" },
           { text: "DuckDB", link: "/supported-sources/duckdb.md" },
           { text: "Google BigQuery", link: "/supported-sources/bigquery.md" },
           { text: "Local CSV Files", link: "/supported-sources/csv.md" },
           { text: "Microsoft SQL Server", link: "/supported-sources/mssql.md" },
+          { text: "MongoDB", link: "/supported-sources/mongodb.md" },
           { text: "MySQL", link: "/supported-sources/mysql.md" },
           { text: "Oracle", link: "/supported-sources/oracle.md" },
+          { text: "Overview", link: "/supported-sources/overview.md" },
           { text: "Postgres", link: "/supported-sources/postgres.md" },
           { text: "Snowflake", link: "/supported-sources/snowflake.md" },
           { text: "SQLite", link: "/supported-sources/sqlite.md" },

ingestr-0.2.1/docs/supported-sources/mongodb.md

@@ -0,0 +1,24 @@
+# MongoDB
+MongoDB is a popular, open-source NoSQL database known for its flexibility, scalability, and wide adoption in a variety of applications.
+
+ingestr supports MongoDB as a source.
+
+## URI Format
+The URI format for MongoDB is as follows:
+
+```plaintext
+mongodb://user:password@host:port
+```
+
+URI parameters:
+- `user`: the user name to connect to the database
+- `password`: the password for the user
+- `host`: the host address of the database server
+- `port`: the port number the database server is listening on (default is 27017 for MongoDB)
+
+
+> [!CAUTION]
+> Do not put the database name at the end of the URI for MongoDB, instead make it a part of `--source-table` option as `database.collection` format.
+
+
+You can read more about MongoDB's connection string format [here](https://docs.mongodb.com/manual/reference/connection-string/).

{ingestr-0.1.4 → ingestr-0.2.1}/docs/supported-sources/overview.md

@@ -11,6 +11,7 @@ ingestr supports the following sources and destinations:
 | DuckDB               | ✅      | ✅         |
 | Microsoft SQL Server | ✅      | ✅         |
 | Local CSV file       | ✅      | ✅         |
+| MongoDB              | ✅      | ❌         |
 | Oracle               | ✅      | ❌         |
 | SQLite               | ✅      | ❌         |
 | MySQL                | ✅      | ❌         |

{ingestr-0.1.4 → ingestr-0.2.1}/ingestr/src/factory.py

@@ -13,7 +13,7 @@ from ingestr.src.destinations import (
     RedshiftDestination,
     SnowflakeDestination,
 )
-from ingestr.src.sources import LocalCsvSource, SqlSource
+from ingestr.src.sources import LocalCsvSource, MongoDbSource, SqlSource
 
 SQL_SOURCE_SCHEMES = [
     "bigquery",
@@ -77,6 +77,8 @@ class SourceDestinationFactory:
             return SqlSource()
         elif self.source_scheme == "csv":
             return LocalCsvSource()
+        elif self.source_scheme == "mongodb":
+            return MongoDbSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")
 

ingestr-0.2.1/ingestr/src/factory_test.py

@@ -0,0 +1,13 @@
+from ingestr.src.factory import parse_scheme_from_uri
+
+
+def test_scheme_is_parsed_from_uri_correctly():
+    assert parse_scheme_from_uri("bigquery://my-project") == "bigquery"
+    assert parse_scheme_from_uri("http://localhost:8080") == "http"
+    assert parse_scheme_from_uri("file:///tmp/myfile") == "file"
+    assert parse_scheme_from_uri("https://example.com?query=123") == "https"
+    assert parse_scheme_from_uri("ftp://ftp.example.com/downloads/file.zip") == "ftp"
+    assert (
+        parse_scheme_from_uri("redshift+psycopg2://user:pw@host") == "redshift+psycopg2"
+    )
+    assert parse_scheme_from_uri("mysql+pymysql://user:pw@host") == "mysql+pymysql"

ingestr-0.2.1/ingestr/src/mongodb/__init__.py

@@ -0,0 +1,103 @@
+"""Source that loads collections form any a mongo database, supports incremental loads."""
+
+from typing import Any, Iterable, List, Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import (
+    MongoDbCollectionConfiguration,
+    MongoDbCollectionResourceConfiguration,
+    client_from_credentials,
+    collection_documents,
+)
+
+
+@dlt.source
+def mongodb(
+    connection_url: str = dlt.secrets.value,
+    database: Optional[str] = dlt.config.value,
+    collection_names: Optional[List[str]] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    parallel: Optional[bool] = dlt.config.value,
+) -> Iterable[DltResource]:
+    """
+    A DLT source which loads data from a mongo database using PyMongo.
+    Resources are automatically created for each collection in the database or from the given list of collection.
+
+    Args:
+        connection_url (str): Database connection_url.
+        database (Optional[str]): Selected database name, it will use the default database if not passed.
+        collection_names (Optional[List[str]]): The list of collections `pymongo.collection.Collection` to load.
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading for the collection.
+            E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
+        write_disposition (str): Write disposition of the resource.
+        parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+    Returns:
+        Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
+    """
+
+    # set up mongo client
+    client = client_from_credentials(connection_url)
+    if not database:
+        mongo_database = client.get_default_database()
+    else:
+        mongo_database = client[database]
+
+    # use provided collection or all conllections
+    if not collection_names:
+        collection_names = mongo_database.list_collection_names()
+
+    collection_list = [mongo_database[collection] for collection in collection_names]
+
+    for collection in collection_list:
+        yield dlt.resource(  # type: ignore
+            collection_documents,
+            name=collection.name,
+            primary_key="_id",
+            write_disposition=write_disposition,
+            spec=MongoDbCollectionConfiguration,
+        )(client, collection, incremental=incremental, parallel=parallel)
+
+
+@dlt.common.configuration.with_config(
+    sections=("sources", "mongodb"), spec=MongoDbCollectionResourceConfiguration
+)
+def mongodb_collection(
+    connection_url: str = dlt.secrets.value,
+    database: Optional[str] = dlt.config.value,
+    collection: str = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    parallel: Optional[bool] = dlt.config.value,
+) -> Any:
+    """
+    A DLT source which loads a collection from a mongo database using PyMongo.
+
+    Args:
+        connection_url (str): Database connection_url.
+        database (Optional[str]): Selected database name, it will use the default database if not passed.
+        collection (str): The collection name to load.
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading for the collection.
+            E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
+        write_disposition (str): Write disposition of the resource.
+        parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+    Returns:
+        Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
+    """
+    # set up mongo client
+    client = client_from_credentials(connection_url)
+    if not database:
+        mongo_database = client.get_default_database()
+    else:
+        mongo_database = client[database]
+
+    collection_obj = mongo_database[collection]
+
+    return dlt.resource(  # type: ignore
+        collection_documents,
+        name=collection_obj.name,
+        primary_key="_id",
+        write_disposition=write_disposition,
+    )(client, collection_obj, incremental=incremental, parallel=parallel)

ingestr-0.2.1/ingestr/src/mongodb/helpers.py

@@ -0,0 +1,166 @@
+"""Mongo database source helpers"""
+
+from itertools import islice
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple
+
+import dlt
+from bson.decimal128 import Decimal128
+from bson.objectid import ObjectId
+from dlt.common.configuration.specs import BaseConfiguration, configspec
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.common.utils import map_nested_in_place
+from pendulum import _datetime
+from pymongo import ASCENDING, DESCENDING, MongoClient
+from pymongo.collection import Collection
+from pymongo.cursor import Cursor
+
+if TYPE_CHECKING:
+    TMongoClient = MongoClient[Any]
+    TCollection = Collection[Any]  # type: ignore
+    TCursor = Cursor[Any]
+else:
+    TMongoClient = Any
+    TCollection = Any
+    TCursor = Any
+
+CHUNK_SIZE = 10000
+
+
+class CollectionLoader:
+    def __init__(
+        self,
+        client: TMongoClient,
+        collection: TCollection,
+        incremental: Optional[dlt.sources.incremental[Any]] = None,
+    ) -> None:
+        self.client = client
+        self.collection = collection
+        self.incremental = incremental
+        if incremental:
+            self.cursor_field = incremental.cursor_path
+            self.last_value = incremental.last_value
+        else:
+            self.cursor_column = None
+            self.last_value = None
+
+    @property
+    def _filter_op(self) -> Dict[str, Any]:
+        if not self.incremental or not self.last_value:
+            return {}
+        if self.incremental.last_value_func is max:
+            return {self.cursor_field: {"$gte": self.last_value}}
+        elif self.incremental.last_value_func is min:
+            return {self.cursor_field: {"$lt": self.last_value}}
+        return {}
+
+    def load_documents(self) -> Iterator[TDataItem]:
+        cursor = self.collection.find(self._filter_op)
+        while docs_slice := list(islice(cursor, CHUNK_SIZE)):
+            yield map_nested_in_place(convert_mongo_objs, docs_slice)
+
+
+class CollectionLoaderParallell(CollectionLoader):
+    @property
+    def _sort_op(self) -> List[Optional[Tuple[str, int]]]:
+        if not self.incremental or not self.last_value:
+            return []
+        if self.incremental.last_value_func is max:
+            return [(self.cursor_field, ASCENDING)]
+        elif self.incremental.last_value_func is min:
+            return [(self.cursor_field, DESCENDING)]
+        return []
+
+    def _get_document_count(self) -> int:
+        return self.collection.count_documents(filter=self._filter_op)
+
+    def _create_batches(self) -> List[Dict[str, int]]:
+        doc_count = self._get_document_count()
+        return [
+            dict(skip=sk, limit=CHUNK_SIZE) for sk in range(0, doc_count, CHUNK_SIZE)
+        ]
+
+    def _get_cursor(self) -> TCursor:
+        cursor = self.collection.find(filter=self._filter_op)
+        if self._sort_op:
+            cursor = cursor.sort(self._sort_op)  # type: ignore
+        return cursor
+
+    @dlt.defer
+    def _run_batch(self, cursor: TCursor, batch: Dict[str, int]) -> TDataItem:
+        cursor = cursor.clone()
+
+        data = []
+        for document in cursor.skip(batch["skip"]).limit(batch["limit"]):
+            data.append(map_nested_in_place(convert_mongo_objs, document))
+        return data
+
+    def _get_all_batches(self) -> Iterator[TDataItem]:
+        batches = self._create_batches()
+        cursor = self._get_cursor()
+
+        for batch in batches:
+            yield self._run_batch(cursor=cursor, batch=batch)
+
+    def load_documents(self) -> Iterator[TDataItem]:
+        for document in self._get_all_batches():
+            yield document
+
+
+def collection_documents(
+    client: TMongoClient,
+    collection: TCollection,
+    incremental: Optional[dlt.sources.incremental[Any]] = None,
+    parallel: bool = False,
+) -> Iterator[TDataItem]:
+    """
+    A DLT source which loads data from a Mongo database using PyMongo.
+    Resources are automatically created for the collection.
+
+    Args:
+        client (MongoClient): The PyMongo client `pymongo.MongoClient` instance.
+        collection (Collection): The collection `pymongo.collection.Collection` to load.
+        incremental (Optional[dlt.sources.incremental[Any]]): The incremental configuration.
+        parallel (bool): Option to enable parallel loading for the collection. Default is False.
+
+    Returns:
+        Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
+    """
+    LoaderClass = CollectionLoaderParallell if parallel else CollectionLoader
+
+    loader = LoaderClass(client, collection, incremental=incremental)
+    for data in loader.load_documents():
+        yield data
+
+
+def convert_mongo_objs(value: Any) -> Any:
+    if isinstance(value, (ObjectId, Decimal128)):
+        return str(value)
+    if isinstance(value, _datetime.datetime):
+        return ensure_pendulum_datetime(value)
+    return value
+
+
+def client_from_credentials(connection_url: str) -> TMongoClient:
+    client: TMongoClient = MongoClient(
+        connection_url, uuidRepresentation="standard", tz_aware=True
+    )
+    return client
+
+
+@configspec
+class MongoDbCollectionConfiguration(BaseConfiguration):
+    incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]
+
+
+@configspec
+class MongoDbCollectionResourceConfiguration(BaseConfiguration):
+    connection_url: str
+    database: Optional[str]
+    collection: str
+    incremental: Optional[dlt.sources.incremental] = None  # type: ignore[type-arg]
+    write_disposition: Optional[str] = None
+    parallel: Optional[bool] = False
+
+
+__source_name__ = "mongodb"

{ingestr-0.1.4 → ingestr-0.2.1}/ingestr/src/sources.py

@@ -3,6 +3,7 @@ from typing import Callable
 
 import dlt
 
+from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.sql_database import sql_table
 
 
@@ -15,7 +16,7 @@ class SqlSource:
     def dlt_source(self, uri: str, table: str, **kwargs):
         table_fields = table.split(".")
         if len(table_fields) != 2:
-            raise ValueError("Table name must be in the format schema.<table>")
+            raise ValueError("Table name must be in the format schema.table")
 
         incremental = None
         if kwargs.get("incremental_key"):
@@ -29,6 +30,9 @@ class SqlSource:
                 end_value=end_value,
             )
 
+        if uri.startswith("mysql://"):
+            uri = uri.replace("mysql://", "mysql+pymysql://")
+
         table_instance = self.table_builder(
             credentials=uri,
             schema=table_fields[-2],
@@ -40,6 +44,39 @@ class SqlSource:
         return table_instance
 
 
+class MongoDbSource:
+    table_builder: Callable
+
+    def __init__(self, table_builder=mongodb_collection) -> None:
+        self.table_builder = table_builder
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format schema.table")
+
+        incremental = None
+        if kwargs.get("incremental_key"):
+            start_value = kwargs.get("interval_start")
+            end_value = kwargs.get("interval_end")
+
+            incremental = dlt.sources.incremental(
+                kwargs.get("incremental_key", ""),
+                initial_value=start_value,
+                end_value=end_value,
+            )
+
+        table_instance = self.table_builder(
+            connection_url=uri,
+            database=table_fields[-2],
+            collection=table_fields[-1],
+            parallel=True,
+            incremental=incremental,
+        )
+
+        return table_instance
+
+
 class LocalCsvSource:
     def dlt_source(self, uri: str, table: str, **kwargs):
         def csv_file():

ingestr-0.2.1/ingestr/src/sources_test.py

@@ -0,0 +1,102 @@
+import unittest
+
+import dlt
+import pytest
+
+from ingestr.src.sources import MongoDbSource, SqlSource
+
+
+class SqlSourceTest(unittest.TestCase):
+    def test_sql_source_requires_two_fields_in_table(self):
+        source = SqlSource()
+        with pytest.raises(ValueError):
+            uri = "bigquery://my-project"
+            source.dlt_source(uri, "onetable")
+
+        with pytest.raises(ValueError):
+            uri = "bigquery://my-project"
+            source.dlt_source(uri, "onetable.with.too.many.fields")
+
+    def test_table_instance_is_created(self):
+        uri = "bigquery://my-project"
+        table = "schema.table"
+
+        # monkey patch the sql_table function
+        def sql_table(credentials, schema, table, incremental, merge_key):
+            self.assertEqual(credentials, uri)
+            self.assertEqual(schema, "schema")
+            self.assertEqual(table, "table")
+            self.assertIsNone(incremental)
+            self.assertIsNone(merge_key)
+            return dlt.resource()
+
+        source = SqlSource(table_builder=sql_table)
+        res = source.dlt_source(uri, table)
+        self.assertIsNotNone(res)
+
+    def test_table_instance_is_created_with_incremental(self):
+        uri = "bigquery://my-project"
+        table = "schema.table"
+        incremental_key = "id"
+
+        # monkey patch the sql_table function
+        def sql_table(credentials, schema, table, incremental, merge_key):
+            self.assertEqual(credentials, uri)
+            self.assertEqual(schema, "schema")
+            self.assertEqual(table, "table")
+            self.assertIsInstance(incremental, dlt.sources.incremental)
+            self.assertEqual(incremental.cursor_path, incremental_key)
+            self.assertIsNone(merge_key)
+            return dlt.resource()
+
+        source = SqlSource(table_builder=sql_table)
+        res = source.dlt_source(uri, table, incremental_key=incremental_key)
+        self.assertIsNotNone(res)
+
+
+class MongoDbSourceTest(unittest.TestCase):
+    def test_sql_source_requires_two_fields_in_table(self):
+        source = MongoDbSource()
+        with pytest.raises(ValueError):
+            uri = "mongodb://my-project"
+            source.dlt_source(uri, "onetable")
+
+        with pytest.raises(ValueError):
+            uri = "mongodb://my-project"
+            source.dlt_source(uri, "onetable.with.too.many.fields")
+
+    def test_table_instance_is_created(self):
+        uri = "mongodb://my-project"
+        table = "schema.table"
+
+        # monkey patch the mongo function
+        def mongo(connection_url, database, collection, incremental, parallel):
+            self.assertEqual(connection_url, uri)
+            self.assertEqual(database, "schema")
+            self.assertEqual(collection, "table")
+            self.assertIsNone(incremental)
+            self.assertTrue(parallel)
+            return dlt.resource()
+
+        source = MongoDbSource(table_builder=mongo)
+        res = source.dlt_source(uri, table)
+        self.assertIsNotNone(res)
+
+    def test_table_instance_is_created_with_incremental(self):
+        uri = "mongodb://my-project"
+        table = "schema.table"
+        incremental_key = "id"
+
+        # monkey patch the mongo function
+        def mongo(connection_url, database, collection, incremental, parallel):
+            self.assertEqual(connection_url, uri)
+            self.assertEqual(database, "schema")
+            self.assertEqual(collection, "table")
+            self.assertIsInstance(incremental, dlt.sources.incremental)
+            self.assertEqual(incremental.cursor_path, incremental_key)
+            self.assertTrue(parallel)
+            return dlt.resource()
+
+        source = MongoDbSource(table_builder=mongo)
+        res = source.dlt_source(uri, table, incremental_key=incremental_key)
+        self.assertIsNotNone(res)

ingestr-0.2.1/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

{ingestr-0.1.4 → ingestr-0.2.1}/requirements.txt

@@ -7,6 +7,7 @@ google-cloud-bigquery-storage==2.24.0
 pendulum==3.0.0
 psycopg2-binary==2.9.9
 py-machineid==0.5.1
+pymongo==4.6.2
 pymysql==1.1.0
 pyodbc==5.1.0
 redshift-connector==2.1.0

ingestr-0.1.4/ingestr/src/sources_test.py

@@ -1,54 +0,0 @@
-import unittest
-
-import dlt
-import pytest
-
-from ingestr.src.sources import SqlSource
-
-
-class SqlSourceTest(unittest.TestCase):
-    def test_sql_source_requires_two_fields_in_table(self):
-        source = SqlSource()
-        with pytest.raises(ValueError):
-            uri = "bigquery://my-project"
-            source.dlt_source(uri, "onetable")
-
-        with pytest.raises(ValueError):
-            uri = "bigquery://my-project"
-            source.dlt_source(uri, "onetable.with.too.many.fields")
-
-    def test_table_instance_is_created(self):
-        uri = "bigquery://my-project"
-        table = "schema.table"
-
-        # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key):
-            self.assertEqual(credentials, uri)
-            self.assertEqual(schema, "schema")
-            self.assertEqual(table, "table")
-            self.assertIsNone(incremental)
-            self.assertIsNone(merge_key)
-            return dlt.resource()
-
-        source = SqlSource(table_builder=sql_table)
-        res = source.dlt_source(uri, table)
-        self.assertIsNotNone(res)
-
-    def test_table_instance_is_created_with_incremental(self):
-        uri = "bigquery://my-project"
-        table = "schema.table"
-        incremental_key = "id"
-
-        # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key):
-            self.assertEqual(credentials, uri)
-            self.assertEqual(schema, "schema")
-            self.assertEqual(table, "table")
-            self.assertIsInstance(incremental, dlt.sources.incremental)
-            self.assertEqual(incremental.cursor_path, incremental_key)
-            self.assertIsNone(merge_key)
-            return dlt.resource()
-
-        source = SqlSource(table_builder=sql_table)
-        res = source.dlt_source(uri, table, incremental_key=incremental_key)
-        self.assertIsNotNone(res)

ingestr-0.1.4/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.1.4"