ingestr 0.13.91__tar.gz → 0.13.93__tar.gz

{ingestr-0.13.91 → ingestr-0.13.93}/.github/workflows/tests.yml

@@ -19,7 +19,12 @@ jobs:
         # I tried running stuff on macOS but it was too slow and unreliable.
         # I also tried windows runners but couldn't get Docker to work there, so I gave up.
         os: [ubuntu-latest]
-        python-version: ['3.10', '3.11', '3.12']
+        python-version: [
+          '3.10',
+          '3.11',
+          '3.12',
+          '3.13',
+        ]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

{ingestr-0.13.91 → ingestr-0.13.93}/Dockerfile

@@ -1,14 +1,27 @@
 FROM python:3.11-slim-trixie
 
+# Guidelines that have been followed.
+# - https://hynek.me/articles/docker-uv/
+
+# Install the `uv` package manager.
+# Security-conscious organizations should package/review uv themselves.
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+# - Tell uv to byte-compile packages for faster application startups.
+# - Silence uv complaining about not being able to use hard links.
+# - Prevent uv from accidentally downloading isolated Python builds.
+# - Install packages into the system Python environment.
+ENV \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_SYSTEM_PYTHON=1
+
 WORKDIR /app
 
-COPY ./requirements.txt /app/requirements.txt
-COPY ./requirements_arm64.txt /app/requirements_arm64.txt
-RUN if [ "$(uname -m)" = "aarch64" ]; then \
-    cp /app/requirements_arm64.txt /app/requirements.txt; \
-fi
 
-# Setup dependencies for pyodbc
+# Install all prerequisites.
+
 RUN \
   export ACCEPT_EULA='Y' && \
   # Install build dependencies
@@ -28,17 +41,18 @@ RUN \
   sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
   mv -f /tmp/temp.ini /etc/odbcinst.ini
 
-ENV VIRTUAL_ENV=/usr/local
-ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
-RUN /install.sh && rm /install.sh
 
-RUN $HOME/.local/bin/uv pip install --system --no-cache -r requirements.txt
+# Install application.
 
+# Copy sources and activate platform-specific requirements file.
 COPY . /app
 RUN if [ "$(uname -m)" = "aarch64" ]; then \
     cp /app/requirements_arm64.txt /app/requirements.txt; \
 fi
 
-RUN $HOME/.local/bin/uv pip install --system . && $HOME/.local/bin/uv pip install --system pyodbc
+# Install all required packages and the application.
+RUN uv pip install --requirement requirements.txt pyodbc .
+
 
-ENTRYPOINT ["ingestr"]
+# Ready.
+ENTRYPOINT ["ingestr"]

{ingestr-0.13.91 → ingestr-0.13.93}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.91
+Version: 0.13.93
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -47,7 +47,7 @@ Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
-Requires-Dist: dlt-cratedb==0.0.1
+Requires-Dist: dlt-cratedb==0.0.2
 Requires-Dist: dlt==1.11.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0

{ingestr-0.13.91 → ingestr-0.13.93}/docs/supported-sources/salesforce.md

@@ -14,7 +14,7 @@ URI parameters:
 - `username` is your Salesforce account username.
 - `password` is your Salesforce account password.
 - `token` is your Salesforce security token.
-- `domain` is your Salesforce instance domain.
+- `domain` is your Salesforce instance domain (for example, `login`, `test`, or `your-domain.my`). Do **not** include `.salesforce.com`.
 
 You can obtain your security token by logging into your Salesforce account and navigating to the user settings under "Reset My Security Token."
 
@@ -28,14 +28,14 @@ Let's say:
 * Your Salesforce username is `user`.
 * Your password is `password123`.
 * Your security token is `fake_token`.
-* Your domain is `test.salesforce.com`.
+* Your domain is `your-domain.my`.
 * You want to ingest `account` data from your salesforce account
 * You want to save this data in a duckdb database `sf.db` under the table `public.account`
 
 You can run the following command to achieve this:
 ```sh
 ingestr ingest \
-  --source-uri "salesforce://?username=user&password=password123&token=fake_token&domain=test.salesforce.com" \
+  --source-uri "salesforce://?username=user&password=password123&token=fake_token&domain=your-domain.my" \
   --source-table "account" \
   --dest-uri "duckdb:///sf.db" \
   --dest-table "public.account"

ingestr-0.13.93/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.93"

{ingestr-0.13.91 → ingestr-0.13.93}/ingestr/src/mongodb/helpers.py

@@ -1,5 +1,6 @@
 """Mongo database source helpers"""
 
+import re
 from itertools import islice
 from typing import (
     TYPE_CHECKING,
@@ -866,4 +867,127 @@ class MongoDbCollectionResourceConfiguration(BaseConfiguration):
     projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value
 
 
+def convert_mongo_shell_to_extended_json(query_string: str) -> str:
+    """
+    Convert MongoDB shell syntax to MongoDB Extended JSON v2 format.
+    
+    This function handles common MongoDB shell constructs like ISODate, ObjectId,
+    NumberLong, NumberDecimal, etc. and converts them to their Extended JSON equivalents
+    that can be parsed by bson.json_util.
+    
+    Args:
+        query_string: A string containing MongoDB shell syntax
+        
+    Returns:
+        A string with MongoDB Extended JSON v2 format
+        
+    Examples:
+        >>> convert_mongo_shell_to_extended_json('ISODate("2010-01-01T00:00:00.000Z")')
+        '{"$date": "2010-01-01T00:00:00.000Z"}'
+        
+        >>> convert_mongo_shell_to_extended_json('ObjectId("507f1f77bcf86cd799439011")')
+        '{"$oid": "507f1f77bcf86cd799439011"}'
+    """
+    converted = query_string
+    
+    # Convert ISODate("...") to {"$date": "..."}
+    # Pattern matches ISODate("2010-01-01T00:00:00.000+0000") or similar
+    converted = re.sub(
+        r'ISODate\("([^"]+)"\)',
+        r'{"$date": "\1"}',
+        converted
+    )
+    
+    # Convert ObjectId("...") to {"$oid": "..."}
+    converted = re.sub(
+        r'ObjectId\("([^"]+)"\)',
+        r'{"$oid": "\1"}',
+        converted
+    )
+    
+    # Convert NumberLong(...) to {"$numberLong": "..."}
+    # Note: NumberLong can have quotes or not: NumberLong(123) or NumberLong("123")
+    converted = re.sub(
+        r'NumberLong\("([^"]+)"\)',
+        r'{"$numberLong": "\1"}',
+        converted
+    )
+    converted = re.sub(
+        r'NumberLong\(([^)]+)\)',
+        r'{"$numberLong": "\1"}',
+        converted
+    )
+    
+    # Convert NumberInt(...) to {"$numberInt": "..."}
+    converted = re.sub(
+        r'NumberInt\("([^"]+)"\)',
+        r'{"$numberInt": "\1"}',
+        converted
+    )
+    converted = re.sub(
+        r'NumberInt\(([^)]+)\)',
+        r'{"$numberInt": "\1"}',
+        converted
+    )
+    
+    # Convert NumberDecimal("...") to {"$numberDecimal": "..."}
+    converted = re.sub(
+        r'NumberDecimal\("([^"]+)"\)',
+        r'{"$numberDecimal": "\1"}',
+        converted
+    )
+    
+    # Convert Timestamp(..., ...) to {"$timestamp": {"t": ..., "i": ...}}
+    # Timestamp(1234567890, 1) -> {"$timestamp": {"t": 1234567890, "i": 1}}
+    converted = re.sub(
+        r'Timestamp\((\d+),\s*(\d+)\)',
+        r'{"$timestamp": {"t": \1, "i": \2}}',
+        converted
+    )
+    
+    # Convert BinData(..., "...") to {"$binary": {"base64": "...", "subType": "..."}}
+    converted = re.sub(
+        r'BinData\((\d+),\s*"([^"]+)"\)',
+        r'{"$binary": {"base64": "\2", "subType": "\1"}}',
+        converted
+    )
+    
+    # Convert MinKey() to {"$minKey": 1}
+    converted = re.sub(
+        r'MinKey\(\)',
+        r'{"$minKey": 1}',
+        converted
+    )
+    
+    # Convert MaxKey() to {"$maxKey": 1}
+    converted = re.sub(
+        r'MaxKey\(\)',
+        r'{"$maxKey": 1}',
+        converted
+    )
+    
+    # Convert UUID("...") to {"$uuid": "..."}
+    converted = re.sub(
+        r'UUID\("([^"]+)"\)',
+        r'{"$uuid": "\1"}',
+        converted
+    )
+    
+    # Convert DBRef("collection", "id") to {"$ref": "collection", "$id": "id"}
+    converted = re.sub(
+        r'DBRef\("([^"]+)",\s*"([^"]+)"\)',
+        r'{"$ref": "\1", "$id": "\2"}',
+        converted
+    )
+    
+    # Convert Code("...") to {"$code": "..."}
+    converted = re.sub(
+        r'Code\("([^"]+)"\)',
+        r'{"$code": "\1"}',
+        converted
+    )
+    
+    return converted
+
+
 __source_name__ = "mongodb"

{ingestr-0.13.91 → ingestr-0.13.93}/ingestr/src/smartsheets/__init__.py

@@ -3,6 +3,22 @@ from typing import Iterable
 import dlt
 import smartsheet  # type: ignore
 from dlt.extract import DltResource
+from smartsheet.models.enums import ColumnType  # type: ignore
+from smartsheet.models.sheet import Sheet  # type: ignore
+
+TYPE_MAPPING = {
+    ColumnType.TEXT_NUMBER: "text",
+    ColumnType.DATE: "date",
+    ColumnType.DATETIME: "timestamp",
+    ColumnType.CONTACT_LIST: "text",
+    ColumnType.CHECKBOX: "bool",
+    ColumnType.PICKLIST: "text",
+    ColumnType.DURATION: "text",
+    ColumnType.PREDECESSOR: "text",
+    ColumnType.ABSTRACT_DATETIME: "timestamp",
+    ColumnType.MULTI_CONTACT_LIST: "text",
+    ColumnType.MULTI_PICKLIST: "text",
+}
 
 
 @dlt.source
@@ -34,21 +50,33 @@ def smartsheet_source(
     )
     sheet_name = sheet_details.name
     resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
+    sheet = smartsheet_client.Sheets.get_sheet(sheet_id_int)
 
     yield dlt.resource(
-        _get_sheet_data(smartsheet_client, sheet_id_int),
+        _get_sheet_data(sheet),
         name=resource_name,
+        columns=_generate_type_hints(sheet),
         write_disposition="replace",
     )
 
 
-def _get_sheet_data(smartsheet_client: smartsheet.Smartsheet, sheet_id: int):
+def _get_sheet_data(sheet: Sheet):
     """Helper function to get all rows from a sheet."""
-    sheet = smartsheet_client.Sheets.get_sheet(sheet_id)
-    # Transform rows to a list of dictionaries
+
     column_titles = [col.title for col in sheet.columns]
     for row in sheet.rows:
         row_data = {"_row_id": row.id}
         for i, cell in enumerate(row.cells):
             row_data[column_titles[i]] = cell.value
         yield row_data
+
+
+def _generate_type_hints(sheet: Sheet):
+    return {
+        col.title: {
+            "data_type": TYPE_MAPPING.get(col.type.value),
+            "nullable": True,
+        }
+        for col in sheet.columns
+        if col.type.value in TYPE_MAPPING
+    }

{ingestr-0.13.91 → ingestr-0.13.93}/ingestr/src/sources.py

@@ -427,14 +427,19 @@ class MongoDbSource:
         if ":" in table:
             collection_name, query_json = table.split(":", 1)
 
-            # Parse and validate the query
+            # Parse the query using MongoDB's extended JSON parser
+            # First, convert MongoDB shell syntax to Extended JSON format
+            from bson import json_util
+            from ingestr.src.mongodb.helpers import convert_mongo_shell_to_extended_json
+            
+            # Convert MongoDB shell constructs to Extended JSON v2 format
+            converted_query = convert_mongo_shell_to_extended_json(query_json)
+            
             try:
-                import json
-
-                query = json.loads(query_json)
-            except json.JSONDecodeError as e:
-                raise ValueError(f"Invalid JSON query format: {e}")
-
+                query = json_util.loads(converted_query)
+            except Exception as e:
+                raise ValueError(f"Invalid MongoDB query format: {e}")
+            
             # Validate that it's a list for aggregation pipeline
             if not isinstance(query, list):
                 raise ValueError(

{ingestr-0.13.91 → ingestr-0.13.93}/ingestr/tests/unit/test_smartsheets.py

@@ -84,7 +84,6 @@ class TestSmartsheetSource(unittest.TestCase):
             list(source)
 
     def test_get_sheet_data(self):
-        mock_smartsheet_client_instance = MagicMock()
         mock_sheet = Sheet(
             {
                 "id": 456,
@@ -121,15 +120,13 @@ class TestSmartsheetSource(unittest.TestCase):
                 ],
             }
         )
-        mock_smartsheet_client_instance.Sheets.get_sheet.return_value = mock_sheet
 
-        data_generator = _get_sheet_data(mock_smartsheet_client_instance, 456)
+        data_generator = _get_sheet_data(mock_sheet)
         data = list(data_generator)
 
         self.assertEqual(len(data), 2)
         self.assertEqual(data[0], {"_row_id": 201, "ID": 1, "Value": "Alpha"})
         self.assertEqual(data[1], {"_row_id": 202, "ID": 2, "Value": "Beta"})
-        mock_smartsheet_client_instance.Sheets.get_sheet.assert_called_once_with(456)
 
 
 if __name__ == "__main__":

{ingestr-0.13.91 → ingestr-0.13.93}/requirements.in

@@ -4,7 +4,7 @@ databricks-sql-connector==4.0.5
 databricks-sqlalchemy==1.0.2
 dataclasses-json==0.6.7
 dlt==1.11.0
-dlt-cratedb>=0.0.1
+dlt-cratedb==0.0.2
 duckdb_engine==0.17.0
 duckdb==1.2.1
 google-ads==25.1.0

{ingestr-0.13.91 → ingestr-0.13.93}/requirements.txt

@@ -100,7 +100,7 @@ dlt==1.11.0
     # via
     #   -r requirements.in
     #   dlt-cratedb
-dlt-cratedb==0.0.1
+dlt-cratedb==0.0.2
     # via -r requirements.in
 dnspython==2.7.0
     # via pymongo

{ingestr-0.13.91 → ingestr-0.13.93}/requirements_arm64.txt

@@ -100,7 +100,7 @@ dlt==1.11.0
     # via
     #   -r requirements.in
     #   dlt-cratedb
-dlt-cratedb==0.0.1
+dlt-cratedb==0.0.2
     # via -r requirements.in
 dnspython==2.7.0
     # via pymongo

ingestr-0.13.91/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.91"