ingestr 0.13.91__tar.gz → 0.13.92__tar.gz

{ingestr-0.13.91 → ingestr-0.13.92}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.91
+Version: 0.13.92
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.91 → ingestr-0.13.92}/docs/supported-sources/salesforce.md

@@ -14,7 +14,7 @@ URI parameters:
 - `username` is your Salesforce account username.
 - `password` is your Salesforce account password.
 - `token` is your Salesforce security token.
-- `domain` is your Salesforce instance domain.
+- `domain` is your Salesforce instance domain (for example, `login`, `test`, or `your-domain.my`). Do **not** include `.salesforce.com`.
 
 You can obtain your security token by logging into your Salesforce account and navigating to the user settings under "Reset My Security Token."
 
@@ -28,14 +28,14 @@ Let's say:
 * Your Salesforce username is `user`.
 * Your password is `password123`.
 * Your security token is `fake_token`.
-* Your domain is `test.salesforce.com`.
+* Your domain is `your-domain.my`.
 * You want to ingest `account` data from your salesforce account
 * You want to save this data in a duckdb database `sf.db` under the table `public.account`
 
 You can run the following command to achieve this:
 ```sh
 ingestr ingest \
-  --source-uri "salesforce://?username=user&password=password123&token=fake_token&domain=test.salesforce.com" \
+  --source-uri "salesforce://?username=user&password=password123&token=fake_token&domain=your-domain.my" \
   --source-table "account" \
   --dest-uri "duckdb:///sf.db" \
   --dest-table "public.account"

ingestr-0.13.92/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.92"

{ingestr-0.13.91 → ingestr-0.13.92}/ingestr/src/mongodb/helpers.py

@@ -1,5 +1,6 @@
 """Mongo database source helpers"""
 
+import re
 from itertools import islice
 from typing import (
     TYPE_CHECKING,
@@ -866,4 +867,127 @@ class MongoDbCollectionResourceConfiguration(BaseConfiguration):
     projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value
 
 
+def convert_mongo_shell_to_extended_json(query_string: str) -> str:
+    """
+    Convert MongoDB shell syntax to MongoDB Extended JSON v2 format.
+    
+    This function handles common MongoDB shell constructs like ISODate, ObjectId,
+    NumberLong, NumberDecimal, etc. and converts them to their Extended JSON equivalents
+    that can be parsed by bson.json_util.
+    
+    Args:
+        query_string: A string containing MongoDB shell syntax
+        
+    Returns:
+        A string with MongoDB Extended JSON v2 format
+        
+    Examples:
+        >>> convert_mongo_shell_to_extended_json('ISODate("2010-01-01T00:00:00.000Z")')
+        '{"$date": "2010-01-01T00:00:00.000Z"}'
+        
+        >>> convert_mongo_shell_to_extended_json('ObjectId("507f1f77bcf86cd799439011")')
+        '{"$oid": "507f1f77bcf86cd799439011"}'
+    """
+    converted = query_string
+    
+    # Convert ISODate("...") to {"$date": "..."}
+    # Pattern matches ISODate("2010-01-01T00:00:00.000+0000") or similar
+    converted = re.sub(
+        r'ISODate\("([^"]+)"\)',
+        r'{"$date": "\1"}',
+        converted
+    )
+    
+    # Convert ObjectId("...") to {"$oid": "..."}
+    converted = re.sub(
+        r'ObjectId\("([^"]+)"\)',
+        r'{"$oid": "\1"}',
+        converted
+    )
+    
+    # Convert NumberLong(...) to {"$numberLong": "..."}
+    # Note: NumberLong can have quotes or not: NumberLong(123) or NumberLong("123")
+    converted = re.sub(
+        r'NumberLong\("([^"]+)"\)',
+        r'{"$numberLong": "\1"}',
+        converted
+    )
+    converted = re.sub(
+        r'NumberLong\(([^)]+)\)',
+        r'{"$numberLong": "\1"}',
+        converted
+    )
+    
+    # Convert NumberInt(...) to {"$numberInt": "..."}
+    converted = re.sub(
+        r'NumberInt\("([^"]+)"\)',
+        r'{"$numberInt": "\1"}',
+        converted
+    )
+    converted = re.sub(
+        r'NumberInt\(([^)]+)\)',
+        r'{"$numberInt": "\1"}',
+        converted
+    )
+    
+    # Convert NumberDecimal("...") to {"$numberDecimal": "..."}
+    converted = re.sub(
+        r'NumberDecimal\("([^"]+)"\)',
+        r'{"$numberDecimal": "\1"}',
+        converted
+    )
+    
+    # Convert Timestamp(..., ...) to {"$timestamp": {"t": ..., "i": ...}}
+    # Timestamp(1234567890, 1) -> {"$timestamp": {"t": 1234567890, "i": 1}}
+    converted = re.sub(
+        r'Timestamp\((\d+),\s*(\d+)\)',
+        r'{"$timestamp": {"t": \1, "i": \2}}',
+        converted
+    )
+    
+    # Convert BinData(..., "...") to {"$binary": {"base64": "...", "subType": "..."}}
+    converted = re.sub(
+        r'BinData\((\d+),\s*"([^"]+)"\)',
+        r'{"$binary": {"base64": "\2", "subType": "\1"}}',
+        converted
+    )
+    
+    # Convert MinKey() to {"$minKey": 1}
+    converted = re.sub(
+        r'MinKey\(\)',
+        r'{"$minKey": 1}',
+        converted
+    )
+    
+    # Convert MaxKey() to {"$maxKey": 1}
+    converted = re.sub(
+        r'MaxKey\(\)',
+        r'{"$maxKey": 1}',
+        converted
+    )
+    
+    # Convert UUID("...") to {"$uuid": "..."}
+    converted = re.sub(
+        r'UUID\("([^"]+)"\)',
+        r'{"$uuid": "\1"}',
+        converted
+    )
+    
+    # Convert DBRef("collection", "id") to {"$ref": "collection", "$id": "id"}
+    converted = re.sub(
+        r'DBRef\("([^"]+)",\s*"([^"]+)"\)',
+        r'{"$ref": "\1", "$id": "\2"}',
+        converted
+    )
+    
+    # Convert Code("...") to {"$code": "..."}
+    converted = re.sub(
+        r'Code\("([^"]+)"\)',
+        r'{"$code": "\1"}',
+        converted
+    )
+    
+    return converted
+
+
 __source_name__ = "mongodb"

{ingestr-0.13.91 → ingestr-0.13.92}/ingestr/src/smartsheets/__init__.py

@@ -3,6 +3,22 @@ from typing import Iterable
 import dlt
 import smartsheet  # type: ignore
 from dlt.extract import DltResource
+from smartsheet.models.enums import ColumnType  # type: ignore
+from smartsheet.models.sheet import Sheet  # type: ignore
+
+TYPE_MAPPING = {
+    ColumnType.TEXT_NUMBER: "text",
+    ColumnType.DATE: "date",
+    ColumnType.DATETIME: "timestamp",
+    ColumnType.CONTACT_LIST: "text",
+    ColumnType.CHECKBOX: "bool",
+    ColumnType.PICKLIST: "text",
+    ColumnType.DURATION: "text",
+    ColumnType.PREDECESSOR: "text",
+    ColumnType.ABSTRACT_DATETIME: "timestamp",
+    ColumnType.MULTI_CONTACT_LIST: "text",
+    ColumnType.MULTI_PICKLIST: "text",
+}
 
 
 @dlt.source
@@ -34,21 +50,33 @@ def smartsheet_source(
     )
     sheet_name = sheet_details.name
     resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
+    sheet = smartsheet_client.Sheets.get_sheet(sheet_id_int)
 
     yield dlt.resource(
-        _get_sheet_data(smartsheet_client, sheet_id_int),
+        _get_sheet_data(sheet),
         name=resource_name,
+        columns=_generate_type_hints(sheet),
         write_disposition="replace",
     )
 
 
-def _get_sheet_data(smartsheet_client: smartsheet.Smartsheet, sheet_id: int):
+def _get_sheet_data(sheet: Sheet):
     """Helper function to get all rows from a sheet."""
-    sheet = smartsheet_client.Sheets.get_sheet(sheet_id)
-    # Transform rows to a list of dictionaries
+
     column_titles = [col.title for col in sheet.columns]
     for row in sheet.rows:
         row_data = {"_row_id": row.id}
         for i, cell in enumerate(row.cells):
             row_data[column_titles[i]] = cell.value
         yield row_data
+
+
+def _generate_type_hints(sheet: Sheet):
+    return {
+        col.title: {
+            "data_type": TYPE_MAPPING.get(col.type.value),
+            "nullable": True,
+        }
+        for col in sheet.columns
+        if col.type.value in TYPE_MAPPING
+    }

{ingestr-0.13.91 → ingestr-0.13.92}/ingestr/src/sources.py

@@ -427,14 +427,19 @@ class MongoDbSource:
         if ":" in table:
             collection_name, query_json = table.split(":", 1)
 
-            # Parse and validate the query
+            # Parse the query using MongoDB's extended JSON parser
+            # First, convert MongoDB shell syntax to Extended JSON format
+            from bson import json_util
+            from ingestr.src.mongodb.helpers import convert_mongo_shell_to_extended_json
+            
+            # Convert MongoDB shell constructs to Extended JSON v2 format
+            converted_query = convert_mongo_shell_to_extended_json(query_json)
+            
             try:
-                import json
-
-                query = json.loads(query_json)
-            except json.JSONDecodeError as e:
-                raise ValueError(f"Invalid JSON query format: {e}")
-
+                query = json_util.loads(converted_query)
+            except Exception as e:
+                raise ValueError(f"Invalid MongoDB query format: {e}")
+            
             # Validate that it's a list for aggregation pipeline
             if not isinstance(query, list):
                 raise ValueError(

{ingestr-0.13.91 → ingestr-0.13.92}/ingestr/tests/unit/test_smartsheets.py

@@ -84,7 +84,6 @@ class TestSmartsheetSource(unittest.TestCase):
             list(source)
 
     def test_get_sheet_data(self):
-        mock_smartsheet_client_instance = MagicMock()
         mock_sheet = Sheet(
             {
                 "id": 456,
@@ -121,15 +120,13 @@ class TestSmartsheetSource(unittest.TestCase):
                 ],
             }
         )
-        mock_smartsheet_client_instance.Sheets.get_sheet.return_value = mock_sheet
 
-        data_generator = _get_sheet_data(mock_smartsheet_client_instance, 456)
+        data_generator = _get_sheet_data(mock_sheet)
         data = list(data_generator)
 
         self.assertEqual(len(data), 2)
         self.assertEqual(data[0], {"_row_id": 201, "ID": 1, "Value": "Alpha"})
         self.assertEqual(data[1], {"_row_id": 202, "ID": 2, "Value": "Beta"})
-        mock_smartsheet_client_instance.Sheets.get_sheet.assert_called_once_with(456)
 
 
 if __name__ == "__main__":

ingestr-0.13.91/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.91"