ingestr 0.13.90__py3-none-any.whl → 0.13.92__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.90"
+version = "v0.13.92"

ingestr/src/mongodb/helpers.py

@@ -1,5 +1,6 @@
 """Mongo database source helpers"""
 
+import re
 from itertools import islice
 from typing import (
     TYPE_CHECKING,
@@ -866,4 +867,127 @@ class MongoDbCollectionResourceConfiguration(BaseConfiguration):
     projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value
 
 
+def convert_mongo_shell_to_extended_json(query_string: str) -> str:
+    """
+    Convert MongoDB shell syntax to MongoDB Extended JSON v2 format.
+    
+    This function handles common MongoDB shell constructs like ISODate, ObjectId,
+    NumberLong, NumberDecimal, etc. and converts them to their Extended JSON equivalents
+    that can be parsed by bson.json_util.
+    
+    Args:
+        query_string: A string containing MongoDB shell syntax
+        
+    Returns:
+        A string with MongoDB Extended JSON v2 format
+        
+    Examples:
+        >>> convert_mongo_shell_to_extended_json('ISODate("2010-01-01T00:00:00.000Z")')
+        '{"$date": "2010-01-01T00:00:00.000Z"}'
+        
+        >>> convert_mongo_shell_to_extended_json('ObjectId("507f1f77bcf86cd799439011")')
+        '{"$oid": "507f1f77bcf86cd799439011"}'
+    """
+    converted = query_string
+    
+    # Convert ISODate("...") to {"$date": "..."}
+    # Pattern matches ISODate("2010-01-01T00:00:00.000+0000") or similar
+    converted = re.sub(
+        r'ISODate\("([^"]+)"\)',
+        r'{"$date": "\1"}',
+        converted
+    )
+    
+    # Convert ObjectId("...") to {"$oid": "..."}
+    converted = re.sub(
+        r'ObjectId\("([^"]+)"\)',
+        r'{"$oid": "\1"}',
+        converted
+    )
+    
+    # Convert NumberLong(...) to {"$numberLong": "..."}
+    # Note: NumberLong can have quotes or not: NumberLong(123) or NumberLong("123")
+    converted = re.sub(
+        r'NumberLong\("([^"]+)"\)',
+        r'{"$numberLong": "\1"}',
+        converted
+    )
+    converted = re.sub(
+        r'NumberLong\(([^)]+)\)',
+        r'{"$numberLong": "\1"}',
+        converted
+    )
+    
+    # Convert NumberInt(...) to {"$numberInt": "..."}
+    converted = re.sub(
+        r'NumberInt\("([^"]+)"\)',
+        r'{"$numberInt": "\1"}',
+        converted
+    )
+    converted = re.sub(
+        r'NumberInt\(([^)]+)\)',
+        r'{"$numberInt": "\1"}',
+        converted
+    )
+    
+    # Convert NumberDecimal("...") to {"$numberDecimal": "..."}
+    converted = re.sub(
+        r'NumberDecimal\("([^"]+)"\)',
+        r'{"$numberDecimal": "\1"}',
+        converted
+    )
+    
+    # Convert Timestamp(..., ...) to {"$timestamp": {"t": ..., "i": ...}}
+    # Timestamp(1234567890, 1) -> {"$timestamp": {"t": 1234567890, "i": 1}}
+    converted = re.sub(
+        r'Timestamp\((\d+),\s*(\d+)\)',
+        r'{"$timestamp": {"t": \1, "i": \2}}',
+        converted
+    )
+    
+    # Convert BinData(..., "...") to {"$binary": {"base64": "...", "subType": "..."}}
+    converted = re.sub(
+        r'BinData\((\d+),\s*"([^"]+)"\)',
+        r'{"$binary": {"base64": "\2", "subType": "\1"}}',
+        converted
+    )
+    
+    # Convert MinKey() to {"$minKey": 1}
+    converted = re.sub(
+        r'MinKey\(\)',
+        r'{"$minKey": 1}',
+        converted
+    )
+    
+    # Convert MaxKey() to {"$maxKey": 1}
+    converted = re.sub(
+        r'MaxKey\(\)',
+        r'{"$maxKey": 1}',
+        converted
+    )
+    
+    # Convert UUID("...") to {"$uuid": "..."}
+    converted = re.sub(
+        r'UUID\("([^"]+)"\)',
+        r'{"$uuid": "\1"}',
+        converted
+    )
+    
+    # Convert DBRef("collection", "id") to {"$ref": "collection", "$id": "id"}
+    converted = re.sub(
+        r'DBRef\("([^"]+)",\s*"([^"]+)"\)',
+        r'{"$ref": "\1", "$id": "\2"}',
+        converted
+    )
+    
+    # Convert Code("...") to {"$code": "..."}
+    converted = re.sub(
+        r'Code\("([^"]+)"\)',
+        r'{"$code": "\1"}',
+        converted
+    )
+    
+    return converted
+
+
 __source_name__ = "mongodb"

ingestr/src/salesforce/__init__.py

@@ -14,6 +14,7 @@ def salesforce_source(
     password: str,
     token: str,
     domain: str,
+    custom_object: str = None,
 ) -> Iterable[DltResource]:
     """
     Retrieves data from Salesforce using the Salesforce API.
@@ -131,6 +132,10 @@ def salesforce_source(
     ) -> Iterable[TDataItem]:
         yield get_records(client, "Event", last_timestamp.last_value, "SystemModstamp")
 
+    @dlt.resource(write_disposition="replace")
+    def custom() -> Iterable[TDataItem]:
+        yield get_records(client, custom_object)
+
     return (
         user,
         user_role,
@@ -147,4 +152,5 @@ def salesforce_source(
         pricebook_entry,
         task,
         event,
+        custom,
     )

ingestr/src/smartsheets/__init__.py

@@ -3,6 +3,22 @@ from typing import Iterable
 import dlt
 import smartsheet  # type: ignore
 from dlt.extract import DltResource
+from smartsheet.models.enums import ColumnType  # type: ignore
+from smartsheet.models.sheet import Sheet  # type: ignore
+
+TYPE_MAPPING = {
+    ColumnType.TEXT_NUMBER: "text",
+    ColumnType.DATE: "date",
+    ColumnType.DATETIME: "timestamp",
+    ColumnType.CONTACT_LIST: "text",
+    ColumnType.CHECKBOX: "bool",
+    ColumnType.PICKLIST: "text",
+    ColumnType.DURATION: "text",
+    ColumnType.PREDECESSOR: "text",
+    ColumnType.ABSTRACT_DATETIME: "timestamp",
+    ColumnType.MULTI_CONTACT_LIST: "text",
+    ColumnType.MULTI_PICKLIST: "text",
+}
 
 
 @dlt.source
@@ -34,21 +50,33 @@ def smartsheet_source(
     )
     sheet_name = sheet_details.name
     resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
+    sheet = smartsheet_client.Sheets.get_sheet(sheet_id_int)
 
     yield dlt.resource(
-        _get_sheet_data(smartsheet_client, sheet_id_int),
+        _get_sheet_data(sheet),
         name=resource_name,
+        columns=_generate_type_hints(sheet),
         write_disposition="replace",
     )
 
 
-def _get_sheet_data(smartsheet_client: smartsheet.Smartsheet, sheet_id: int):
+def _get_sheet_data(sheet: Sheet):
     """Helper function to get all rows from a sheet."""
-    sheet = smartsheet_client.Sheets.get_sheet(sheet_id)
-    # Transform rows to a list of dictionaries
+
     column_titles = [col.title for col in sheet.columns]
     for row in sheet.rows:
         row_data = {"_row_id": row.id}
         for i, cell in enumerate(row.cells):
             row_data[column_titles[i]] = cell.value
         yield row_data
+
+
+def _generate_type_hints(sheet: Sheet):
+    return {
+        col.title: {
+            "data_type": TYPE_MAPPING.get(col.type.value),
+            "nullable": True,
+        }
+        for col in sheet.columns
+        if col.type.value in TYPE_MAPPING
+    }

ingestr/src/sources.py

@@ -427,14 +427,19 @@ class MongoDbSource:
         if ":" in table:
             collection_name, query_json = table.split(":", 1)
 
-            # Parse and validate the query
+            # Parse the query using MongoDB's extended JSON parser
+            # First, convert MongoDB shell syntax to Extended JSON format
+            from bson import json_util
+            from ingestr.src.mongodb.helpers import convert_mongo_shell_to_extended_json
+            
+            # Convert MongoDB shell constructs to Extended JSON v2 format
+            converted_query = convert_mongo_shell_to_extended_json(query_json)
+            
             try:
-                import json
-
-                query = json.loads(query_json)
-            except json.JSONDecodeError as e:
-                raise ValueError(f"Invalid JSON query format: {e}")
-
+                query = json_util.loads(converted_query)
+            except Exception as e:
+                raise ValueError(f"Invalid MongoDB query format: {e}")
+            
             # Validate that it's a list for aggregation pipeline
             if not isinstance(query, list):
                 raise ValueError(
@@ -2525,9 +2530,14 @@ class SalesforceSource:
 
         src = salesforce_source(**creds)  # type: ignore
 
+        if table.startswith("custom:"):
+            custom_object = table.split(":")[1]
+            src = salesforce_source(**creds, custom_object=custom_object)
+            return src.with_resources("custom")
+
         if table not in src.resources:
             raise UnsupportedResourceError(table, "Salesforce")
-
+        
         return src.with_resources(table)
 
 

ingestr/tests/unit/test_smartsheets.py

@@ -84,7 +84,6 @@ class TestSmartsheetSource(unittest.TestCase):
             list(source)
 
     def test_get_sheet_data(self):
-        mock_smartsheet_client_instance = MagicMock()
         mock_sheet = Sheet(
             {
                 "id": 456,
@@ -121,15 +120,13 @@ class TestSmartsheetSource(unittest.TestCase):
                 ],
             }
         )
-        mock_smartsheet_client_instance.Sheets.get_sheet.return_value = mock_sheet
 
-        data_generator = _get_sheet_data(mock_smartsheet_client_instance, 456)
+        data_generator = _get_sheet_data(mock_sheet)
         data = list(data_generator)
 
         self.assertEqual(len(data), 2)
         self.assertEqual(data[0], {"_row_id": 201, "ID": 1, "Value": "Alpha"})
         self.assertEqual(data[1], {"_row_id": 202, "ID": 2, "Value": "Beta"})
-        mock_smartsheet_client_instance.Sheets.get_sheet.assert_called_once_with(456)
 
 
 if __name__ == "__main__":

{ingestr-0.13.90.dist-info → ingestr-0.13.92.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.90
+Version: 0.13.92
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.90.dist-info → ingestr-0.13.92.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=ZctKjmmFfAq8G28jd1zT9vTB-KNWs9X_92vQvWFmF_8,21
+ingestr/src/buildinfo.py,sha256=bR_SHBOnG2JaQD4K3ZvZC_ctkUAC7obqll7Reuy7YHE,21
 ingestr/src/destinations.py,sha256=QNT2rm91cZmY1_Zyj4VnbI14qGmZOUQOQUg9xUTVVYs,23799
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=hC5E_XgrgTHMqwqPc6ihUYvRGTGMTzdPfQhrgPyD0tY,6945
@@ -12,7 +12,7 @@ ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=MM_-6ZmIwFLS_L4kBkwJJc0XDyjDyHUkxMMnQaRfuRA,125176
+ingestr/src/sources.py,sha256=5S-RDTrAD6ZfpzTC4m7RW3WPKY2gBWCrPRC9esEsO5U,125785
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -96,7 +96,7 @@ ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnb
 ingestr/src/mixpanel/__init__.py,sha256=s1QtqMP0BTGW6YtdCabJFWj7lEn7KujzELwGpBOQgfs,1796
 ingestr/src/mixpanel/client.py,sha256=c_reouegOVYBOwHLfgYFwpmkba0Sxro1Zkml07NCYf0,3602
 ingestr/src/mongodb/__init__.py,sha256=wu3KJ3VH5FF67gctJqm4T3ZTdBOQam1u6xuFBohq7bs,7486
-ingestr/src/mongodb/helpers.py,sha256=TmEbQ-Rz5ajxmaMgZa7nrI13-L7Z_ClbFCFPnmPIrgE,31739
+ingestr/src/mongodb/helpers.py,sha256=-CZo96tRPy5SvJVMejJ1F72F-PJhyVJ3Rfpk0o2M9rQ,35243
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
 ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -116,7 +116,7 @@ ingestr/src/pipedrive/helpers/pages.py,sha256=Klpjw2OnMuhzit3PpiHKsfzGcJ3rQPSQBl
 ingestr/src/quickbooks/__init__.py,sha256=cZUuVCOTGPHTscRj6i0DytO63_fWF-4ieMxoU4PcyTg,3727
 ingestr/src/revenuecat/__init__.py,sha256=5HbyZuEOekkbeeT72sM_bnGygSyYdmd_vczfAUz7xoM,4029
 ingestr/src/revenuecat/helpers.py,sha256=CYU6l79kplnfL87GfdxyGeEBrBSWEZfGP0GyjPHuVDk,9619
-ingestr/src/salesforce/__init__.py,sha256=HVHY8pDngB498B6g6KDzwq-q2KPU4PxuEd9Y_8tDDFs,4716
+ingestr/src/salesforce/__init__.py,sha256=Ijveo8gyo_wLzQRBklxIm3RV0y2Gta9-mR44RbJljpI,4901
 ingestr/src/salesforce/helpers.py,sha256=QTdazBt-qRTBbCQMZnyclIaDQFmBixBy_RDKD00Lt-8,2492
 ingestr/src/shopify/__init__.py,sha256=RzSSG93g-Qlkz6TAxi1XasFDdxxtVXIo53ZTtjGczW4,62602
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
@@ -125,7 +125,7 @@ ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSs
 ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM,10113
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
-ingestr/src/smartsheets/__init__.py,sha256=3_Kz3AW68UhRg8WvppOeIcQYw04XlyrIHLx2A2pCx-o,1640
+ingestr/src/smartsheets/__init__.py,sha256=RIEfN1T2TMFg8T0RvN4o6sqC58YusJRDrmE9Isos5P4,2375
 ingestr/src/solidgate/__init__.py,sha256=Ts83j-JSnFsFuF4tDhVOfZKg7H0-bIpfn3kg1ZOR58A,8003
 ingestr/src/solidgate/helpers.py,sha256=mAsW_1hpD7ab3Y2vw8fxHi4yD3aT1geLdIYZ7ycyxBc,5690
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -157,9 +157,9 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr/tests/unit/test_smartsheets.py,sha256=i9diA7mXXwQLKGTcalD89OEDx8rqZi8pxtcE23ncgRQ,5122
-ingestr-0.13.90.dist-info/METADATA,sha256=i2rKDULepQJ4dBoMOr-PHvDI_DAEcEC4NSy8zknxoRc,15182
-ingestr-0.13.90.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.90.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.90.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.90.dist-info/RECORD,,
+ingestr/tests/unit/test_smartsheets.py,sha256=djU1TFD0-Zr7h-WNhm281DoyGApOpNOrZcN4r-U9Pks,4873
+ingestr-0.13.92.dist-info/METADATA,sha256=m59fSWl6js1EilscWOHeNyTe1p7mfKor2Xc2qvElXwA,15182
+ingestr-0.13.92.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.92.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.92.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.92.dist-info/RECORD,,