aio-sf 0.1.0b5__py3-none-any.whl → 0.1.0b7__py3-none-any.whl

aio_sf/__init__.py

@@ -14,9 +14,8 @@ from .api.auth import (  # noqa: F401
     SfdxCliAuth,
 )
 
-# Core package only exports client functionality
-# Users import exporter functions directly: from aio_sf.exporter import bulk_query
-
+# Core package exports client functionality
+# Exporter functionality is included by default, but gracefully handles missing deps
 __all__ = [
     "SalesforceClient",
     "SalesforceAuthError",
@@ -26,3 +25,37 @@ __all__ = [
     "StaticTokenAuth",
     "SfdxCliAuth",
 ]
+
+# Try to import exporter functionality if dependencies are available
+try:
+    from .exporter import (  # noqa: F401
+        bulk_query,
+        get_bulk_fields,
+        resume_from_locator,
+        write_records_to_csv,
+        QueryResult,
+        batch_records_async,
+        ParquetWriter,
+        create_schema_from_metadata,
+        write_query_to_parquet,
+        salesforce_to_arrow_type,
+    )
+
+    __all__.extend(
+        [
+            "bulk_query",
+            "get_bulk_fields",
+            "resume_from_locator",
+            "write_records_to_csv",
+            "QueryResult",
+            "batch_records_async",
+            "ParquetWriter",
+            "create_schema_from_metadata",
+            "write_query_to_parquet",
+            "salesforce_to_arrow_type",
+        ]
+    )
+
+except ImportError:
+    # Exporter dependencies not available - this is fine for core-only installs
+    pass

aio_sf/exporter/bulk_export.py

@@ -320,7 +320,7 @@ async def get_bulk_fields(fields_metadata: List[FieldInfo]) -> List[FieldInfo]:
     queryable_fields = [
         field
         for field in fields_metadata
-        if field.get("type") not in ["address", "location"]
+        if field.get("type") not in ["address", "location", "base64"]
     ]
 
     return queryable_fields

aio_sf/exporter/parquet_writer.py

@@ -16,24 +16,20 @@ from .bulk_export import QueryResult, batch_records_async
 
 
 def salesforce_to_arrow_type(
-    sf_type: str, convert_datetime_to_timestamp: bool = True
+    sf_type: str, type_mapping_overrides: Optional[Dict[str, pa.DataType]] = None
 ) -> pa.DataType:
     """Convert Salesforce data types to Arrow data types.
 
     :param sf_type: Salesforce field type
-    :param convert_datetime_to_timestamp: If True, datetime fields use timestamp type, otherwise string
+    :param type_mapping_overrides: Optional dict to override default type mappings
     """
-    type_mapping = {
+    default_type_mapping = {
         "string": pa.string(),
         "boolean": pa.bool_(),
         "int": pa.int64(),
         "double": pa.float64(),
-        "date": pa.string(),  # Always store as string since SF returns ISO format
-        "datetime": (
-            pa.timestamp("us", tz="UTC")
-            if convert_datetime_to_timestamp
-            else pa.string()
-        ),
+        "date": pa.date32(),  # Store as proper date type
+        "datetime": pa.timestamp("us", tz="UTC"),
         "currency": pa.float64(),
         "reference": pa.string(),
         "picklist": pa.string(),
@@ -48,20 +44,27 @@ def salesforce_to_arrow_type(
         "base64": pa.string(),
         "anyType": pa.string(),
     }
+
+    # Apply overrides if provided
+    if type_mapping_overrides:
+        type_mapping = {**default_type_mapping, **type_mapping_overrides}
+    else:
+        type_mapping = default_type_mapping
+
     return type_mapping.get(sf_type.lower(), pa.string())
 
 
 def create_schema_from_metadata(
     fields_metadata: List[FieldInfo],
     column_formatter: Optional[Callable[[str], str]] = None,
-    convert_datetime_to_timestamp: bool = True,
+    type_mapping_overrides: Optional[Dict[str, pa.DataType]] = None,
 ) -> pa.Schema:
     """
     Create a PyArrow schema from Salesforce field metadata.
 
     :param fields_metadata: List of field metadata dictionaries from Salesforce
     :param column_formatter: Optional function to format column names
-    :param convert_datetime_to_timestamp: If True, datetime fields use timestamp type, otherwise string
+    :param type_mapping_overrides: Optional dict to override default type mappings
     :returns: PyArrow schema
     """
     arrow_fields = []
@@ -70,7 +73,7 @@ def create_schema_from_metadata(
         if column_formatter:
             field_name = column_formatter(field_name)
         sf_type = field.get("type", "string")
-        arrow_type = salesforce_to_arrow_type(sf_type, convert_datetime_to_timestamp)
+        arrow_type = salesforce_to_arrow_type(sf_type, type_mapping_overrides)
         # All fields are nullable since Salesforce can return empty values
         arrow_fields.append(pa.field(field_name, arrow_type, nullable=True))
 
@@ -90,7 +93,7 @@ class ParquetWriter:
         batch_size: int = 10000,
         convert_empty_to_null: bool = True,
         column_formatter: Optional[Callable[[str], str]] = None,
-        convert_datetime_to_timestamp: bool = True,
+        type_mapping_overrides: Optional[Dict[str, pa.DataType]] = None,
     ):
         """
         Initialize ParquetWriter.
@@ -100,14 +103,14 @@ class ParquetWriter:
         :param batch_size: Number of records to process in each batch
         :param convert_empty_to_null: Convert empty strings to null values
         :param column_formatter: Optional function to format column names. If None, no formatting is applied
-        :param convert_datetime_to_timestamp: If True, datetime fields are converted to timestamps, otherwise stored as strings
+        :param type_mapping_overrides: Optional dict to override default type mappings
         """
         self.file_path = file_path
         self.schema = schema
         self.batch_size = batch_size
         self.convert_empty_to_null = convert_empty_to_null
         self.column_formatter = column_formatter
-        self.convert_datetime_to_timestamp = convert_datetime_to_timestamp
+        self.type_mapping_overrides = type_mapping_overrides
         self._writer = None
         self._schema_finalized = False
 
@@ -248,6 +251,11 @@ class ParquetWriter:
                     df[field_name] = self._convert_datetime_strings_to_timestamps(
                         datetime_series
                     )
+            elif pa.types.is_date(field.type):
+                # Convert Salesforce ISO date strings to dates
+                date_series = df[field_name]
+                if isinstance(date_series, pd.Series):
+                    df[field_name] = self._convert_date_strings_to_dates(date_series)
 
             # Replace empty strings with None for non-string fields
             if not pa.types.is_string(field.type):
@@ -290,6 +298,36 @@ class ParquetWriter:
             # This shouldn't happen, but handle it gracefully
             return pd.Series(result, index=series.index)
 
+    def _convert_date_strings_to_dates(self, series: pd.Series) -> pd.Series:
+        """
+        Convert Salesforce ISO date strings to pandas date objects.
+
+        Salesforce returns date in ISO format like '2025-10-01'.
+        """
+
+        def parse_sf_date(date_str):
+            if pd.isna(date_str) or date_str == "" or date_str is None:
+                return pd.NaT
+
+            try:
+                # Handle Salesforce date format (YYYY-MM-DD)
+                date_str = str(date_str).strip()
+
+                # Use pandas to_datetime for date parsing, then convert to date
+                return pd.to_datetime(date_str, format="%Y-%m-%d").date()
+
+            except (ValueError, TypeError) as e:
+                logging.warning(f"Failed to parse date string '{date_str}': {e}")
+                return pd.NaT
+
+        # Apply the conversion function to the series
+        result = series.apply(parse_sf_date)
+        if isinstance(result, pd.Series):
+            return result
+        else:
+            # This shouldn't happen, but handle it gracefully
+            return pd.Series(result, index=series.index)
+
     def close(self) -> None:
         """Close the parquet writer."""
         if self._writer:
@@ -305,7 +343,7 @@ async def write_query_to_parquet(
     batch_size: int = 10000,
     convert_empty_to_null: bool = True,
     column_formatter: Optional[Callable[[str], str]] = None,
-    convert_datetime_to_timestamp: bool = True,
+    type_mapping_overrides: Optional[Dict[str, pa.DataType]] = None,
 ) -> None:
     """
     Convenience function to write a QueryResult to a parquet file (async version).
@@ -317,14 +355,14 @@ async def write_query_to_parquet(
     :param batch_size: Number of records to process in each batch
     :param convert_empty_to_null: Convert empty strings to null values
     :param column_formatter: Optional function to format column names
-    :param convert_datetime_to_timestamp: If True, datetime fields are converted to timestamps, otherwise stored as strings
+    :param type_mapping_overrides: Optional dict to override default type mappings
     """
     effective_schema = None
     if schema:
         effective_schema = schema
     elif fields_metadata:
         effective_schema = create_schema_from_metadata(
-            fields_metadata, column_formatter, convert_datetime_to_timestamp
+            fields_metadata, column_formatter, type_mapping_overrides
         )
 
     writer = ParquetWriter(
@@ -333,7 +371,7 @@ async def write_query_to_parquet(
         batch_size=batch_size,
         convert_empty_to_null=convert_empty_to_null,
         column_formatter=column_formatter,
-        convert_datetime_to_timestamp=convert_datetime_to_timestamp,
+        type_mapping_overrides=type_mapping_overrides,
     )
 
     await writer.write_query_result(query_result)

{aio_sf-0.1.0b5.dist-info → aio_sf-0.1.0b7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aio-sf
-Version: 0.1.0b5
+Version: 0.1.0b7
 Summary: Async Salesforce library for Python
 Project-URL: Homepage, https://github.com/callawaycloud/aio-salesforce
 Project-URL: Repository, https://github.com/callawaycloud/aio-salesforce
@@ -35,13 +35,16 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.11
+Requires-Dist: boto3>=1.34.0
 Requires-Dist: httpx>=0.25.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=10.0.0
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: python-dotenv>=1.0.0
-Provides-Extra: all
-Requires-Dist: boto3>=1.34.0; extra == 'all'
-Requires-Dist: pandas>=2.0.0; extra == 'all'
-Requires-Dist: pyarrow>=10.0.0; extra == 'all'
+Provides-Extra: core
+Requires-Dist: httpx>=0.25.0; extra == 'core'
+Requires-Dist: pydantic>=2.0.0; extra == 'core'
+Requires-Dist: python-dotenv>=1.0.0; extra == 'core'
 Provides-Extra: dev
 Requires-Dist: black>=23.0.0; extra == 'dev'
 Requires-Dist: mypy>=1.5.0; extra == 'dev'
@@ -88,16 +91,16 @@ An async Salesforce library for Python.
 
 ## Installation
 
-### Core (Connection Only)
+### Full Package (Default - Includes Everything)
 ```bash
 uv add aio-sf
 # or: pip install aio-sf
 ```
 
-### With Export Capabilities
+### Core Only (Minimal Dependencies)
 ```bash
-uv add "aio-sf[exporter]"
-# or: pip install "aio-sf[exporter]"
+uv add "aio-sf[core]"
+# or: pip install "aio-sf[core]"
 ```
 
 ## Quick Start
@@ -157,7 +160,11 @@ The Exporter library contains a streamlined and "opinionated" way to export data
 
 ### 3. Export to Parquet
 ```python
-from aio_sf.exporter import bulk_query, write_query_to_parquet
+# With full installation (default), you can import directly from aio_sf
+from aio_sf import SalesforceClient, ClientCredentialsAuth, bulk_query, write_query_to_parquet
+
+# Or import from the exporter module (both work)
+# from aio_sf.exporter import bulk_query, write_query_to_parquet
 
 async def main():
     # ... authentication code from above ...

{aio_sf-0.1.0b5.dist-info → aio_sf-0.1.0b7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-aio_sf/__init__.py,sha256=p1WqfaXQ0ldxDkLCK2hc07rMeq9ywenUYUBQc2SfQOE,707
+aio_sf/__init__.py,sha256=jWZkLTyHyOyHDJBfZORC2KcdtTBbI9pyVoSRxMbcJ1U,1603
 aio_sf/api/__init__.py,sha256=gLa2cbEvXPMLdcZVV7vhUQ-I9yJJitSaRN9mLn3Qhas,2299
 aio_sf/api/client.py,sha256=2epNf31HN1WpAVRSV0MtYthFkXlYiwwgMJ1Qms8-LKc,9750
 aio_sf/api/types.py,sha256=hpdCfzOzq32ESStQAewJc4j4NXQluENKjJEhUeGEI1Y,7738
@@ -21,9 +21,9 @@ aio_sf/api/query/__init__.py,sha256=OMh9g9PAPBFyQtUJc4aua1nKAiiIFYouZfaa5Zw4yZU,
 aio_sf/api/query/client.py,sha256=E9NTFgfAv01SDrOvZqufsM0GkQN4joHRRUp7-WMzBdk,8108
 aio_sf/api/query/types.py,sha256=Wfk75kJpNDCGpTHonCbzjWvayy8guA3eyZp3hE7nBt0,845
 aio_sf/exporter/__init__.py,sha256=waTegrvw_SvJzREAWD4twSDldSL-HfvhLTLLT1o765o,771
-aio_sf/exporter/bulk_export.py,sha256=JmlVwDcXqvBYYY9o7DjdfcJ9jIKiVAPrcnk5Bz4sLoU,13159
-aio_sf/exporter/parquet_writer.py,sha256=EysoeFI8TptcAfhor8FeJsJO0eNAuw9i250JdlHPoug,13078
-aio_sf-0.1.0b5.dist-info/METADATA,sha256=GL__z4LmVTaxTO_soOtE0JwR95tl2hxyDzMAMj7MkNk,6439
-aio_sf-0.1.0b5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-aio_sf-0.1.0b5.dist-info/licenses/LICENSE,sha256=gu0Cbpiqs-vX7YgJJhGI1jH1mHup3dZMrZc-gmpEG60,1071
-aio_sf-0.1.0b5.dist-info/RECORD,,
+aio_sf/exporter/bulk_export.py,sha256=2GtiwXChf7dq7dByGLPDhIJJg-yq9eyoE57H4Ekqaus,13169
+aio_sf/exporter/parquet_writer.py,sha256=jGiLooxyaqciSDUbXj5F_4uWoR_YrQaB-PrDfRuXR3Y,14495
+aio_sf-0.1.0b7.dist-info/METADATA,sha256=EB5BfuZ2Td8dNAJThB3ed1VuNHH0LbwupD32sVIzKjA,6780
+aio_sf-0.1.0b7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+aio_sf-0.1.0b7.dist-info/licenses/LICENSE,sha256=gu0Cbpiqs-vX7YgJJhGI1jH1mHup3dZMrZc-gmpEG60,1071
+aio_sf-0.1.0b7.dist-info/RECORD,,