aio-sf 0.1.0b1__py3-none-any.whl

aio_salesforce/exporter/parquet_writer.py.backup

@@ -0,0 +1,326 @@
+"""
+Parquet writer module for converting Salesforce QueryResult to Parquet format.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from pathlib import Path
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+    import pandas as pd
+    import pyarrow.parquet as pq
+
+try:
+    import pyarrow as pa
+    import pandas as pd
+    import pyarrow.parquet as pq
+
+    _DEPENDENCIES_AVAILABLE = True
+except ImportError as e:
+    _DEPENDENCIES_AVAILABLE = False
+    _MISSING_DEPS = str(e)
+    # Create dummy objects to prevent import errors
+    pa = None
+    pd = None
+    pq = None
+
+from ..api.types import FieldInfo
+
+from .bulk_export import QueryResult, batch_records, batch_records_async
+
+
+def _check_dependencies():
+    """Check if required dependencies are available and raise helpful error if not."""
+    if not _DEPENDENCIES_AVAILABLE:
+        raise ImportError(
+            f"Missing required dependencies for exporter functionality: {_MISSING_DEPS}. "
+            "Install with: pip install 'aio-salesforce[exporter]' or uv add 'aio-salesforce[exporter]'"
+        )
+
+
+def salesforce_to_arrow_type(sf_type: str) -> pa.DataType:
+    """Convert Salesforce data types to Arrow data types."""
+    _check_dependencies()
+    type_mapping = {
+        "string": pa.string(),
+        "boolean": pa.bool_(),
+        "int": pa.int64(),
+        "double": pa.float64(),
+        "date": pa.string(),  # Store as string since SF returns ISO format
+        "datetime": pa.string(),  # Store as string since SF returns ISO format
+        "currency": pa.float64(),
+        "reference": pa.string(),
+        "picklist": pa.string(),
+        "multipicklist": pa.string(),
+        "textarea": pa.string(),
+        "phone": pa.string(),
+        "url": pa.string(),
+        "email": pa.string(),
+        "combobox": pa.string(),
+        "percent": pa.float64(),
+        "id": pa.string(),
+        "base64": pa.string(),
+        "anyType": pa.string(),
+    }
+    return type_mapping.get(sf_type.lower(), pa.string())
+
+
+def create_schema_from_metadata(fields_metadata: List[FieldInfo]) -> pa.Schema:
+    """
+    Create a PyArrow schema from Salesforce field metadata.
+
+    :param fields_metadata: List of field metadata dictionaries from Salesforce
+    :returns: PyArrow schema
+    """
+    _check_dependencies()
+    arrow_fields = []
+    for field in fields_metadata:
+        field_name = field.get("name", "").lower()  # Normalize to lowercase
+        sf_type = field.get("type", "string")
+        arrow_type = salesforce_to_arrow_type(sf_type)
+        # All fields are nullable since Salesforce can return empty values
+        arrow_fields.append(pa.field(field_name, arrow_type, nullable=True))
+
+    return pa.schema(arrow_fields)
+
+
+class ParquetWriter:
+    """
+    Writer class for converting Salesforce QueryResult to Parquet format.
+    Supports streaming writes and optional schema from field metadata.
+    """
+
+    def __init__(
+        self,
+        file_path: str,
+        schema: Optional[pa.Schema] = None,
+        batch_size: int = 10000,
+        convert_empty_to_null: bool = True,
+    ):
+        """
+        Initialize ParquetWriter.
+
+        :param file_path: Path to output parquet file
+        :param schema: Optional PyArrow schema. If None, will be inferred from first batch
+        :param batch_size: Number of records to process in each batch
+        :param convert_empty_to_null: Convert empty strings to null values
+        """
+        _check_dependencies()
+        self.file_path = file_path
+        self.schema = schema
+        self.batch_size = batch_size
+        self.convert_empty_to_null = convert_empty_to_null
+        self._writer = None
+        self._schema_finalized = False
+
+        # Ensure parent directory exists
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+
+    def write_query_result(self, query_result: QueryResult) -> None:
+        """
+        Write all records from a QueryResult to the parquet file.
+
+        :param query_result: QueryResult to write
+        """
+        try:
+            for batch in batch_records(query_result, self.batch_size):
+                self._write_batch(batch)
+        finally:
+            self.close()
+
+    async def write_query_result_async(self, query_result: QueryResult) -> None:
+        """
+        Write all records from a QueryResult to the parquet file (async version).
+
+        :param query_result: QueryResult to write
+        """
+        try:
+            async for batch in batch_records_async(query_result, self.batch_size):
+                self._write_batch(batch)
+        finally:
+            self.close()
+
+    def _write_batch(self, batch: List[Dict[str, Any]]) -> None:
+        """Write a batch of records to the parquet file."""
+        if not batch:
+            return
+
+        # Convert field names to lowercase for consistency
+        converted_batch = []
+        for record in batch:
+            converted_record = {k.lower(): v for k, v in record.items()}
+            converted_batch.append(converted_record)
+
+        # Create DataFrame
+        df = pd.DataFrame(converted_batch)
+
+        # If schema not finalized, create it from first batch
+        if not self._schema_finalized:
+            if self.schema is None:
+                self.schema = self._infer_schema_from_dataframe(df)
+            else:
+                # Filter schema to only include fields that are actually in the data
+                self.schema = self._filter_schema_to_data(self.schema, df.columns)
+            self._schema_finalized = True
+
+        # Apply data type conversions based on schema
+        self._convert_dataframe_types(df)
+
+        # Create Arrow table
+        table = pa.Table.from_pandas(df, schema=self.schema)
+
+        # Initialize writer if needed
+        if self._writer is None:
+            self._writer = pq.ParquetWriter(self.file_path, self.schema)
+
+        # Write the table
+        self._writer.write_table(table)
+
+    def _infer_schema_from_dataframe(self, df: pd.DataFrame) -> pa.Schema:
+        """Infer schema from the first DataFrame."""
+        fields = []
+        for col_name, dtype in df.dtypes.items():
+            if dtype == "object":
+                arrow_type = pa.string()
+            elif dtype == "bool":
+                arrow_type = pa.bool_()
+            elif dtype in ["int64", "int32"]:
+                arrow_type = pa.int64()
+            elif dtype in ["float64", "float32"]:
+                arrow_type = pa.float64()
+            else:
+                arrow_type = pa.string()
+
+            fields.append(pa.field(col_name, arrow_type, nullable=True))
+
+        return pa.schema(fields)
+
+    def _filter_schema_to_data(
+        self, schema: pa.Schema, data_columns: List[str]
+    ) -> pa.Schema:
+        """Filter schema to only include fields that are present in the data."""
+        # Convert data columns to set for faster lookup
+        data_columns_set = set(data_columns)
+
+        # Filter schema fields to only those present in data
+        filtered_fields = []
+        for field in schema:
+            if field.name in data_columns_set:
+                filtered_fields.append(field)
+
+        if len(filtered_fields) != len(data_columns_set):
+            # Log fields that are in data but not in schema (shouldn't happen normally)
+            missing_in_schema = data_columns_set - {f.name for f in filtered_fields}
+            if missing_in_schema:
+                logging.warning(
+                    f"Fields in data but not in schema: {missing_in_schema}"
+                )
+
+        return pa.schema(filtered_fields)
+
+    def _convert_dataframe_types(self, df: pd.DataFrame) -> None:
+        """Convert DataFrame types based on the schema."""
+        for field in self.schema:
+            field_name = field.name
+            if field_name not in df.columns:
+                continue
+
+            # Convert empty strings to null if requested
+            if self.convert_empty_to_null:
+                df[field_name] = df[field_name].replace({"": None})
+
+            # Apply type-specific conversions
+            if pa.types.is_boolean(field.type):
+                # Convert string 'true'/'false' to boolean
+                df[field_name] = (
+                    df[field_name]
+                    .map({"true": True, "false": False, None: None})
+                    .fillna(df[field_name])
+                )  # Keep original values for non-string booleans
+            elif pa.types.is_integer(field.type):
+                df[field_name] = pd.to_numeric(df[field_name], errors="coerce").astype(
+                    "Int64"
+                )  # Nullable integer
+            elif pa.types.is_floating(field.type):
+                df[field_name] = pd.to_numeric(df[field_name], errors="coerce")
+
+            # Replace empty strings with None for non-string fields
+            if not pa.types.is_string(field.type):
+                df[field_name] = df[field_name].replace("", pd.NA)
+
+    def close(self) -> None:
+        """Close the parquet writer."""
+        if self._writer:
+            self._writer.close()
+            self._writer = None
+
+
+def write_query_to_parquet(
+    query_result: QueryResult,
+    file_path: str,
+    fields_metadata: Optional[List[FieldInfo]] = None,
+    schema: Optional[pa.Schema] = None,
+    batch_size: int = 10000,
+    convert_empty_to_null: bool = True,
+) -> None:
+    """
+    Convenience function to write a QueryResult to a parquet file.
+
+    :param query_result: QueryResult to write
+    :param file_path: Path to output parquet file
+    :param fields_metadata: Optional Salesforce field metadata for schema creation
+    :param schema: Optional pre-created PyArrow schema (takes precedence over fields_metadata)
+    :param batch_size: Number of records to process in each batch
+    :param convert_empty_to_null: Convert empty strings to null values
+    """
+    effective_schema = None
+    if schema:
+        effective_schema = schema
+    elif fields_metadata:
+        effective_schema = create_schema_from_metadata(fields_metadata)
+
+    writer = ParquetWriter(
+        file_path=file_path,
+        schema=effective_schema,
+        batch_size=batch_size,
+        convert_empty_to_null=convert_empty_to_null,
+    )
+
+    writer.write_query_result(query_result)
+
+
+async def write_query_to_parquet_async(
+    query_result: QueryResult,
+    file_path: str,
+    fields_metadata: Optional[List[Dict[str, Any]]] = None,
+    schema: Optional[pa.Schema] = None,
+    batch_size: int = 10000,
+    convert_empty_to_null: bool = True,
+) -> None:
+    """
+    Convenience function to write a QueryResult to a parquet file (async version).
+
+    :param query_result: QueryResult to write
+    :param file_path: Path to output parquet file
+    :param fields_metadata: Optional Salesforce field metadata for schema creation
+    :param schema: Optional pre-created PyArrow schema (takes precedence over fields_metadata)
+    :param batch_size: Number of records to process in each batch
+    :param convert_empty_to_null: Convert empty strings to null values
+    """
+    effective_schema = None
+    if schema:
+        effective_schema = schema
+    elif fields_metadata:
+        effective_schema = create_schema_from_metadata(fields_metadata)
+
+    writer = ParquetWriter(
+        file_path=file_path,
+        schema=effective_schema,
+        batch_size=batch_size,
+        convert_empty_to_null=convert_empty_to_null,
+    )
+
+    await writer.write_query_result_async(query_result)

aio_sf-0.1.0b1.dist-info/METADATA

@@ -0,0 +1,198 @@
+Metadata-Version: 2.4
+Name: aio-sf
+Version: 0.1.0b1
+Summary: Async Salesforce library for Python with Bulk API 2.0 support
+Project-URL: Homepage, https://github.com/callawaycloud/aio-salesforce
+Project-URL: Repository, https://github.com/callawaycloud/aio-salesforce
+Project-URL: Issues, https://github.com/callawaycloud/aio-salesforce/issues
+Author-email: Jonas <charlie@callaway.cloud>
+License: MIT License
+        
+        Copyright (c) 2025 Callaway Cloud
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Provides-Extra: all
+Requires-Dist: boto3>=1.34.0; extra == 'all'
+Requires-Dist: pandas>=2.0.0; extra == 'all'
+Requires-Dist: pyarrow>=10.0.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: black>=23.0.0; extra == 'dev'
+Requires-Dist: mypy>=1.5.0; extra == 'dev'
+Requires-Dist: pre-commit>=3.0.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Requires-Dist: types-requests>=2.31.0; extra == 'dev'
+Provides-Extra: exporter
+Requires-Dist: boto3>=1.34.0; extra == 'exporter'
+Requires-Dist: pandas>=2.0.0; extra == 'exporter'
+Requires-Dist: pyarrow>=10.0.0; extra == 'exporter'
+Description-Content-Type: text/markdown
+
+# aio-sf
+
+An async Salesforce library for Python with Bulk API 2.0 support.
+
+## Features
+
+### ✅ Supported APIs
+- [x] **OAuth Client Credentials Flow** - Automatic authentication
+- [x] **Bulk API 2.0** - Efficient querying of large datasets
+- [x] **Describe API** - Field metadata and object descriptions
+- [x] **SOQL Query API** - Standard Salesforce queries
+
+### 🔄 Planned APIs
+- [ ] **SObjects API** - Standard CRUD operations
+- [ ] **Tooling API** - Development and deployment tools
+- [ ] **Bulk API 1.0** - Legacy bulk operations
+- [ ] **Streaming API** - Real-time event streaming
+
+### 🚀 Export Features
+- [x] **Parquet Export** - Efficient columnar storage with schema mapping
+- [x] **CSV Export** - Simple text format export
+- [x] **Resume Support** - Resume interrupted queries using job IDs
+- [x] **Streaming Processing** - Memory-efficient processing of large datasets
+- [x] **Type Mapping** - Automatic Salesforce to PyArrow type conversion
+
+## Installation
+
+### Core (Connection Only)
+```bash
+uv add aio-sf
+# or: pip install aio-sf
+```
+
+### With Export Capabilities
+```bash
+uv add "aio-sf[exporter]"
+# or: pip install "aio-sf[exporter]"
+```
+
+## Quick Start
+
+### Authentication & Connection
+```python
+import asyncio
+import os
+from aio_salesforce import SalesforceConnection, ClientCredentialsAuth
+
+async def main():
+    auth = ClientCredentialsAuth(
+        client_id=os.getenv('SF_CLIENT_ID'),
+        client_secret=os.getenv('SF_CLIENT_SECRET'),
+        instance_url=os.getenv('SF_INSTANCE_URL'),
+    )
+    
+    async with SalesforceConnection(auth_strategy=auth) as sf:
+        print(f"✅ Connected to: {sf.instance_url}")
+
+        sobjects = await sf.describe.list_sobjects()
+        print(sobjects[0]["name"])
+
+        contact_describe = await sf.describe.sobject("Contact")
+
+        # retrieve first 5 "creatable" fields on contact
+        queryable_fields = [
+            field.get("name", "")
+            for field in contact_describe["fields"]
+            if field.get("createable")
+        ][:5]
+
+        query = f"SELECT {', '.join(queryable_fields)} FROM Contact LIMIT 5"
+        print(query)
+
+        query_result = await sf.query.soql(query)
+        # Loop over records using async iteration
+        async for record in query_result:
+            print(record.get("AccountId"))
+
+asyncio.run(main())
+```
+
+
+
+
+## Exporter
+
+The Exporter library contains a streamlined and "opinionated" way to export data from Salesforce to various formats.  
+
+### 2. Query Records
+```python
+from aio_salesforce.exporter import bulk_query
+
+async def main():
+    # ... authentication code from above ...
+    
+    async with SalesforceConnection(auth_strategy=auth) as sf:
+        # Execute bulk query
+        query_result = await bulk_query(
+            sf=sf,
+            soql_query="SELECT Id, Name, Email FROM Contact LIMIT 1000"
+        )
+        
+        # Process records
+        count = 0
+        async for record in query_result:
+            print(f"Contact: {record['Name']} - {record['Email']}")
+            count += 1
+            
+        print(f"Processed {count} records")
+```
+
+### 3. Export to Parquet
+```python
+from aio_salesforce.exporter import bulk_query, write_query_to_parquet
+
+async def main():
+    # ... authentication code from above ...
+    
+    async with SalesforceConnection(auth_strategy=auth) as sf:
+        # Query with proper schema
+        query_result = await bulk_query(
+            sf=sf,
+            soql_query="SELECT Id, Name, Email, CreatedDate FROM Contact"
+        )
+        
+        # Export to Parquet
+        write_query_to_parquet(
+            query_result=query_result,
+            file_path="contacts.parquet"
+        )
+        
+        print(f"✅ Exported {len(query_result)} contacts to Parquet")
+```
+
+
+## License
+
+MIT License

aio_sf-0.1.0b1.dist-info/RECORD

@@ -0,0 +1,22 @@
+aio_salesforce/__init__.py,sha256=KChsKtzT6q_p_-kRf13pibWucHOSYF2hQYv1gHePq2A,688
+aio_salesforce/connection.py,sha256=-t02A45nekySuGaEwT2O_sHNzoxkYP6LzuR8fd5QjNs,18411
+aio_salesforce/api/README.md,sha256=T-LWAnhq-m_Cvsj7GVJZvBdiZk5n3MAmKwuWHlW84J8,3512
+aio_salesforce/api/__init__.py,sha256=kEA4UrX4DncSetO9s0agfKUnf0UxTzBfeYqP6T_SPEk,1328
+aio_salesforce/api/types.py,sha256=5-HcYJVD0-qskpUXCS5EwLuoCJzYOk-hy0KvnVlG3VE,6761
+aio_salesforce/api/bulk_v2/__init__.py,sha256=TxNM9dFmRX5k57Wj_JnposeEqqd9xcJx78wI-d6VH0o,315
+aio_salesforce/api/bulk_v2/client.py,sha256=x9Yd_OsGaOW4FGIHTCO0LB9wmoJPcm6JhSZyD4I9QFg,6878
+aio_salesforce/api/bulk_v2/types.py,sha256=18TN_VMisKJVCvo39q36moYXQhGLRUs73CKWIxBeAcs,1471
+aio_salesforce/api/describe/__init__.py,sha256=Gog7UMc1v474mCOmEhnds6zUXPor-fDLNgy4vsLfBos,528
+aio_salesforce/api/describe/client.py,sha256=9e3JVe35B_25UQ5RJ60NV7e3uTS_byME_MNpJjMApGI,3052
+aio_salesforce/api/describe/types.py,sha256=5-HcYJVD0-qskpUXCS5EwLuoCJzYOk-hy0KvnVlG3VE,6761
+aio_salesforce/api/query/__init__.py,sha256=OMh9g9PAPBFyQtUJc4aua1nKAiiIFYouZfaa5Zw4yZU,335
+aio_salesforce/api/query/client.py,sha256=CS9d73cSVdhry08os__nx6YoP3bM9uQqHZjdSkWVbhY,7164
+aio_salesforce/api/query/types.py,sha256=Wfk75kJpNDCGpTHonCbzjWvayy8guA3eyZp3hE7nBt0,845
+aio_salesforce/exporter/__init__.py,sha256=ae2q2t9F8mSalbM-UT-fAnLV67USL0gG2u4Jky7Ou6o,881
+aio_salesforce/exporter/bulk_export.py,sha256=pK2BNTL8WThLBbjWaHvzIJZoT7KdwfGi7GTguVSkShk,13563
+aio_salesforce/exporter/parquet_writer.py,sha256=BD191vKi9t7qvccCugaL7Sz8qpaKOZzF6BbKsfJ1chI,10620
+aio_salesforce/exporter/parquet_writer.py.backup,sha256=SyWRJyDaE9BIr8_mU2_El3x8b_lpt_Tvkebja0sq7kY,11465
+aio_sf-0.1.0b1.dist-info/METADATA,sha256=T0tMOIyhbVIqwS0v2omrWidDYD5FvhmKnr1sXFXOr1s,6720
+aio_sf-0.1.0b1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+aio_sf-0.1.0b1.dist-info/licenses/LICENSE,sha256=gu0Cbpiqs-vX7YgJJhGI1jH1mHup3dZMrZc-gmpEG60,1071
+aio_sf-0.1.0b1.dist-info/RECORD,,

aio_sf-0.1.0b1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

aio_sf-0.1.0b1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Callaway Cloud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.