atlan-application-sdk 0.1.1rc39__py3-none-any.whl → 0.1.1rc41__py3-none-any.whl

application_sdk/outputs/json.py

@@ -93,6 +93,7 @@ class JsonOutput(Output):
         path_gen: Callable[[int | None, int], str] = path_gen,
         start_marker: Optional[str] = None,
         end_marker: Optional[str] = None,
+        retain_local_copy: bool = False,
         **kwargs: Dict[str, Any],
     ):
         """Initialize the JSON output handler.
@@ -113,6 +114,8 @@ class JsonOutput(Output):
                 Defaults to 0.
             path_gen (Callable, optional): Function to generate file paths.
                 Defaults to path_gen function.
+            retain_local_copy (bool, optional): Whether to retain the local copy of the files.
+                Defaults to False.
         """
         self.output_path = output_path
         self.output_suffix = output_suffix
@@ -133,6 +136,7 @@ class JsonOutput(Output):
         self.start_marker = start_marker
         self.end_marker = end_marker
         self.metrics = get_metrics()
+        self.retain_local_copy = retain_local_copy
 
         if not self.output_path:
             raise ValueError("output_path is required")
@@ -282,6 +286,7 @@ class JsonOutput(Output):
             await ObjectStore.upload_prefix(
                 source=self.output_path,
                 destination=get_object_store_prefix(self.output_path),
+                retain_local_copy=self.retain_local_copy,
             )
 
         except Exception as e:
@@ -367,6 +372,7 @@ class JsonOutput(Output):
                 await ObjectStore.upload_file(
                     source=output_file_name,
                     destination=get_object_store_prefix(output_file_name),
+                    retain_local_copy=self.retain_local_copy,
                 )
 
             self.buffer.clear()

application_sdk/outputs/parquet.py

@@ -60,6 +60,7 @@ class ParquetOutput(Output):
         chunk_start: Optional[int] = None,
         start_marker: Optional[str] = None,
         end_marker: Optional[str] = None,
+        retain_local_copy: bool = False,
     ):
         """Initialize the Parquet output handler.
 
@@ -79,6 +80,8 @@ class ParquetOutput(Output):
                 Defaults to None.
             end_marker (Optional[str], optional): End marker for query extraction.
                 Defaults to None.
+            retain_local_copy (bool, optional): Whether to retain the local copy of the files.
+                Defaults to False.
         """
         self.output_path = output_path
         self.output_suffix = output_suffix
@@ -99,6 +102,7 @@ class ParquetOutput(Output):
         self.end_marker = end_marker
         self.statistics = []
         self.metrics = get_metrics()
+        self.retain_local_copy = retain_local_copy
 
         # Create output directory
         self.output_path = os.path.join(self.output_path, self.output_suffix)
@@ -295,13 +299,19 @@ class ParquetOutput(Output):
             #  Upload the entire directory (contains multiple parquet files created by Daft)
             if write_mode == WriteMode.OVERWRITE:
                 # Delete the directory from object store
-                await ObjectStore.delete_prefix(
-                    prefix=get_object_store_prefix(self.output_path)
-                )
+                try:
+                    await ObjectStore.delete_prefix(
+                        prefix=get_object_store_prefix(self.output_path)
+                    )
+                except FileNotFoundError as e:
+                    logger.info(
+                        f"No files found under prefix {get_object_store_prefix(self.output_path)}: {str(e)}"
+                    )
 
             await ObjectStore.upload_prefix(
                 source=self.output_path,
                 destination=get_object_store_prefix(self.output_path),
+                retain_local_copy=self.retain_local_copy,
             )
 
         except Exception as e:

application_sdk/server/.cursor/BUGBOT.md

@@ -0,0 +1,442 @@
+# Server Code Review Guidelines - FastAPI Applications
+
+## Context-Specific Patterns
+
+This directory contains FastAPI server implementations, middleware, routers, and API endpoint definitions. These components handle HTTP requests, authentication, and API responses.
+
+### Phase 1: Critical Server Safety Issues
+
+**API Security Requirements:**
+
+- All endpoints must have proper input validation using Pydantic models
+- Authentication and authorization must be enforced on protected endpoints
+- No sensitive data in API responses (passwords, tokens, internal IDs)
+- Request rate limiting must be implemented for public endpoints
+- CORS configuration must be explicit and restrictive
+
+**Input Validation and Sanitization:**
+
+- All request bodies must use Pydantic models for validation
+- Query parameters must be validated with proper types
+- File uploads must have size and type restrictions
+- SQL injection prevention in any database queries
+- No raw user input in log messages
+
+```python
+# ✅ DO: Proper input validation
+from pydantic import BaseModel, Field, validator
+
+class CreateUserRequest(BaseModel):
+    username: str = Field(..., min_length=3, max_length=50, regex="^[a-zA-Z0-9_]+$")
+    email: str = Field(..., regex=r'^[\w\.-]+@[\w\.-]+\.\w+$')
+    age: int = Field(..., ge=18, le=120)
+
+    @validator('username')
+    def username_must_not_contain_prohibited_words(cls, v):
+        prohibited = ['admin', 'root', 'system']
+        if any(word in v.lower() for word in prohibited):
+            raise ValueError('Username contains prohibited words')
+        return v
+
+@app.post("/users/", response_model=UserResponse)
+async def create_user(user_data: CreateUserRequest):
+    # Input is already validated by Pydantic
+    return await user_service.create_user(user_data)
+
+# ❌ NEVER: Raw input without validation
+@app.post("/users/")
+async def bad_create_user(request: dict):  # No validation!
+    username = request.get("username")  # Could be anything
+    return await user_service.create_user(username)
+```
+
+### Phase 2: FastAPI Architecture Patterns
+
+**Router Organization:**
+
+- Group related endpoints in separate router modules
+- Use consistent URL patterns and naming conventions
+- Implement proper HTTP status codes for all responses
+- Use response models for all endpoint returns
+- Implement proper error handling with HTTP exceptions
+
+**Dependency Injection:**
+
+- Use FastAPI's dependency injection for database connections
+- Implement proper dependency scoping (request, application)
+- Create reusable dependencies for authentication, logging, etc.
+- Use dependency override for testing
+- Implement proper cleanup for dependencies
+
+**Async Pattern Enforcement:**
+
+- **Always use async/await for I/O operations**: Database queries, external API calls, file operations
+- **Non-blocking operations**: Ensure that async endpoints don't accidentally use blocking operations
+- **Proper context switching**: Use async context managers for resource management
+- **Background task usage**: Use FastAPI BackgroundTasks for non-critical operations that shouldn't block responses
+
+```python
+# ✅ DO: Proper async patterns
+from fastapi import BackgroundTasks
+
+@router.post("/users/", response_model=UserResponse)
+async def create_user_async(
+    user_data: CreateUserRequest,
+    background_tasks: BackgroundTasks,
+    db: AsyncConnection = Depends(get_async_db)
+) -> UserResponse:
+    """Create user with proper async patterns."""
+
+    # Main operation - blocking response
+    async with db.transaction():
+        user = await user_service.create_user_async(db, user_data)
+
+    # Non-critical operations in background (don't block response)
+    background_tasks.add_task(send_welcome_email, user.email)
+    background_tasks.add_task(update_analytics, "user_created")
+
+    return user
+
+# ❌ REJECT: Mixed async/sync patterns
+@router.post("/users/")
+async def bad_async_patterns(user_data: dict):
+    # Blocking database call in async function
+    user = sync_db_connection.execute(f"INSERT INTO users...")  # Blocks event loop
+
+    # Synchronous email sending that blocks response
+    email_client.send_email(user.email, "Welcome")  # Should be background task
+
+    return {"status": "created"}
+```
+
+```python
+# ✅ DO: Proper FastAPI router with dependencies
+from fastapi import APIRouter, Depends, HTTPException, status
+from typing import List
+
+router = APIRouter(prefix="/api/v1/users", tags=["users"])
+
+async def get_db_connection():
+    async with database_pool.acquire() as conn:
+        try:
+            yield conn
+        finally:
+            # Connection automatically returned to pool
+            pass
+
+async def get_current_user(token: str = Depends(oauth2_scheme)):
+    user = await auth_service.get_user_from_token(token)
+    if not user:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authentication credentials",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    return user
+
+@router.get("/{user_id}", response_model=UserResponse)
+async def get_user(
+    user_id: int,
+    current_user: User = Depends(get_current_user),
+    db: AsyncConnection = Depends(get_db_connection)
+):
+    if user_id != current_user.id and not current_user.is_admin:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Not authorized to access this user"
+        )
+
+    user = await user_service.get_user(db, user_id)
+    if not user:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="User not found"
+        )
+
+    return user
+```
+
+**Logging Standards:**
+
+- **Appropriate log levels**: Use correct log levels for different types of messages
+
+  - DEBUG: Development/debugging information
+  - INFO: General operational information, successful operations
+  - WARNING: Potentially problematic situations that don't prevent operation
+  - ERROR: Error conditions that may still allow operation to continue
+  - CRITICAL: Serious errors that may prevent program from continuing
+
+- **Context inclusion**: Include request IDs, user information, and operation context
+- **Structured logging**: Use consistent log formats that can be parsed by log aggregation tools
+- **No sensitive data**: Never log passwords, tokens, or personal information
+
+```python
+# ✅ DO: Proper logging levels and context
+import logging
+
+logger = logging.getLogger(__name__)
+
+@router.post("/users/{user_id}/reset-password")
+async def reset_password(user_id: int, request_id: str = Depends(get_request_id)):
+    """Reset user password with proper logging."""
+
+    # INFO: Normal operation
+    logger.info(f"Password reset requested for user {user_id}", extra={
+        "request_id": request_id,
+        "user_id": user_id,
+        "operation": "password_reset"
+    })
+
+    try:
+        await password_service.reset_password(user_id)
+
+        # INFO: Successful completion
+        logger.info(f"Password reset completed for user {user_id}", extra={
+            "request_id": request_id,
+            "user_id": user_id,
+            "status": "success"
+        })
+
+    except UserNotFoundError:
+        # WARNING: Expected error that doesn't prevent system operation
+        logger.warning(f"Password reset attempted for non-existent user {user_id}", extra={
+            "request_id": request_id,
+            "user_id": user_id,
+            "error_type": "user_not_found"
+        })
+        raise HTTPException(status_code=404, detail="User not found")
+
+    except DatabaseConnectionError as e:
+        # ERROR: Unexpected error that prevents operation but system can continue
+        logger.error(f"Database connection failed during password reset", extra={
+            "request_id": request_id,
+            "user_id": user_id,
+            "error": str(e),
+            "operation": "password_reset"
+        })
+        raise HTTPException(status_code=500, detail="Service temporarily unavailable")
+
+# ❌ REJECT: Inappropriate log levels and missing context
+@router.post("/users/login")
+async def bad_logging_example(credentials: dict):
+    logger.error("User login attempted")  # Should be INFO, not ERROR
+
+    if not credentials.get("username"):
+        logger.debug("Login failed - no username")  # Should be WARNING with context
+        return {"error": "Bad request"}
+
+    logger.critical("Processing login")  # Should be DEBUG or INFO, not CRITICAL
+
+    # No context, wrong levels, missing request tracking
+```
+
+### Phase 3: Server Testing Requirements
+
+**API Testing Standards:**
+
+- Use FastAPI's TestClient for endpoint testing
+- Test all HTTP status codes (success, client errors, server errors)
+- Test authentication and authorization scenarios
+- Test input validation with invalid data
+- Mock external dependencies in API tests
+- Include integration tests with real database
+
+**Request/Response Testing:**
+
+- Test request body validation with Pydantic models
+- Test query parameter validation
+- Test response model serialization
+- Test error response formats
+- Test file upload functionality
+- Include performance tests for API endpoints
+
+### Phase 4: Performance and Scalability
+
+**API Performance:**
+
+- Use async/await for all I/O operations
+- Implement proper database connection pooling
+- Use response caching where appropriate
+- Implement request batching for bulk operations
+- Monitor API response times and error rates
+
+**Middleware and Request Processing:**
+
+- Implement request logging middleware with correlation IDs
+- Use compression middleware for large responses
+- Implement proper timeout handling for long-running operations
+- Use background tasks for non-critical operations
+- Monitor memory usage and connection counts
+
+```python
+# ✅ DO: Efficient async endpoint with proper error handling
+@router.post("/users/bulk", response_model=List[UserResponse])
+async def create_users_bulk(
+    users_data: List[CreateUserRequest],
+    background_tasks: BackgroundTasks,
+    db: AsyncConnection = Depends(get_db_connection),
+    current_user: User = Depends(get_admin_user)
+):
+    if len(users_data) > 100:  # Prevent abuse
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Cannot create more than 100 users at once"
+        )
+
+    try:
+        # Batch operation for better performance
+        created_users = await user_service.create_users_batch(db, users_data)
+
+        # Non-critical operation in background
+        background_tasks.add_task(
+            send_welcome_emails,
+            [user.email for user in created_users]
+        )
+
+        return created_users
+
+    except ValidationError as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Validation failed: {e}"
+        )
+    except Exception as e:
+        logger.error(f"Bulk user creation failed: {e}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Internal server error"
+        )
+```
+
+### Phase 5: Server Maintainability
+
+**API Documentation and Versioning:**
+
+- Use OpenAPI tags for endpoint organization
+- Document all endpoints with proper descriptions
+- Implement API versioning strategy
+- Use response examples in OpenAPI documentation
+- Document all possible error responses
+
+**Configuration and Environment:**
+
+- Externalize all server configuration
+- Use environment-specific settings
+- Implement proper CORS configuration
+- Configure security headers
+- Document all configuration options
+
+---
+
+## Server-Specific Anti-Patterns
+
+**Always Reject:**
+
+- Endpoints without input validation
+- Missing authentication on protected endpoints
+- Raw dictionaries instead of Pydantic models
+- Generic exception handling without proper HTTP responses
+- Hardcoded configuration values
+- Missing CORS configuration
+- Endpoints without proper HTTP status codes
+- Blocking operations in async endpoints
+
+**Logging Anti-Patterns:**
+
+- **Wrong log levels**: Using ERROR for normal operations, DEBUG for production warnings
+- **Missing context**: Log messages without request IDs, user context, or operation details
+- **Sensitive data**: Logging passwords, tokens, personal information
+- **Inconsistent formats**: Different log formats that can't be parsed consistently
+
+**Async Pattern Anti-Patterns:**
+
+- **Blocking in async**: Using synchronous database calls or file operations in async endpoints
+- **Missing background tasks**: Long-running operations that block API responses
+- **Sync/async mixing**: Inconsistent use of async patterns within the same service
+
+**Input Validation Anti-Patterns:**
+
+```python
+# ❌ REJECT: No input validation
+@app.post("/users/")
+async def bad_endpoint(data: dict):  # No validation
+    username = data["username"]  # Could fail with KeyError
+    # No type checking, no sanitization
+    return {"status": "created"}
+
+# ✅ REQUIRE: Proper validation with Pydantic
+@app.post("/users/", response_model=UserResponse)
+async def good_endpoint(user_data: CreateUserRequest):
+    # Pydantic automatically validates input
+    validated_user = await user_service.create_user(user_data)
+    return validated_user
+```
+
+**Error Handling Anti-Patterns:**
+
+```python
+# ❌ REJECT: Poor error handling and logging
+@app.get("/users/{user_id}")
+async def bad_get_user(user_id: int):
+    logger.error(f"Getting user {user_id}")  # Wrong log level
+    try:
+        user = await user_service.get_user(user_id)
+        return user  # No response model
+    except Exception as e:
+        logger.debug(f"User lookup failed: {e}")  # Should be ERROR with context
+        return {"error": str(e)}  # Wrong HTTP status, exposes internals
+
+# ✅ REQUIRE: Proper error handling and logging
+@app.get("/users/{user_id}", response_model=UserResponse)
+async def good_get_user(user_id: int, request_id: str = Depends(get_request_id)):
+    logger.info(f"Retrieving user {user_id}", extra={"request_id": request_id})
+
+    try:
+        user = await user_service.get_user(user_id)
+        if not user:
+            logger.warning(f"User {user_id} not found", extra={
+                "request_id": request_id,
+                "user_id": user_id
+            })
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail="User not found"
+            )
+        return user
+    except ValidationError as e:
+        logger.warning(f"Invalid user ID format: {user_id}", extra={
+            "request_id": request_id,
+            "error": str(e)
+        })
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Invalid request: {e}"
+        )
+    except Exception as e:
+        logger.error(f"User retrieval failed for {user_id}: {e}", extra={
+            "request_id": request_id,
+            "user_id": user_id
+        }, exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Internal server error"
+        )
+```
+
+## Educational Context for Server Reviews
+
+When reviewing server code, emphasize:
+
+1. **Security Impact**: "API endpoints are the primary attack surface. Proper input validation and authentication aren't just good practices - they're essential for preventing data breaches and unauthorized access."
+
+2. **Performance Impact**: "Server performance directly affects user experience. Blocking operations in async endpoints can cause cascading slowdowns that affect all API users."
+
+3. **Reliability Impact**: "Proper error handling in APIs determines whether clients can gracefully handle failures or crash unexpectedly. Clear error responses help clients implement proper retry logic."
+
+4. **Maintainability Impact**: "Well-structured FastAPI applications with proper dependency injection and router organization make it easier for teams to add features and maintain the codebase as it grows."
+
+5. **Observability Impact**: "API logging and monitoring are critical for debugging production issues. Proper request correlation IDs and structured logging make the difference between quick problem resolution and extended outages."
+
+6. **Async Pattern Impact**: "Proper async patterns are essential for handling concurrent requests efficiently. Blocking operations in async code can degrade performance for all users and cause connection pool exhaustion."
+
+7. **Logging Quality Impact**: "Appropriate log levels and structured context are crucial for operational visibility. Wrong log levels create noise and hide real issues, while missing context makes debugging nearly impossible."

application_sdk/server/fastapi/__init__.py

@@ -1,3 +1,4 @@
+import os
 import time
 from typing import Any, Callable, List, Optional, Type
 
@@ -32,6 +33,7 @@ from application_sdk.server import ServerInterface
 from application_sdk.server.fastapi.middleware.logmiddleware import LogMiddleware
 from application_sdk.server.fastapi.middleware.metrics import MetricsMiddleware
 from application_sdk.server.fastapi.models import (
+    ConfigMapResponse,
     EventWorkflowRequest,
     EventWorkflowResponse,
     EventWorkflowTrigger,
@@ -95,6 +97,8 @@ class APIServer(ServerInterface):
     docs_directory_path: str = "docs"
     docs_export_path: str = "dist"
 
+    frontend_assets_path: str = "frontend/static"
+
     workflows: List[WorkflowInterface] = []
     event_triggers: List[EventWorkflowTrigger] = []
 
@@ -107,6 +111,7 @@ class APIServer(ServerInterface):
         workflow_client: Optional[WorkflowClient] = None,
         frontend_templates_path: str = "frontend/templates",
         ui_enabled: bool = True,
+        has_configmap: bool = False,
     ):
         """Initialize the FastAPI application.
 
@@ -121,6 +126,7 @@ class APIServer(ServerInterface):
         self.templates = Jinja2Templates(directory=frontend_templates_path)
         self.duckdb_ui = DuckDBUI()
         self.ui_enabled = ui_enabled
+        self.has_configmap = has_configmap
 
         # Create the FastAPI app using the renamed import
         if isinstance(lifespan, Callable):
@@ -177,6 +183,20 @@ class APIServer(ServerInterface):
         except Exception as e:
             logger.warning(str(e))
 
+    def frontend_home(self, request: Request) -> HTMLResponse:
+        frontend_html_path = os.path.join(
+            self.frontend_assets_path,
+            "index.html",
+        )
+
+        if not os.path.exists(frontend_html_path) or not self.has_configmap:
+            return self.fallback_home(request)
+
+        with open(frontend_html_path, "r", encoding="utf-8") as file:
+            contents = file.read()
+
+        return HTMLResponse(content=contents)
+
     def register_routers(self):
         """Register all routers with the FastAPI application.
 
@@ -195,7 +215,7 @@ class APIServer(ServerInterface):
         self.app.include_router(self.dapr_router, prefix="/dapr")
         self.app.include_router(self.events_router, prefix="/events/v1")
 
-    async def home(self, request: Request) -> HTMLResponse:
+    def fallback_home(self, request: Request) -> HTMLResponse:
         return self.templates.TemplateResponse(
             "index.html",
             {
@@ -328,7 +348,6 @@ class APIServer(ServerInterface):
             methods=["GET"],
             response_class=RedirectResponse,
         )
-
         self.workflow_router.add_api_route(
             "/auth",
             self.test_auth,
@@ -374,6 +393,13 @@ class APIServer(ServerInterface):
             methods=["POST"],
         )
 
+        self.workflow_router.add_api_route(
+            "/configmap/{config_map_id}",
+            self.get_configmap,
+            methods=["GET"],
+            response_model=ConfigMapResponse,
+        )
+
         self.dapr_router.add_api_route(
             "/subscribe",
             self.get_dapr_subscriptions,
@@ -390,7 +416,8 @@ class APIServer(ServerInterface):
 
     def register_ui_routes(self):
         """Register the UI routes for the FastAPI application."""
-        self.app.get("/")(self.home)
+        self.app.get("/")(self.frontend_home)
+
         # Mount static files
         self.app.mount("/", StaticFiles(directory="frontend/static"), name="static")
 
@@ -587,6 +614,35 @@ class APIServer(ServerInterface):
             )
             raise e
 
+    async def get_configmap(self, config_map_id: str) -> ConfigMapResponse:
+        """Get a configuration map by its ID.
+
+        Args:
+            config_map_id (str): The ID of the configuration map to retrieve.
+
+        Returns:
+            ConfigMapResponse: Response containing the configuration map.
+        """
+        try:
+            if not self.handler:
+                raise Exception("Handler not initialized")
+
+            # Call the getConfigmap method on the workflow class
+            config_map_data = await self.handler.get_configmap(config_map_id)
+
+            return ConfigMapResponse(
+                success=True,
+                message="Configuration map fetched successfully",
+                data=config_map_data,
+            )
+        except Exception as e:
+            logger.error(f"Error fetching configuration map: {e}")
+            return ConfigMapResponse(
+                success=False,
+                message=f"Failed to fetch configuration map: {str(e)}",
+                data={},
+            )
+
     async def get_workflow_config(
         self, config_id: str, type: str = "workflows"
     ) -> WorkflowConfigResponse:

application_sdk/server/fastapi/models.py

@@ -195,6 +195,33 @@ class WorkflowConfigResponse(BaseModel):
         }
 
 
+class ConfigMapResponse(BaseModel):
+    success: bool = Field(
+        ..., description="Indicates whether the operation was successful"
+    )
+    message: str = Field(
+        ..., description="Message describing the result of the operation"
+    )
+    data: Dict[str, Any] = Field(..., description="Configuration map object")
+
+    class Config:
+        schema_extra = {
+            "example": {
+                "success": True,
+                "message": "Configuration map fetched successfully",
+                "data": {
+                    "config_map_id": "pikachu-config-001",
+                    "name": "Pikachu Configuration",
+                    "settings": {
+                        "electric_type": True,
+                        "level": 25,
+                        "moves": ["Thunderbolt", "Quick Attack"],
+                    },
+                },
+            }
+        }
+
+
 class WorkflowTrigger(BaseModel):
     workflow_class: Optional[Type[WorkflowInterface]] = None
     model_config = {"arbitrary_types_allowed": True}