julee 0.1.0__py3-none-any.whl

julee/worker.py

@@ -0,0 +1,211 @@
+"""
+Temporal worker for julee domain workflows and activities.
+
+This worker runs workflows and activities for document processing,
+assembly, and knowledge service operations within the julee domain.
+"""
+
+import asyncio
+import logging
+import os
+from temporalio.client import Client
+from temporalio.service import RPCError
+from temporalio.worker import Worker
+from julee.util.repos.temporal.data_converter import temporal_data_converter
+
+from julee.workflows import (
+    ExtractAssembleWorkflow,
+    ValidateDocumentWorkflow,
+)
+from julee.repositories.temporal.activities import (
+    TemporalMinioAssemblyRepository,
+    TemporalMinioAssemblySpecificationRepository,
+    TemporalMinioDocumentRepository,
+    TemporalMinioKnowledgeServiceConfigRepository,
+    TemporalMinioKnowledgeServiceQueryRepository,
+    TemporalMinioPolicyRepository,
+    TemporalMinioDocumentPolicyValidationRepository,
+)
+from julee.services.temporal.activities import (
+    TemporalKnowledgeService,
+)
+from minio import Minio
+from julee.repositories.minio.client import MinioClient
+from julee.util.temporal.activities import collect_activities_from_instances
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging() -> None:
+    """Configure logging based on environment variables"""
+    log_level = os.environ.get("LOG_LEVEL", "INFO").upper()
+    log_format = os.environ.get(
+        "LOG_FORMAT", "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+
+    # Validate log level
+    numeric_level = getattr(logging, log_level, None)
+    if not isinstance(numeric_level, int):
+        print(f"Invalid log level: {log_level}, defaulting to INFO")
+        numeric_level = logging.INFO
+
+    logging.basicConfig(
+        level=numeric_level,
+        format=log_format,
+        force=True,  # Override any existing configuration
+    )
+
+    logger.info(
+        "Logging configured",
+        extra={"log_level": log_level, "numeric_level": numeric_level},
+    )
+
+
+async def get_temporal_client_with_retries(
+    endpoint: str, attempts: int = 10, delay: int = 5
+) -> Client:
+    """Attempt to connect to Temporal with retries."""
+    logger.debug(
+        "Attempting to connect to Temporal",
+        extra={
+            "endpoint": endpoint,
+            "max_attempts": attempts,
+            "delay_seconds": delay,
+        },
+    )
+
+    for attempt in range(attempts):
+        try:
+            # Use the proper Pydantic v2 data converter and connect to the
+            # 'default' namespace
+            client = await Client.connect(
+                endpoint,
+                data_converter=temporal_data_converter,
+                namespace="default",
+            )
+            logger.info(
+                "Successfully connected to Temporal",
+                extra={
+                    "endpoint": endpoint,
+                    "attempt": attempt + 1,
+                    "data_converter_type": type(client.data_converter).__name__,
+                },
+            )
+            return client
+        except RPCError as e:
+            logger.warning(
+                "Failed to connect to Temporal",
+                extra={
+                    "endpoint": endpoint,
+                    "attempt": attempt + 1,
+                    "max_attempts": attempts,
+                    "error": str(e),
+                    "retry_in_seconds": delay,
+                },
+            )
+            if attempt + 1 == attempts:
+                logger.error(
+                    "All connection attempts to Temporal failed",
+                    extra={"endpoint": endpoint, "total_attempts": attempts},
+                )
+                raise
+            await asyncio.sleep(delay)
+
+    # This should never be reached due to the raise in the loop, but mypy
+    # needs it
+    raise RuntimeError("Failed to connect to Temporal after all attempts")
+
+
+async def run_worker() -> None:
+    """Run the Temporal worker for julee domain"""
+    # Setup logging first
+    setup_logging()
+
+    # Connect to Temporal server using environment variable
+    temporal_endpoint = os.environ.get("TEMPORAL_ENDPOINT", "localhost:7234")
+    logger.info(
+        "Starting julee Temporal worker",
+        extra={"temporal_endpoint": temporal_endpoint},
+    )
+
+    client = await get_temporal_client_with_retries(temporal_endpoint)
+
+    # Get Minio endpoint and create client for repositories
+    logger.debug("Preparing repository configurations")
+    minio_endpoint = os.environ.get("MINIO_ENDPOINT", "localhost:9000")
+
+    # Create Minio client for repositories
+    # minio.Minio implements the MinioClient protocol
+    minio_client: MinioClient = Minio(  # type: ignore[assignment]
+        endpoint=minio_endpoint,
+        access_key="minioadmin",
+        secret_key="minioadmin",
+        secure=False,
+    )
+
+    # Instantiate temporal repository classes for activity registration
+    logger.debug("Creating Temporal Activity repository implementations")
+    temporal_assembly_repo = TemporalMinioAssemblyRepository(client=minio_client)
+    temporal_assembly_spec_repo = TemporalMinioAssemblySpecificationRepository(
+        client=minio_client
+    )
+    temporal_document_repo = TemporalMinioDocumentRepository(client=minio_client)
+    temporal_knowledge_config_repo = TemporalMinioKnowledgeServiceConfigRepository(
+        client=minio_client
+    )
+    temporal_knowledge_query_repo = TemporalMinioKnowledgeServiceQueryRepository(
+        client=minio_client
+    )
+
+    # Create policy repositories for validation workflow
+    temporal_policy_repo = TemporalMinioPolicyRepository(client=minio_client)
+    temporal_document_policy_validation_repo = (
+        TemporalMinioDocumentPolicyValidationRepository(client=minio_client)
+    )
+
+    # Create temporal knowledge service for activity registration
+    # Pass the document repository for dependency injection
+    temporal_knowledge_service = TemporalKnowledgeService(
+        document_repo=temporal_document_repo
+    )
+
+    # Automatically collect all activities from decorated instances
+    # This uses the same _discover_protocol_methods that the decorator uses,
+    # ensuring we never miss activities and eliminating boilerplate
+    activities = collect_activities_from_instances(
+        temporal_assembly_repo,
+        temporal_assembly_spec_repo,
+        temporal_document_repo,
+        temporal_knowledge_config_repo,
+        temporal_knowledge_query_repo,
+        temporal_policy_repo,
+        temporal_document_policy_validation_repo,
+        temporal_knowledge_service,
+    )
+
+    logger.info(
+        "Creating Temporal worker for julee domain",
+        extra={
+            "task_queue": "julee-extract-assemble-queue",
+            "workflow_count": 2,
+            "activity_count": len(activities),
+            "data_converter_type": type(client.data_converter).__name__,
+        },
+    )
+
+    # Create worker with workflow retry policy
+    worker = Worker(
+        client,
+        task_queue="julee-extract-assemble-queue",
+        workflows=[ExtractAssembleWorkflow, ValidateDocumentWorkflow],
+        activities=activities,  # type: ignore[arg-type]
+    )
+
+    logger.info("Starting julee worker execution")
+
+    # Run the worker
+    await worker.run()
+
+
+if __name__ == "__main__":
+    asyncio.run(run_worker())

julee/workflows/__init__.py

@@ -0,0 +1,26 @@
+"""
+Temporal workflows for the julee domain.
+
+This package contains Temporal workflow definitions that orchestrate
+use cases with durability guarantees, retry logic, and state management.
+
+Workflows in this package:
+- ExtractAssembleWorkflow: Orchestrates document extraction and assembly
+- ValidateDocumentWorkflow: Orchestrates document validation against policies
+"""
+
+from .extract_assemble import (
+    ExtractAssembleWorkflow,
+    EXTRACT_ASSEMBLE_RETRY_POLICY,
+)
+from .validate_document import (
+    ValidateDocumentWorkflow,
+    VALIDATE_DOCUMENT_RETRY_POLICY,
+)
+
+__all__ = [
+    "ExtractAssembleWorkflow",
+    "EXTRACT_ASSEMBLE_RETRY_POLICY",
+    "ValidateDocumentWorkflow",
+    "VALIDATE_DOCUMENT_RETRY_POLICY",
+]

julee/workflows/extract_assemble.py

@@ -0,0 +1,215 @@
+"""
+Temporal workflow for extract and assemble data operations.
+
+This workflow orchestrates the ExtractAssembleDataUseCase with Temporal's
+durability guarantees, providing retry logic, state management, and
+compensation for the complex document assembly process.
+"""
+
+import logging
+from temporalio import workflow
+from temporalio.common import RetryPolicy
+from datetime import timedelta
+
+from julee.domain.models.assembly import Assembly
+from julee.domain.use_cases import ExtractAssembleDataUseCase
+from julee.repositories.temporal.proxies import (
+    WorkflowAssemblyRepositoryProxy,
+    WorkflowAssemblySpecificationRepositoryProxy,
+    WorkflowDocumentRepositoryProxy,
+    WorkflowKnowledgeServiceConfigRepositoryProxy,
+    WorkflowKnowledgeServiceQueryRepositoryProxy,
+)
+from julee.services.temporal.proxies import (
+    WorkflowKnowledgeServiceProxy,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@workflow.defn
+class ExtractAssembleWorkflow:
+    """
+    Temporal workflow for document extract and assemble operations.
+
+    This workflow:
+    1. Receives document_id and assembly_specification_id
+    2. Orchestrates the ExtractAssembleDataUseCase with workflow-safe proxies
+    3. Provides durability and retry logic for long-running assembly
+    4. Returns the completed Assembly object
+
+    The workflow remains framework-agnostic by delegating all business logic
+    to the use case, while providing Temporal-specific orchestration concerns
+    like retry policies, timeouts, and state management.
+    """
+
+    def __init__(self) -> None:
+        self.current_step = "initialized"
+        self.assembly_id: str | None = None
+
+    @workflow.query
+    def get_current_step(self) -> str:
+        """Query method to get the current workflow step"""
+        return self.current_step
+
+    @workflow.query
+    def get_assembly_id(self) -> str | None:
+        """Query method to get the assembly ID once created"""
+        return self.assembly_id
+
+    @workflow.run
+    async def run(self, document_id: str, assembly_specification_id: str) -> Assembly:
+        """
+        Execute the extract and assemble workflow.
+
+        Args:
+            document_id: ID of the document to assemble
+            assembly_specification_id: ID of the specification to use
+
+        Returns:
+            Completed Assembly object with assembled document
+
+        Raises:
+            ValueError: If required entities are not found
+            RuntimeError: If assembly processing fails after retries
+        """
+        workflow.logger.info(
+            "Starting extract assemble workflow",
+            extra={
+                "document_id": document_id,
+                "assembly_specification_id": assembly_specification_id,
+                "workflow_id": workflow.info().workflow_id,
+                "run_id": workflow.info().run_id,
+            },
+        )
+
+        self.current_step = "initializing_repositories"
+
+        try:
+            # Create workflow-safe repository proxies
+            # These proxy all calls through Temporal activities for durability
+            document_repo = WorkflowDocumentRepositoryProxy()  # type: ignore[abstract]
+            assembly_repo = WorkflowAssemblyRepositoryProxy()  # type: ignore[abstract]
+            assembly_specification_repo = (
+                WorkflowAssemblySpecificationRepositoryProxy()  # type: ignore[abstract]
+            )
+            knowledge_service_query_repo = (
+                WorkflowKnowledgeServiceQueryRepositoryProxy()  # type: ignore[abstract]
+            )
+            knowledge_service_config_repo = (
+                WorkflowKnowledgeServiceConfigRepositoryProxy()  # type: ignore[abstract]
+            )
+
+            workflow.logger.debug(
+                "Repository proxies created",
+                extra={
+                    "document_id": document_id,
+                    "assembly_specification_id": assembly_specification_id,
+                },
+            )
+
+            self.current_step = "creating_use_case"
+
+            # Create workflow-safe knowledge service proxy
+            knowledge_service = WorkflowKnowledgeServiceProxy()  # type: ignore[abstract]
+
+            # Create the use case with workflow-safe repositories
+            # The use case remains completely unaware it's running in workflow
+            use_case = ExtractAssembleDataUseCase(
+                document_repo=document_repo,
+                assembly_repo=assembly_repo,
+                assembly_specification_repo=assembly_specification_repo,
+                knowledge_service_query_repo=knowledge_service_query_repo,
+                knowledge_service_config_repo=knowledge_service_config_repo,
+                knowledge_service=knowledge_service,
+                now_fn=workflow.now,
+            )
+
+            workflow.logger.debug(
+                "Use case created successfully",
+                extra={
+                    "document_id": document_id,
+                    "assembly_specification_id": assembly_specification_id,
+                },
+            )
+
+            self.current_step = "executing_assembly"
+
+            # Execute the assembly process with workflow durability
+            # All repository calls inside the use case will be executed as
+            # Temporal activities with automatic retry and state persistence
+            assembly = await use_case.assemble_data(
+                document_id=document_id,
+                assembly_specification_id=assembly_specification_id,
+                workflow_id=workflow.info().workflow_id,
+            )
+
+            # Store the assembly ID for queries
+            self.assembly_id = assembly.assembly_id
+
+            self.current_step = "completed"
+
+            workflow.logger.info(
+                "Extract assemble workflow completed successfully",
+                extra={
+                    "document_id": document_id,
+                    "assembly_specification_id": assembly_specification_id,
+                    "assembly_id": assembly.assembly_id,
+                    "assembled_document_id": assembly.assembled_document_id,
+                    "status": assembly.status.value,
+                },
+            )
+
+            return assembly
+
+        except Exception as e:
+            self.current_step = "failed"
+
+            workflow.logger.error(
+                "Extract assemble workflow failed",
+                extra={
+                    "document_id": document_id,
+                    "assembly_specification_id": assembly_specification_id,
+                    "assembly_id": self.assembly_id,
+                    "error": str(e),
+                    "error_type": type(e).__name__,
+                },
+                exc_info=True,
+            )
+
+            # Re-raise to let Temporal handle retry logic
+            raise
+
+    @workflow.signal
+    async def cancel_assembly(self, reason: str) -> None:
+        """
+        Signal handler to cancel the assembly process.
+
+        Args:
+            reason: Reason for cancellation
+
+        Note:
+            This is a placeholder for future cancellation logic.
+            Currently, we rely on Temporal's built-in workflow cancellation.
+        """
+        workflow.logger.info(
+            "Assembly cancellation requested",
+            extra={
+                "assembly_id": self.assembly_id,
+                "reason": reason,
+                "current_step": self.current_step,
+            },
+        )
+
+        # Future: Implement graceful cancellation logic here
+        # For now, let the workflow be cancelled naturally by Temporal
+
+
+# Workflow configuration with retry policies optimized for document processing
+EXTRACT_ASSEMBLE_RETRY_POLICY = RetryPolicy(
+    initial_interval=timedelta(seconds=1),
+    backoff_coefficient=2.0,
+    maximum_interval=timedelta(minutes=5),
+    maximum_attempts=5,
+    non_retryable_error_types=["ValueError"],  # Don't retry validation errors
+)

julee/workflows/validate_document.py

@@ -0,0 +1,228 @@
+"""
+Temporal workflow for document validation operations.
+
+This workflow orchestrates the ValidateDocumentUseCase with Temporal's
+durability guarantees, providing retry logic, state management, and
+compensation for the document validation process.
+"""
+
+import logging
+from temporalio import workflow
+from temporalio.common import RetryPolicy
+from datetime import timedelta
+
+from julee.domain.models.policy import DocumentPolicyValidation
+from julee.domain.use_cases import ValidateDocumentUseCase
+from julee.repositories.temporal.proxies import (
+    WorkflowDocumentRepositoryProxy,
+    WorkflowKnowledgeServiceConfigRepositoryProxy,
+    WorkflowKnowledgeServiceQueryRepositoryProxy,
+)
+from julee.services.temporal.proxies import (
+    WorkflowKnowledgeServiceProxy,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@workflow.defn
+class ValidateDocumentWorkflow:
+    """
+    Temporal workflow for document validation operations.
+
+    This workflow:
+    1. Receives document_id and policy_id
+    2. Orchestrates the ValidateDocumentUseCase with workflow-safe proxies
+    3. Provides durability and retry logic for validation processing
+    4. Returns the completed DocumentPolicyValidation object
+
+    The workflow remains framework-agnostic by delegating all business logic
+    to the use case, while providing Temporal-specific orchestration concerns
+    like retry policies, timeouts, and state management.
+    """
+
+    def __init__(self) -> None:
+        self.current_step = "initialized"
+        self.validation_id: str | None = None
+
+    @workflow.query
+    def get_current_step(self) -> str:
+        """Query method to get the current workflow step"""
+        return self.current_step
+
+    @workflow.query
+    def get_validation_id(self) -> str | None:
+        """Query method to get the validation ID once created"""
+        return self.validation_id
+
+    @workflow.run
+    async def run(self, document_id: str, policy_id: str) -> DocumentPolicyValidation:
+        """
+        Execute the document validation workflow.
+
+        Args:
+            document_id: ID of the document to validate
+            policy_id: ID of the policy to validate against
+
+        Returns:
+            Completed DocumentPolicyValidation object with validation results
+
+        Raises:
+            ValueError: If required entities are not found
+            RuntimeError: If validation processing fails after retries
+        """
+        workflow.logger.info(
+            "Starting document validation workflow",
+            extra={
+                "document_id": document_id,
+                "policy_id": policy_id,
+                "workflow_id": workflow.info().workflow_id,
+                "run_id": workflow.info().run_id,
+            },
+        )
+
+        self.current_step = "initializing_repositories"
+
+        try:
+            # Create workflow-safe repository proxies
+            # These proxy all calls through Temporal activities for durability
+            document_repo = WorkflowDocumentRepositoryProxy()  # type: ignore[abstract]
+            knowledge_service_query_repo = (
+                WorkflowKnowledgeServiceQueryRepositoryProxy()  # type: ignore[abstract]
+            )
+            knowledge_service_config_repo = (
+                WorkflowKnowledgeServiceConfigRepositoryProxy()  # type: ignore[abstract]
+            )
+
+            workflow.logger.debug(
+                "Repository proxies created",
+                extra={
+                    "document_id": document_id,
+                    "policy_id": policy_id,
+                },
+            )
+
+            self.current_step = "creating_use_case"
+
+            # Create workflow-safe knowledge service proxy
+            knowledge_service = WorkflowKnowledgeServiceProxy()  # type: ignore[abstract]
+
+            # Import policy repository proxy (assuming it exists)
+            try:
+                from julee.repositories.temporal.proxies import (
+                    WorkflowPolicyRepositoryProxy,
+                    WorkflowDocumentPolicyValidationRepositoryProxy,
+                )
+
+                policy_repo = WorkflowPolicyRepositoryProxy()  # type: ignore[abstract]
+                document_policy_validation_repo = (
+                    WorkflowDocumentPolicyValidationRepositoryProxy()  # type: ignore[abstract]
+                )
+            except ImportError:
+                # Fallback if proxies don't exist yet
+                workflow.logger.warning(
+                    "Policy repository proxies not found, workflow may fail"
+                )
+                raise ValueError(
+                    "Policy repository proxies required for validation " "workflow"
+                )
+
+            # Create the use case with workflow-safe repositories
+            # The use case remains completely unaware it's running in workflow
+            use_case = ValidateDocumentUseCase(
+                document_repo=document_repo,
+                knowledge_service_query_repo=knowledge_service_query_repo,
+                knowledge_service_config_repo=knowledge_service_config_repo,
+                policy_repo=policy_repo,
+                document_policy_validation_repo=document_policy_validation_repo,
+                knowledge_service=knowledge_service,
+                now_fn=workflow.now,
+            )
+
+            workflow.logger.debug(
+                "Use case created successfully",
+                extra={
+                    "document_id": document_id,
+                    "policy_id": policy_id,
+                },
+            )
+
+            self.current_step = "executing_validation"
+
+            # Execute the validation process with workflow durability
+            # All repository calls inside the use case will be executed as
+            # Temporal activities with automatic retry and state persistence
+            validation = await use_case.validate_document(
+                document_id=document_id,
+                policy_id=policy_id,
+            )
+
+            # Store the validation ID for queries
+            self.validation_id = validation.validation_id
+
+            self.current_step = "completed"
+
+            workflow.logger.info(
+                "Document validation workflow completed successfully",
+                extra={
+                    "document_id": document_id,
+                    "policy_id": policy_id,
+                    "validation_id": validation.validation_id,
+                    "status": validation.status.value,
+                    "passed": validation.passed,
+                },
+            )
+
+            return validation
+
+        except Exception as e:
+            self.current_step = "failed"
+
+            workflow.logger.error(
+                "Document validation workflow failed",
+                extra={
+                    "document_id": document_id,
+                    "policy_id": policy_id,
+                    "validation_id": self.validation_id,
+                    "error": str(e),
+                    "error_type": type(e).__name__,
+                },
+                exc_info=True,
+            )
+
+            # Re-raise to let Temporal handle retry logic
+            raise
+
+    @workflow.signal
+    async def cancel_validation(self, reason: str) -> None:
+        """
+        Signal handler to cancel the validation process.
+
+        Args:
+            reason: Reason for cancellation
+
+        Note:
+            This is a placeholder for future cancellation logic.
+            Currently, we rely on Temporal's built-in workflow cancellation.
+        """
+        workflow.logger.info(
+            "Validation cancellation requested",
+            extra={
+                "validation_id": self.validation_id,
+                "reason": reason,
+                "current_step": self.current_step,
+            },
+        )
+
+        # Future: Implement graceful cancellation logic here
+        # For now, let the workflow be cancelled naturally by Temporal
+
+
+# Workflow configuration with retry policies optimized for document validation
+VALIDATE_DOCUMENT_RETRY_POLICY = RetryPolicy(
+    initial_interval=timedelta(seconds=1),
+    backoff_coefficient=2.0,
+    maximum_interval=timedelta(minutes=5),
+    maximum_attempts=3,
+    non_retryable_error_types=["ValueError"],  # Don't retry validation errors
+)