julee 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

julee/domain/use_cases/extract_assemble_data.py

@@ -10,8 +10,9 @@ instances following the Clean Architecture principles.
 import hashlib
 import json
 import logging
+from collections.abc import Callable
 from datetime import datetime, timezone
-from typing import Any, Callable, Dict
+from typing import Any
 
 import jsonpointer  # type: ignore
 import jsonschema
@@ -256,8 +257,8 @@ class ExtractAssembleDataUseCase:
     async def _register_document_with_services(
         self,
         document: Document,
-        queries: Dict[str, KnowledgeServiceQuery],
-    ) -> Dict[str, str]:
+        queries: dict[str, KnowledgeServiceQuery],
+    ) -> dict[str, str]:
         """
         Register the document with all knowledge services needed for assembly.
 
@@ -301,7 +302,7 @@ class ExtractAssembleDataUseCase:
     @try_use_case_step("queries_retrieval")
     async def _retrieve_all_queries(
         self, assembly_specification: AssemblySpecification
-    ) -> Dict[str, KnowledgeServiceQuery]:
+    ) -> dict[str, KnowledgeServiceQuery]:
         """Retrieve all knowledge service queries needed for this assembly."""
         query_ids = list(assembly_specification.knowledge_service_queries.values())
 
@@ -351,8 +352,8 @@ class ExtractAssembleDataUseCase:
         self,
         document: Document,
         assembly_specification: AssemblySpecification,
-        document_registrations: Dict[str, str],
-        queries: Dict[str, KnowledgeServiceQuery],
+        document_registrations: dict[str, str],
+        queries: dict[str, KnowledgeServiceQuery],
     ) -> str:
         """
         Perform a single assembly iteration using knowledge services.
@@ -379,7 +380,7 @@ class ExtractAssembleDataUseCase:
 
         """
         # Initialize the result data structure
-        assembled_data: Dict[str, Any] = {}
+        assembled_data: dict[str, Any] = {}
 
         # Process each knowledge service query
         # TODO: This is where we may want to fan-out/fan-in to do these
@@ -470,7 +471,7 @@ class ExtractAssembleDataUseCase:
         return document
 
     def _extract_schema_section(
-        self, jsonschema: Dict[str, Any], schema_pointer: str
+        self, jsonschema: dict[str, Any], schema_pointer: str
     ) -> Any:
         """Extract relevant section of JSON schema using JSON Pointer."""
         if not schema_pointer:
@@ -495,7 +496,7 @@ Please structure your response according to this JSON schema:
 Return only valid JSON that conforms to this schema, without any surrounding
 text or markdown formatting."""
 
-    def _parse_query_result(self, result_data: Dict[str, Any]) -> Any:
+    def _parse_query_result(self, result_data: dict[str, Any]) -> Any:
         """Parse the query result to extract the JSON response."""
         response_text = result_data.get("response", "")
         if not response_text:
@@ -514,7 +515,7 @@ text or markdown formatting."""
 
     def _store_result_in_assembled_data(
         self,
-        assembled_data: Dict[str, Any],
+        assembled_data: dict[str, Any],
         schema_pointer: str,
         result_data: Any,
     ) -> None:
@@ -572,7 +573,7 @@ text or markdown formatting."""
     @try_use_case_step("assembled_document_creation")
     async def _create_assembled_document(
         self,
-        assembled_data: Dict[str, Any],
+        assembled_data: dict[str, Any],
         assembly_specification: AssemblySpecification,
     ) -> str:
         """Create and store the assembled document."""
@@ -593,7 +594,7 @@ text or markdown formatting."""
             size_bytes=len(content_bytes),
             content_multihash=self._calculate_multihash_from_content(content_bytes),
             status=DocumentStatus.ASSEMBLED,
-            content_string=assembled_content,  # Use content_string for small
+            content_bytes=assembled_content,
             created_at=self.now_fn(),
             updated_at=self.now_fn(),
         )
@@ -605,7 +606,7 @@ text or markdown formatting."""
 
     def _validate_assembled_data(
         self,
-        assembled_data: Dict[str, Any],
+        assembled_data: dict[str, Any],
         assembly_specification: AssemblySpecification,
     ) -> None:
         """Validate that the assembled data conforms to the JSON schema."""

julee/domain/use_cases/initialize_system_data.py

@@ -13,10 +13,11 @@ The use case follows clean architecture principles:
 """
 
 import hashlib
+import json
 import logging
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any
 
 import yaml
 
@@ -192,7 +193,7 @@ class InitializeSystemDataUseCase:
             )
             raise
 
-    def _load_fixture_configurations(self) -> List[Dict[str, Any]]:
+    def _load_fixture_configurations(self) -> list[dict[str, Any]]:
         """
         Load knowledge service configurations from the YAML fixture file.
 
@@ -217,7 +218,7 @@ class InitializeSystemDataUseCase:
             )
 
         try:
-            with open(fixture_path, "r", encoding="utf-8") as f:
+            with open(fixture_path, encoding="utf-8") as f:
                 fixture_data = yaml.safe_load(f)
 
             if not fixture_data or "knowledge_services" not in fixture_data:
@@ -240,7 +241,7 @@ class InitializeSystemDataUseCase:
             raise yaml.YAMLError(f"Invalid YAML in fixture file: {e}")
 
     def _create_config_from_fixture_data(
-        self, config_data: Dict[str, Any]
+        self, config_data: dict[str, Any]
     ) -> KnowledgeServiceConfig:
         """
         Create a KnowledgeServiceConfig from fixture data.
@@ -363,7 +364,7 @@ class InitializeSystemDataUseCase:
             )
             raise
 
-    def _load_fixture_queries(self) -> List[Dict[str, Any]]:
+    def _load_fixture_queries(self) -> list[dict[str, Any]]:
         """
         Load knowledge service queries from the YAML fixture file.
 
@@ -388,7 +389,7 @@ class InitializeSystemDataUseCase:
             )
 
         try:
-            with open(fixture_path, "r", encoding="utf-8") as f:
+            with open(fixture_path, encoding="utf-8") as f:
                 fixture_data = yaml.safe_load(f)
 
             if not fixture_data or "knowledge_service_queries" not in fixture_data:
@@ -413,7 +414,7 @@ class InitializeSystemDataUseCase:
             raise yaml.YAMLError(f"Invalid YAML in queries fixture file: {e}")
 
     def _create_query_from_fixture_data(
-        self, query_data: Dict[str, Any]
+        self, query_data: dict[str, Any]
     ) -> KnowledgeServiceQuery:
         """
         Create a KnowledgeServiceQuery from fixture data.
@@ -533,9 +534,9 @@ class InitializeSystemDataUseCase:
             )
             raise
 
-    def _load_fixture_assembly_specifications(self) -> List[Dict[str, Any]]:
+    def _load_fixture_assembly_specifications(self) -> list[dict[str, Any]]:
         """
-        Load assembly specifications from the YAML fixture file.
+        Load assembly specifications from a YAML or JSON fixture file.
 
         Returns:
             List of specification dictionaries from the fixture file
@@ -543,23 +544,34 @@ class InitializeSystemDataUseCase:
         Raises:
             FileNotFoundError: If the fixture file doesn't exist
             yaml.YAMLError: If the fixture file is invalid YAML
+            json.JSONDecodeError: If the fixture file is invalid JSON
             KeyError: If required fields are missing from the fixture
+            ValueError: If the specification section is malformed
         """
-        fixture_path = self._get_demo_fixture_path("assembly_specifications.yaml")
+        # Accept both .yaml and .json files
+        fixture_path = None
+        for ext in ("json", "yaml"):
+            candidate = self._get_demo_fixture_path(f"assembly_specifications.{ext}")
+            if candidate.exists():
+                fixture_path = candidate
+                break
+
+        if fixture_path is None:
+            raise FileNotFoundError(
+                "Assembly specifications fixture file not found (.yaml or .json)"
+            )
 
         self.logger.debug(
             "Loading assembly specifications fixture file",
             extra={"fixture_path": str(fixture_path)},
         )
 
-        if not fixture_path.exists():
-            raise FileNotFoundError(
-                f"Assembly specifications fixture file not found: {fixture_path}"
-            )
-
         try:
-            with open(fixture_path, "r", encoding="utf-8") as f:
-                fixture_data = yaml.safe_load(f)
+            with open(fixture_path, encoding="utf-8") as f:
+                if fixture_path.suffix.lower() == ".json":
+                    fixture_data = json.load(f)
+                else:
+                    fixture_data = yaml.safe_load(f)
 
             if not fixture_data or "assembly_specifications" not in fixture_data:
                 raise KeyError(
@@ -569,8 +581,7 @@ class InitializeSystemDataUseCase:
             specs = fixture_data["assembly_specifications"]
             if not isinstance(specs, list):
                 raise ValueError(
-                    "'assembly_specifications' must be a list of "
-                    "specification configurations"
+                    "'assembly_specifications' must be a list of specification configurations"
                 )
 
             self.logger.debug(
@@ -585,8 +596,15 @@ class InitializeSystemDataUseCase:
                 f"Invalid YAML in assembly specifications fixture file: {e}"
             )
 
+        except json.JSONDecodeError as e:
+            raise json.JSONDecodeError(
+                f"Invalid JSON in assembly specifications fixture file: {e}",
+                e.doc,
+                e.pos,
+            )
+
     def _create_assembly_spec_from_fixture_data(
-        self, spec_data: Dict[str, Any]
+        self, spec_data: dict[str, Any]
     ) -> AssemblySpecification:
         """
         Create an AssemblySpecification from fixture data.
@@ -719,7 +737,7 @@ class InitializeSystemDataUseCase:
             )
             raise
 
-    def _load_fixture_documents(self) -> List[Dict[str, Any]]:
+    def _load_fixture_documents(self) -> list[dict[str, Any]]:
         """
         Load documents from the YAML fixture file.
 
@@ -742,7 +760,7 @@ class InitializeSystemDataUseCase:
             raise FileNotFoundError(f"Documents fixture file not found: {fixture_path}")
 
         try:
-            with open(fixture_path, "r", encoding="utf-8") as f:
+            with open(fixture_path, encoding="utf-8") as f:
                 fixture_data = yaml.safe_load(f)
 
             if not fixture_data or "documents" not in fixture_data:
@@ -764,7 +782,7 @@ class InitializeSystemDataUseCase:
         except yaml.YAMLError as e:
             raise yaml.YAMLError(f"Invalid YAML in documents fixture file: {e}")
 
-    def _create_document_from_fixture_data(self, doc_data: Dict[str, Any]) -> Document:
+    def _create_document_from_fixture_data(self, doc_data: dict[str, Any]) -> Document:
         """
         Create a Document from fixture data.
 
@@ -782,24 +800,62 @@ class InitializeSystemDataUseCase:
             "document_id",
             "original_filename",
             "content_type",
-            "content",
         ]
 
-        # Validate required fields
         for field in required_fields:
             if field not in doc_data:
                 raise KeyError(f"Required field '{field}' missing from document")
 
-        # Get content and calculate hash
-        content = doc_data["content"]
-        content_bytes = content.encode("utf-8")
-        size_bytes = len(content_bytes)
+        content_type = doc_data["content_type"]
+        is_text = content_type.startswith("text/") or content_type in {
+            "application/json",
+            "application/xml",
+            "application/javascript",
+        }
+
+        if "content" in doc_data:
+            content = doc_data["content"]
+
+            if isinstance(content, bytes):
+                content_bytes = content
+            elif isinstance(content, str):
+                content_bytes = content.encode("utf-8")
+            else:
+                raise TypeError(
+                    f"Unsupported type for 'content': {type(content)!r}. Expected str or bytes."
+                )
+        else:
+            current_file = Path(__file__)
+            julee_dir = current_file.parent.parent.parent
+            fixture_path = julee_dir / "fixtures" / doc_data["original_filename"]
 
-        # Create multihash (using SHA-256)
+            open_mode = "r" if is_text else "rb"
+            encoding = "utf-8" if is_text else None
+
+            try:
+                with fixture_path.open(open_mode, encoding=encoding) as f:
+                    content = f.read()
+            except FileNotFoundError as e:
+                self.logger.error(
+                    "Fixture file not found for document",
+                    extra={
+                        "document_id": doc_data["document_id"],
+                        "fixture_path": str(fixture_path),
+                    },
+                )
+                raise FileNotFoundError(
+                    f"Fixture file '{fixture_path}' not found for document "
+                    f"{doc_data['document_id']}"
+                ) from e
+
+            content_bytes = content.encode("utf-8") if is_text else content
+
+            self.logger.info(content_bytes)
+
+        size_bytes = len(content_bytes)
         sha256_hash = hashlib.sha256(content_bytes).hexdigest()
         content_multihash = f"sha256-{sha256_hash}"
 
-        # Parse status
         status = DocumentStatus.CAPTURED
         if "status" in doc_data:
             try:
@@ -809,12 +865,10 @@ class InitializeSystemDataUseCase:
                     f"Invalid status '{doc_data['status']}', using default 'captured'"
                 )
 
-        # Get optional fields
         knowledge_service_id = doc_data.get("knowledge_service_id")
         assembly_types = doc_data.get("assembly_types", [])
         additional_metadata = doc_data.get("additional_metadata", {})
 
-        # Create document
         document = Document(
             document_id=doc_data["document_id"],
             original_filename=doc_data["original_filename"],
@@ -827,7 +881,7 @@ class InitializeSystemDataUseCase:
             created_at=datetime.now(timezone.utc),
             updated_at=datetime.now(timezone.utc),
             additional_metadata=additional_metadata,
-            content_string=content,  # Store content as string for fixtures
+            content_bytes=content_bytes,
         )
 
         self.logger.debug(

julee/domain/use_cases/tests/test_extract_assemble_data.py

@@ -8,35 +8,35 @@ following the Clean Architecture principles.
 
 import io
 import json
-import pytest
-
-from unittest.mock import AsyncMock
 from datetime import datetime, timezone
+from unittest.mock import AsyncMock
+
+import pytest
 
-from julee.domain.use_cases import ExtractAssembleDataUseCase
 from julee.domain.models import (
     Assembly,
+    AssemblySpecification,
+    AssemblySpecificationStatus,
     AssemblyStatus,
+    ContentStream,
     Document,
     DocumentStatus,
-    ContentStream,
-    AssemblySpecification,
-    AssemblySpecificationStatus,
-    KnowledgeServiceQuery,
     KnowledgeServiceConfig,
+    KnowledgeServiceQuery,
 )
 from julee.domain.models.knowledge_service_config import ServiceApi
+from julee.domain.use_cases import ExtractAssembleDataUseCase
 from julee.repositories.memory import (
-    MemoryDocumentRepository,
     MemoryAssemblyRepository,
     MemoryAssemblySpecificationRepository,
+    MemoryDocumentRepository,
     MemoryKnowledgeServiceConfigRepository,
     MemoryKnowledgeServiceQueryRepository,
 )
+from julee.services.knowledge_service import QueryResult
 from julee.services.knowledge_service.memory import (
     MemoryKnowledgeService,
 )
-from julee.services.knowledge_service import QueryResult
 
 
 class TestExtractAssembleDataUseCase:

julee/domain/use_cases/tests/test_initialize_system_data.py

@@ -86,7 +86,7 @@ def fixture_configs() -> list[dict]:
 
     assert fixture_path.exists(), f"Fixture file not found: {fixture_path}"
 
-    with open(fixture_path, "r", encoding="utf-8") as f:
+    with open(fixture_path, encoding="utf-8") as f:
         fixture_data = yaml.safe_load(f)
 
     assert "knowledge_services" in fixture_data
@@ -320,7 +320,7 @@ class TestYamlFixtureIntegration:
         assert fixture_path.exists(), f"Fixture file not found: {fixture_path}"
 
         # Verify file can be parsed
-        with open(fixture_path, "r", encoding="utf-8") as f:
+        with open(fixture_path, encoding="utf-8") as f:
             fixture_data = yaml.safe_load(f)
 
         # Verify structure

julee/domain/use_cases/tests/test_validate_document.py

@@ -7,38 +7,38 @@ following the Clean Architecture principles.
 """
 
 import io
-import pytest
-from unittest.mock import AsyncMock
 from datetime import datetime, timezone
-from pydantic import ValidationError
+from unittest.mock import AsyncMock
 
-from julee.domain.use_cases import ValidateDocumentUseCase
+import pytest
+from pydantic import ValidationError
 
 from julee.domain.models import (
+    ContentStream,
     Document,
     DocumentStatus,
-    ContentStream,
     KnowledgeServiceConfig,
     KnowledgeServiceQuery,
 )
+from julee.domain.models.knowledge_service_config import ServiceApi
 from julee.domain.models.policy import (
-    Policy,
-    PolicyStatus,
     DocumentPolicyValidation,
     DocumentPolicyValidationStatus,
+    Policy,
+    PolicyStatus,
 )
-from julee.domain.models.knowledge_service_config import ServiceApi
+from julee.domain.use_cases import ValidateDocumentUseCase
 from julee.repositories.memory import (
+    MemoryDocumentPolicyValidationRepository,
     MemoryDocumentRepository,
-    MemoryKnowledgeServiceQueryRepository,
     MemoryKnowledgeServiceConfigRepository,
+    MemoryKnowledgeServiceQueryRepository,
     MemoryPolicyRepository,
-    MemoryDocumentPolicyValidationRepository,
 )
+from julee.services.knowledge_service import QueryResult
 from julee.services.knowledge_service.memory import (
     MemoryKnowledgeService,
 )
-from julee.services.knowledge_service import QueryResult
 
 
 class TestValidateDocumentUseCase:

julee/domain/use_cases/validate_document.py

@@ -11,8 +11,8 @@ import hashlib
 import io
 import json
 import logging
+from collections.abc import Callable
 from datetime import datetime
-from typing import Callable, Dict, List, Tuple
 
 import multihash
 
@@ -397,13 +397,13 @@ class ValidateDocumentUseCase:
     @try_use_case_step("all_queries_retrieval")
     async def _retrieve_all_queries(
         self, policy: Policy
-    ) -> Dict[str, KnowledgeServiceQuery]:
+    ) -> dict[str, KnowledgeServiceQuery]:
         """Retrieve all knowledge service queries needed for validation and
         transformation."""
         all_queries = {}
 
         # Get validation queries
-        for query_id, required_score in policy.validation_scores:
+        for query_id, _required_score in policy.validation_scores:
             query = await self.knowledge_service_query_repo.get(query_id)
             if not query:
                 raise ValueError(f"Validation query not found: {query_id}")
@@ -423,8 +423,8 @@ class ValidateDocumentUseCase:
     async def _register_document_with_services(
         self,
         document: Document,
-        queries: Dict[str, KnowledgeServiceQuery],
-    ) -> Dict[str, str]:
+        queries: dict[str, KnowledgeServiceQuery],
+    ) -> dict[str, str]:
         """
         Register the document with all knowledge services needed for
         validation.
@@ -464,9 +464,9 @@ class ValidateDocumentUseCase:
         self,
         document: Document,
         policy: Policy,
-        document_registrations: Dict[str, str],
-        queries: Dict[str, KnowledgeServiceQuery],
-    ) -> List[Tuple[str, int]]:
+        document_registrations: dict[str, str],
+        queries: dict[str, KnowledgeServiceQuery],
+    ) -> list[tuple[str, int]]:
         """
         Execute all validation queries and return the actual scores achieved.
 
@@ -528,7 +528,7 @@ class ValidateDocumentUseCase:
 
         return validation_scores
 
-    def _extract_score_from_result(self, result_data: Dict) -> int:
+    def _extract_score_from_result(self, result_data: dict) -> int:
         """
         Extract a numeric score from the knowledge service query result.
 
@@ -551,8 +551,8 @@ class ValidateDocumentUseCase:
 
     def _determine_validation_result(
         self,
-        actual_scores: List[Tuple[str, int]],
-        required_scores: List[Tuple[str, int]],
+        actual_scores: list[tuple[str, int]],
+        required_scores: list[tuple[str, int]],
     ) -> bool:
         """
         Determine if validation passed based on actual vs required scores.
@@ -591,8 +591,8 @@ class ValidateDocumentUseCase:
         self,
         document: Document,
         policy: Policy,
-        all_queries: Dict[str, KnowledgeServiceQuery],
-        document_registrations: Dict[str, str],
+        all_queries: dict[str, KnowledgeServiceQuery],
+        document_registrations: dict[str, str],
     ) -> Document:
         """
         Apply transformation queries to a document and return the
@@ -714,7 +714,7 @@ class ValidateDocumentUseCase:
 
         return transformed_document
 
-    def _extract_transformed_content(self, result_data: Dict) -> str:
+    def _extract_transformed_content(self, result_data: dict) -> str:
         """
         Extract transformed document content from knowledge service result.
 

julee/fixtures/documents.yaml

@@ -9,6 +9,10 @@
 # repository if they don't already exist (idempotent loading).
 
 documents:
+  - document_id: "product-spec-sheet"
+    original_filename: "Spec-Sheet-BondorPanel-v17.pdf"
+    content_type: "application/pdf"
+    status: "captured"
   - document_id: "meeting-transcript-q1-planning"
     original_filename: "q1_planning_meeting.txt"
     content_type: "text/plain"
@@ -19,49 +23,6 @@ documents:
       attendee_count: 3
       duration_minutes: 90
       department: "product"
-    content: |
-      Meeting Transcript - Q1 Planning Session
-      Date: March 15, 2024
-      Time: 2:00 PM - 3:30 PM
-      Attendees: Sarah Chen (Product Manager), Mike Rodriguez (Engineering Lead),
-      Lisa Wang (Designer)
-
-      Sarah: Thanks everyone for joining. Let's kick off our Q1 planning. Mike,
-      can you give us an update on the current sprint?
-
-      Mike: Sure, we're about 80% through sprint 23. We've completed the user
-      authentication module and are working on the data migration tool. Should be
-      done by Friday.
-
-      Lisa: Great! I've finished the mockups for the dashboard redesign. Sarah,
-      have you had a chance to review them?
-
-      Sarah: Yes, they look fantastic. I especially like the new navigation
-      structure. When can we start implementation?
-
-      Mike: I'd estimate 2 weeks for the frontend work, plus another week for
-      backend API changes.
-
-      Lisa: I can start on the component library updates while Mike works on the
-      APIs.
-
-      Sarah: Perfect. Let's also discuss the customer feedback integration. We had
-      47 responses to our survey.
-
-      Mike: The main requests were for better reporting and mobile optimization.
-
-      Sarah: Those should be our next priorities then. Lisa, can you start
-      sketching mobile designs?
-
-      Lisa: Absolutely. I'll have initial concepts by next Tuesday.
-
-      Sarah: Excellent. Any other items?
-
-      Mike: Just a heads up that we'll need to schedule downtime for the database
-      migration, probably next weekend.
-
-      Sarah: Noted. I'll coordinate with support. Meeting adjourned at 3:30 PM.
-
   - document_id: "customer-feedback-survey-q4"
     original_filename: "customer_survey_results_q4_2023.txt"
     content_type: "text/plain"

julee/fixtures/knowledge_service_queries.yaml

@@ -1,4 +1,13 @@
 knowledge_service_queries:
+  - query_id: "generate-dpp"
+    name: "Generate Digital Product Passport"
+    knowledge_service_id: "anthropic-4.5-as-a-knowledge-service"
+    prompt: "From this product specification sheet, extract the product information to generate a Digital Product Passport, that conforms to the provided schema, including the issuer, the credential subject and the validation dates. Please make sure that the DPP conforms to the provided schema and types and that you don't add any other fields."
+    assistant_prompt: "Looking at the product specification sheet, here's the digital product passport that conforms to the provided schema, without surrounding ```json ... ``` markers:"
+    query_metadata:
+      max_tokens: 3000
+      temperature: 0.1
+
   - query_id: "extract-meeting-info-query"
     name: "Extract Meeting Information"
     knowledge_service_id: "anthropic-4.5-as-a-knowledge-service"

julee/maintenance/__init__.py

@@ -0,0 +1 @@
+"""Maintenance utilities for julee and julee-based solutions."""