parallex 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

parallex/ai/open_ai_client.py

@@ -10,14 +10,19 @@ from parallex.utils.logger import logger
 
 # Exceptions for missing keys, etc
 class OpenAIClient:
-    def __init__(self, model: str, remote_file_handler: RemoteFileHandler):
-        self.model = model
+    def __init__(
+        self,
+        remote_file_handler: RemoteFileHandler,
+        azure_endpoint_env_name: str,
+        azure_api_key_env_name: str,
+        azure_api_version_env_name: str,
+    ):
         self.file_handler = remote_file_handler
 
         self._client = AsyncAzureOpenAI(
-            azure_endpoint=os.getenv("AZURE_API_BASE"),
-            api_key=os.getenv("AZURE_API_KEY"),
-            api_version=os.getenv("AZURE_API_VERSION"),
+            azure_endpoint=os.getenv(azure_endpoint_env_name),
+            api_key=os.getenv(azure_api_key_env_name),
+            api_version=os.getenv(azure_api_version_env_name),
         )
 
     async def upload(self, file_path: str) -> FileObject:

parallex/ai/uploader.py

@@ -21,6 +21,7 @@ async def upload_images_for_processing(
     image_files: list[ImageFile],
     temp_directory: str,
     prompt_text: str,
+    azure_api_deployment_env_name: str,
     model: Optional[type[BaseModel]] = None,
 ) -> list[BatchFile]:
     """Base64 encodes image, converts to expected jsonl format and uploads"""
@@ -48,7 +49,13 @@ async def upload_images_for_processing(
         prompt_custom_id = (
             f"{image_file.trace_id}{CUSTOM_ID_DELINEATOR}{image_file.page_number}.jsonl"
         )
-        jsonl = _image_jsonl_format(prompt_custom_id, base64_encoded_image, prompt_text, model)
+        jsonl = _image_jsonl_format(
+            prompt_custom_id,
+            base64_encoded_image,
+            prompt_text,
+            azure_api_deployment_env_name,
+            model
+        )
         with open(upload_file_location, "a") as jsonl_file:
             jsonl_file.write(json.dumps(jsonl) + "\n")
     batch_file = await _create_batch_file(client, trace_id, upload_file_location)
@@ -60,7 +67,8 @@ async def upload_prompts_for_processing(
     client: OpenAIClient,
     prompts: list[str], temp_directory: str,
     trace_id: UUID,
-    model: Optional[type[BaseModel]] = None
+    azure_api_deployment_env_name: str,
+    model: Optional[type[BaseModel]] = None,
 ) -> list[BatchFile]:
     """Creates jsonl file and uploads for processing"""
     current_index = 0
@@ -81,7 +89,12 @@ async def upload_prompts_for_processing(
             )
 
         prompt_custom_id = f"{trace_id}{CUSTOM_ID_DELINEATOR}{index}.jsonl"
-        jsonl = _simple_jsonl_format(prompt_custom_id, prompt, model)
+        jsonl = _simple_jsonl_format(
+            prompt_custom_id,
+            prompt,
+            azure_api_deployment_env_name,
+            model
+        )
         with open(upload_file_location, "a") as jsonl_file:
             jsonl_file.write(json.dumps(jsonl) + "\n")
     batch_file = await _create_batch_file(client, trace_id, upload_file_location)
@@ -139,13 +152,18 @@ def _response_format(model: type[BaseModel]) -> dict:
     }
 
 
-def _simple_jsonl_format(prompt_custom_id: str, prompt_text: str, model: Optional[type[BaseModel]]) -> dict:
+def _simple_jsonl_format(
+    prompt_custom_id: str,
+    prompt_text: str,
+    azure_api_deployment_env_name: str,
+    model: Optional[type[BaseModel]]
+) -> dict:
     payload = {
         "custom_id": prompt_custom_id,
         "method": "POST",
         "url": "/chat/completions",
         "body": {
-            "model": os.getenv("AZURE_API_DEPLOYMENT"),
+            "model": os.getenv(azure_api_deployment_env_name),
             "messages": [{"role": "user", "content": prompt_text}],
             "temperature": 0.0, # TODO make configurable
         },
@@ -155,13 +173,19 @@ def _simple_jsonl_format(prompt_custom_id: str, prompt_text: str, model: Optiona
     return payload
 
 
-def _image_jsonl_format(prompt_custom_id: str, encoded_image: str, prompt_text: str, model: Optional[type[BaseModel]] = None) -> dict:
+def _image_jsonl_format(
+    prompt_custom_id: str,
+    encoded_image: str,
+    prompt_text: str,
+    azure_api_deployment_env_name: str,
+    model: Optional[type[BaseModel]] = None
+) -> dict:
     payload = {
         "custom_id": prompt_custom_id,
         "method": "POST",
         "url": "/chat/completions",
         "body": {
-            "model": os.getenv("AZURE_API_DEPLOYMENT"),
+            "model": os.getenv(azure_api_deployment_env_name),
             "messages": [
                 {
                     "role": "user",

parallex/parallex.py

@@ -35,10 +35,19 @@ async def parallex(
     prompt_text: Optional[str] = DEFAULT_PROMPT,
     log_level: Optional[str] = "ERROR",
     response_model: Optional[type[BaseModel]] = None,
+    azure_endpoint_env_name: Optional[str] = "AZURE_API_BASE",
+    azure_api_key_env_name: Optional[str] = "AZURE_API_KEY",
+    azure_api_version_env_name: Optional[str] = "AZURE_API_VERSION",
+    azure_api_deployment_env_name: Optional[str] = "AZURE_API_DEPLOYMENT",
 ) -> ParallexCallableOutput:
     setup_logger(log_level)
     remote_file_handler = RemoteFileHandler()
-    open_ai_client = OpenAIClient(model=model, remote_file_handler=remote_file_handler)
+    open_ai_client = OpenAIClient(
+        remote_file_handler=remote_file_handler,
+        azure_endpoint_env_name=azure_endpoint_env_name,
+        azure_api_key_env_name=azure_api_key_env_name,
+        azure_api_version_env_name=azure_api_version_env_name,
+    )
     try:
         return await _execute(
             open_ai_client=open_ai_client,
@@ -46,6 +55,7 @@ async def parallex(
             post_process_callable=post_process_callable,
             concurrency=concurrency,
             prompt_text=prompt_text,
+            azure_api_deployment_env_name=azure_api_deployment_env_name,
             model=response_model
         )
     except Exception as e:
@@ -56,16 +66,24 @@ async def parallex(
 
 
 async def parallex_simple_prompts(
-    model: str,
     prompts: list[str],
     post_process_callable: Optional[Callable[..., None]] = None,
     log_level: Optional[str] = "ERROR",
     concurrency: Optional[int] = 20,
     response_model: Optional[type[BaseModel]] = None,
+    azure_endpoint_env_name: Optional[str] = "AZURE_API_BASE",
+    azure_api_key_env_name: Optional[str] = "AZURE_API_KEY",
+    azure_api_version_env_name: Optional[str] = "AZURE_API_VERSION",
+    azure_api_deployment_env_name: Optional[str] = "AZURE_API_DEPLOYMENT",
 ) -> ParallexPromptsCallableOutput:
     setup_logger(log_level)
     remote_file_handler = RemoteFileHandler()
-    open_ai_client = OpenAIClient(model=model, remote_file_handler=remote_file_handler)
+    open_ai_client = OpenAIClient(
+        remote_file_handler=remote_file_handler,
+        azure_endpoint_env_name=azure_endpoint_env_name,
+        azure_api_key_env_name=azure_api_key_env_name,
+        azure_api_version_env_name=azure_api_version_env_name,
+    )
     try:
         return await _prompts_execute(
             open_ai_client=open_ai_client,
@@ -73,6 +91,7 @@ async def parallex_simple_prompts(
             post_process_callable=post_process_callable,
             concurrency=concurrency,
             model=response_model,
+            azure_api_deployment_env_name=azure_api_deployment_env_name
         )
     except Exception as e:
         logger.error(f"Error occurred: {e}")
@@ -84,6 +103,7 @@ async def parallex_simple_prompts(
 async def _prompts_execute(
     open_ai_client: OpenAIClient,
     prompts: list[str],
+    azure_api_deployment_env_name: str,
     post_process_callable: Optional[Callable[..., None]] = None,
     concurrency: Optional[int] = 20,
     model: Optional[type[BaseModel]] = None,
@@ -95,6 +115,7 @@ async def _prompts_execute(
             prompts=prompts,
             temp_directory=temp_directory,
             trace_id=trace_id,
+            azure_api_deployment_env_name=azure_api_deployment_env_name,
             model=model,
         )
         start_batch_semaphore = asyncio.Semaphore(concurrency)
@@ -139,6 +160,7 @@ async def _prompts_execute(
 async def _execute(
     open_ai_client: OpenAIClient,
     pdf_source_url: str,
+    azure_api_deployment_env_name: str,
     post_process_callable: Optional[Callable[..., None]] = None,
     concurrency: Optional[int] = 20,
     prompt_text: Optional[str] = DEFAULT_PROMPT,
@@ -158,6 +180,8 @@ async def _execute(
             image_files=image_files,
             temp_directory=temp_directory,
             prompt_text=prompt_text,
+            model=model,
+            azure_api_deployment_env_name=azure_api_deployment_env_name
         )
         start_batch_semaphore = asyncio.Semaphore(concurrency)
         start_batch_tasks = []

{parallex-0.4.0.dist-info → parallex-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: parallex
-Version: 0.4.0
+Version: 0.5.0
 Summary: PDF to markdown using Azure OpenAI batch processing
 Home-page: https://github.com/Summed-AI/parallex
 Author: Jeff Hostetler

{parallex-0.4.0.dist-info → parallex-0.5.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
 parallex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parallex/ai/batch_processor.py,sha256=O5q_jaIU0VI93p7Riq4aZ_qUiN9Omxp5GOfn0IqEYgo,1361
-parallex/ai/open_ai_client.py,sha256=TRH78oYod_EWpp3hjEh097OT7hwsQmtv44_j3X9Frxo,2047
+parallex/ai/open_ai_client.py,sha256=Nkl8F4EaHQbkNtgOXjLAtynWr10w9Q1Ym3TDisjtIiw,2168
 parallex/ai/output_processor.py,sha256=kd50DwB2txhzz4_MPYl97bPOtLMl0KV2UP_eFmUtq34,2087
-parallex/ai/uploader.py,sha256=FKleSK8GWextqpUUAthvTtxGHSwN-aYF127t1YmGOcw,6375
+parallex/ai/uploader.py,sha256=t_R-3FMX3OVo90EQRCGL0VqBn3vKKn5iUe3qoWVIbMM,6772
 parallex/file_management/converter.py,sha256=Rj-93LXNl2gCY-XUOCZv7DdCNI2-GyRpS5FobnTqwzo,1111
 parallex/file_management/file_finder.py,sha256=BPvrkxZlwOYmRXzzS138wGTsVzuhDIKfQZn0CISUj3o,1598
 parallex/file_management/remote_file_handler.py,sha256=jsI9NhOrKQR8K3yo536lGplVBGis9XY0G4dRpumgWFM,213
@@ -15,10 +15,10 @@ parallex/models/parallex_prompts_callable_output.py,sha256=IlNX9627_E8aXWQ-vDBuv
 parallex/models/prompt_response.py,sha256=2Zmnwlj8Ou2VgEHmi1VZrlnv5XRzw5VLMEkpQ1VelQQ,242
 parallex/models/raw_file.py,sha256=Nlv6u_jlDCXDgU2_Ff7DRbDCx27pB1NZugNhEoaBMQU,483
 parallex/models/upload_batch.py,sha256=jrnds9ryXg9drL4TF8TGimMVTCDfKaWsBzFv_ed0i88,2068
-parallex/parallex.py,sha256=JogDmjB-HdsauCis6hyfSBF_tQi2IdmXfltK72roi28,9322
+parallex/parallex.py,sha256=uP36YPJkWhaSgfrXcOLprea2W-9ZwQ-MXmU7liE-aKk,10591
 parallex/utils/constants.py,sha256=508ieZLZ5kse0T4_QyNJp57Aq0DMNFjjyFlsKa0xtek,366
 parallex/utils/logger.py,sha256=i3ZZ7YTUmhUStbvVME67F9ffnkLOv5ijm7wVUyJT8Ys,440
-parallex-0.4.0.dist-info/LICENSE,sha256=wPwCqGrisXnEcpaUxSO79C2mdOUTbtjhLjyy8mVW6p8,1046
-parallex-0.4.0.dist-info/METADATA,sha256=Hdq1xbDWVVPhR-61O88E9Glv7rn3LzKfz72--rzJovo,4461
-parallex-0.4.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-parallex-0.4.0.dist-info/RECORD,,
+parallex-0.5.0.dist-info/LICENSE,sha256=wPwCqGrisXnEcpaUxSO79C2mdOUTbtjhLjyy8mVW6p8,1046
+parallex-0.5.0.dist-info/METADATA,sha256=0Mm0BYWvEGpYr5SNBkQw2qLjI2TGKWRVUDVw_e8XyMo,4461
+parallex-0.5.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+parallex-0.5.0.dist-info/RECORD,,