unstructured-platform-plugins 0.0.3__tar.gz → 0.0.6__tar.gz

{unstructured-platform-plugins-0.0.3/unstructured_platform_plugins.egg-info → unstructured-platform-plugins-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-platform-plugins
-Version: 0.0.3
+Version: 0.0.6
 Summary: A utility library that provides tools around unstructured plugin development
 Home-page: https://github.com/Unstructured-IO/unstructured-platform-plugins
 License: Apache-2.0

{unstructured-platform-plugins-0.0.3 → unstructured-platform-plugins-0.0.6}/test/test_schema.py

@@ -566,6 +566,10 @@ def test_file_data():
                         "items": {"key": {"type": "string"}, "value": {}},
                     },
                     "reprocess": {"type": "boolean", "default": False},
+                    "local_download_path": {
+                        "anyOf": [{"type": "string"}, {"type": "null"}],
+                        "default": None,
+                    },
                 },
                 "required": ["identifier", "connector_type", "metadata", "additional_metadata"],
             }
@@ -659,6 +663,10 @@ def test_file_data():
                     "items": {"key": {"type": "string"}, "value": {}},
                 },
                 "reprocess": {"type": "boolean", "default": False},
+                "local_download_path": {
+                    "anyOf": [{"type": "string"}, {"type": "null"}],
+                    "default": None,
+                },
             },
             "required": ["identifier", "connector_type", "metadata", "additional_metadata"],
         },

unstructured-platform-plugins-0.0.6/unstructured_platform_plugins/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.0.6"  # pragma: no cover

{unstructured-platform-plugins-0.0.3 → unstructured-platform-plugins-0.0.6}/unstructured_platform_plugins/etl_uvicorn/api_generator.py

@@ -3,12 +3,15 @@ import hashlib
 import inspect
 import json
 import logging
+from functools import partial
 from typing import Any, Callable, Optional
 
 from fastapi import FastAPI, status
+from fastapi.responses import StreamingResponse
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from pydantic import BaseModel
 from starlette.responses import RedirectResponse
+from uvicorn.config import LOG_LEVELS
 from uvicorn.importer import import_from_string
 
 from unstructured_platform_plugins.etl_uvicorn.otel import get_metric_provider, get_trace_provider
@@ -28,12 +31,28 @@ from unstructured_platform_plugins.schema.usage import UsageData
 logger = logging.getLogger("uvicorn.error")
 
 
+def log_func_and_body(func: Callable, body: Optional[str] = None) -> None:
+    msg = None
+    if logger.level == LOG_LEVELS.get("debug", logging.NOTSET):
+        if not body:
+            msg = f"invoking function without inputs: {func.__name__}"
+        else:
+            msg = f"invoking function {func.__name__} with body"
+    elif logger.level == LOG_LEVELS.get("trace", logging.NOTSET):
+        if not body:
+            msg = f"invoking function without inputs: {func}"
+        else:
+            msg = f"invoking function {func} with body: {body}"
+    if msg:
+        logger.log(level=logger.level, msg=msg)
+
+
 async def invoke_func(func: Callable, kwargs: Optional[dict[str, Any]] = None) -> Any:
     kwargs = kwargs or {}
     if inspect.iscoroutinefunction(func):
         return await func(**kwargs)
     else:
-        return func(**kwargs)
+        return await asyncio.get_event_loop().run_in_executor(None, partial(func, **kwargs))
 
 
 def check_precheck_func(precheck_func: Callable):
@@ -92,7 +111,9 @@ def generate_fast_api(
 
     logging.getLogger("etl_uvicorn.fastapi")
 
-    async def wrap_fn(func: Callable, kwargs: Optional[dict[str, Any]] = None) -> InvokeResponse:
+    ResponseType = StreamingResponse if inspect.isasyncgenfunction(func) else InvokeResponse
+
+    async def wrap_fn(func: Callable, kwargs: Optional[dict[str, Any]] = None) -> ResponseType:
         usage: list[UsageData] = []
         request_dict = kwargs if kwargs else {}
         if "usage" in inspect.signature(func).parameters:
@@ -100,8 +121,19 @@ def generate_fast_api(
         else:
             logger.warning("usage data not an expected parameter, omitting")
         try:
-            output = await invoke_func(func=func, kwargs=request_dict)
-            return InvokeResponse(usage=usage, status_code=status.HTTP_200_OK, output=output)
+            if inspect.isasyncgenfunction(func):
+                # Stream response if function is an async generator
+
+                async def _stream_response():
+                    async for output in func(**(request_dict or {})):
+                        yield InvokeResponse(
+                            usage=usage, status_code=status.HTTP_200_OK, output=output
+                        ).model_dump_json() + "\n"
+
+                return StreamingResponse(_stream_response(), media_type="application/x-ndjson")
+            else:
+                output = await invoke_func(func=func, kwargs=request_dict)
+                return InvokeResponse(usage=usage, status_code=status.HTTP_200_OK, output=output)
         except Exception as invoke_error:
             logger.error(f"failed to invoke plugin: {invoke_error}", exc_info=True)
             return InvokeResponse(
@@ -114,19 +146,20 @@ def generate_fast_api(
     if input_schema_model.model_fields:
 
         @fastapi_app.post("/invoke", response_model=InvokeResponse)
-        async def run_job(request: input_schema_model) -> InvokeResponse:
-            logger.debug(f"invoking function {func} with input: {request.model_dump()}")
+        async def run_job(request: input_schema_model) -> ResponseType:
+            log_func_and_body(func=func, body=request.json())
             # Create dictionary from pydantic model while preserving underlying types
             request_dict = {f: getattr(request, f) for f in request.model_fields}
             map_inputs(func=func, raw_inputs=request_dict)
-            logger.debug(f"passing inputs to function: {request_dict}")
+            if logger.level == LOG_LEVELS.get("trace", logging.NOTSET):
+                logger.log(level=logger.level, msg=f"passing inputs to function: {request_dict}")
             return await wrap_fn(func=func, kwargs=request_dict)
 
     else:
 
         @fastapi_app.post("/invoke", response_model=InvokeResponse)
-        async def run_job() -> InvokeResponse:
-            logger.debug(f"invoking function without inputs: {func}")
+        async def run_job() -> ResponseType:
+            log_func_and_body(func=func)
             return await wrap_fn(
                 func=func,
             )

{unstructured-platform-plugins-0.0.3 → unstructured-platform-plugins-0.0.6/unstructured_platform_plugins.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-platform-plugins
-Version: 0.0.3
+Version: 0.0.6
 Summary: A utility library that provides tools around unstructured plugin development
 Home-page: https://github.com/Unstructured-IO/unstructured-platform-plugins
 License: Apache-2.0

unstructured-platform-plugins-0.0.3/unstructured_platform_plugins/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.0.3"  # pragma: no cover

{unstructured-platform-plugins-0.0.3 → unstructured-platform-plugins-0.0.6}/unstructured_platform_plugins.egg-info/requires.txt

@@ -1,8 +1,8 @@
-opentelemetry-instrumentation-fastapi
-uvicorn
 unstructured-ingest
+opentelemetry-instrumentation-fastapi
 opentelemetry-exporter-otlp-proto-grpc
+click
+uvicorn
 fastapi
 click
 requests
-click