unstructured-platform-plugins 0.0.1__tar.gz → 0.0.3__tar.gz

{unstructured-platform-plugins-0.0.1/unstructured_platform_plugins.egg-info → unstructured-platform-plugins-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-platform-plugins
-Version: 0.0.1
+Version: 0.0.3
 Summary: A utility library that provides tools around unstructured plugin development
 Home-page: https://github.com/Unstructured-IO/unstructured-platform-plugins
 License: Apache-2.0

{unstructured-platform-plugins-0.0.1 → unstructured-platform-plugins-0.0.3}/test/test_schema.py

@@ -7,7 +7,7 @@ from typing import Any, Optional, Union
 import pytest
 from pydantic import BaseModel
 from typing_extensions import TypedDict
-from unstructured.ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData
 
 import unstructured_platform_plugins.schema.json_schema as js
 from unstructured_platform_plugins.etl_uvicorn.utils import get_input_schema
@@ -507,7 +507,7 @@ def test_file_data():
                         ],
                         "default": None,
                     },
-                    "doc_type": {"type": "string", "enum": ["batch", "file"], "default": "file"},
+                    "doc_type": {"type": "string", "default": "file"},
                     "metadata": {
                         "type": "object",
                         "properties": {
@@ -554,6 +554,10 @@ def test_file_data():
                                 ],
                                 "default": None,
                             },
+                            "filesize_bytes": {
+                                "anyOf": [{"type": "integer"}, {"type": "null"}],
+                                "default": None,
+                            },
                         },
                         "required": [],
                     },
@@ -599,7 +603,7 @@ def test_file_data():
                     ],
                     "default": None,
                 },
-                "doc_type": {"type": "string", "enum": ["batch", "file"], "default": "file"},
+                "doc_type": {"type": "string", "default": "file"},
                 "metadata": {
                     "type": "object",
                     "properties": {
@@ -643,6 +647,10 @@ def test_file_data():
                             ],
                             "default": None,
                         },
+                        "filesize_bytes": {
+                            "anyOf": [{"type": "integer"}, {"type": "null"}],
+                            "default": None,
+                        },
                     },
                     "required": [],
                 },

{unstructured-platform-plugins-0.0.1 → unstructured-platform-plugins-0.0.3}/test/test_utils.py

@@ -4,7 +4,7 @@ from enum import Enum
 
 import pytest
 from pydantic import BaseModel
-from unstructured.ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData
 from uvicorn.importer import import_from_string
 
 from unstructured_platform_plugins.etl_uvicorn import utils

unstructured-platform-plugins-0.0.3/unstructured_platform_plugins/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.0.3"  # pragma: no cover

{unstructured-platform-plugins-0.0.1 → unstructured-platform-plugins-0.0.3}/unstructured_platform_plugins/etl_uvicorn/api_generator.py

@@ -6,10 +6,12 @@ import logging
 from typing import Any, Callable, Optional
 
 from fastapi import FastAPI, status
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from pydantic import BaseModel
 from starlette.responses import RedirectResponse
 from uvicorn.importer import import_from_string
 
+from unstructured_platform_plugins.etl_uvicorn.otel import get_metric_provider, get_trace_provider
 from unstructured_platform_plugins.etl_uvicorn.utils import (
     get_func,
     get_input_schema,
@@ -122,8 +124,8 @@ def generate_fast_api(
 
     else:
 
-        @fastapi_app.post("/invoke", response_model=response_type)
-        async def run_job() -> response_type:
+        @fastapi_app.post("/invoke", response_model=InvokeResponse)
+        async def run_job() -> InvokeResponse:
             logger.debug(f"invoking function without inputs: {func}")
             return await wrap_fn(
                 func=func,
@@ -170,4 +172,8 @@ def generate_fast_api(
     except TypeError as e:
         raise TypeError(f"failed to validate function schema: {e}") from e
 
+    FastAPIInstrumentor.instrument_app(
+        fastapi_app, tracer_provider=get_trace_provider(), meter_provider=get_metric_provider()
+    )
+
     return fastapi_app

unstructured-platform-plugins-0.0.3/unstructured_platform_plugins/etl_uvicorn/otel.py

@@ -0,0 +1,107 @@
+import os
+from typing import Literal, TypedDict
+
+from opentelemetry.environment_variables import OTEL_METRICS_EXPORTER, OTEL_TRACES_EXPORTER
+from opentelemetry.sdk.environment_variables import OTEL_SERVICE_NAME
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    ConsoleMetricExporter,
+    MetricReader,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    ConsoleSpanExporter,
+    SimpleSpanProcessor,
+)
+
+TraceExporterType = Literal["otlp", "jaeger", "zipkin", "console"]
+MetricExporterType = Literal["otlp", "prometheus", "none"]
+
+
+class OtelSettings(TypedDict):
+    service_name: str
+    trace_exporters: list[TraceExporterType]
+    metric_exporters: list[MetricExporterType]
+
+
+def get_settings() -> OtelSettings:
+    service_name = os.environ.get(OTEL_SERVICE_NAME, "unknown_service")
+    trace_exporters = os.environ.get(OTEL_TRACES_EXPORTER)
+    trace_exporters = trace_exporters.split(",") if trace_exporters else []
+
+    metric_exporters = os.environ.get(OTEL_METRICS_EXPORTER)
+    metric_exporters = metric_exporters.split(",") if metric_exporters else []
+    return OtelSettings(
+        service_name=service_name,
+        trace_exporters=trace_exporters,
+        metric_exporters=metric_exporters,
+    )
+
+
+def get_trace_provider() -> TracerProvider:
+    settings = get_settings()
+    print(settings)
+    provider = TracerProvider(resource=Resource({SERVICE_NAME: settings["service_name"]}))
+
+    for trace_exporter_type in settings["trace_exporters"]:
+        _add_trace_exporter(exporter_type=trace_exporter_type, provider=provider)
+
+    return provider
+
+
+def get_metric_provider() -> MeterProvider:
+    settings = get_settings()
+    readers = []
+    for metric_exporter_type in settings["metric_exporters"]:
+        readers.append(_get_metrics_reader(exporter_type=metric_exporter_type))
+    return MeterProvider(
+        resource=Resource({SERVICE_NAME: settings["service_name"]}), metric_readers=readers
+    )
+
+
+def _add_trace_exporter(exporter_type: TraceExporterType, provider: TracerProvider):
+    if exporter_type == "otlp":
+        _add_traces_otlp_exporter(
+            provider,
+        )
+
+    elif exporter_type == "console":
+        _add_traces_console_exporter(provider)
+    else:
+        raise NotImplementedError(f"{exporter_type} implementation not supported yet")
+
+
+def _get_metrics_reader(exporter_type: MetricExporterType) -> MetricReader:
+    if exporter_type == "otlp":
+        return _get_metric_otlp_reader()
+    if exporter_type == "console":
+        return _get_metric_console_reader()
+    raise NotImplementedError(f"{exporter_type} implementation not supported yet")
+
+
+def _add_traces_console_exporter(provider: TracerProvider) -> None:
+    exporter = ConsoleSpanExporter()
+    processor = SimpleSpanProcessor(exporter)
+    provider.add_span_processor(processor)
+
+
+def _add_traces_otlp_exporter(provider: TracerProvider) -> None:
+    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+    exporter = OTLPSpanExporter()
+    processor = SimpleSpanProcessor(exporter)
+    provider.add_span_processor(processor)
+
+
+def _get_metric_otlp_reader() -> MetricReader:
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+
+    exporter = OTLPMetricExporter()
+    return PeriodicExportingMetricReader(exporter)
+
+
+def _get_metric_console_reader() -> MetricReader:
+    exporter = ConsoleMetricExporter()
+    return PeriodicExportingMetricReader(exporter)

{unstructured-platform-plugins-0.0.1 → unstructured-platform-plugins-0.0.3}/unstructured_platform_plugins/etl_uvicorn/utils.py

@@ -1,6 +1,7 @@
 import inspect
 from dataclasses import is_dataclass
 from enum import EnumMeta
+from inspect import Parameter
 from types import GenericAlias, NoneType
 from typing import Any, Callable, Optional
 
@@ -58,6 +59,10 @@ def get_input_schema(func: Callable, omit: Optional[list[str]] = None) -> dict:
     parameters = get_typed_parameters(func)
     if omit:
         parameters = [p for p in parameters if p.name not in omit]
+    # Omit self if wrapping method
+    parameters = [
+        p for p in parameters if not (p.param_type is Parameter.empty and p.name == "self")
+    ]
     return parameters_to_json_schema(parameters)
 
 

{unstructured-platform-plugins-0.0.1 → unstructured-platform-plugins-0.0.3}/unstructured_platform_plugins/schema/json_schema.py

@@ -4,11 +4,11 @@ from enum import Enum, EnumMeta
 from inspect import Parameter
 from pathlib import Path
 from types import GenericAlias, NoneType, UnionType
-from typing import Any, Optional, Type, Union, _UnionGenericAlias
+from typing import Any, Literal, Optional, Type, Union, _UnionGenericAlias
 
 from pydantic import BaseModel, create_model
 from pydantic.fields import FieldInfo, PydanticUndefined
-from unstructured.ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData
 
 from unstructured_platform_plugins.schema.utils import TypedParameter
 from unstructured_platform_plugins.type_hints import get_type_hints
@@ -37,6 +37,13 @@ def is_typed_dict(val: Any) -> bool:
 
 
 def type_to_json_schema(t: Type, args: Optional[tuple[Any, ...]] = None) -> dict:
+    if t is Literal:
+        arg_types = [type(a) for a in args]
+        if len(set(arg_types)) != 1:
+            raise TypeError(
+                "Literal must be of a single type. Defined with: {}".format(", ".join(arg_types))
+            )
+        t = arg_types[0]
     resp = {"type": types_map[t]}
     if t is list and args:
         list_type = args[0]

{unstructured-platform-plugins-0.0.1 → unstructured-platform-plugins-0.0.3/unstructured_platform_plugins.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-platform-plugins
-Version: 0.0.1
+Version: 0.0.3
 Summary: A utility library that provides tools around unstructured plugin development
 Home-page: https://github.com/Unstructured-IO/unstructured-platform-plugins
 License: Apache-2.0

{unstructured-platform-plugins-0.0.1 → unstructured-platform-plugins-0.0.3}/unstructured_platform_plugins.egg-info/SOURCES.txt

@@ -31,6 +31,7 @@ unstructured_platform_plugins.egg-info/top_level.txt
 unstructured_platform_plugins/etl_uvicorn/__init__.py
 unstructured_platform_plugins/etl_uvicorn/api_generator.py
 unstructured_platform_plugins/etl_uvicorn/main.py
+unstructured_platform_plugins/etl_uvicorn/otel.py
 unstructured_platform_plugins/etl_uvicorn/utils.py
 unstructured_platform_plugins/schema/__init__.py
 unstructured_platform_plugins/schema/json_schema.py

unstructured-platform-plugins-0.0.3/unstructured_platform_plugins.egg-info/requires.txt

@@ -0,0 +1,8 @@
+opentelemetry-instrumentation-fastapi
+uvicorn
+unstructured-ingest
+opentelemetry-exporter-otlp-proto-grpc
+fastapi
+click
+requests
+click

unstructured-platform-plugins-0.0.1/unstructured_platform_plugins/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.0.1"  # pragma: no cover

unstructured-platform-plugins-0.0.1/unstructured_platform_plugins.egg-info/requires.txt

@@ -1,6 +0,0 @@
-fastapi
-unstructured
-click
-uvicorn
-requests
-click