UncountablePythonSDK 0.0.24__py3-none-any.whl → 0.0.131__py3-none-any.whl

uncountable/integration/telemetry.py

@@ -0,0 +1,251 @@
+import functools
+import json
+import os
+import time
+import traceback
+import typing
+from contextlib import contextmanager
+from enum import StrEnum
+from typing import Generator, assert_never, cast
+
+from opentelemetry import _logs, trace
+from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk._logs import Logger as OTELLogger
+from opentelemetry.sdk._logs import (
+    LoggerProvider,
+    LogRecord,
+)
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, ConsoleLogExporter
+from opentelemetry.sdk.resources import Attributes, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    SimpleSpanProcessor,
+)
+from opentelemetry.trace import DEFAULT_TRACE_OPTIONS, Span, Tracer
+
+from uncountable.core.environment import (
+    get_otel_enabled,
+    get_server_env,
+    get_version,
+)
+from uncountable.types import base_t, job_definition_t
+
+
+def _cast_attributes(attributes: dict[str, base_t.JsonValue]) -> Attributes:
+    return cast(Attributes, attributes)
+
+
+def one_line_formatter(record: LogRecord) -> str:
+    json_data = record.to_json()
+    return json.dumps(json.loads(json_data), separators=(",", ":")) + "\n"
+
+
+@functools.cache
+def get_otel_resource() -> Resource:
+    attributes: dict[str, base_t.JsonValue] = {
+        "service.name": "integration-server",
+        "sdk.version": get_version(),
+    }
+    unc_version = os.environ.get("UNC_VERSION")
+    if unc_version is not None:
+        attributes["service.version"] = unc_version
+    unc_env = get_server_env()
+    if unc_env is not None:
+        attributes["deployment.environment"] = unc_env
+    resource = Resource.create(attributes=_cast_attributes(attributes))
+    return resource
+
+
+@functools.cache
+def get_otel_tracer() -> Tracer:
+    provider = TracerProvider(resource=get_otel_resource())
+    if get_otel_enabled():
+        provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter()))
+    trace.set_tracer_provider(provider)
+    return provider.get_tracer("integration.telemetry")
+
+
+@functools.cache
+def get_otel_logger() -> OTELLogger:
+    provider = LoggerProvider(resource=get_otel_resource())
+    provider.add_log_record_processor(
+        BatchLogRecordProcessor(ConsoleLogExporter(formatter=one_line_formatter))
+    )
+    if get_otel_enabled():
+        provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
+    _logs.set_logger_provider(provider)
+    return provider.get_logger("integration.telemetry")
+
+
+class LogSeverity(StrEnum):
+    INFO = "Info"
+    WARN = "Warn"
+    ERROR = "Error"
+
+
+class Logger:
+    current_span: Span
+
+    def __init__(self, base_span: Span) -> None:
+        self.current_span = base_span
+
+    @property
+    def current_span_id(self) -> int:
+        return self.current_span.get_span_context().span_id
+
+    @property
+    def current_trace_id(self) -> int | None:
+        return self.current_span.get_span_context().trace_id
+
+    def _patch_attributes(
+        self,
+        attributes: Attributes | None,
+        *,
+        message: str | None = None,
+        severity: LogSeverity | None = None,
+    ) -> Attributes:
+        patched_attributes = {**(attributes if attributes is not None else {})}
+        if message is not None:
+            patched_attributes["message"] = message
+        elif "body" in patched_attributes:
+            patched_attributes["message"] = patched_attributes["body"]
+
+        if severity is not None:
+            patched_attributes["status"] = severity.lower()
+        elif "severity_text" in patched_attributes and isinstance(
+            patched_attributes["severity_text"], str
+        ):
+            patched_attributes["status"] = patched_attributes["severity_text"].lower()
+
+        return patched_attributes
+
+    def _emit_log(
+        self, message: str, *, severity: LogSeverity, attributes: Attributes | None
+    ) -> None:
+        otel_logger = get_otel_logger()
+        log_record = LogRecord(
+            body=message,
+            severity_text=severity,
+            timestamp=time.time_ns(),
+            attributes=self._patch_attributes(
+                message=message, severity=severity, attributes=attributes
+            ),
+            span_id=self.current_span_id,
+            trace_id=self.current_trace_id,
+            trace_flags=DEFAULT_TRACE_OPTIONS,
+            severity_number=_logs.SeverityNumber.UNSPECIFIED,
+            resource=get_otel_resource(),
+        )
+        otel_logger.emit(log_record)
+
+    def log_info(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(
+            message=message, severity=LogSeverity.INFO, attributes=attributes
+        )
+
+    def log_warning(
+        self, message: str, *, attributes: Attributes | None = None
+    ) -> None:
+        self._emit_log(
+            message=message, severity=LogSeverity.WARN, attributes=attributes
+        )
+
+    def log_error(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(
+            message=message, severity=LogSeverity.ERROR, attributes=attributes
+        )
+
+    def log_exception(
+        self,
+        exception: BaseException,
+        *,
+        message: str | None = None,
+        attributes: Attributes | None = None,
+    ) -> None:
+        traceback_str = "".join(traceback.format_exception(exception))
+        patched_attributes = self._patch_attributes(
+            message=message, severity=LogSeverity.ERROR, attributes=attributes
+        )
+        self.current_span.record_exception(
+            exception=exception, attributes=patched_attributes
+        )
+        self.log_error(
+            message=f"error: {message}\nexception: {exception}{traceback_str}",
+            attributes=patched_attributes,
+        )
+
+    @contextmanager
+    def push_scope(
+        self, scope_name: str, *, attributes: Attributes | None = None
+    ) -> Generator[typing.Self, None, None]:
+        with get_otel_tracer().start_as_current_span(
+            scope_name, attributes=self._patch_attributes(attributes)
+        ):
+            yield self
+
+
+class JobLogger(Logger):
+    def __init__(
+        self,
+        *,
+        base_span: Span,
+        profile_metadata: job_definition_t.ProfileMetadata,
+        job_definition: job_definition_t.JobDefinition,
+    ) -> None:
+        self.profile_metadata = profile_metadata
+        self.job_definition = job_definition
+        super().__init__(base_span)
+
+    def _patch_attributes(
+        self,
+        attributes: Attributes | None,
+        *,
+        message: str | None = None,
+        severity: LogSeverity | None = None,
+    ) -> Attributes:
+        patched_attributes: dict[str, base_t.JsonValue] = {
+            **super()._patch_attributes(
+                attributes=attributes, message=message, severity=severity
+            )
+        }
+        patched_attributes["profile.name"] = self.profile_metadata.name
+        patched_attributes["profile.base_url"] = self.profile_metadata.base_url
+        patched_attributes["job.name"] = self.job_definition.name
+        patched_attributes["job.id"] = self.job_definition.id
+        patched_attributes["job.definition_type"] = self.job_definition.type
+        match self.job_definition:
+            case job_definition_t.CronJobDefinition():
+                patched_attributes["job.definition.cron_spec"] = (
+                    self.job_definition.cron_spec
+                )
+            case job_definition_t.HttpJobDefinitionBase():
+                pass
+            case _:
+                assert_never(self.job_definition)
+        patched_attributes["job.definition.executor.type"] = (
+            self.job_definition.executor.type
+        )
+        match self.job_definition.executor:
+            case job_definition_t.JobExecutorScript():
+                patched_attributes["job.definition.executor.import_path"] = (
+                    self.job_definition.executor.import_path
+                )
+            case job_definition_t.JobExecutorGenericUpload():
+                patched_attributes["job.definition.executor.data_source.type"] = (
+                    self.job_definition.executor.data_source.type
+                )
+            case _:
+                assert_never(self.job_definition.executor)
+        return _cast_attributes(patched_attributes)
+
+
+@contextmanager
+def push_scope_optional(
+    logger: Logger | None, scope_name: str, *, attributes: Attributes | None = None
+) -> Generator[None, None, None]:
+    if logger is None:
+        yield
+    else:
+        with logger.push_scope(scope_name, attributes=attributes):
+            yield

uncountable/integration/webhook_server/entrypoint.py

@@ -0,0 +1,97 @@
+import base64
+
+import flask
+from flask.typing import ResponseReturnValue
+from opentelemetry.trace import get_current_span
+from uncountable.core.environment import (
+    get_http_server_port,
+    get_server_env,
+)
+from uncountable.integration.executors.script_executor import resolve_script_executor
+from uncountable.integration.http_server import GenericHttpRequest, HttpException
+from uncountable.integration.job import CustomHttpJob, WebhookJob
+from uncountable.integration.scan_profiles import load_profiles
+from uncountable.integration.telemetry import Logger
+from uncountable.types import job_definition_t
+
+app = flask.Flask(__name__)
+
+
+def register_route(
+    *,
+    server_logger: Logger,
+    profile_meta: job_definition_t.ProfileMetadata,
+    job: job_definition_t.HttpJobDefinitionBase,
+) -> None:
+    route = f"/{profile_meta.name}/{job.id}"
+
+    def handle_request() -> ResponseReturnValue:
+        with server_logger.push_scope(route):
+            try:
+                if not isinstance(job.executor, job_definition_t.JobExecutorScript):
+                    raise HttpException.configuration_error(
+                        message="[internal] http job must use a script executor"
+                    )
+                job_instance = resolve_script_executor(
+                    executor=job.executor, profile_metadata=profile_meta
+                )
+                if not isinstance(job_instance, (CustomHttpJob, WebhookJob)):
+                    raise HttpException.configuration_error(
+                        message="[internal] http job must descend from CustomHttpJob"
+                    )
+                http_request = GenericHttpRequest(
+                    body_base64=base64.b64encode(flask.request.get_data()).decode(),
+                    headers=dict(flask.request.headers),
+                )
+                job_instance.validate_request(
+                    request=http_request, job_definition=job, profile_meta=profile_meta
+                )
+                http_response = job_instance.handle_request(
+                    request=http_request, job_definition=job, profile_meta=profile_meta
+                )
+
+                return flask.make_response(
+                    http_response.response,
+                    http_response.status_code,
+                    http_response.headers,
+                )
+            except HttpException as e:
+                server_logger.log_exception(e)
+                return e.make_error_response()
+            except Exception as e:
+                server_logger.log_exception(e)
+                return HttpException.unknown_error().make_error_response()
+
+    app.add_url_rule(
+        route,
+        endpoint=f"handle_request_{job.id}",
+        view_func=handle_request,
+        methods=["POST"],
+    )
+
+    server_logger.log_info(f"job {job.id} webhook registered at: {route}")
+
+
+def main() -> None:
+    app.add_url_rule("/health", "health", lambda: ("OK", 200))
+
+    profiles = load_profiles()
+    for profile_metadata in profiles:
+        server_logger = Logger(get_current_span())
+        for job in profile_metadata.jobs:
+            if isinstance(job, job_definition_t.HttpJobDefinitionBase):
+                register_route(
+                    server_logger=server_logger, profile_meta=profile_metadata, job=job
+                )
+
+
+main()
+
+
+if __name__ == "__main__":
+    app.run(
+        host="0.0.0.0",
+        port=get_http_server_port(),
+        debug=get_server_env() == "playground",
+        exclude_patterns=[],
+    )

uncountable/types/__init__.py

@@ -1,31 +1,50 @@
 # DO NOT MODIFY -- This file is generated by type_spec
-# flake8: noqa: F821
-# ruff: noqa: E402
+# ruff: noqa: E402 Q003
 # fmt: off
 # isort: skip_file
+from .api.notebooks import add_notebook_content as add_notebook_content_t
+from .api.recipes import add_recipe_to_project as add_recipe_to_project_t
+from .api.recipes import add_time_series_data as add_time_series_data_t
 from .api.recipes import archive_recipes as archive_recipes_t
+from .api.equipment import associate_equipment_input as associate_equipment_input_t
 from .api.recipes import associate_recipe_as_input as associate_recipe_as_input_t
 from .api.recipes import associate_recipe_as_lot as associate_recipe_as_lot_t
-from . import async_batch as async_batch_t
-from . import base as base_t
-from . import calculations as calculations_t
-from . import chemical_structure as chemical_structure_t
+from . import async_batch_t as async_batch_t
+from . import async_jobs_t as async_jobs_t
+from . import auth_retrieval_t as auth_retrieval_t
+from . import base_t as base_t
+from . import calculations_t as calculations_t
+from . import chemical_structure_t as chemical_structure_t
+from .api.recipes import clear_recipe_outputs as clear_recipe_outputs_t
+from . import client_config_t as client_config_t
+from .api.uploader import complete_async_parse as complete_async_parse_t
+from .api.runsheet import complete_async_upload as complete_async_upload_t
 from .api.chemical import convert_chemical_formats as convert_chemical_formats_t
 from .api.entity import create_entities as create_entities_t
 from .api.entity import create_entity as create_entity_t
 from .api.inputs import create_inputs as create_inputs_t
+from .api.recipes import create_mix_order as create_mix_order_t
+from .api.entity import create_or_update_entity as create_or_update_entity_t
 from .api.recipes import create_recipe as create_recipe_t
 from .api.recipe_links import create_recipe_link as create_recipe_link_t
 from .api.recipes import create_recipes as create_recipes_t
-from . import curves as curves_t
+from . import curves_t as curves_t
+from . import data_t as data_t
 from .api.recipes import disassociate_recipe_as_input as disassociate_recipe_as_input_t
+from .api.files import download_file as download_file_t
 from .api.recipes import edit_recipe_inputs as edit_recipe_inputs_t
-from . import entity as entity_t
+from . import entity_t as entity_t
 from .api.batch import execute_batch as execute_batch_t
 from .api.batch import execute_batch_load_async as execute_batch_load_async_t
-from . import experiment_groups as experiment_groups_t
-from . import field_values as field_values_t
-from . import fields as fields_t
+from . import experiment_groups_t as experiment_groups_t
+from .api.entity import export_entities as export_entities_t
+from . import exports_t as exports_t
+from .api.listing import fetch_listing as fetch_listing_t
+from . import field_values_t as field_values_t
+from . import fields_t as fields_t
+from . import generic_upload_t as generic_upload_t
+from .api.recipes import get_column_calculation_values as get_column_calculation_values_t
+from .api.user import get_current_user_info as get_current_user_info_t
 from .api.recipes import get_curve as get_curve_t
 from .api.entity import get_entities_data as get_entities_data_t
 from .api.inputs import get_input_data as get_input_data_t
@@ -34,6 +53,7 @@ from .api.inputs import get_input_names as get_input_names_t
 from .api.inputs import get_inputs_data as get_inputs_data_t
 from .api.outputs import get_output_data as get_output_data_t
 from .api.outputs import get_output_names as get_output_names_t
+from .api.outputs import get_output_organization as get_output_organization_t
 from .api.project import get_projects as get_projects_t
 from .api.project import get_projects_data as get_projects_data_t
 from .api.recipes import get_recipe_calculations as get_recipe_calculations_t
@@ -42,65 +62,124 @@ from .api.recipe_metadata import get_recipe_metadata_data as get_recipe_metadata
 from .api.recipes import get_recipe_names as get_recipe_names_t
 from .api.recipes import get_recipe_output_metadata as get_recipe_output_metadata_t
 from .api.recipes import get_recipes_data as get_recipes_data_t
-from . import id_source as id_source_t
-from . import identifier as identifier_t
-from . import input_attributes as input_attributes_t
-from . import inputs as inputs_t
+from .api.entity import grant_entity_permissions as grant_entity_permissions_t
+from . import id_source_t as id_source_t
+from . import identifier_t as identifier_t
+from . import input_attributes_t as input_attributes_t
+from . import inputs_t as inputs_t
+from . import integration_server_t as integration_server_t
+from . import integration_session_t as integration_session_t
+from . import integrations_t as integrations_t
+from .api.uploader import invoke_uploader as invoke_uploader_t
+from . import job_definition_t as job_definition_t
+from .api.entity import list_aggregate as list_aggregate_t
 from .api.entity import list_entities as list_entities_t
 from .api.id_source import list_id_source as list_id_source_t
+from . import listing_t as listing_t
+from .api.entity import lock_entity as lock_entity_t
+from .api.recipes import lock_recipes as lock_recipes_t
+from .api.entity import lookup_entity as lookup_entity_t
 from .api.id_source import match_id_source as match_id_source_t
-from . import outputs as outputs_t
-from . import permissions as permissions_t
-from . import phases as phases_t
-from . import post_base as post_base_t
-from . import recipe_identifiers as recipe_identifiers_t
-from . import recipe_inputs as recipe_inputs_t
-from . import recipe_links as recipe_links_t
-from . import recipe_metadata as recipe_metadata_t
-from . import recipe_output_metadata as recipe_output_metadata_t
-from . import recipe_tags as recipe_tags_t
-from . import recipe_workflow_steps as recipe_workflow_steps_t
+from . import notices_t as notices_t
+from . import notifications_t as notifications_t
+from . import outputs_t as outputs_t
+from . import overrides_t as overrides_t
+from . import permissions_t as permissions_t
+from . import phases_t as phases_t
+from . import post_base_t as post_base_t
+from .api.integrations import publish_realtime_data as publish_realtime_data_t
+from .api.integrations import push_notification as push_notification_t
+from . import queued_job_t as queued_job_t
+from . import recipe_identifiers_t as recipe_identifiers_t
+from . import recipe_inputs_t as recipe_inputs_t
+from . import recipe_links_t as recipe_links_t
+from . import recipe_metadata_t as recipe_metadata_t
+from . import recipe_output_metadata_t as recipe_output_metadata_t
+from . import recipe_tags_t as recipe_tags_t
+from . import recipe_workflow_steps_t as recipe_workflow_steps_t
+from . import recipes_t as recipes_t
+from .api.integrations import register_sockets_token as register_sockets_token_t
+from .api.recipes import remove_recipe_from_project as remove_recipe_from_project_t
+from .api.recipe_links import remove_recipe_link as remove_recipe_link_t
 from .api.entity import resolve_entity_ids as resolve_entity_ids_t
 from .api.outputs import resolve_output_conditions as resolve_output_conditions_t
-from . import response as response_t
+from . import response_t as response_t
 from .api.triggers import run_trigger as run_trigger_t
+from . import secret_retrieval_t as secret_retrieval_t
 from .api.permissions import set_core_permissions as set_core_permissions_t
+from .api.entity import set_entity_field_values as set_entity_field_values_t
 from .api.inputs import set_input_attribute_values as set_input_attribute_values_t
+from .api.inputs import set_input_category as set_input_category_t
+from .api.inputs import set_input_subcategories as set_input_subcategories_t
+from .api.inputs import set_intermediate_type as set_intermediate_type_t
 from .api.recipes import set_recipe_inputs as set_recipe_inputs_t
 from .api.recipes import set_recipe_metadata as set_recipe_metadata_t
+from .api.recipes import set_recipe_output_annotations as set_recipe_output_annotations_t
+from .api.recipes import set_recipe_output_file as set_recipe_output_file_t
 from .api.recipes import set_recipe_outputs as set_recipe_outputs_t
 from .api.recipes import set_recipe_tags as set_recipe_tags_t
+from .api.recipes import set_recipe_total as set_recipe_total_t
 from .api.entity import set_values as set_values_t
+from . import sockets_t as sockets_t
+from . import structured_filters_t as structured_filters_t
 from .api.entity import transition_entity_phase as transition_entity_phase_t
-from . import units as units_t
-from . import users as users_t
-from . import workflows as workflows_t
+from .api.recipes import unarchive_recipes as unarchive_recipes_t
+from . import units_t as units_t
+from .api.entity import unlock_entity as unlock_entity_t
+from .api.recipes import unlock_recipes as unlock_recipes_t
+from .api.material_families import update_entity_material_families as update_entity_material_families_t
+from . import uploader_t as uploader_t
+from .api.condition_parameters import upsert_condition_match as upsert_condition_match_t
+from .api.field_options import upsert_field_options as upsert_field_options_t
+from . import users_t as users_t
+from . import webhook_job_t as webhook_job_t
+from . import workflows_t as workflows_t
 
 
 __all__: list[str] = [
+    "add_notebook_content_t",
+    "add_recipe_to_project_t",
+    "add_time_series_data_t",
     "archive_recipes_t",
+    "associate_equipment_input_t",
     "associate_recipe_as_input_t",
     "associate_recipe_as_lot_t",
     "async_batch_t",
+    "async_jobs_t",
+    "auth_retrieval_t",
     "base_t",
     "calculations_t",
     "chemical_structure_t",
+    "clear_recipe_outputs_t",
+    "client_config_t",
+    "complete_async_parse_t",
+    "complete_async_upload_t",
     "convert_chemical_formats_t",
     "create_entities_t",
     "create_entity_t",
     "create_inputs_t",
+    "create_mix_order_t",
+    "create_or_update_entity_t",
     "create_recipe_t",
     "create_recipe_link_t",
     "create_recipes_t",
     "curves_t",
+    "data_t",
     "disassociate_recipe_as_input_t",
+    "download_file_t",
     "edit_recipe_inputs_t",
     "entity_t",
     "execute_batch_t",
     "execute_batch_load_async_t",
     "experiment_groups_t",
+    "export_entities_t",
+    "exports_t",
+    "fetch_listing_t",
     "field_values_t",
     "fields_t",
+    "generic_upload_t",
+    "get_column_calculation_values_t",
+    "get_current_user_info_t",
     "get_curve_t",
     "get_entities_data_t",
     "get_input_data_t",
@@ -109,6 +188,7 @@ __all__: list[str] = [
     "get_inputs_data_t",
     "get_output_data_t",
     "get_output_names_t",
+    "get_output_organization_t",
     "get_projects_t",
     "get_projects_data_t",
     "get_recipe_calculations_t",
@@ -117,17 +197,34 @@ __all__: list[str] = [
     "get_recipe_names_t",
     "get_recipe_output_metadata_t",
     "get_recipes_data_t",
+    "grant_entity_permissions_t",
     "id_source_t",
     "identifier_t",
     "input_attributes_t",
     "inputs_t",
+    "integration_server_t",
+    "integration_session_t",
+    "integrations_t",
+    "invoke_uploader_t",
+    "job_definition_t",
+    "list_aggregate_t",
     "list_entities_t",
     "list_id_source_t",
+    "listing_t",
+    "lock_entity_t",
+    "lock_recipes_t",
+    "lookup_entity_t",
     "match_id_source_t",
+    "notices_t",
+    "notifications_t",
     "outputs_t",
+    "overrides_t",
     "permissions_t",
     "phases_t",
     "post_base_t",
+    "publish_realtime_data_t",
+    "push_notification_t",
+    "queued_job_t",
     "recipe_identifiers_t",
     "recipe_inputs_t",
     "recipe_links_t",
@@ -135,19 +232,41 @@ __all__: list[str] = [
     "recipe_output_metadata_t",
     "recipe_tags_t",
     "recipe_workflow_steps_t",
+    "recipes_t",
+    "register_sockets_token_t",
+    "remove_recipe_from_project_t",
+    "remove_recipe_link_t",
     "resolve_entity_ids_t",
     "resolve_output_conditions_t",
     "response_t",
     "run_trigger_t",
+    "secret_retrieval_t",
     "set_core_permissions_t",
+    "set_entity_field_values_t",
     "set_input_attribute_values_t",
+    "set_input_category_t",
+    "set_input_subcategories_t",
+    "set_intermediate_type_t",
     "set_recipe_inputs_t",
     "set_recipe_metadata_t",
+    "set_recipe_output_annotations_t",
+    "set_recipe_output_file_t",
     "set_recipe_outputs_t",
     "set_recipe_tags_t",
+    "set_recipe_total_t",
     "set_values_t",
+    "sockets_t",
+    "structured_filters_t",
     "transition_entity_phase_t",
+    "unarchive_recipes_t",
     "units_t",
+    "unlock_entity_t",
+    "unlock_recipes_t",
+    "update_entity_material_families_t",
+    "uploader_t",
+    "upsert_condition_match_t",
+    "upsert_field_options_t",
     "users_t",
+    "webhook_job_t",
     "workflows_t",
 ]