UncountablePythonSDK 0.0.181__py3-none-any.whl → 0.0.182__py3-none-any.whl

examples/integration-server/jobs/materials_auto/profile.yaml

@@ -1,4 +1,5 @@
 base_url: http://host.docker.internal:5000
+integration_server_base_url: http://localhost:5001
 auth_retrieval:
   type: basic
   api_id_secret:
@@ -60,6 +61,13 @@ jobs:
     executor:
       type: script
       import_path: example_wh
+  - id: example_unc_wh
+    type: uncountable_webhook
+    name: Uncountable Webhook
+    purpose: trigger
+    executor:
+      type: script
+      import_path: example_wh
   - id: example_http
     type: custom_http
     name: Custom HTTP

uncountable/integration/entrypoint.py

@@ -1,11 +1,14 @@
 from uncountable.integration.db.connect import IntegrationDBService, create_db_engine
 from uncountable.integration.scan_profiles import load_profiles
 from uncountable.integration.server import IntegrationServer
+from uncountable.integration.webhook_bootstrap import start_webhook_entity_bootstrap
 
 
 def main() -> None:
+    profiles = load_profiles()
     with IntegrationServer(create_db_engine(IntegrationDBService.CRON)) as server:
-        server.register_jobs(load_profiles())
+        start_webhook_entity_bootstrap(profiles)
+        server.register_jobs(profiles)
         server.serve_forever()
 
 

uncountable/integration/http_server/types.py

@@ -28,6 +28,10 @@ class HttpException(Exception):
     def body_parse_error() -> "HttpException":
         return HttpException(error_code=400, message="body parse error")
 
+    @staticmethod
+    def bad_request(message: str = "bad request") -> "HttpException":
+        return HttpException(error_code=400, message=message)
+
     @staticmethod
     def unauthorized() -> "HttpException":
         return HttpException(error_code=401, message="unauthorized")

uncountable/integration/job.py

@@ -27,7 +27,7 @@ from uncountable.integration.queue_runner.command_server.types import (
 from uncountable.integration.request_context import extract_request_context
 from uncountable.integration.secret_retrieval import retrieve_secret
 from uncountable.integration.telemetry import JobLogger
-from uncountable.integration.webhook_signature_key import (
+from uncountable.integration.webhook_entity import (
     webhook_signature_key_retrieval,
 )
 from uncountable.types import (

uncountable/integration/queue_runner/command_server/provision_webhook.py

@@ -3,9 +3,7 @@ from opentelemetry.trace import get_current_span
 from uncountable.integration.construct_client import construct_uncountable_client
 from uncountable.integration.scan_profiles import load_profiles
 from uncountable.integration.telemetry import Logger
-from uncountable.integration.webhook_signature_key import (
-    bootstrap_webhook_signature_key,
-)
+from uncountable.integration.webhook_entity import create_missing_webhook_entities
 from uncountable.types import job_definition_t
 
 
@@ -17,12 +15,10 @@ def provision_webhook(*, job_id: str) -> str | None:
                 isinstance(job, job_definition_t.UncountableWebhookJobDefinition)
                 and job.id == job_id
             ):
-                client = construct_uncountable_client(profile_metadata, logger)
-                bootstrap_webhook_signature_key(
-                    client=client,
+                create_missing_webhook_entities(
+                    client=construct_uncountable_client(profile_metadata, logger),
                     profile_metadata=profile_metadata,
-                    job_id=job.id,
-                    purpose=job.purpose,
+                    jobs=[job],
                 )
                 return profile_metadata.name
     return None

uncountable/integration/webhook_bootstrap.py

@@ -0,0 +1,67 @@
+import threading
+
+from opentelemetry.trace import get_current_span
+
+from uncountable.integration.construct_client import construct_uncountable_client
+from uncountable.integration.telemetry import Logger
+from uncountable.integration.webhook_entity import create_missing_webhook_entities
+from uncountable.types import job_definition_t
+
+_BOOTSTRAP_SCOPE_NAME = "bootstrap_webhook_entities"
+_BOOTSTRAP_THREAD_NAME = "webhook-entity-bootstrap"
+
+
+def _bootstrap_profile_webhook_entities(
+    *,
+    profile_metadata: job_definition_t.ProfileMetadata,
+    jobs: list[job_definition_t.UncountableWebhookJobDefinition],
+    logger: Logger,
+) -> None:
+    created_job_ids = create_missing_webhook_entities(
+        client=construct_uncountable_client(profile_metadata, logger),
+        profile_metadata=profile_metadata,
+        jobs=jobs,
+    )
+    logger.log_debug(
+        f"bootstrapped {len(created_job_ids)} webhook entities for profile {profile_metadata.name}",
+        attributes={
+            "profile.name": profile_metadata.name,
+            "webhook.created_job_ids": created_job_ids,
+        },
+    )
+
+
+def _bootstrap_webhook_entities(
+    profiles: list[job_definition_t.ProfileMetadata],
+) -> None:
+    logger = Logger(get_current_span())
+    with logger.push_scope(_BOOTSTRAP_SCOPE_NAME):
+        for profile_metadata in profiles:
+            webhook_jobs = [
+                job
+                for job in profile_metadata.jobs
+                if isinstance(job, job_definition_t.UncountableWebhookJobDefinition)
+            ]
+            if len(webhook_jobs) == 0:
+                continue
+            try:
+                _bootstrap_profile_webhook_entities(
+                    profile_metadata=profile_metadata, jobs=webhook_jobs, logger=logger
+                )
+            except Exception as e:
+                logger.log_exception(
+                    e,
+                    message=f"failed to bootstrap webhook entities for profile {profile_metadata.name}",
+                    attributes={"profile.name": profile_metadata.name},
+                )
+
+
+def start_webhook_entity_bootstrap(
+    profiles: list[job_definition_t.ProfileMetadata],
+) -> None:
+    threading.Thread(
+        target=_bootstrap_webhook_entities,
+        args=(profiles,),
+        name=_BOOTSTRAP_THREAD_NAME,
+        daemon=True,
+    ).start()

uncountable/integration/webhook_entity.py

@@ -0,0 +1,147 @@
+import secrets
+from collections.abc import Sequence
+
+from uncountable.core.client import Client
+from uncountable.core.query.builder import QueryBuilder
+from uncountable.core.query.column import (
+    NullableColumn,
+    TextColumn,
+)
+from uncountable.core.query.row import QueryRow
+from uncountable.types import (
+    create_or_update_entities_t,
+    entity_t,
+    field_values_t,
+    identifier_t,
+    listing_t,
+)
+from uncountable.types.job_definition_t import (
+    ProfileMetadata,
+    UncountableWebhookJobDefinition,
+)
+from uncountable.types.secret_retrieval_t import SecretRetrievalEntityColumn
+
+_WEBHOOK_DEFINITION_REF_NAME = "unc_webhook_definition"
+_WEBHOOK_REF_NAME_FIELD_REF_NAME = "refName"
+_WEBHOOK_NAME_FIELD_REF_NAME = "name"
+_WEBHOOK_URL_FIELD_REF_NAME = "core_webhook_url"
+_WEBHOOK_SIGNATURE_KEY_FIELD_REF_NAME = "core_webhook_signatureKey"
+_WEBHOOK_PURPOSE_FIELD_REF_NAME = "core_webhook_purpose"
+_SIGNATURE_KEY_NUM_BYTES = 32
+
+
+class _WebhookEntityQueryRow(QueryRow):
+    ref_name = TextColumn(
+        identifier=listing_t.ColumnIdentifierEntityRefName(
+            entity_type=entity_t.EntityType.WEBHOOK,
+            ref_name=_WEBHOOK_REF_NAME_FIELD_REF_NAME,
+        ),
+        nullable=NullableColumn,
+    )
+
+
+def webhook_signature_key_retrieval(*, job_id: str) -> SecretRetrievalEntityColumn:
+    return SecretRetrievalEntityColumn(
+        entity_key=identifier_t.IdentifierKeyRefName(ref_name=job_id),
+        column_identifier=listing_t.ColumnIdentifierEntityRefName(
+            entity_type=entity_t.EntityType.WEBHOOK,
+            ref_name=_WEBHOOK_SIGNATURE_KEY_FIELD_REF_NAME,
+        ),
+    )
+
+
+def _resolve_webhook_url(*, profile_metadata: ProfileMetadata, job_id: str) -> str:
+    integration_server_base_url = profile_metadata.integration_server_base_url
+    if integration_server_base_url is None:
+        raise ValueError(
+            "integration_server_base_url is required to resolve the webhook entity "
+            f"for {profile_metadata.name}/{job_id}"
+        )
+    return f"{integration_server_base_url.rstrip('/')}/{profile_metadata.name}/{job_id}"
+
+
+def _fetch_existing_webhook_ref_names(
+    *, client: Client, job_ids: Sequence[str]
+) -> set[str]:
+    existing_webhooks = (
+        QueryBuilder(client=client, model=_WebhookEntityQueryRow)
+        .filter(_WebhookEntityQueryRow.ref_name.includes(tuple(job_ids)))
+        .all()
+    )
+    return {
+        webhook.ref_name
+        for webhook in existing_webhooks
+        if webhook.ref_name is not None
+    }
+
+
+def _text_field_value(
+    *, field_ref_name: str, value: str
+) -> field_values_t.FieldArgumentValue:
+    return field_values_t.FieldArgumentValue(
+        field_key=identifier_t.IdentifierKeyRefName(ref_name=field_ref_name),
+        value=field_values_t.FieldValueText(value=value),
+    )
+
+
+def _build_webhook_operation(
+    *, profile_metadata: ProfileMetadata, job: UncountableWebhookJobDefinition
+) -> create_or_update_entities_t.EntityToCreateOrUpdate:
+    # entity_key is deliberately omitted so the operation always takes the create
+    # path. Supplying a ref_name key that resolves to nothing raises not-found
+    # rather than creating the entity, and callers only pass jobs already known
+    # to have no webhook entity.
+    return create_or_update_entities_t.EntityToCreateOrUpdate(
+        field_values=[],
+        on_create_init_field_values=[
+            _text_field_value(
+                field_ref_name=_WEBHOOK_REF_NAME_FIELD_REF_NAME, value=job.id
+            ),
+            _text_field_value(
+                field_ref_name=_WEBHOOK_NAME_FIELD_REF_NAME,
+                value=f"{profile_metadata.name}/{job.id}",
+            ),
+            _text_field_value(
+                field_ref_name=_WEBHOOK_URL_FIELD_REF_NAME,
+                value=_resolve_webhook_url(
+                    profile_metadata=profile_metadata, job_id=job.id
+                ),
+            ),
+            _text_field_value(
+                field_ref_name=_WEBHOOK_SIGNATURE_KEY_FIELD_REF_NAME,
+                value=secrets.token_urlsafe(_SIGNATURE_KEY_NUM_BYTES),
+            ),
+            field_values_t.FieldArgumentValue(
+                field_key=identifier_t.IdentifierKeyRefName(
+                    ref_name=_WEBHOOK_PURPOSE_FIELD_REF_NAME
+                ),
+                value=field_values_t.FieldValueFieldOption(value=job.purpose.value),
+            ),
+        ],
+    )
+
+
+def create_missing_webhook_entities(
+    *,
+    client: Client,
+    profile_metadata: ProfileMetadata,
+    jobs: Sequence[UncountableWebhookJobDefinition],
+) -> list[str]:
+    existing_ref_names = _fetch_existing_webhook_ref_names(
+        client=client, job_ids=[job.id for job in jobs]
+    )
+    missing_jobs = [job for job in jobs if job.id not in existing_ref_names]
+    if len(missing_jobs) == 0:
+        return []
+
+    client.create_or_update_entities(
+        entity_type=entity_t.EntityType.WEBHOOK,
+        definition_key=identifier_t.IdentifierKeyRefName(
+            ref_name=_WEBHOOK_DEFINITION_REF_NAME
+        ),
+        operations=[
+            _build_webhook_operation(profile_metadata=profile_metadata, job=job)
+            for job in missing_jobs
+        ],
+    )
+    return [job.id for job in missing_jobs]

uncountable/integration/webhook_server/admin_routes/__init__.py

@@ -0,0 +1,3 @@
+from uncountable.integration.webhook_server.admin_routes.register import (
+    register_admin_routes as register_admin_routes,
+)

uncountable/integration/webhook_server/admin_routes/_boundary.py

@@ -0,0 +1,39 @@
+import functools
+from collections.abc import Callable
+
+import flask
+from flask.typing import ResponseReturnValue
+from uncountable.integration.http_server import HttpException
+from uncountable.integration.queue_runner.command_server.types import (
+    CommandServerException,
+)
+from uncountable.integration.telemetry import Logger
+
+
+def register_admin_route(
+    *,
+    app: flask.Flask,
+    logger: Logger,
+    route: str,
+    endpoint: str,
+    view: Callable[..., ResponseReturnValue],
+    methods: list[str],
+) -> None:
+    @functools.wraps(view)
+    def wrapped(*args: object, **kwargs: object) -> ResponseReturnValue:
+        with logger.push_scope(route):
+            try:
+                return view(*args, **kwargs)
+            except HttpException as e:
+                return e.make_error_response()
+            except CommandServerException as e:
+                logger.log_exception(
+                    e,
+                    message=f"admin route {route} could not reach the command server",
+                )
+                return HttpException.bad_gateway().make_error_response()
+            except Exception as e:
+                logger.log_exception(e, message=f"unexpected error on {route}")
+                return HttpException.unknown_error().make_error_response()
+
+    app.add_url_rule(route, endpoint=endpoint, view_func=wrapped, methods=methods)

uncountable/integration/webhook_server/admin_routes/health.py

@@ -0,0 +1,30 @@
+import flask
+from flask.typing import ResponseReturnValue
+from uncountable.core.environment import get_local_admin_server_port
+from uncountable.integration.queue_runner.command_server.command_client import (
+    check_health,
+)
+from uncountable.integration.telemetry import Logger
+from uncountable.integration.webhook_server.admin_auth import require_admin_auth
+from uncountable.integration.webhook_server.admin_routes._boundary import (
+    register_admin_route,
+)
+
+_ADMIN_HEALTH_ROUTE = "/admin/health"
+
+
+@require_admin_auth
+def _handle_admin_health() -> ResponseReturnValue:
+    healthy = check_health(port=get_local_admin_server_port())
+    return flask.jsonify({"healthy": healthy})
+
+
+def register_health_routes(*, app: flask.Flask, logger: Logger) -> None:
+    register_admin_route(
+        app=app,
+        logger=logger,
+        route=_ADMIN_HEALTH_ROUTE,
+        endpoint="admin_health",
+        view=_handle_admin_health,
+        methods=["GET"],
+    )

uncountable/integration/webhook_server/admin_routes/jobs.py

@@ -0,0 +1,147 @@
+import datetime
+
+import flask
+from flask.typing import ResponseReturnValue
+from uncountable.core.environment import get_local_admin_server_port
+from uncountable.integration.http_server import HttpException
+from uncountable.integration.queue_runner.command_server.command_client import (
+    send_job_cancellation_message,
+    send_job_queue_message,
+    send_list_queued_jobs_message,
+    send_retry_job_message,
+)
+from uncountable.integration.queue_runner.command_server.protocol.command_server_pb2 import (
+    ListQueuedJobsResult,
+)
+from uncountable.integration.telemetry import Logger
+from uncountable.integration.webhook_server.admin_auth import require_admin_auth
+from uncountable.integration.webhook_server.admin_routes._boundary import (
+    register_admin_route,
+)
+from uncountable.types import base_t, queued_job_t
+
+from pkgs.serialization_util import serialize_for_api
+
+_ADMIN_JOBS_ROUTE = "/admin/jobs"
+_ADMIN_ENQUEUE_JOB_ROUTE = "/admin/jobs/enqueue"
+_ADMIN_RETRY_JOB_ROUTE = "/admin/jobs/<job_uuid>/retry"
+_ADMIN_CANCEL_JOB_ROUTE = "/admin/jobs/<job_uuid>/cancel"
+_DEFAULT_JOBS_LIMIT = 100
+_VALID_JOB_STATUSES = {status.value for status in queued_job_t.JobStatus}
+
+
+def _parse_job_status(raw_status: str | None) -> queued_job_t.JobStatus | None:
+    if raw_status is None:
+        return None
+    if raw_status not in _VALID_JOB_STATUSES:
+        raise HttpException.bad_request(f"invalid status: {raw_status}")
+    return queued_job_t.JobStatus(raw_status)
+
+
+def _parse_non_negative_int(raw_value: str | None, *, default: int) -> int:
+    if raw_value is None:
+        return default
+    if not raw_value.isdigit():
+        raise HttpException.bad_request(
+            f"expected a non-negative integer, got: {raw_value}"
+        )
+    return int(raw_value)
+
+
+def _serialize_queued_job(
+    item: ListQueuedJobsResult.ListQueuedJobsResultItem,
+) -> dict[str, base_t.JsonValue]:
+    metadata = queued_job_t.QueuedJobMetadata(
+        queued_job_uuid=item.uuid,
+        job_ref_name=item.job_ref_name,
+        num_attempts=item.num_attempts,
+        status=queued_job_t.JobStatus(item.status),
+        submitted_at=item.submitted_at.ToDatetime(datetime.timezone.utc),
+    )
+    return serialize_for_api(metadata)
+
+
+@require_admin_auth
+def _handle_list_jobs() -> ResponseReturnValue:
+    status = _parse_job_status(flask.request.args.get("status"))
+    offset = _parse_non_negative_int(flask.request.args.get("offset"), default=0)
+    limit = _parse_non_negative_int(
+        flask.request.args.get("limit"), default=_DEFAULT_JOBS_LIMIT
+    )
+    jobs = send_list_queued_jobs_message(
+        status=status, offset=offset, limit=limit, port=get_local_admin_server_port()
+    )
+    return flask.jsonify({"jobs": [_serialize_queued_job(job) for job in jobs]})
+
+
+@require_admin_auth
+def _handle_enqueue_job() -> ResponseReturnValue:
+    body = flask.request.get_json(silent=True)
+    if not isinstance(body, dict):
+        raise HttpException.bad_request("expected a JSON object body")
+    job_ref_name = body.get("job_ref_name")
+    if not isinstance(job_ref_name, str):
+        raise HttpException.bad_request("job_ref_name is required")
+    webhook_payload = body.get("webhook_payload")
+    invocation_context: queued_job_t.InvocationContext = (
+        queued_job_t.InvocationContextManual()
+        if webhook_payload is None
+        else queued_job_t.InvocationContextWebhook(webhook_payload=webhook_payload)
+    )
+    queued_job_uuid = send_job_queue_message(
+        job_ref_name=job_ref_name,
+        payload=queued_job_t.QueuedJobPayload(invocation_context=invocation_context),
+        port=get_local_admin_server_port(),
+    )
+    return flask.jsonify({"queuedJobUuid": queued_job_uuid})
+
+
+@require_admin_auth
+def _handle_retry_job(job_uuid: str) -> ResponseReturnValue:
+    queued_job_uuid = send_retry_job_message(
+        job_uuid=job_uuid, port=get_local_admin_server_port()
+    )
+    return flask.jsonify({"queuedJobUuid": queued_job_uuid})
+
+
+@require_admin_auth
+def _handle_cancel_job(job_uuid: str) -> ResponseReturnValue:
+    status = send_job_cancellation_message(
+        queued_job_uuid=job_uuid, port=get_local_admin_server_port()
+    )
+    return flask.jsonify({"status": status.value})
+
+
+def register_job_routes(*, app: flask.Flask, logger: Logger) -> None:
+    register_admin_route(
+        app=app,
+        logger=logger,
+        route=_ADMIN_JOBS_ROUTE,
+        endpoint="admin_list_jobs",
+        view=_handle_list_jobs,
+        methods=["GET"],
+    )
+    register_admin_route(
+        app=app,
+        logger=logger,
+        route=_ADMIN_ENQUEUE_JOB_ROUTE,
+        endpoint="admin_enqueue_job",
+        view=_handle_enqueue_job,
+        methods=["POST"],
+    )
+    register_admin_route(
+        app=app,
+        logger=logger,
+        route=_ADMIN_RETRY_JOB_ROUTE,
+        endpoint="admin_retry_job",
+        view=_handle_retry_job,
+        methods=["POST"],
+    )
+    register_admin_route(
+        app=app,
+        logger=logger,
+        route=_ADMIN_CANCEL_JOB_ROUTE,
+        endpoint="admin_cancel_job",
+        view=_handle_cancel_job,
+        methods=["POST"],
+    )

uncountable/integration/webhook_server/admin_routes/register.py

@@ -0,0 +1,13 @@
+import flask
+from opentelemetry.trace import get_current_span
+from uncountable.integration.telemetry import Logger
+from uncountable.integration.webhook_server.admin_routes.health import (
+    register_health_routes,
+)
+from uncountable.integration.webhook_server.admin_routes.jobs import register_job_routes
+
+
+def register_admin_routes(app: flask.Flask) -> None:
+    logger = Logger(get_current_span())
+    register_health_routes(app=app, logger=logger)
+    register_job_routes(app=app, logger=logger)

uncountable/types/generic_upload.py

@@ -9,6 +9,7 @@ from .generic_upload_t import UploadDestinationBase as UploadDestinationBase
 from .generic_upload_t import UploadDestinationProject as UploadDestinationProject
 from .generic_upload_t import UploadDestinationMaterialFamily as UploadDestinationMaterialFamily
 from .generic_upload_t import UploadDestinationRecipe as UploadDestinationRecipe
+from .generic_upload_t import UploadDestinationAutoMatch as UploadDestinationAutoMatch
 from .generic_upload_t import UploadDestination as UploadDestination
 from .generic_upload_t import GenericUploadStrategy as GenericUploadStrategy
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/generic_upload_t.py

@@ -16,6 +16,7 @@ __all__: list[str] = [
     "GenericRemoteDirectoryScope",
     "GenericUploadStrategy",
     "UploadDestination",
+    "UploadDestinationAutoMatch",
     "UploadDestinationBase",
     "UploadDestinationMaterialFamily",
     "UploadDestinationProject",
@@ -47,6 +48,7 @@ class UploadDestinationType(StrEnum):
     PROJECT = "project"
     MATERIAL_FAMILY = "material_family"
     RECIPE = "recipe"
+    AUTO_MATCH = "auto_match"
 
 
 # DO NOT MODIFY -- This file is generated by type_spec
@@ -91,9 +93,19 @@ class UploadDestinationRecipe(UploadDestinationBase):
     recipe_key: identifier_t.IdentifierKey
 
 
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    named_type_path="sdk.generic_upload.UploadDestinationAutoMatch",
+    parse_require={"type"},
+)
+@dataclasses.dataclass(slots=True, kw_only=True)
+class UploadDestinationAutoMatch(UploadDestinationBase):
+    type: typing.Literal[UploadDestinationType.AUTO_MATCH] = UploadDestinationType.AUTO_MATCH
+
+
 # DO NOT MODIFY -- This file is generated by type_spec
 UploadDestination = typing.Annotated[
-    UploadDestinationProject | UploadDestinationMaterialFamily | UploadDestinationRecipe,
+    UploadDestinationProject | UploadDestinationMaterialFamily | UploadDestinationRecipe | UploadDestinationAutoMatch,
     serial_union_annotation(
     named_type_path="sdk.generic_upload.UploadDestination",
         discriminator="type",
@@ -101,6 +113,7 @@ UploadDestination = typing.Annotated[
             "project": UploadDestinationProject,
             "material_family": UploadDestinationMaterialFamily,
             "recipe": UploadDestinationRecipe,
+            "auto_match": UploadDestinationAutoMatch,
         },
     ),
 ]

{uncountablepythonsdk-0.0.181.dist-info → uncountablepythonsdk-0.0.182.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: UncountablePythonSDK
-Version: 0.0.181
+Version: 0.0.182
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{uncountablepythonsdk-0.0.181.dist-info → uncountablepythonsdk-0.0.182.dist-info}/RECORD

@@ -42,7 +42,7 @@ examples/integration-server/jobs/materials_auto/example_parse.py,sha256=Z3iqAUdI
 examples/integration-server/jobs/materials_auto/example_predictions.py,sha256=5fO4rqRa80_968A1uVZn2TlMOUib54A8rumGW02sIMM,2112
 examples/integration-server/jobs/materials_auto/example_runsheet_wh.py,sha256=7FWDz3QpueVcf83fPrwGSqxs9pX06iKcQFte_vz3dC8,2944
 examples/integration-server/jobs/materials_auto/example_wh.py,sha256=PN-skP27yJwDZboWk5g5EZEc3AKfVayQLfnopjsDKJc,659
-examples/integration-server/jobs/materials_auto/profile.yaml,sha256=ywDrDRAyqiUdj_HvosNP5bXBL8mCWsvdJ1eYQd-mGYo,2369
+examples/integration-server/jobs/materials_auto/profile.yaml,sha256=N9kX-xczfgBSqnfY5qsxmjxXkBXLhMnZU1TOMWI1hfg,2587
 pkgs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pkgs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pkgs/argument_parser/__init__.py,sha256=cEEOAQS8UJ4Kbuo0oCVAHHhkbYCfrH6cUTTt2Mq3PmI,1445
@@ -134,14 +134,15 @@ uncountable/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 uncountable/integration/cli.py,sha256=5RzlfScaZHapiXp4XoaYh0QqSenIjkEf-kyjEue1ycg,9301
 uncountable/integration/construct_client.py,sha256=9c8ZqMjyXebsptiGkJbg53_af4TLOumY1hOQtIbC15w,2229
 uncountable/integration/cron.py,sha256=6eH-kIs3sdYPCyb62_L2M7U_uQTdMTdwY5hreEJb0hw,887
-uncountable/integration/entrypoint.py,sha256=BHOYPQgKvZE6HG8Rv15MkdYl8lRkvfDgv1OdLo0oQ9Q,433
-uncountable/integration/job.py,sha256=eKaRiVagBR0qKUp96jR2NZdxOGKWQbJpBrkcRWoJcXw,9194
+uncountable/integration/entrypoint.py,sha256=seXMXxUPjBLzpP6UAB6fRVodkgGea6vnl2oy4FWvCyE,591
+uncountable/integration/job.py,sha256=Jeiiwvk-T3TxNYpesGbyTuQirw8gX1qQlmCIZ7r7Wuc,9187
 uncountable/integration/request_context.py,sha256=N_FJJxqvfUJ0yV9h3I3vFTGNJiDfyLYObczcYa44pw8,999
 uncountable/integration/scan_profiles.py,sha256=86CQgrQ-zXl3p41ERazzbzhTIIjIh_BprWVfYTSHHE4,3259
 uncountable/integration/scheduler.py,sha256=Z_CL2a-kOAJFpMWzFKtuvRz4AlCIRYbxWdN8fpM0RSQ,9150
 uncountable/integration/server.py,sha256=wnqpowr3xdAXfi4gCTdfFBvmQVyTro439kqrXewBauM,6002
 uncountable/integration/telemetry.py,sha256=W7z2h47Rrn7F3gF1aazbBdthNbSVi4dkQVbZFIs99-c,16155
-uncountable/integration/webhook_signature_key.py,sha256=vXlNtj0qLqZxjKDLmRvlChvLS5xoPAGmYcYcs4WIVaM,3882
+uncountable/integration/webhook_bootstrap.py,sha256=RlMsdfoWhsOX8eQCVz9XhyQLqfr0hPPwW6pdGhhmfeI,2332
+uncountable/integration/webhook_entity.py,sha256=As4x_W_zXuJ4uCAoVZwUKHPJILxP12ow0psPEv5sv9M,5220
 uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/db/connect.py,sha256=-xYWcJYxwKJMBvdQJ5J0-4vwBU-m24uTBtE-VrjwgKw,1479
 uncountable/integration/db/session.py,sha256=z9kor6wy8fdHhHSmdBj9HVvDOQWvs1QM_0kisD1A06E,702
@@ -150,7 +151,7 @@ uncountable/integration/executors/executors.py,sha256=E3a0_yrQesfigDzjhUAwD7-l-o
 uncountable/integration/executors/generic_upload_executor.py,sha256=MuNzrMnAaj6PBhrfTKX6e6vyQhwoDeBrYLz96ai653k,12149
 uncountable/integration/executors/script_executor.py,sha256=hKq09fOG89OAZ1Iq7wm2OgpUz1bh7DIYdZ2yn6ySr10,881
 uncountable/integration/http_server/__init__.py,sha256=ld7hD3wOGwAbnVRBrkTUCP5zNG-kptMd4wGJyQvUWVc,170
-uncountable/integration/http_server/types.py,sha256=eCdYZdEWjdFj9JDTQvhLF3W83HkdRowgX3X_axbA6-0,2274
+uncountable/integration/http_server/types.py,sha256=9Hnt_rvrhGh0MCj1O2k2KHF81U85jURUnG6vt0boPDk,2425
 uncountable/integration/queue_runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/queue_runner/job_process.py,sha256=OONJ7aLsE80qIJ2ABLY3HYyKI7VdpBjI_pPQ3GCC13U,2586
 uncountable/integration/queue_runner/job_runner.py,sha256=Yh9hYfEYOya3BR6ypsal3LKPKrrEb4VQwfGuh4qyU2w,6115
@@ -162,7 +163,7 @@ uncountable/integration/queue_runner/command_server/__init__.py,sha256=hMCDLWct8
 uncountable/integration/queue_runner/command_server/command_client.py,sha256=hvioJc3XcJ4HsJmEwOFpQ2VZ56IqXHldZdR5yvWo39U,6361
 uncountable/integration/queue_runner/command_server/command_server.py,sha256=IsM_4YfuRqr8Ev2tbk6TpfdkBWh9mOTu-8dPUijfVQ8,8581
 uncountable/integration/queue_runner/command_server/constants.py,sha256=7J9mQIAMOfV50wnwpn7HgrPFEi3Ritj6HwrGYwxGLoU,88
-uncountable/integration/queue_runner/command_server/provision_webhook.py,sha256=yPYEqCi9lDFYOeGBJgwcLS3IdfYGcaZFEYo6_8sFiS8,1102
+uncountable/integration/queue_runner/command_server/provision_webhook.py,sha256=TqH_6YjRZz8EHMKxOcCxvmK6vweR8nuY8ilWNEQoogk,1010
 uncountable/integration/queue_runner/command_server/types.py,sha256=ZNqJE6b6rfMZGdcPS-7umB_8x2a7dzTfRBsdGoCzDjY,2077
 uncountable/integration/queue_runner/command_server/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/queue_runner/command_server/protocol/command_server.proto,sha256=mvgCHAIQocdtou7e8SZmmJIBK7jIzJ2fT4yDz6Bur3c,1869
@@ -180,8 +181,12 @@ uncountable/integration/secret_retrieval/__init__.py,sha256=sisf8cxCYcJmCXCKjEPo
 uncountable/integration/secret_retrieval/basic_secret.py,sha256=PfpeaEDwdi8nFlHUJnzCob9GYgGG5O_3mmcaxPPY6zI,3272
 uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=c0VkCvppWKXwgxdblQuj2WhRaEGBPa7vLDoS6mlNOAQ,3919
 uncountable/integration/webhook_server/admin_auth.py,sha256=RYyK54wH5SHfzAVS4CeGgaKHeD4qAhqsvNZqkWuN34Q,1234
-uncountable/integration/webhook_server/admin_routes.py,sha256=S73fIQu7-0U8RKHqvgKIkRMzkEcaYKwplKSexWA5vIQ,1862
 uncountable/integration/webhook_server/entrypoint.py,sha256=LT0IDQtCqNoZBtzb-x6Ev96e1Xug3GRv4hn5NUbG_A4,4385
+uncountable/integration/webhook_server/admin_routes/__init__.py,sha256=iAgtrlljGe-yBCFNMwz8ZkH_KCC_DgW_hazAZgHb8I8,129
+uncountable/integration/webhook_server/admin_routes/_boundary.py,sha256=RKM5LfnMKslRvOBFVi_5iIKDB-3t1RpAVDO5sV1wLIo,1371
+uncountable/integration/webhook_server/admin_routes/health.py,sha256=OJ3fwzchLLeS33rIU23cdKnB3rtTxKlHZqqcpUA3Frc,963
+uncountable/integration/webhook_server/admin_routes/jobs.py,sha256=TR9VeHB1nZQ2KHTipSXWTxvELWRj9f1KxIOOtRTO3Q0,5112
+uncountable/integration/webhook_server/admin_routes/register.py,sha256=Jr1qVEhQgzp8T8zLifAQ5dDAlorFad-1su5kdyvtYWg,501
 uncountable/types/__init__.py,sha256=7fIYhDsz995jFGK7wBXIVMzIxUJbltUmdqm9U6HA_QQ,14367
 uncountable/types/async_batch.py,sha256=yCCWrrLQfxXVqZp-KskxLBNkNmuELdz4PJjx8ULppgs,662
 uncountable/types/async_batch_processor.py,sha256=6Z1qx_H9IqLqp0rzpCtrRvw0DqQ2O5ctkEnw0yNVaV8,48727
@@ -217,8 +222,8 @@ uncountable/types/field_values.py,sha256=iG4TvITLnlz023GuhFrlDwXB7oov5DPpAs_FBaM
 uncountable/types/field_values_t.py,sha256=jfvHmnMLnPz_q6bjdJvWi204uKN2F_KtIzXrfU701Jg,9096
 uncountable/types/fields.py,sha256=M0_ZZr0QdNLXkdHAGo5mfU90kEtHedCSKrcod-FG30Y,245
 uncountable/types/fields_t.py,sha256=MbV4S0hX137FZLe-fXEMRuTo53wjdy1TxodtJmjXbiI,656
-uncountable/types/generic_upload.py,sha256=bNep2nT0fbKAlJaGvHWPmuvfX5KtS8kgTqTh8FQk1NA,858
-uncountable/types/generic_upload_t.py,sha256=prMriVeukeiJa5gWfIOHuFad9rz_18FiOtlNjhb2WZ0,3828
+uncountable/types/generic_upload.py,sha256=XhPnQDxsCqQec3lfq9Afs9t8lzQmVJYmY97Bg5CxvTE,945
+uncountable/types/generic_upload_t.py,sha256=MAsNFqfcsbdCEsFkB4dQx7k7WyW0jX683e18eHVeGUo,4346
 uncountable/types/id_source.py,sha256=sBlDfUwHQ7bGWMschSD_aPQL7LVnCPiV2RAlPLXrAqk,546
 uncountable/types/id_source_t.py,sha256=KhOK1wXgVgbRy1vSdEdbwyz9z4prWRcAx8hOJyjCvyQ,1886
 uncountable/types/identifier.py,sha256=J-ptCFE0_R0bBAvrYp-gvHk8H9Qq9rhbmeyXgwb9nos,482
@@ -434,7 +439,7 @@ uncountable/types/api/uploader/complete_async_parse.py,sha256=Os1HAubxTlbut6Gylj
 uncountable/types/api/uploader/invoke_uploader.py,sha256=QU1KmAFgGHSWskZEKfaAww8rCj-hVNP0i4kMQE3x3Fc,1822
 uncountable/types/api/user/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/user/get_current_user_info.py,sha256=BT7bGp5SOHz3XdKWhhdZ6acnInafT2PgdJ-H4YH3W7c,1181
-uncountablepythonsdk-0.0.181.dist-info/METADATA,sha256=3K5y5CfPXJkAFZQvtT-N4ziTKT3hNqTUovVVjbaucfg,2211
-uncountablepythonsdk-0.0.181.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-uncountablepythonsdk-0.0.181.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
-uncountablepythonsdk-0.0.181.dist-info/RECORD,,
+uncountablepythonsdk-0.0.182.dist-info/METADATA,sha256=oGJSxcjq_RKLaOjhKMpBEnj3_JRLoewFIegeuf9_ZWI,2211
+uncountablepythonsdk-0.0.182.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+uncountablepythonsdk-0.0.182.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
+uncountablepythonsdk-0.0.182.dist-info/RECORD,,

uncountable/integration/webhook_server/admin_routes.py

@@ -1,49 +0,0 @@
-import flask
-from flask.typing import ResponseReturnValue
-from opentelemetry.trace import get_current_span
-from uncountable.core.environment import get_local_admin_server_port
-from uncountable.integration.http_server import HttpException
-from uncountable.integration.queue_runner.command_server.command_client import (
-    check_health,
-)
-from uncountable.integration.queue_runner.command_server.types import (
-    CommandServerException,
-)
-from uncountable.integration.telemetry import Logger
-from uncountable.integration.webhook_server.admin_auth import require_admin_auth
-
-_ADMIN_HEALTH_ROUTE = "/admin/health"
-
-
-def register_admin_routes(app: flask.Flask) -> None:
-    server_logger = Logger(get_current_span())
-
-    @require_admin_auth
-    def handle_admin_health() -> ResponseReturnValue:
-        healthy = check_health(port=get_local_admin_server_port())
-        return flask.jsonify({"healthy": healthy})
-
-    def admin_health() -> ResponseReturnValue:
-        with server_logger.push_scope(_ADMIN_HEALTH_ROUTE):
-            try:
-                return handle_admin_health()
-            except HttpException as e:
-                return e.make_error_response()
-            except CommandServerException as e:
-                server_logger.log_exception(
-                    e,
-                    message="admin health check could not reach the command server",
-                )
-                return HttpException.bad_gateway().make_error_response()
-            except Exception as e:
-                server_logger.log_exception(
-                    e, message=f"unexpected error on GET {_ADMIN_HEALTH_ROUTE}"
-                )
-                return HttpException.unknown_error().make_error_response()
-
-    app.add_url_rule(
-        _ADMIN_HEALTH_ROUTE,
-        endpoint="admin_health",
-        view_func=admin_health,
-        methods=["GET"],
-    )

uncountable/integration/webhook_signature_key.py

@@ -1,110 +0,0 @@
-import secrets
-
-from uncountable.core.client import Client
-from uncountable.core.query.builder import QueryBuilder
-from uncountable.core.query.column import (
-    NonNullableColumn,
-    TextColumn,
-)
-from uncountable.core.query.row import QueryRow
-from uncountable.types import (
-    entity_t,
-    field_values_t,
-    identifier_t,
-    listing_t,
-    webhook_t,
-)
-from uncountable.types.job_definition_t import ProfileMetadata
-from uncountable.types.secret_retrieval_t import SecretRetrievalEntityColumn
-
-_WEBHOOK_DEFINITION_REF_NAME = "unc_webhook_definition"
-_WEBHOOK_REF_NAME_FIELD_REF_NAME = "refName"
-_WEBHOOK_NAME_FIELD_REF_NAME = "name"
-_WEBHOOK_URL_FIELD_REF_NAME = "core_webhook_url"
-_WEBHOOK_SIGNATURE_KEY_FIELD_REF_NAME = "core_webhook_signatureKey"
-_WEBHOOK_PURPOSE_FIELD_REF_NAME = "core_webhook_purpose"
-
-
-class _WebhookEntityQueryRow(QueryRow):
-    ref_name = TextColumn(
-        identifier=listing_t.ColumnIdentifierEntityRefName(
-            entity_type=entity_t.EntityType.WEBHOOK,
-            ref_name=_WEBHOOK_REF_NAME_FIELD_REF_NAME,
-        ),
-        nullable=NonNullableColumn,
-    )
-    signature_key = TextColumn(
-        identifier=listing_t.ColumnIdentifierEntityRefName(
-            entity_type=entity_t.EntityType.WEBHOOK,
-            ref_name=_WEBHOOK_SIGNATURE_KEY_FIELD_REF_NAME,
-        ),
-        nullable=NonNullableColumn,
-    )
-
-
-def webhook_signature_key_retrieval(*, job_id: str) -> SecretRetrievalEntityColumn:
-    return SecretRetrievalEntityColumn(
-        entity_key=identifier_t.IdentifierKeyRefName(ref_name=job_id),
-        column_identifier=listing_t.ColumnIdentifierEntityRefName(
-            entity_type=entity_t.EntityType.WEBHOOK,
-            ref_name=_WEBHOOK_SIGNATURE_KEY_FIELD_REF_NAME,
-        ),
-    )
-
-
-def _resolve_webhook_url(*, profile_metadata: ProfileMetadata, job_id: str) -> str:
-    integration_server_base_url = profile_metadata.integration_server_base_url
-    if integration_server_base_url is None:
-        raise ValueError(
-            "integration_server_base_url is required to resolve the webhook entity "
-            f"for {profile_metadata.name}/{job_id}"
-        )
-    return f"{integration_server_base_url.rstrip('/')}/{profile_metadata.name}/{job_id}"
-
-
-def bootstrap_webhook_signature_key(
-    *,
-    client: Client,
-    profile_metadata: ProfileMetadata,
-    job_id: str,
-    purpose: webhook_t.WebhookPurpose,
-) -> str:
-    existing_webhook = (
-        QueryBuilder(client=client, model=_WebhookEntityQueryRow)
-        .filter(_WebhookEntityQueryRow.ref_name == job_id)
-        .one_or_none()
-    )
-    if existing_webhook is not None:
-        return existing_webhook.signature_key
-
-    webhook_url = _resolve_webhook_url(profile_metadata=profile_metadata, job_id=job_id)
-    signature_key = secrets.token_urlsafe(32)
-    client.create_entity(
-        entity_type=entity_t.EntityType.WEBHOOK,
-        definition_key=identifier_t.IdentifierKeyRefName(
-            ref_name=_WEBHOOK_DEFINITION_REF_NAME
-        ),
-        field_values=[
-            field_values_t.FieldRefNameValue(
-                field_ref_name=_WEBHOOK_REF_NAME_FIELD_REF_NAME,
-                value=job_id,
-            ),
-            field_values_t.FieldRefNameValue(
-                field_ref_name=_WEBHOOK_NAME_FIELD_REF_NAME,
-                value=f"{profile_metadata.name}/{job_id}",
-            ),
-            field_values_t.FieldRefNameValue(
-                field_ref_name=_WEBHOOK_URL_FIELD_REF_NAME,
-                value=webhook_url,
-            ),
-            field_values_t.FieldRefNameValue(
-                field_ref_name=_WEBHOOK_SIGNATURE_KEY_FIELD_REF_NAME,
-                value=signature_key,
-            ),
-            field_values_t.FieldRefNameValue(
-                field_ref_name=_WEBHOOK_PURPOSE_FIELD_REF_NAME,
-                value=purpose.value,
-            ),
-        ],
-    )
-    return signature_key