UncountablePythonSDK 0.0.59__py3-none-any.whl → 0.0.60__py3-none-any.whl

{UncountablePythonSDK-0.0.59.dist-info → UncountablePythonSDK-0.0.60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.59
+Version: 0.0.60
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -33,6 +33,7 @@ Requires-Dist: opentelemetry-exporter-otlp-proto-http ==1.*
 Requires-Dist: opentelemetry-sdk ==1.*
 Requires-Dist: paramiko ==3.*
 Requires-Dist: boto3 ==1.*
+Requires-Dist: flask ==3.*
 Provides-Extra: test
 Requires-Dist: mypy ==1.* ; extra == 'test'
 Requires-Dist: ruff ==0.* ; extra == 'test'

{UncountablePythonSDK-0.0.59.dist-info → UncountablePythonSDK-0.0.60.dist-info}/RECORD

@@ -74,27 +74,28 @@ uncountable/__init__.py,sha256=8l8XWNCKsu7TG94c-xa2KHpDegvxDC2FyQISdWC763Y,89
 uncountable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/core/__init__.py,sha256=RFv0kO6rKFf1PtBPu83hCGmxqkJamRtsgQ9_-ztw7tA,341
 uncountable/core/async_batch.py,sha256=Gur0VOS0AH2ugwvk65hwoX-iqwQAAyJaejY_LyAZZPo,1210
-uncountable/core/client.py,sha256=kKd9MvvlSKDWh69iZ6K2IfbLxMMQ8l0tFLk6p0YG6GM,10622
+uncountable/core/client.py,sha256=UaeQx8Tdif2BMIKO5FOHdpaCr6ySGmEplXSDT9wQls8,10626
+uncountable/core/environment.py,sha256=4rLWeUIDYB6w1iqrAiSEaD_OqMoo_ed3xcjz5jM4HGU,572
 uncountable/core/file_upload.py,sha256=qR7BBBWVxFNrb1_WICreo3dkZygE9lcE1fmZCQrDZU0,3469
 uncountable/core/types.py,sha256=s2CjqYJpsmbC7xMwxxT7kJ_V9bwokrjjWVVjpMcQpKI,333
-uncountable/core/version.py,sha256=SqQIHLhiVZXQBeOwygS2FRZ4WEO27JmWhse0lKm7fgU,274
 uncountable/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/construct_client.py,sha256=I2XTamht13vs-JYkV4PpNS_Pc4FJm-KVYqNNvxI4qNk,1916
-uncountable/integration/cron.py,sha256=e5456IYJF2ipiSsd1R2T334lfe7mtp-gwP7JpS645L0,1858
+uncountable/integration/cron.py,sha256=U6aVIKwLy11n2_ez4MQo5XNZsGjb_Ykh582WG02SWCs,1424
 uncountable/integration/entrypoint.py,sha256=wgOXhTzErttRjOzV4rS4psZW5qUKIa5ez89QndQl61k,785
-uncountable/integration/job.py,sha256=UTzcMes2KrBBRLOM3u94imMKLLnv50glqOkNf8-JOZw,1022
+uncountable/integration/job.py,sha256=UrsZHWXKE2wD5M3lFKJCXvDdWj7QMDtAREKv6RBnT3Q,1548
 uncountable/integration/scan_profiles.py,sha256=3o_h5Ta8ZQEX1epWyXhtyEof0M1b7MobVG7bRKcsFuM,1166
-uncountable/integration/server.py,sha256=bmX-ukLiNDq0ThVB2lUyXl-vtID5HI4gqJHxhsVNG3w,4440
-uncountable/integration/telemetry.py,sha256=lTaTwhf-suKM8AXSeOe3wdUG1mdV7gbi88UBIDkzs9o,6036
+uncountable/integration/server.py,sha256=EhzfOyXVNOKUmS5_XN4TZVSke1n1vnGLfuDrVUlo1j4,4528
+uncountable/integration/telemetry.py,sha256=9vIyV_4Hfxdh-EEP5dH3SalQOSCHjXchwgSB34M2ndI,7127
 uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/db/connect.py,sha256=YtQHJ1DBGPhxKFRCfiXqohOYUceKSxMVOJ88aPI48Ug,181
 uncountable/integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-uncountable/integration/executors/executors.py,sha256=v5ClGVUlvrZcMdmGQa8Ll668G_HGTnKpGOnTM7UMZCQ,956
-uncountable/integration/executors/generic_upload_executor.py,sha256=nB-kgLWXePbl8u6UwCKFXRrXnFpguMR-91ylnfeWPSA,10280
+uncountable/integration/executors/executors.py,sha256=CbwatKkHrLhnqYr_nsBjr0KYeOn8KqijxrZPHbxChBY,1961
+uncountable/integration/executors/generic_upload_executor.py,sha256=PpxZmGUac4PtwlEJi7-S7scSa3y6HrdBPs_p2QDyqZs,10286
 uncountable/integration/executors/script_executor.py,sha256=OmSBOtU48G3mqza9c2lCm84pGGyaDk-ZBJCx3RsdJXc,846
 uncountable/integration/secret_retrieval/__init__.py,sha256=3QXVj35w8rRMxVvmmsViFYDi3lcb3g70incfalOEm6o,87
 uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=eoPWbkUtCn_63A4TFlK_nvEDvfm4u2fiOoglmAkBG3U,3004
-uncountable/types/__init__.py,sha256=0KN0QKnwQgEHP90-BfnW67OF77L4Ino-hN4ea-Tx1M0,8324
+uncountable/integration/webhook_server/entrypoint.py,sha256=gmbGgHpQqrxrfE4GHl-uXNNjET8QVO7Ym5usjyY8MgU,5749
+uncountable/types/__init__.py,sha256=zejeAhEYwf5Jbs-n7S9Wr6NPFXfhWIFdyARB1v40scc,8390
 uncountable/types/async_batch.py,sha256=_OhT25_dEVts_z_n1kqfJH3xlZg3btLqR6TNkfFLlXE,609
 uncountable/types/async_batch_processor.py,sha256=obVzN-PcYLV2pHScszfCGjSq6-Xc34WM1ysx6Fv6tZk,11293
 uncountable/types/async_batch_t.py,sha256=ipSGz93O1KB-WE2dvlvflTKS51rJrf3bJkUojyxos7I,2193
@@ -110,7 +111,7 @@ uncountable/types/client_config_t.py,sha256=_HdS37gMSTIiD4qLnW9dIgt8_Rt5A6xhwMGG
 uncountable/types/curves.py,sha256=W6uMpG5SyW1MS82szNpxkFEn1MnxNpBFyFbQb2Ysfng,366
 uncountable/types/curves_t.py,sha256=TDpsThz4lKmiBmS9b4ItUSCp64TGv8-qDkxb4B2RoTo,1314
 uncountable/types/entity.py,sha256=3XhLteFDRDZvHejDuYh-KvB65hpwrBygljFfiUcOAM8,315
-uncountable/types/entity_t.py,sha256=kH3RyuAXEs6moMKwDGSQokzHaRBDcfuO-tHkFbre2zk,14537
+uncountable/types/entity_t.py,sha256=D-Z92DPWObIYdqaQE4UDQX2RB3GBUuyAgcAkKaSBuWw,14599
 uncountable/types/experiment_groups.py,sha256=_0OXcPzSAbkE-rfKt5tPx178YJ4pcEKZvrCxUHgDnvw,309
 uncountable/types/experiment_groups_t.py,sha256=0IGAXwkYiwdjj6aFjLMihxwauACQTyuRU_1usJTeUg4,593
 uncountable/types/field_values.py,sha256=uuIWX-xmfvcinYPdfkWJeb56zzQY01mc9rmotMPMh24,503
@@ -128,7 +129,7 @@ uncountable/types/input_attributes_t.py,sha256=wE1ekiQfb72Z9VpF5SHipKJkgaJFUHJrN
 uncountable/types/inputs.py,sha256=6RIEFfCxLqpeHEGOpu63O4i8zPogjGeB7wiV_rPBw_g,404
 uncountable/types/inputs_t.py,sha256=RW7gF9zTOwByu-nMTMVuBabLOuWKx4O1nvfgvx_R55o,1611
 uncountable/types/job_definition.py,sha256=HXfaYl5Nafm9C0teQLBtqzroe1HlfKJtfGVm2-40hvg,1937
-uncountable/types/job_definition_t.py,sha256=0BKIqg1fSJKXHxLAWm11bgz7OC4ye1k4EPaY439_Mns,7549
+uncountable/types/job_definition_t.py,sha256=frtTpcJJCNXzdDlz8OWLMBFvsnS9nmqJVn9l7fBI7uc,7616
 uncountable/types/outputs.py,sha256=sUZx_X-TKCZtLm1YCEH8OISX9DdPlv9ZuUfM3-askCc,281
 uncountable/types/outputs_t.py,sha256=2aORUOr0ls1ZYo-ddkWax3D1ZndmQsWtHfJxpYozlhg,656
 uncountable/types/overrides.py,sha256=Mv-smwK1B3pvbt48fNOiqkeQn9wMgYlBFJKUBOJqceE,431
@@ -163,6 +164,8 @@ uncountable/types/units.py,sha256=R_TBhxWCIWSSXK9J3S0Omtj3t5BZNK9C80MyqFjMO7k,27
 uncountable/types/units_t.py,sha256=w_k7SQ_J_-XTGY-BMcWC5sQi2z3uInqtrRUPHgniuew,559
 uncountable/types/users.py,sha256=YEk8v0vDOBFvmOQMQw7MAOicSGzMui8Hb9hdFX2Vw3E,275
 uncountable/types/users_t.py,sha256=ATsF1EGKw49jfcspf9OWHL0tSgOR76x80XloKk624fk,582
+uncountable/types/webhook_job.py,sha256=Y8IVmL311Hd4Jvdl1d7ND_OCygyC0dAoV-_E4A-lI6A,363
+uncountable/types/webhook_job_t.py,sha256=MM-0epv4nIp4lgtLBTr-gs51PVjBuAMqpHfCPXDnYc0,847
 uncountable/types/workflows.py,sha256=uSZWsdDe2jpXnBnRunEen7_7pbKBrE0ds_ez98EzyPg,353
 uncountable/types/workflows_t.py,sha256=Nz9z_eerEDEP_Ieuz7gQEv1SabkFdCiZmu6RL8w3kyE,831
 uncountable/types/api/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
@@ -244,7 +247,7 @@ uncountable/types/api/triggers/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr
 uncountable/types/api/triggers/run_trigger.py,sha256=_Rpha9nxXI3Xr17CrGDtofg4HZ81x2lt0rMZ6As0qfE,893
 uncountable/types/api/uploader/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/uploader/invoke_uploader.py,sha256=Rc77y5q-3R9-SNQgm8P35zKaW2D1Hbtm7PDixnOn1G0,1025
-UncountablePythonSDK-0.0.59.dist-info/METADATA,sha256=NUmxAfNfwlnajLwx1OOVQiFEI0FZ7Crm7z2GP1oL3zI,1934
-UncountablePythonSDK-0.0.59.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-UncountablePythonSDK-0.0.59.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
-UncountablePythonSDK-0.0.59.dist-info/RECORD,,
+UncountablePythonSDK-0.0.60.dist-info/METADATA,sha256=bKGoizXKBfPpFqoKEjCvCBGjvN49fQf77Gn7pP25u0o,1961
+UncountablePythonSDK-0.0.60.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+UncountablePythonSDK-0.0.60.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
+UncountablePythonSDK-0.0.60.dist-info/RECORD,,

uncountable/core/client.py

@@ -14,7 +14,7 @@ from requests.exceptions import JSONDecodeError
 from pkgs.argument_parser import CachedParser
 from pkgs.serialization_util import serialize_for_api
 from pkgs.serialization_util.serialization_helpers import JsonValue
-from uncountable.core.version import get_version
+from uncountable.core.environment import get_version
 from uncountable.integration.telemetry import JobLogger
 from uncountable.types.client_base import APIRequest, ClientMethods
 from uncountable.types.client_config import ClientConfigOptions

uncountable/core/environment.py

@@ -0,0 +1,24 @@
+import functools
+import os
+from importlib.metadata import PackageNotFoundError, version
+
+
+@functools.cache
+def get_version() -> str:
+    try:
+        version_str = version("UncountablePythonSDK")
+    except PackageNotFoundError:
+        version_str = "unknown"
+    return version_str
+
+
+def get_integration_env() -> str | None:
+    return os.environ.get("UNC_INTEGRATION_ENV")
+
+
+def get_webhook_server_port() -> int:
+    return int(os.environ.get("UNC_WEBHOOK_SERVER_PORT") or 5001)
+
+
+def get_otel_enabled() -> bool:
+    return os.environ.get("UNC_OTEL_ENABLED") == "true"

uncountable/integration/cron.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass
 from pkgs.argument_parser import CachedParser
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.integration.construct_client import construct_uncountable_client
-from uncountable.integration.executors.executors import resolve_executor
+from uncountable.integration.executors.executors import execute_job
 from uncountable.integration.job import CronJobArguments
 from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import JobDefinition, ProfileMetadata
@@ -36,20 +36,8 @@ def cron_job_executor(**kwargs: dict) -> None:
         logger=job_logger,
     )
 
-    with job_logger.push_scope(args_passed.definition.name) as job_logger:
-        job = resolve_executor(
-            args_passed.definition.executor, args_passed.profile_metadata
-        )
-
-        job_logger.log_info("running job")
-
-        job.run(args=args)
-
-        if batch_processor.current_queue_size() != 0:
-            batch_processor.send()
-
-        submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
-        job_logger.log_info(
-            "completed job",
-            attributes={"submitted_batch_job_ids": submitted_batch_job_ids},
-        )
+    execute_job(
+        args=args,
+        profile_metadata=args_passed.profile_metadata,
+        job_definition=args_passed.definition,
+    )

uncountable/integration/executors/executors.py

@@ -2,7 +2,7 @@ from typing import assert_never
 
 from uncountable.integration.executors.generic_upload_executor import GenericUploadJob
 from uncountable.integration.executors.script_executor import resolve_script_executor
-from uncountable.integration.job import Job
+from uncountable.integration.job import Job, JobArguments
 from uncountable.types import job_definition_t
 
 
@@ -22,3 +22,35 @@ def resolve_executor(
                 data_source=job_executor.data_source,
             )
     assert_never(job_executor)
+
+
+def execute_job(
+    *,
+    job_definition: job_definition_t.JobDefinition,
+    profile_metadata: job_definition_t.ProfileMetadata,
+    args: JobArguments,
+) -> job_definition_t.JobResult:
+    with args.logger.push_scope(job_definition.name) as job_logger:
+        job = resolve_executor(job_definition.executor, profile_metadata)
+
+        job_logger.log_info("running job")
+
+        try:
+            result = job.run_outer(args=args)
+        except Exception as e:
+            job_logger.log_exception(e)
+            return job_definition_t.JobResult(success=False)
+
+        if args.batch_processor.current_queue_size() != 0:
+            args.batch_processor.send()
+
+        submitted_batch_job_ids = args.batch_processor.get_submitted_job_ids()
+        job_logger.log_info(
+            "completed job",
+            attributes={
+                "submitted_batch_job_ids": submitted_batch_job_ids,
+                "success": result.success,
+            },
+        )
+
+        return result

uncountable/integration/executors/generic_upload_executor.py

@@ -226,7 +226,7 @@ class GenericUploadJob(Job):
 
                 return S3Session(s3_config=s3_config)
 
-    def run(self, args: JobArguments) -> JobResult:
+    def run_outer(self, args: JobArguments) -> JobResult:
         client = args.client
         batch_processor = args.batch_processor
         logger = args.logger

uncountable/integration/job.py

@@ -4,10 +4,11 @@ from dataclasses import dataclass
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.client import Client
 from uncountable.integration.telemetry import JobLogger
+from uncountable.types import webhook_job_t
 from uncountable.types.job_definition_t import JobDefinition, JobResult, ProfileMetadata
 
 
-@dataclass
+@dataclass(kw_only=True)
 class JobArgumentsBase:
     job_definition: JobDefinition
     profile_metadata: ProfileMetadata
@@ -16,27 +17,44 @@ class JobArgumentsBase:
     logger: JobLogger
 
 
-@dataclass
+@dataclass(kw_only=True)
 class CronJobArguments(JobArgumentsBase):
-    # can imagine passing additional data such as in the sftp or webhook cases
     pass
 
 
-JobArguments = CronJobArguments
+@dataclass(kw_only=True)
+class WebhookJobArguments(JobArgumentsBase):
+    payload: webhook_job_t.WebhookEventBody
+
+
+JobArguments = CronJobArguments | WebhookJobArguments
 
 
 class Job(ABC):
     _unc_job_registered: bool = False
 
     @abstractmethod
-    def run(self, args: JobArguments) -> JobResult: ...
+    def run_outer(self, args: JobArguments) -> JobResult: ...
 
 
 class CronJob(Job):
+    def run_outer(self, args: JobArguments) -> JobResult:
+        assert isinstance(args, CronJobArguments)
+        return self.run(args)
+
     @abstractmethod
     def run(self, args: CronJobArguments) -> JobResult: ...
 
 
+class WebhookJob(Job):
+    def run_outer(self, args: JobArguments) -> JobResult:
+        assert isinstance(args, WebhookJobArguments)
+        return self.run(args)
+
+    @abstractmethod
+    def run(self, args: WebhookJobArguments) -> JobResult: ...
+
+
 def register_job(cls: type[Job]) -> type[Job]:
     cls._unc_job_registered = True
     return cls

uncountable/integration/server.py

@@ -19,6 +19,7 @@ from uncountable.types.job_definition_t import (
     CronJobDefinition,
     JobDefinition,
     ProfileMetadata,
+    WebhookJobDefinition,
 )
 
 _MAX_APSCHEDULER_CONCURRENT_JOBS = 1
@@ -97,8 +98,10 @@ class IntegrationServer:
                             kwargs=job_kwargs,
                             **job_opts,
                         )
+                case WebhookJobDefinition():
+                    pass
                 case _:
-                    assert_never(job_defn.trigger)
+                    assert_never(job_defn)
 
     def serve_forever(self) -> None:
         signal.pause()

uncountable/integration/telemetry.py

@@ -1,6 +1,7 @@
 import functools
 import os
 import time
+import typing
 from contextlib import contextmanager
 from enum import StrEnum
 from typing import Generator, assert_never, cast
@@ -16,9 +17,13 @@ from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import (
     SimpleSpanProcessor,
 )
-from opentelemetry.trace import DEFAULT_TRACE_OPTIONS, Tracer
+from opentelemetry.trace import DEFAULT_TRACE_OPTIONS, Span, Tracer
 
-from uncountable.core.version import get_version
+from uncountable.core.environment import (
+    get_integration_env,
+    get_otel_enabled,
+    get_version,
+)
 from uncountable.types import base_t, job_definition_t
 
 
@@ -35,7 +40,7 @@ def get_otel_resource() -> Resource:
     unc_version = os.environ.get("UNC_VERSION")
     if unc_version is not None:
         attributes["service.version"] = unc_version
-    unc_env = os.environ.get("UNC_INTEGRATION_ENV")
+    unc_env = get_integration_env()
     if unc_env is not None:
         attributes["deployment.environment"] = unc_env
     resource = Resource.create(attributes=_cast_attributes(attributes))
@@ -45,7 +50,8 @@ def get_otel_resource() -> Resource:
 @functools.cache
 def get_otel_tracer() -> Tracer:
     provider = TracerProvider(resource=get_otel_resource())
-    provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter()))
+    if get_otel_enabled():
+        provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter()))
     trace.set_tracer_provider(provider)
     return provider.get_tracer("integration.telemetry")
 
@@ -54,7 +60,8 @@ def get_otel_tracer() -> Tracer:
 def get_otel_logger() -> OTELLogger:
     provider = LoggerProvider(resource=get_otel_resource())
     provider.add_log_record_processor(BatchLogRecordProcessor(ConsoleLogExporter()))
-    provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
+    if get_otel_enabled():
+        provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
     _logs.set_logger_provider(provider)
     return provider.get_logger("integration.telemetry")
 
@@ -66,8 +73,19 @@ class LogSeverity(StrEnum):
 
 
 class Logger:
-    current_span_id: int | None = None
-    current_trace_id: int | None = None
+    current_span: Span | None = None
+
+    @property
+    def current_span_id(self) -> int | None:
+        if self.current_span is None:
+            return None
+        return self.current_span.get_span_context().span_id
+
+    @property
+    def current_trace_id(self) -> int | None:
+        if self.current_span is None:
+            return None
+        return self.current_span.get_span_context().trace_id
 
     def _patch_attributes(self, attributes: Attributes | None) -> Attributes:
         return attributes or {}
@@ -84,6 +102,8 @@ class Logger:
             span_id=self.current_span_id,
             trace_id=self.current_trace_id,
             trace_flags=DEFAULT_TRACE_OPTIONS,
+            severity_number=_logs.SeverityNumber.UNSPECIFIED,
+            resource=get_otel_resource(),
         )
         otel_logger.emit(log_record)
 
@@ -104,6 +124,33 @@ class Logger:
             message=message, severity=LogSeverity.ERROR, attributes=attributes
         )
 
+    def log_exception(
+        self,
+        exception: BaseException,
+        *,
+        message: str | None = None,
+        attributes: Attributes | None = None,
+    ) -> None:
+        patched_attributes = self._patch_attributes(attributes)
+        if self.current_span is not None:
+            self.current_span.record_exception(
+                exception=exception, attributes=patched_attributes
+            )
+        self.log_error(
+            message=f"error: {message}\nexception: {exception}",
+            attributes=patched_attributes,
+        )
+
+    @contextmanager
+    def push_scope(
+        self, scope_name: str, *, attributes: Attributes | None = None
+    ) -> Generator[typing.Self, None, None]:
+        with get_otel_tracer().start_as_current_span(
+            scope_name, attributes=self._patch_attributes(attributes)
+        ) as span:
+            self.current_span = span
+            yield self
+
 
 class JobLogger(Logger):
     def __init__(
@@ -129,6 +176,8 @@ class JobLogger(Logger):
                 patched_attributes["job.definition.cron_spec"] = (
                     self.job_definition.cron_spec
                 )
+            case job_definition_t.WebhookJobDefinition():
+                pass
             case _:
                 assert_never(self.job_definition)
         patched_attributes["job.definition.executor.type"] = (
@@ -146,14 +195,3 @@ class JobLogger(Logger):
             case _:
                 assert_never(self.job_definition.executor)
         return _cast_attributes(patched_attributes)
-
-    @contextmanager
-    def push_scope(
-        self, scope_name: str, *, attributes: Attributes | None = None
-    ) -> Generator["JobLogger", None, None]:
-        with get_otel_tracer().start_as_current_span(
-            scope_name, attributes=self._patch_attributes(attributes)
-        ) as span:
-            self.current_span_id = span.get_span_context().span_id
-            self.current_trace_id = span.get_span_context().trace_id
-            yield self

uncountable/integration/webhook_server/entrypoint.py

@@ -0,0 +1,164 @@
+import hmac
+from dataclasses import dataclass
+
+import flask
+import simplejson
+from flask.typing import ResponseReturnValue
+from flask.wrappers import Response
+from uncountable.core.async_batch import AsyncBatchProcessor
+from uncountable.core.environment import get_integration_env, get_webhook_server_port
+from uncountable.integration.construct_client import construct_uncountable_client
+from uncountable.integration.executors.executors import execute_job
+from uncountable.integration.job import WebhookJobArguments
+from uncountable.integration.scan_profiles import load_profiles
+from uncountable.integration.secret_retrieval.retrieve_secret import retrieve_secret
+from uncountable.integration.telemetry import JobLogger, Logger
+from uncountable.types import job_definition_t, webhook_job_t
+
+from pkgs.argument_parser import CachedParser
+
+app = flask.Flask(__name__)
+
+
+@dataclass(kw_only=True)
+class WebhookResponse:
+    pass
+
+
+webhook_payload_parser = CachedParser(webhook_job_t.WebhookEventBody)
+
+
+class WebhookException(BaseException):
+    error_code: int
+    message: str
+
+    def __init__(self, *, error_code: int, message: str) -> None:
+        self.error_code = error_code
+        self.message = message
+
+    @staticmethod
+    def payload_failed_signature() -> "WebhookException":
+        return WebhookException(
+            error_code=401, message="webhook payload did not match signature"
+        )
+
+    @staticmethod
+    def no_signature_passed() -> "WebhookException":
+        return WebhookException(error_code=400, message="missing signature")
+
+    @staticmethod
+    def body_parse_error() -> "WebhookException":
+        return WebhookException(error_code=400, message="body parse error")
+
+    @staticmethod
+    def unknown_error() -> "WebhookException":
+        return WebhookException(error_code=500, message="internal server error")
+
+    def __str__(self) -> str:
+        return f"[{self.error_code}]: {self.message}"
+
+    def make_error_response(self) -> Response:
+        return Response(
+            status=self.error_code, response={"error": {"message": str(self)}}
+        )
+
+
+def _parse_webhook_payload(
+    *, raw_request_body: bytes, signature_key: str, passed_signature: str
+) -> webhook_job_t.WebhookEventBody:
+    request_body_signature = hmac.new(
+        signature_key.encode("utf-8"), msg=raw_request_body, digestmod="sha256"
+    ).hexdigest()
+
+    if request_body_signature != passed_signature:
+        raise WebhookException.payload_failed_signature()
+
+    try:
+        request_body = simplejson.loads(raw_request_body.decode())
+        return webhook_payload_parser.parse_api(request_body)
+    except (simplejson.JSONDecodeError, ValueError) as e:
+        raise WebhookException.body_parse_error() from e
+
+
+def register_route(
+    *,
+    server_logger: Logger,
+    profile_meta: job_definition_t.ProfileMetadata,
+    job: job_definition_t.WebhookJobDefinition,
+) -> None:
+    route = f"/{profile_meta.name}/{job.id}"
+
+    @app.route(route, methods=["POST"])
+    def handle_webhook() -> ResponseReturnValue:
+        with server_logger.push_scope(route):
+            try:
+                signature_key = retrieve_secret(
+                    profile_metadata=profile_meta,
+                    secret_retrieval=job.signature_key_secret,
+                )
+
+                passed_signature = flask.request.headers.get(
+                    "Uncountable-Webhook-Signature"
+                )
+                if passed_signature is None:
+                    raise WebhookException.no_signature_passed()
+
+                webhook_payload = _parse_webhook_payload(
+                    raw_request_body=flask.request.data,
+                    signature_key=signature_key,
+                    passed_signature=passed_signature,
+                )
+
+                job_logger = JobLogger(
+                    profile_metadata=profile_metadata, job_definition=job
+                )
+                client = construct_uncountable_client(
+                    profile_meta=profile_meta, job_logger=job_logger
+                )
+                execute_job(
+                    job_definition=job,
+                    profile_metadata=profile_meta,
+                    args=WebhookJobArguments(
+                        job_definition=job,
+                        profile_metadata=profile_metadata,
+                        client=client,
+                        batch_processor=AsyncBatchProcessor(client=client),
+                        logger=job_logger,
+                        payload=webhook_payload,
+                    ),
+                )
+
+                return flask.jsonify(WebhookResponse())
+            except WebhookException as e:
+                server_logger.log_exception(e)
+                return e.make_error_response()
+            except Exception as e:
+                server_logger.log_exception(e)
+                return WebhookException.unknown_error().make_error_response()
+
+    server_logger.log_info(f"job {job.id} webhook registered at: {route}")
+
+
+profiles = load_profiles()
+for profile in profiles:
+    server_logger = Logger()
+    profile_metadata = job_definition_t.ProfileMetadata(
+        name=profile.name,
+        auth_retrieval=profile.definition.auth_retrieval,
+        base_url=profile.definition.base_url,
+        client_options=profile.definition.client_options,
+    )
+    for job in profile.definition.jobs:
+        if isinstance(job, job_definition_t.WebhookJobDefinition):
+            register_route(
+                server_logger=server_logger, profile_meta=profile_metadata, job=job
+            )
+
+
+if __name__ == "__main__":
+    app.run(
+        host="0.0.0.0",
+        port=get_webhook_server_port(),
+        debug=get_integration_env() == "playground",
+        exclude_patterns=[],
+    )

uncountable/types/__init__.py

@@ -97,6 +97,7 @@ from .api.recipes import unlock_recipes as unlock_recipes_t
 from .api.material_families import update_entity_material_families as update_entity_material_families_t
 from .api.field_options import upsert_field_options as upsert_field_options_t
 from . import users_t as users_t
+from . import webhook_job_t as webhook_job_t
 from . import workflows_t as workflows_t
 
 
@@ -195,5 +196,6 @@ __all__: list[str] = [
     "update_entity_material_families_t",
     "upsert_field_options_t",
     "users_t",
+    "webhook_job_t",
     "workflows_t",
 ]

uncountable/types/entity_t.py

@@ -25,6 +25,7 @@ __all__: list[str] = [
         "analytical_method_category": "Analytical Method Category",
         "annotation_type": "Annotation Type",
         "approval": "Approval",
+        "async_job": "Async Job",
         "calculation": "Calculation",
         "calendar": "Calendar",
         "calendar_event": "Calendar Event",
@@ -176,6 +177,7 @@ class EntityType(StrEnum):
     ANALYTICAL_METHOD_CATEGORY = "analytical_method_category"
     ANNOTATION_TYPE = "annotation_type"
     APPROVAL = "approval"
+    ASYNC_JOB = "async_job"
     CALCULATION = "calculation"
     CALENDAR = "calendar"
     CALENDAR_EVENT = "calendar_event"

uncountable/types/job_definition_t.py

@@ -191,11 +191,12 @@ class WebhookJobDefinition(JobDefinitionBase):
 
 # DO NOT MODIFY -- This file is generated by type_spec
 JobDefinition = typing.Annotated[
-    typing.Union[CronJobDefinition],
+    typing.Union[CronJobDefinition, WebhookJobDefinition],
     serial_union_annotation(
         discriminator="type",
         discriminator_map={
             "cron": CronJobDefinition,
+            "webhook": WebhookJobDefinition,
         },
     ),
 ]

uncountable/types/webhook_job.py

@@ -0,0 +1,9 @@
+# flake8: noqa: F821
+# ruff: noqa: E402 Q003
+# fmt: off
+# isort: skip_file
+# DO NOT MODIFY -- This file is generated by type_spec
+# Kept only for SDK backwards compatibility
+from .webhook_job_t import WebhookEventPayload as WebhookEventPayload
+from .webhook_job_t import WebhookEventBody as WebhookEventBody
+# DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/webhook_job_t.py

@@ -0,0 +1,33 @@
+# DO NOT MODIFY -- This file is generated by type_spec
+# flake8: noqa: F821
+# ruff: noqa: E402 Q003
+# fmt: off
+# isort: skip_file
+from __future__ import annotations
+import typing  # noqa: F401
+import datetime  # noqa: F401
+from decimal import Decimal  # noqa: F401
+import dataclasses
+from pkgs.serialization import serial_class
+from . import base_t
+
+__all__: list[str] = [
+    "WebhookEventBody",
+    "WebhookEventPayload",
+]
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    unconverted_values={"data"},
+)
+@dataclasses.dataclass(kw_only=True)
+class WebhookEventPayload:
+    data: dict[str, base_t.JsonValue]
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@dataclasses.dataclass(kw_only=True)
+class WebhookEventBody(WebhookEventPayload):
+    event_id: str
+# DO NOT MODIFY -- This file is generated by type_spec

uncountable/core/version.py

@@ -1,11 +0,0 @@
-import functools
-from importlib.metadata import PackageNotFoundError, version
-
-
-@functools.cache
-def get_version() -> str:
-    try:
-        version_str = version("UncountablePythonSDK")
-    except PackageNotFoundError:
-        version_str = "unknown"
-    return version_str