UncountablePythonSDK 0.0.49__py3-none-any.whl → 0.0.51__py3-none-any.whl

{UncountablePythonSDK-0.0.49.dist-info → UncountablePythonSDK-0.0.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.49
+Version: 0.0.51
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -27,6 +27,10 @@ Requires-Dist: PyYAML ==6.*
 Requires-Dist: google-api-python-client ==2.*
 Requires-Dist: tqdm ==4.*
 Requires-Dist: pysftp ==0.*
+Requires-Dist: opentelemetry-api ==1.*
+Requires-Dist: opentelemetry-exporter-otlp-proto-common ==1.*
+Requires-Dist: opentelemetry-exporter-otlp-proto-http ==1.*
+Requires-Dist: opentelemetry-sdk ==1.*
 Requires-Dist: paramiko ==3.*
 Requires-Dist: boto3 ==1.*
 Provides-Extra: test

{UncountablePythonSDK-0.0.49.dist-info → UncountablePythonSDK-0.0.51.dist-info}/RECORD

@@ -3,7 +3,7 @@ docs/conf.py,sha256=YF5J-9g_Wg8wXmyHsGaE8xYlDEzqocNl3UWUmP0CwBg,1702
 docs/index.md,sha256=eEdirX_Ds6ICTRtIS5iT4irCquHcQyKN7E4M5QP9T8A,257
 docs/justfile,sha256=cvNcpb-ByPOF2aCrFlg3DDZBoYMx5W8xGdr13m9HcnI,215
 docs/quickstart.md,sha256=3GuJ0MB1O5kjlsrgAmdSkDq0rYqATrYy-tzEHDy8H-c,422
-docs/requirements.txt,sha256=Y6wdRPGnrd-HMShTgy9Fgydz868Bnyl007P9HCUKXhw,139
+docs/requirements.txt,sha256=YDDAaHfuLxkdLhrjEUJeHDE-NSmD5chTgVTIO7BEeto,139
 docs/static/logo_blue.png,sha256=SyYpMTVhhBbhF5Wl8lWaVwz-_p1MIR6dW6bVhufQRME,46708
 docs/static/favicons/android-chrome-192x192.png,sha256=XoF-AhD55JlSBDGsEPJKfT_VeXT-awhwKyZnxLhrwvk,1369
 docs/static/favicons/android-chrome-512x512.png,sha256=1S4xwY9YtJQ5ifFsZ-DOzssoyBYs0t9uwdOUmYx0Xso,3888
@@ -74,26 +74,28 @@ uncountable/__init__.py,sha256=8l8XWNCKsu7TG94c-xa2KHpDegvxDC2FyQISdWC763Y,89
 uncountable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/core/__init__.py,sha256=RFv0kO6rKFf1PtBPu83hCGmxqkJamRtsgQ9_-ztw7tA,341
 uncountable/core/async_batch.py,sha256=Gur0VOS0AH2ugwvk65hwoX-iqwQAAyJaejY_LyAZZPo,1210
-uncountable/core/client.py,sha256=2QRLy0GcSamQDqUABQq8R7D_wGofYewSoTQiZzOyZXk,10168
+uncountable/core/client.py,sha256=C0hJ0_SGL5WEhPuAWDSj4ShjjIiQasxpfpnisTi-Uag,10554
 uncountable/core/file_upload.py,sha256=TkQ0fKbbYrPgns1Jh51JU35DUqZHB3ljOaVgjSlBx9Y,3149
 uncountable/core/types.py,sha256=s2CjqYJpsmbC7xMwxxT7kJ_V9bwokrjjWVVjpMcQpKI,333
+uncountable/core/version.py,sha256=SqQIHLhiVZXQBeOwygS2FRZ4WEO27JmWhse0lKm7fgU,274
 uncountable/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-uncountable/integration/construct_client.py,sha256=9ZsfkA0oZIE04mPnwGlt6EcrlnpaUPA93K100FS1F7A,1546
-uncountable/integration/cron.py,sha256=n4HjW-KNruYQ_09iwNBd9i4-Juy_9ddzbf71_wnydHA,1674
+uncountable/integration/construct_client.py,sha256=e1uAMVp4FTbValCJ3gfaZFuObKxHbTXq496-T-KMOG4,1899
+uncountable/integration/cron.py,sha256=e5456IYJF2ipiSsd1R2T334lfe7mtp-gwP7JpS645L0,1858
 uncountable/integration/entrypoint.py,sha256=9rk06gBTsCqytIs8Shsnlf6ir_4Uq5d5rfP1veiSLzc,1437
-uncountable/integration/job.py,sha256=5HJcjtwj_pGFYAZ6SK7xa9RqDXCoBb5rAb_OHdTR61Q,1463
-uncountable/integration/server.py,sha256=85yzQrrpp8OEG37CTN9wtgB9UrzWnDLuDkY0nWcp--k,3548
+uncountable/integration/job.py,sha256=UTzcMes2KrBBRLOM3u94imMKLLnv50glqOkNf8-JOZw,1022
+uncountable/integration/server.py,sha256=JlnbidtiSLCEod0DzLKIzTCxP1qgaHBIBolUvnihJn8,3980
+uncountable/integration/telemetry.py,sha256=H5XQnRTszDL6Nl_oQYuWRr_oNYMocDaCCpKlwll-qZI,5127
 uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/db/connect.py,sha256=YtQHJ1DBGPhxKFRCfiXqohOYUceKSxMVOJ88aPI48Ug,181
 uncountable/integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/executors/executors.py,sha256=v5ClGVUlvrZcMdmGQa8Ll668G_HGTnKpGOnTM7UMZCQ,956
-uncountable/integration/executors/generic_upload_executor.py,sha256=HR5bZE1DMVLRB-lwmsfZnoLEsaL66aHZM0xCEO_D3-4,10238
+uncountable/integration/executors/generic_upload_executor.py,sha256=wafNY_gpbUiQhvkFPDw-GGiJLmDVtTRgH_5jwMLy2Z4,10283
 uncountable/integration/executors/script_executor.py,sha256=OmSBOtU48G3mqza9c2lCm84pGGyaDk-ZBJCx3RsdJXc,846
 uncountable/integration/secret_retrieval/__init__.py,sha256=3QXVj35w8rRMxVvmmsViFYDi3lcb3g70incfalOEm6o,87
-uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=M0qXVJpD8hMYIFypHFeyh598sqmIDX8ZOyXK23CluF0,1323
+uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=iDqLbYKKLv8Wl5hNSKl7b4hKJnDshFe6tgdAJmQgQyA,2205
 uncountable/types/__init__.py,sha256=95iOd3WXWoI_4a461IS2ieWRic3zRyNaCYzfTpX764o,8162
 uncountable/types/async_batch.py,sha256=ihCv5XWSTTPmuO-GMPn1EACGI2CBUIJTATZ3aPgsNBA,523
-uncountable/types/async_batch_processor.py,sha256=VO0P-oSLAvzBK_Rk-KPS74AhSyrkj5ILZrAu-gs7Lpc,8564
+uncountable/types/async_batch_processor.py,sha256=R--exgi4Gw0HWCnh8M-3_2PqG2ByTBtdyuSQ2eYtYn8,8671
 uncountable/types/async_batch_t.py,sha256=9jp9rOyetRdD5aQVyijzQggTyYU4021PBVGXk0ooBCQ,1911
 uncountable/types/base.py,sha256=xVSjWvA_fUUnkCg83EjoYEFvAfmskinKFMeYFOxNc9E,359
 uncountable/types/base_t.py,sha256=XXjZXexx0xWFUxMMhW8i9nIL6n8dsZVsHwdgnhZ0zJ4,2714
@@ -101,7 +103,7 @@ uncountable/types/calculations.py,sha256=FFO_D3BbKoGDZnqWvTKpW4KF359i2vrKjpdFCLY
 uncountable/types/calculations_t.py,sha256=7GTSi2L8NYjzjUJJx3cmtVkK9uD-uhfYvIFK-ffQj-8,556
 uncountable/types/chemical_structure.py,sha256=E-LnikTFDoVQ1b2zKaVUIO_PAKm-7aZZYJi8I8SDSic,302
 uncountable/types/chemical_structure_t.py,sha256=aFsTkkbzy6Gvyde3qrrEYD95gcYhxkgKMiDRaRE0o-Y,760
-uncountable/types/client_base.py,sha256=3CKr_v3bXez7ZjUg41q09pju5tgYGMsbgJAoYMffh-c,65240
+uncountable/types/client_base.py,sha256=db6V-Jea8iWT4kz2hXc5c_doLJKLZTd5T-F4ehobWMs,65431
 uncountable/types/client_config.py,sha256=4h5Liko9uKCo9_0gdbPhoK6Jr2Kv7tioLiQ8iKeq-_4,301
 uncountable/types/client_config_t.py,sha256=_HdS37gMSTIiD4qLnW9dIgt8_Rt5A6xhwMGGga7vnLg,625
 uncountable/types/curves.py,sha256=W6uMpG5SyW1MS82szNpxkFEn1MnxNpBFyFbQb2Ysfng,366
@@ -124,8 +126,8 @@ uncountable/types/input_attributes.py,sha256=IrIKQnHqHdS1Utdfzr9GnOe17a8riaqYcO1
 uncountable/types/input_attributes_t.py,sha256=wE1ekiQfb72Z9VpF5SHipKJkgaJFUHJrNkkJdmuhF9w,820
 uncountable/types/inputs.py,sha256=6RIEFfCxLqpeHEGOpu63O4i8zPogjGeB7wiV_rPBw_g,404
 uncountable/types/inputs_t.py,sha256=RW7gF9zTOwByu-nMTMVuBabLOuWKx4O1nvfgvx_R55o,1611
-uncountable/types/job_definition.py,sha256=sCQqtyHI3hksc5pVpk5tqbG55F91ST4FoDwr2TmTOuQ,1787
-uncountable/types/job_definition_t.py,sha256=DpqTYDocJ6n6kVq5PBFhFb5Dlb2vpHdp9IZfFb1Qcpo,6590
+uncountable/types/job_definition.py,sha256=Cgwop6iNcWEsEpfjMl1EtkRES_3-SVzpMe-w73V4G_U,1862
+uncountable/types/job_definition_t.py,sha256=pL-0YVyR0oz6-5wWXJsX1UyYo4R_FPwhOYFj9iMKgvo,7170
 uncountable/types/outputs.py,sha256=sUZx_X-TKCZtLm1YCEH8OISX9DdPlv9ZuUfM3-askCc,281
 uncountable/types/outputs_t.py,sha256=2aORUOr0ls1ZYo-ddkWax3D1ZndmQsWtHfJxpYozlhg,656
 uncountable/types/permissions.py,sha256=1mRnSsmRgjuLgp6pylTwwACD_YRIcmlqxHkufwZtMns,297
@@ -167,8 +169,8 @@ uncountable/types/api/batch/execute_batch_load_async.py,sha256=3ptjtokj7eZ_A7OUX
 uncountable/types/api/chemical/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/chemical/convert_chemical_formats.py,sha256=-FKBOcg1jteFu920NM-0lBk90pfucpcg2WAsaddfDc8,1323
 uncountable/types/api/entity/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
-uncountable/types/api/entity/create_entities.py,sha256=wYxj0Dsi6nRy94qzTUkZYWHkFFpTC8Gt2sHOuwCosVs,1738
-uncountable/types/api/entity/create_entity.py,sha256=pqN0VNcWlaHFxOtuPUjE2DsONVDYCkq-gUEAV3R0Ds0,1908
+uncountable/types/api/entity/create_entities.py,sha256=qvra6BustbxWmt7_c83mCMkfxaBJ4B8xayJUvk6Ab9E,1780
+uncountable/types/api/entity/create_entity.py,sha256=vfbJiHOMEEzJFZNNiHbIBqisytYdKa7rTOraWdZrXpw,1950
 uncountable/types/api/entity/get_entities_data.py,sha256=_dyDJ8Aukeijj2U3ZEQYNECoI3TC4phn8RhgJQP1e1s,1162
 uncountable/types/api/entity/list_entities.py,sha256=93J8jbHOdBL7Ee2_z_M57JpIVsmI2RddDMpprTakYks,1717
 uncountable/types/api/entity/lock_entity.py,sha256=twQ-f61AKS_NrKP-TzqHkMmzTJGMc4yxZFl-2JctOg4,1004
@@ -237,8 +239,8 @@ uncountable/types/api/recipes/unlock_recipes.py,sha256=AvzQeZCLs9i7CuhMs3Xltdi4n
 uncountable/types/api/triggers/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/triggers/run_trigger.py,sha256=_Rpha9nxXI3Xr17CrGDtofg4HZ81x2lt0rMZ6As0qfE,893
 uncountable/types/api/uploader/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
-uncountable/types/api/uploader/invoke_uploader.py,sha256=w7NVSpTpKxsF2TgLzVrji1Ql5Z8QWTz6d5OvXpRtyDo,992
-UncountablePythonSDK-0.0.49.dist-info/METADATA,sha256=YzHISP7OjFmFRJWuVPWl6jgdlvsE6W7glCma0mhU7sc,1734
-UncountablePythonSDK-0.0.49.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-UncountablePythonSDK-0.0.49.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
-UncountablePythonSDK-0.0.49.dist-info/RECORD,,
+uncountable/types/api/uploader/invoke_uploader.py,sha256=4zOcB_38uT73Jm3-XqkG40fBM1R5vpvPpGAg-U4lzxY,1059
+UncountablePythonSDK-0.0.51.dist-info/METADATA,sha256=AzvBprRKL--Fkabx8qrLx9P07XsCyTu1BqpTGl9AMFg,1934
+UncountablePythonSDK-0.0.51.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
+UncountablePythonSDK-0.0.51.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
+UncountablePythonSDK-0.0.51.dist-info/RECORD,,

{UncountablePythonSDK-0.0.49.dist-info → UncountablePythonSDK-0.0.51.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: setuptools (73.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

docs/requirements.txt

@@ -2,6 +2,6 @@ furo==2024.7.18
 myst-parser==3.0.1
 sphinx-autoapi==3.2.0
 sphinx-copybutton==0.5.2
-Sphinx==7.4.4
+Sphinx==8.0.0
 sphinx_design==0.6.0
 sphinx-favicon==1.0.1

uncountable/core/client.py

@@ -4,16 +4,18 @@ import typing
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import StrEnum
-from importlib.metadata import PackageNotFoundError, version
 from urllib.parse import urljoin
 from uuid import uuid4
 
 import requests
+from opentelemetry.sdk.resources import Attributes
 from requests.exceptions import JSONDecodeError
 
 from pkgs.argument_parser import CachedParser
 from pkgs.serialization_util import serialize_for_api
 from pkgs.serialization_util.serialization_helpers import JsonValue
+from uncountable.core.version import get_version
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.client_base import APIRequest, ClientMethods
 from uncountable.types.client_config import ClientConfigOptions
 
@@ -25,12 +27,6 @@ UNC_REQUEST_ID_HEADER = "X-UNC-REQUEST-ID"
 UNC_SDK_VERSION_HEADER = "X-UNC-SDK-VERSION"
 
 
-try:
-    __version__ = version("UncountablePythonSDK")
-except PackageNotFoundError:
-    __version__ = "unknown"
-
-
 class EndpointMethod(StrEnum):
     POST = "POST"
     GET = "GET"
@@ -61,6 +57,7 @@ HTTPRequest = HTTPPostRequest | HTTPGetRequest
 @dataclass(kw_only=True)
 class ClientConfig(ClientConfigOptions):
     transform_request: typing.Callable[[requests.Request], requests.Request] | None = None
+    job_logger: typing.Optional[JobLogger] = None
 
 
 OAUTH_REFRESH_WINDOW_SECONDS = 60 * 5
@@ -204,7 +201,15 @@ class Client(ClientMethods):
             case _:
                 typing.assert_never(http_request)
         request.headers = http_request.headers
-        response = self._send_request(request)
+        if self._cfg.job_logger is not None:
+            attributes: Attributes = {
+                "method": http_request.method,
+                "endpoint": api_request.endpoint,
+            }
+            with self._cfg.job_logger.push_scope("api_call", attributes=attributes):
+                response = self._send_request(request)
+        else:
+            response = self._send_request(request)
         response_data = self._get_response_json(response, request_id=request_id)
         cached_parser = self._get_cached_parser(return_type)
         try:
@@ -260,7 +265,7 @@ class Client(ClientMethods):
     ) -> HTTPRequest:
         headers = self._build_auth_headers()
         headers[UNC_REQUEST_ID_HEADER] = request_id
-        headers[UNC_SDK_VERSION_HEADER] = __version__
+        headers[UNC_SDK_VERSION_HEADER] = get_version()
         method = api_request.method.lower()
         data = {"data": json.dumps(serialize_for_api(api_request.args))}
         match method:

uncountable/core/version.py

@@ -0,0 +1,11 @@
+import functools
+from importlib.metadata import PackageNotFoundError, version
+
+
+@functools.cache
+def get_version() -> str:
+    try:
+        version_str = version("UncountablePythonSDK")
+    except PackageNotFoundError:
+        version_str = "unknown"
+    return version_str

uncountable/integration/construct_client.py

@@ -1,47 +1,51 @@
-import os
-
 from uncountable.core import AuthDetailsApiKey, Client
 from uncountable.core.client import ClientConfig
-from uncountable.core.types import AuthDetailsAll
+from uncountable.core.types import AuthDetailsAll, AuthDetailsOAuth
+from uncountable.integration.secret_retrieval.retrieve_secret import retrieve_secret
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import (
-    AuthRetrievalEnv,
+    AuthRetrievalBasic,
+    AuthRetrievalOAuth,
     ProfileMetadata,
 )
 
 
-def _get_env_var(name: str) -> str:
-    value = os.getenv(name)
-    if value is None:
-        raise Exception(f"environment variable {name} is missing")
-    return value
-
-
 def _construct_auth_details(profile_meta: ProfileMetadata) -> AuthDetailsAll:
     match profile_meta.auth_retrieval:
-        case AuthRetrievalEnv():
-            api_id = _get_env_var(f"UNC_PROFILE_{profile_meta.name.upper()}_API_ID")
-            api_secret_key = _get_env_var(
-                f"UNC_PROFILE_{profile_meta.name.upper()}_API_SECRET_KEY"
+        case AuthRetrievalOAuth():
+            refresh_token = retrieve_secret(
+                profile_meta.auth_retrieval.refresh_token_secret,
+                profile_metadata=profile_meta,
+            )
+            return AuthDetailsOAuth(refresh_token=refresh_token)
+        case AuthRetrievalBasic():
+            api_id = retrieve_secret(
+                profile_meta.auth_retrieval.api_id_secret, profile_metadata=profile_meta
+            )
+            api_key = retrieve_secret(
+                profile_meta.auth_retrieval.api_key_secret, profile_metadata=profile_meta
             )
 
-            assert api_id is not None
-            assert api_secret_key is not None
-
-            return AuthDetailsApiKey(api_id=api_id, api_secret_key=api_secret_key)
+            return AuthDetailsApiKey(api_id=api_id, api_secret_key=api_key)
 
 
-def _construct_client_config(profile_meta: ProfileMetadata) -> ClientConfig | None:
+def _construct_client_config(
+    profile_meta: ProfileMetadata, job_logger: JobLogger
+) -> ClientConfig | None:
     if profile_meta.client_options is None:
         return None
     return ClientConfig(
         allow_insecure_tls=profile_meta.client_options.allow_insecure_tls,
         extra_headers=profile_meta.client_options.extra_headers,
+        job_logger=job_logger,
     )
 
 
-def construct_uncountable_client(profile_meta: ProfileMetadata) -> Client:
+def construct_uncountable_client(
+    profile_meta: ProfileMetadata, job_logger: JobLogger
+) -> Client:
     return Client(
         base_url=profile_meta.base_url,
         auth_details=_construct_auth_details(profile_meta),
-        config=_construct_client_config(profile_meta),
+        config=_construct_client_config(profile_meta, job_logger),
     )

uncountable/integration/cron.py

@@ -4,7 +4,8 @@ from pkgs.argument_parser import CachedParser
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.integration.construct_client import construct_uncountable_client
 from uncountable.integration.executors.executors import resolve_executor
-from uncountable.integration.job import CronJobArguments, JobLogger
+from uncountable.integration.job import CronJobArguments
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import JobDefinition, ProfileMetadata
 
 
@@ -19,29 +20,36 @@ cron_args_parser = CachedParser(CronJobArgs)
 
 def cron_job_executor(**kwargs: dict) -> None:
     args_passed = cron_args_parser.parse_storage(kwargs)
-    client = construct_uncountable_client(profile_meta=args_passed.profile_metadata)
+    job_logger = JobLogger(
+        profile_metadata=args_passed.profile_metadata,
+        job_definition=args_passed.definition,
+    )
+    client = construct_uncountable_client(
+        profile_meta=args_passed.profile_metadata, job_logger=job_logger
+    )
     batch_processor = AsyncBatchProcessor(client=client)
     args = CronJobArguments(
         job_definition=args_passed.definition,
         client=client,
         batch_processor=batch_processor,
         profile_metadata=args_passed.profile_metadata,
-        logger=JobLogger(
-            profile_metadata=args_passed.profile_metadata,
-            job_definition=args_passed.definition,
-        ),
+        logger=job_logger,
     )
 
-    job = resolve_executor(args_passed.definition.executor, args_passed.profile_metadata)
+    with job_logger.push_scope(args_passed.definition.name) as job_logger:
+        job = resolve_executor(
+            args_passed.definition.executor, args_passed.profile_metadata
+        )
 
-    print(f"running job {args_passed.definition.name}")
+        job_logger.log_info("running job")
 
-    job.run(args=args)
+        job.run(args=args)
 
-    if batch_processor.current_queue_size() != 0:
-        batch_processor.send()
+        if batch_processor.current_queue_size() != 0:
+            batch_processor.send()
 
-    print(f"completed job {args_passed.definition.name}")
-    submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
-    if len(submitted_batch_job_ids) != 0:
-        print("submitted batch jobs", submitted_batch_job_ids)
+        submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
+        job_logger.log_info(
+            "completed job",
+            attributes={"submitted_batch_job_ids": submitted_batch_job_ids},
+        )

uncountable/integration/executors/generic_upload_executor.py

@@ -17,8 +17,9 @@ from pkgs.filesystem_utils import (
 )
 from pkgs.filesystem_utils.filesystem_session import FileSystemSession
 from uncountable.core.file_upload import DataFileUpload, FileUpload
-from uncountable.integration.job import Job, JobArguments, JobLogger
+from uncountable.integration.job import Job, JobArguments
 from uncountable.integration.secret_retrieval import retrieve_secret
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.generic_upload_t import (
     GenericRemoteDirectoryScope,
     GenericUploadStrategy,

uncountable/integration/job.py

@@ -3,24 +3,10 @@ from dataclasses import dataclass
 
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.client import Client
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import JobDefinition, JobResult, ProfileMetadata
 
 
-class JobLogger:
-    def __init__(
-        self, *, profile_metadata: ProfileMetadata, job_definition: JobDefinition
-    ) -> None:
-        self.profile_metadata = profile_metadata
-        self.job_definition = job_definition
-
-    def log_info(self, *log_objects: object) -> None:
-        # IMPROVE: log a json message with context that can be parsed by OT
-        print(
-            f"[{self.job_definition.id}] in profile ({self.profile_metadata.name}): ",
-            *log_objects,
-        )
-
-
 @dataclass
 class JobArgumentsBase:
     job_definition: JobDefinition

uncountable/integration/secret_retrieval/retrieve_secret.py

@@ -1,5 +1,10 @@
+import base64
+import functools
+import json
 import os
 
+import boto3
+
 from uncountable.types.job_definition_t import ProfileMetadata
 from uncountable.types.secret_retrieval_t import (
     SecretRetrieval,
@@ -22,6 +27,25 @@ class SecretRetrievalError(BaseException):
         return f"{self.secret_retrieval.type} secret retrieval failed{append_message}"
 
 
+@functools.cache
+def _get_aws_secret(*, secret_name: str, region_name: str, sub_key: str | None) -> str:
+    client = boto3.client("secretsmanager", region_name=region_name)
+    response = client.get_secret_value(SecretId=secret_name)
+
+    if "SecretString" in response:
+        secret = response["SecretString"]
+    else:
+        secret = base64.b64decode(response["SecretBinary"])
+
+    value = json.loads(secret)
+
+    if sub_key is not None:
+        assert isinstance(value, dict)
+        return str(value[sub_key])
+    else:
+        return str(value)
+
+
 def retrieve_secret(
     secret_retrieval: SecretRetrieval, profile_metadata: ProfileMetadata
 ) -> str:
@@ -37,4 +61,11 @@ def retrieve_secret(
                 )
             return secret
         case SecretRetrievalAWS():
-            raise NotImplementedError("aws secret retrieval not yet implemented")
+            try:
+                return _get_aws_secret(
+                    secret_name=secret_retrieval.secret_name,
+                    region_name=secret_retrieval.region,
+                    sub_key=secret_retrieval.sub_key,
+                )
+            except Exception as e:
+                raise SecretRetrievalError(secret_retrieval) from e

uncountable/integration/server.py

@@ -11,6 +11,7 @@ from apscheduler.triggers.cron import CronTrigger
 from sqlalchemy.engine.base import Engine
 
 from uncountable.integration.cron import CronJobArgs, cron_job_executor
+from uncountable.types import base_t
 from uncountable.types.client_config_t import ClientConfigOptions
 from uncountable.types.job_definition_t import (
     AuthRetrieval,
@@ -67,7 +68,14 @@ class IntegrationServer:
                         existing_job.reschedule(
                             CronTrigger.from_crontab(job_defn.cron_spec)
                         )
+                        if not job_defn.enabled:
+                            existing_job.pause()
+                        else:
+                            existing_job.resume()
                     else:
+                        job_opts: dict[str, base_t.JsonValue] = {}
+                        if not job_defn.enabled:
+                            job_opts["next_run_time"] = None
                         self._scheduler.add_job(
                             cron_job_executor,
                             # IMPROVE: reconsider these defaults
@@ -77,6 +85,7 @@ class IntegrationServer:
                             name=job_defn.name,
                             id=job_defn.id,
                             kwargs=job_kwargs,
+                            **job_opts,
                         )
                 case _:
                     assert_never(job_defn.trigger)

uncountable/integration/telemetry.py

@@ -0,0 +1,130 @@
+import functools
+import os
+import sys
+import time
+from contextlib import contextmanager
+from enum import StrEnum
+from typing import Generator, TextIO, assert_never, cast
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk._logs import LogRecord
+from opentelemetry.sdk.resources import Attributes, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    SimpleSpanProcessor,
+)
+from opentelemetry.trace import Tracer
+
+from uncountable.core.version import get_version
+from uncountable.types import base_t, job_definition_t
+
+
+def _cast_attributes(attributes: dict[str, base_t.JsonValue]) -> Attributes:
+    return cast(Attributes, attributes)
+
+
+@functools.cache
+def get_tracer() -> Tracer:
+    attributes: dict[str, base_t.JsonValue] = {
+        "service.name": "integration-server",
+        "sdk.version": get_version(),
+    }
+    unc_version = os.environ.get("UNC_VERSION")
+    if unc_version is not None:
+        attributes["service.version"] = unc_version
+    unc_env = os.environ.get("UNC_INTEGRATION_ENV")
+    if unc_env is not None:
+        attributes["deployment.environment"] = unc_env
+    resource = Resource.create(attributes=_cast_attributes(attributes))
+    provider = TracerProvider(resource=resource)
+    provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter()))
+    trace.set_tracer_provider(provider)
+    return provider.get_tracer("integration.telemetry")
+
+
+class LogSeverity(StrEnum):
+    INFO = "Info"
+    WARN = "Warn"
+    ERROR = "Error"
+
+
+class JobLogger:
+    current_span_id: int | None = None
+    current_trace_id: int | None = None
+
+    def __init__(
+        self,
+        *,
+        profile_metadata: job_definition_t.ProfileMetadata,
+        job_definition: job_definition_t.JobDefinition,
+    ) -> None:
+        self.profile_metadata = profile_metadata
+        self.job_definition = job_definition
+
+    def _patch_attributes(self, attributes: Attributes | None) -> Attributes:
+        patched_attributes: dict[str, base_t.JsonValue] = {
+            **(attributes if attributes is not None else {})
+        }
+        patched_attributes["profile.name"] = self.profile_metadata.name
+        patched_attributes["profile.base_url"] = self.profile_metadata.base_url
+        patched_attributes["job.name"] = self.job_definition.name
+        patched_attributes["job.id"] = self.job_definition.id
+        patched_attributes["job.definition_type"] = self.job_definition.type
+        match self.job_definition:
+            case job_definition_t.CronJobDefinition():
+                patched_attributes["job.definition.cron_spec"] = (
+                    self.job_definition.cron_spec
+                )
+            case _:
+                assert_never(self.job_definition)
+        patched_attributes["job.definition.executor.type"] = (
+            self.job_definition.executor.type
+        )
+        match self.job_definition.executor:
+            case job_definition_t.JobExecutorScript():
+                patched_attributes["job.definition.executor.import_path"] = (
+                    self.job_definition.executor.import_path
+                )
+            case job_definition_t.JobExecutorGenericUpload():
+                patched_attributes["job.definition.executor.data_source.type"] = (
+                    self.job_definition.executor.data_source.type
+                )
+            case _:
+                assert_never(self.job_definition.executor)
+        return _cast_attributes(patched_attributes)
+
+    @contextmanager
+    def push_scope(
+        self, scope_name: str, *, attributes: Attributes | None = None
+    ) -> Generator["JobLogger", None, None]:
+        with get_tracer().start_as_current_span(
+            scope_name, attributes=self._patch_attributes(attributes)
+        ) as span:
+            self.current_span_id = span.get_span_context().span_id
+            self.current_trace_id = span.get_span_context().trace_id
+            yield self
+
+    def _emit_log(
+        self, message: str, *, severity: LogSeverity, attributes: Attributes | None
+    ) -> None:
+        log_record = LogRecord(
+            body=message,
+            severity_text=severity,
+            timestamp=time.time_ns(),
+            attributes=self._patch_attributes(attributes),
+            span_id=self.current_span_id,
+            trace_id=self.current_trace_id,
+        )
+        log_file: TextIO = sys.stderr if severity == LogSeverity.ERROR else sys.stdout
+        log_file.write(log_record.to_json())
+        log_file.flush()
+
+    def log_info(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.INFO, attributes=attributes)
+
+    def log_warning(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.WARN, attributes=attributes)
+
+    def log_error(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.ERROR, attributes=attributes)

uncountable/types/api/entity/create_entities.py

@@ -34,7 +34,7 @@ class EntityToCreate:
 @dataclasses.dataclass(kw_only=True)
 class Arguments:
     definition_id: base_t.ObjectId
-    entity_type: typing.Union[typing.Literal[entity_t.EntityType.LAB_REQUEST], typing.Literal[entity_t.EntityType.APPROVAL], typing.Literal[entity_t.EntityType.CUSTOM_ENTITY], typing.Literal[entity_t.EntityType.INVENTORY_AMOUNT], typing.Literal[entity_t.EntityType.TASK], typing.Literal[entity_t.EntityType.PROJECT], typing.Literal[entity_t.EntityType.EQUIPMENT], typing.Literal[entity_t.EntityType.INV_LOCAL_LOCATIONS], typing.Literal[entity_t.EntityType.FIELD_OPTION_SET], typing.Literal[entity_t.EntityType.WEBHOOK], typing.Literal[entity_t.EntityType.SPECS]]
+    entity_type: typing.Union[typing.Literal[entity_t.EntityType.LAB_REQUEST], typing.Literal[entity_t.EntityType.APPROVAL], typing.Literal[entity_t.EntityType.CUSTOM_ENTITY], typing.Literal[entity_t.EntityType.INVENTORY_AMOUNT], typing.Literal[entity_t.EntityType.TASK], typing.Literal[entity_t.EntityType.PROJECT], typing.Literal[entity_t.EntityType.EQUIPMENT], typing.Literal[entity_t.EntityType.INV_LOCAL_LOCATIONS], typing.Literal[entity_t.EntityType.FIELD_OPTION_SET], typing.Literal[entity_t.EntityType.WEBHOOK], typing.Literal[entity_t.EntityType.SPECS], typing.Literal[entity_t.EntityType.GOAL]]
     entities_to_create: list[EntityToCreate]
 
 

uncountable/types/api/entity/create_entity.py

@@ -40,7 +40,7 @@ class EntityFieldInitialValue:
 @dataclasses.dataclass(kw_only=True)
 class Arguments:
     definition_id: base_t.ObjectId
-    entity_type: typing.Union[typing.Literal[entity_t.EntityType.LAB_REQUEST], typing.Literal[entity_t.EntityType.APPROVAL], typing.Literal[entity_t.EntityType.CUSTOM_ENTITY], typing.Literal[entity_t.EntityType.INVENTORY_AMOUNT], typing.Literal[entity_t.EntityType.TASK], typing.Literal[entity_t.EntityType.PROJECT], typing.Literal[entity_t.EntityType.EQUIPMENT], typing.Literal[entity_t.EntityType.INV_LOCAL_LOCATIONS], typing.Literal[entity_t.EntityType.FIELD_OPTION_SET], typing.Literal[entity_t.EntityType.WEBHOOK], typing.Literal[entity_t.EntityType.SPECS]]
+    entity_type: typing.Union[typing.Literal[entity_t.EntityType.LAB_REQUEST], typing.Literal[entity_t.EntityType.APPROVAL], typing.Literal[entity_t.EntityType.CUSTOM_ENTITY], typing.Literal[entity_t.EntityType.INVENTORY_AMOUNT], typing.Literal[entity_t.EntityType.TASK], typing.Literal[entity_t.EntityType.PROJECT], typing.Literal[entity_t.EntityType.EQUIPMENT], typing.Literal[entity_t.EntityType.INV_LOCAL_LOCATIONS], typing.Literal[entity_t.EntityType.FIELD_OPTION_SET], typing.Literal[entity_t.EntityType.WEBHOOK], typing.Literal[entity_t.EntityType.SPECS], typing.Literal[entity_t.EntityType.GOAL]]
     field_values: typing.Optional[typing.Optional[list[field_values_t.FieldRefNameValue]]] = None
 
 

uncountable/types/api/uploader/invoke_uploader.py

@@ -29,6 +29,7 @@ class Arguments:
     file_id: base_t.ObjectId
     uploader_key: identifier_t.IdentifierKey
     material_family_key: identifier_t.IdentifierKey
+    recipe_key: typing.Optional[identifier_t.IdentifierKey] = None
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/async_batch_processor.py

@@ -163,6 +163,7 @@ class AsyncBatchProcessorBase(ABC):
         file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
         material_family_key: identifier_t.IdentifierKey,
+        recipe_key: typing.Optional[identifier_t.IdentifierKey] = None,
         depends_on: typing.Optional[list[str]] = None,
     ) -> async_batch_t.QueuedAsyncBatchRequest:
         """Runs a file through an uploader.
@@ -173,6 +174,7 @@ class AsyncBatchProcessorBase(ABC):
             file_id=file_id,
             uploader_key=uploader_key,
             material_family_key=material_family_key,
+            recipe_key=recipe_key,
         )
         json_data = serialize_for_api(args)
 

uncountable/types/client_base.py

@@ -235,7 +235,7 @@ class ClientMethods(ABC):
         self,
         *,
         definition_id: base_t.ObjectId,
-        entity_type: typing.Union[typing.Literal[entity_t.EntityType.LAB_REQUEST], typing.Literal[entity_t.EntityType.APPROVAL], typing.Literal[entity_t.EntityType.CUSTOM_ENTITY], typing.Literal[entity_t.EntityType.INVENTORY_AMOUNT], typing.Literal[entity_t.EntityType.TASK], typing.Literal[entity_t.EntityType.PROJECT], typing.Literal[entity_t.EntityType.EQUIPMENT], typing.Literal[entity_t.EntityType.INV_LOCAL_LOCATIONS], typing.Literal[entity_t.EntityType.FIELD_OPTION_SET], typing.Literal[entity_t.EntityType.WEBHOOK], typing.Literal[entity_t.EntityType.SPECS]],
+        entity_type: typing.Union[typing.Literal[entity_t.EntityType.LAB_REQUEST], typing.Literal[entity_t.EntityType.APPROVAL], typing.Literal[entity_t.EntityType.CUSTOM_ENTITY], typing.Literal[entity_t.EntityType.INVENTORY_AMOUNT], typing.Literal[entity_t.EntityType.TASK], typing.Literal[entity_t.EntityType.PROJECT], typing.Literal[entity_t.EntityType.EQUIPMENT], typing.Literal[entity_t.EntityType.INV_LOCAL_LOCATIONS], typing.Literal[entity_t.EntityType.FIELD_OPTION_SET], typing.Literal[entity_t.EntityType.WEBHOOK], typing.Literal[entity_t.EntityType.SPECS], typing.Literal[entity_t.EntityType.GOAL]],
         entities_to_create: list[create_entities_t.EntityToCreate],
     ) -> create_entities_t.Data:
         """Creates new Uncountable entities
@@ -260,7 +260,7 @@ class ClientMethods(ABC):
         self,
         *,
         definition_id: base_t.ObjectId,
-        entity_type: typing.Union[typing.Literal[entity_t.EntityType.LAB_REQUEST], typing.Literal[entity_t.EntityType.APPROVAL], typing.Literal[entity_t.EntityType.CUSTOM_ENTITY], typing.Literal[entity_t.EntityType.INVENTORY_AMOUNT], typing.Literal[entity_t.EntityType.TASK], typing.Literal[entity_t.EntityType.PROJECT], typing.Literal[entity_t.EntityType.EQUIPMENT], typing.Literal[entity_t.EntityType.INV_LOCAL_LOCATIONS], typing.Literal[entity_t.EntityType.FIELD_OPTION_SET], typing.Literal[entity_t.EntityType.WEBHOOK], typing.Literal[entity_t.EntityType.SPECS]],
+        entity_type: typing.Union[typing.Literal[entity_t.EntityType.LAB_REQUEST], typing.Literal[entity_t.EntityType.APPROVAL], typing.Literal[entity_t.EntityType.CUSTOM_ENTITY], typing.Literal[entity_t.EntityType.INVENTORY_AMOUNT], typing.Literal[entity_t.EntityType.TASK], typing.Literal[entity_t.EntityType.PROJECT], typing.Literal[entity_t.EntityType.EQUIPMENT], typing.Literal[entity_t.EntityType.INV_LOCAL_LOCATIONS], typing.Literal[entity_t.EntityType.FIELD_OPTION_SET], typing.Literal[entity_t.EntityType.WEBHOOK], typing.Literal[entity_t.EntityType.SPECS], typing.Literal[entity_t.EntityType.GOAL]],
         field_values: typing.Optional[typing.Optional[list[field_values_t.FieldRefNameValue]]] = None,
     ) -> create_entity_t.Data:
         """Creates a new Uncountable entity
@@ -857,6 +857,7 @@ class ClientMethods(ABC):
         file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
         material_family_key: identifier_t.IdentifierKey,
+        recipe_key: typing.Optional[identifier_t.IdentifierKey] = None,
     ) -> invoke_uploader_t.Data:
         """Runs a file through an uploader.
 
@@ -865,6 +866,7 @@ class ClientMethods(ABC):
             file_id=file_id,
             uploader_key=uploader_key,
             material_family_key=material_family_key,
+            recipe_key=recipe_key,
         )
         api_request = APIRequest(
             method=invoke_uploader_t.ENDPOINT_METHOD,

uncountable/types/job_definition.py

@@ -21,7 +21,8 @@ from .job_definition_t import JobDefinitionBase as JobDefinitionBase
 from .job_definition_t import CronJobDefinition as CronJobDefinition
 from .job_definition_t import JobDefinition as JobDefinition
 from .job_definition_t import AuthRetrievalBase as AuthRetrievalBase
-from .job_definition_t import AuthRetrievalEnv as AuthRetrievalEnv
+from .job_definition_t import AuthRetrievalOAuth as AuthRetrievalOAuth
+from .job_definition_t import AuthRetrievalBasic as AuthRetrievalBasic
 from .job_definition_t import AuthRetrieval as AuthRetrieval
 from .job_definition_t import ProfileDefinition as ProfileDefinition
 from .job_definition_t import ProfileMetadata as ProfileMetadata

uncountable/types/job_definition_t.py

@@ -18,7 +18,8 @@ from . import secret_retrieval_t
 __all__: list[str] = [
     "AuthRetrieval",
     "AuthRetrievalBase",
-    "AuthRetrievalEnv",
+    "AuthRetrievalBasic",
+    "AuthRetrievalOAuth",
     "AuthRetrievalType",
     "CronJobDefinition",
     "GenericUploadDataSource",
@@ -54,7 +55,8 @@ class JobExecutorType(StrEnum):
 
 # DO NOT MODIFY -- This file is generated by type_spec
 class AuthRetrievalType(StrEnum):
-    ENV = "env"
+    OAUTH = "oauth"
+    BASIC = "basic"
 
 
 # DO NOT MODIFY -- This file is generated by type_spec
@@ -162,6 +164,7 @@ class JobDefinitionBase:
     id: str
     name: str
     executor: JobExecutor
+    enabled: bool = True
 
 
 # DO NOT MODIFY -- This file is generated by type_spec
@@ -197,17 +200,30 @@ class AuthRetrievalBase:
     parse_require={"type"},
 )
 @dataclasses.dataclass(kw_only=True)
-class AuthRetrievalEnv(AuthRetrievalBase):
-    type: typing.Literal[AuthRetrievalType.ENV] = AuthRetrievalType.ENV
+class AuthRetrievalOAuth(AuthRetrievalBase):
+    type: typing.Literal[AuthRetrievalType.OAUTH] = AuthRetrievalType.OAUTH
+    refresh_token_secret: secret_retrieval_t.SecretRetrieval
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    parse_require={"type"},
+)
+@dataclasses.dataclass(kw_only=True)
+class AuthRetrievalBasic(AuthRetrievalBase):
+    type: typing.Literal[AuthRetrievalType.BASIC] = AuthRetrievalType.BASIC
+    api_id_secret: secret_retrieval_t.SecretRetrieval
+    api_key_secret: secret_retrieval_t.SecretRetrieval
 
 
 # DO NOT MODIFY -- This file is generated by type_spec
 AuthRetrieval = typing.Annotated[
-    typing.Union[AuthRetrievalEnv],
+    typing.Union[AuthRetrievalOAuth, AuthRetrievalBasic],
     serial_union_annotation(
         discriminator="type",
         discriminator_map={
-            "env": AuthRetrievalEnv,
+            "oauth": AuthRetrievalOAuth,
+            "basic": AuthRetrievalBasic,
         },
     ),
 ]