UncountablePythonSDK 0.0.48__py3-none-any.whl → 0.0.50__py3-none-any.whl

{UncountablePythonSDK-0.0.48.dist-info → UncountablePythonSDK-0.0.50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.48
+Version: 0.0.50
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -27,6 +27,10 @@ Requires-Dist: PyYAML ==6.*
 Requires-Dist: google-api-python-client ==2.*
 Requires-Dist: tqdm ==4.*
 Requires-Dist: pysftp ==0.*
+Requires-Dist: opentelemetry-api ==1.*
+Requires-Dist: opentelemetry-exporter-otlp-proto-common ==1.*
+Requires-Dist: opentelemetry-exporter-otlp-proto-http ==1.*
+Requires-Dist: opentelemetry-sdk ==1.*
 Requires-Dist: paramiko ==3.*
 Requires-Dist: boto3 ==1.*
 Provides-Extra: test

{UncountablePythonSDK-0.0.48.dist-info → UncountablePythonSDK-0.0.50.dist-info}/RECORD

@@ -3,7 +3,7 @@ docs/conf.py,sha256=YF5J-9g_Wg8wXmyHsGaE8xYlDEzqocNl3UWUmP0CwBg,1702
 docs/index.md,sha256=eEdirX_Ds6ICTRtIS5iT4irCquHcQyKN7E4M5QP9T8A,257
 docs/justfile,sha256=cvNcpb-ByPOF2aCrFlg3DDZBoYMx5W8xGdr13m9HcnI,215
 docs/quickstart.md,sha256=3GuJ0MB1O5kjlsrgAmdSkDq0rYqATrYy-tzEHDy8H-c,422
-docs/requirements.txt,sha256=Y6wdRPGnrd-HMShTgy9Fgydz868Bnyl007P9HCUKXhw,139
+docs/requirements.txt,sha256=YDDAaHfuLxkdLhrjEUJeHDE-NSmD5chTgVTIO7BEeto,139
 docs/static/logo_blue.png,sha256=SyYpMTVhhBbhF5Wl8lWaVwz-_p1MIR6dW6bVhufQRME,46708
 docs/static/favicons/android-chrome-192x192.png,sha256=XoF-AhD55JlSBDGsEPJKfT_VeXT-awhwKyZnxLhrwvk,1369
 docs/static/favicons/android-chrome-512x512.png,sha256=1S4xwY9YtJQ5ifFsZ-DOzssoyBYs0t9uwdOUmYx0Xso,3888
@@ -25,7 +25,7 @@ pkgs/argument_parser/_is_enum.py,sha256=Gw6jJa8nBwYGqXwwCZbSnWL8Rvr5alkg5lSVAqXt
 pkgs/argument_parser/_is_namedtuple.py,sha256=Rjc1bKanIPPogl3qG5JPBxglG1TqWYOo1nxxhBASQWY,265
 pkgs/argument_parser/argument_parser.py,sha256=S5x4yDpaBqTRkmcOyX2UuBWw9iCE4j2Po5LZPg9jhe4,17308
 pkgs/argument_parser/case_convert.py,sha256=NuJLJUJRbyVb6_Slen4uqaStEHbcOS1d-hBBfDrrw-c,605
-pkgs/filesystem_utils/__init__.py,sha256=yxfwtYFvqq_fjMl-tg2jwa6eNPNZUF4ykqsSzALyNdw,1143
+pkgs/filesystem_utils/__init__.py,sha256=NSsQrUCoGISBCqCCyq6_583sYHTVEQeDjDO8hvZn3ag,1261
 pkgs/filesystem_utils/_gdrive_session.py,sha256=OZudNoP2HikolnpurVJhJdh5fgzqbaZQvn53ReGGXx4,11015
 pkgs/filesystem_utils/_local_session.py,sha256=xFEYhAvNqrOYqwt4jrEYOuYkjJn0zclZhTelW_Q1-rw,2325
 pkgs/filesystem_utils/_s3_session.py,sha256=q4q0MTWXWu5RNRVZ5ibv4M4UXXxWl_J6xCnitvngIMM,3957
@@ -74,26 +74,28 @@ uncountable/__init__.py,sha256=8l8XWNCKsu7TG94c-xa2KHpDegvxDC2FyQISdWC763Y,89
 uncountable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/core/__init__.py,sha256=RFv0kO6rKFf1PtBPu83hCGmxqkJamRtsgQ9_-ztw7tA,341
 uncountable/core/async_batch.py,sha256=Gur0VOS0AH2ugwvk65hwoX-iqwQAAyJaejY_LyAZZPo,1210
-uncountable/core/client.py,sha256=2QRLy0GcSamQDqUABQq8R7D_wGofYewSoTQiZzOyZXk,10168
+uncountable/core/client.py,sha256=C0hJ0_SGL5WEhPuAWDSj4ShjjIiQasxpfpnisTi-Uag,10554
 uncountable/core/file_upload.py,sha256=TkQ0fKbbYrPgns1Jh51JU35DUqZHB3ljOaVgjSlBx9Y,3149
 uncountable/core/types.py,sha256=s2CjqYJpsmbC7xMwxxT7kJ_V9bwokrjjWVVjpMcQpKI,333
+uncountable/core/version.py,sha256=SqQIHLhiVZXQBeOwygS2FRZ4WEO27JmWhse0lKm7fgU,274
 uncountable/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-uncountable/integration/construct_client.py,sha256=9ZsfkA0oZIE04mPnwGlt6EcrlnpaUPA93K100FS1F7A,1546
-uncountable/integration/cron.py,sha256=n4HjW-KNruYQ_09iwNBd9i4-Juy_9ddzbf71_wnydHA,1674
+uncountable/integration/construct_client.py,sha256=m6iFAhyAkD3WETLGayv5t47WpwLP1bct4k1z2a2GPh0,1703
+uncountable/integration/cron.py,sha256=e5456IYJF2ipiSsd1R2T334lfe7mtp-gwP7JpS645L0,1858
 uncountable/integration/entrypoint.py,sha256=9rk06gBTsCqytIs8Shsnlf6ir_4Uq5d5rfP1veiSLzc,1437
-uncountable/integration/job.py,sha256=5HJcjtwj_pGFYAZ6SK7xa9RqDXCoBb5rAb_OHdTR61Q,1463
-uncountable/integration/server.py,sha256=85yzQrrpp8OEG37CTN9wtgB9UrzWnDLuDkY0nWcp--k,3548
+uncountable/integration/job.py,sha256=UTzcMes2KrBBRLOM3u94imMKLLnv50glqOkNf8-JOZw,1022
+uncountable/integration/server.py,sha256=JlnbidtiSLCEod0DzLKIzTCxP1qgaHBIBolUvnihJn8,3980
+uncountable/integration/telemetry.py,sha256=H5XQnRTszDL6Nl_oQYuWRr_oNYMocDaCCpKlwll-qZI,5127
 uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/db/connect.py,sha256=YtQHJ1DBGPhxKFRCfiXqohOYUceKSxMVOJ88aPI48Ug,181
 uncountable/integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/executors/executors.py,sha256=v5ClGVUlvrZcMdmGQa8Ll668G_HGTnKpGOnTM7UMZCQ,956
-uncountable/integration/executors/generic_upload_executor.py,sha256=Ixxg01LLzc9pBpptBqJG6-jG82eltZTF7igFqORujyo,8393
+uncountable/integration/executors/generic_upload_executor.py,sha256=wafNY_gpbUiQhvkFPDw-GGiJLmDVtTRgH_5jwMLy2Z4,10283
 uncountable/integration/executors/script_executor.py,sha256=OmSBOtU48G3mqza9c2lCm84pGGyaDk-ZBJCx3RsdJXc,846
 uncountable/integration/secret_retrieval/__init__.py,sha256=3QXVj35w8rRMxVvmmsViFYDi3lcb3g70incfalOEm6o,87
 uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=M0qXVJpD8hMYIFypHFeyh598sqmIDX8ZOyXK23CluF0,1323
 uncountable/types/__init__.py,sha256=95iOd3WXWoI_4a461IS2ieWRic3zRyNaCYzfTpX764o,8162
 uncountable/types/async_batch.py,sha256=ihCv5XWSTTPmuO-GMPn1EACGI2CBUIJTATZ3aPgsNBA,523
-uncountable/types/async_batch_processor.py,sha256=VO0P-oSLAvzBK_Rk-KPS74AhSyrkj5ILZrAu-gs7Lpc,8564
+uncountable/types/async_batch_processor.py,sha256=R--exgi4Gw0HWCnh8M-3_2PqG2ByTBtdyuSQ2eYtYn8,8671
 uncountable/types/async_batch_t.py,sha256=9jp9rOyetRdD5aQVyijzQggTyYU4021PBVGXk0ooBCQ,1911
 uncountable/types/base.py,sha256=xVSjWvA_fUUnkCg83EjoYEFvAfmskinKFMeYFOxNc9E,359
 uncountable/types/base_t.py,sha256=XXjZXexx0xWFUxMMhW8i9nIL6n8dsZVsHwdgnhZ0zJ4,2714
@@ -101,7 +103,7 @@ uncountable/types/calculations.py,sha256=FFO_D3BbKoGDZnqWvTKpW4KF359i2vrKjpdFCLY
 uncountable/types/calculations_t.py,sha256=7GTSi2L8NYjzjUJJx3cmtVkK9uD-uhfYvIFK-ffQj-8,556
 uncountable/types/chemical_structure.py,sha256=E-LnikTFDoVQ1b2zKaVUIO_PAKm-7aZZYJi8I8SDSic,302
 uncountable/types/chemical_structure_t.py,sha256=aFsTkkbzy6Gvyde3qrrEYD95gcYhxkgKMiDRaRE0o-Y,760
-uncountable/types/client_base.py,sha256=3CKr_v3bXez7ZjUg41q09pju5tgYGMsbgJAoYMffh-c,65240
+uncountable/types/client_base.py,sha256=XjgzUcLIE-Q-Tc30IQ8hv7qw4LF7yQlKS0H_130ruwA,65347
 uncountable/types/client_config.py,sha256=4h5Liko9uKCo9_0gdbPhoK6Jr2Kv7tioLiQ8iKeq-_4,301
 uncountable/types/client_config_t.py,sha256=_HdS37gMSTIiD4qLnW9dIgt8_Rt5A6xhwMGGga7vnLg,625
 uncountable/types/curves.py,sha256=W6uMpG5SyW1MS82szNpxkFEn1MnxNpBFyFbQb2Ysfng,366
@@ -124,8 +126,8 @@ uncountable/types/input_attributes.py,sha256=IrIKQnHqHdS1Utdfzr9GnOe17a8riaqYcO1
 uncountable/types/input_attributes_t.py,sha256=wE1ekiQfb72Z9VpF5SHipKJkgaJFUHJrNkkJdmuhF9w,820
 uncountable/types/inputs.py,sha256=6RIEFfCxLqpeHEGOpu63O4i8zPogjGeB7wiV_rPBw_g,404
 uncountable/types/inputs_t.py,sha256=RW7gF9zTOwByu-nMTMVuBabLOuWKx4O1nvfgvx_R55o,1611
-uncountable/types/job_definition.py,sha256=W2wth9vPRBVL0temwnSmvfxxPMKFxZqbQtsMXn8BU3U,1637
-uncountable/types/job_definition_t.py,sha256=1VZO3l5oGnvtG-ikQB6YuukblHhEwtmJfQpGmo0Gi40,5735
+uncountable/types/job_definition.py,sha256=sCQqtyHI3hksc5pVpk5tqbG55F91ST4FoDwr2TmTOuQ,1787
+uncountable/types/job_definition_t.py,sha256=oDHLHCophse2x8vZOYagQ49yz7GTnCKwY6ecpDQsfRk,6616
 uncountable/types/outputs.py,sha256=sUZx_X-TKCZtLm1YCEH8OISX9DdPlv9ZuUfM3-askCc,281
 uncountable/types/outputs_t.py,sha256=2aORUOr0ls1ZYo-ddkWax3D1ZndmQsWtHfJxpYozlhg,656
 uncountable/types/permissions.py,sha256=1mRnSsmRgjuLgp6pylTwwACD_YRIcmlqxHkufwZtMns,297
@@ -237,8 +239,8 @@ uncountable/types/api/recipes/unlock_recipes.py,sha256=AvzQeZCLs9i7CuhMs3Xltdi4n
 uncountable/types/api/triggers/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/triggers/run_trigger.py,sha256=_Rpha9nxXI3Xr17CrGDtofg4HZ81x2lt0rMZ6As0qfE,893
 uncountable/types/api/uploader/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
-uncountable/types/api/uploader/invoke_uploader.py,sha256=w7NVSpTpKxsF2TgLzVrji1Ql5Z8QWTz6d5OvXpRtyDo,992
-UncountablePythonSDK-0.0.48.dist-info/METADATA,sha256=j72rX6737WefzshiwZNBrM8nqCM55gDs2DH6dH79C24,1734
-UncountablePythonSDK-0.0.48.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-UncountablePythonSDK-0.0.48.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
-UncountablePythonSDK-0.0.48.dist-info/RECORD,,
+uncountable/types/api/uploader/invoke_uploader.py,sha256=4zOcB_38uT73Jm3-XqkG40fBM1R5vpvPpGAg-U4lzxY,1059
+UncountablePythonSDK-0.0.50.dist-info/METADATA,sha256=h6s5vQQCK3UNrcaHgQxLbyyskpShrzKpEToBx0Rx5EQ,1934
+UncountablePythonSDK-0.0.50.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
+UncountablePythonSDK-0.0.50.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
+UncountablePythonSDK-0.0.50.dist-info/RECORD,,

{UncountablePythonSDK-0.0.48.dist-info → UncountablePythonSDK-0.0.50.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: setuptools (73.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

docs/requirements.txt

@@ -2,6 +2,6 @@ furo==2024.7.18
 myst-parser==3.0.1
 sphinx-autoapi==3.2.0
 sphinx-copybutton==0.5.2
-Sphinx==7.4.4
+Sphinx==8.0.0
 sphinx_design==0.6.0
 sphinx-favicon==1.0.1

pkgs/filesystem_utils/__init__.py

@@ -5,12 +5,14 @@ from ._gdrive_session import list_gdrive_files as list_gdrive_files
 from ._gdrive_session import move_gdrive_file as move_gdrive_file
 from ._gdrive_session import upload_file_gdrive as upload_file_gdrive
 from ._local_session import LocalSession as LocalSession
+from ._s3_session import S3Session as S3Session
 from ._sftp_session import SFTPSession as SFTPSession
 from ._sftp_session import list_sftp_files as list_sftp_files
 from ._sftp_session import move_sftp_files as move_sftp_files
 from .file_type_utils import FileObjectData as FileObjectData
 from .file_type_utils import FileSystemFileReference as FileSystemFileReference
 from .file_type_utils import FileSystemObject as FileSystemObject
+from .file_type_utils import FileSystemS3Config as FileSystemS3Config
 from .file_type_utils import FileSystemSFTPConfig as FileSystemSFTPConfig
 from .file_type_utils import FileTransfer as FileTransfer
 from .file_type_utils import IncompatibleFileReference as IncompatibleFileReference

uncountable/core/client.py

@@ -4,16 +4,18 @@ import typing
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import StrEnum
-from importlib.metadata import PackageNotFoundError, version
 from urllib.parse import urljoin
 from uuid import uuid4
 
 import requests
+from opentelemetry.sdk.resources import Attributes
 from requests.exceptions import JSONDecodeError
 
 from pkgs.argument_parser import CachedParser
 from pkgs.serialization_util import serialize_for_api
 from pkgs.serialization_util.serialization_helpers import JsonValue
+from uncountable.core.version import get_version
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.client_base import APIRequest, ClientMethods
 from uncountable.types.client_config import ClientConfigOptions
 
@@ -25,12 +27,6 @@ UNC_REQUEST_ID_HEADER = "X-UNC-REQUEST-ID"
 UNC_SDK_VERSION_HEADER = "X-UNC-SDK-VERSION"
 
 
-try:
-    __version__ = version("UncountablePythonSDK")
-except PackageNotFoundError:
-    __version__ = "unknown"
-
-
 class EndpointMethod(StrEnum):
     POST = "POST"
     GET = "GET"
@@ -61,6 +57,7 @@ HTTPRequest = HTTPPostRequest | HTTPGetRequest
 @dataclass(kw_only=True)
 class ClientConfig(ClientConfigOptions):
     transform_request: typing.Callable[[requests.Request], requests.Request] | None = None
+    job_logger: typing.Optional[JobLogger] = None
 
 
 OAUTH_REFRESH_WINDOW_SECONDS = 60 * 5
@@ -204,7 +201,15 @@ class Client(ClientMethods):
             case _:
                 typing.assert_never(http_request)
         request.headers = http_request.headers
-        response = self._send_request(request)
+        if self._cfg.job_logger is not None:
+            attributes: Attributes = {
+                "method": http_request.method,
+                "endpoint": api_request.endpoint,
+            }
+            with self._cfg.job_logger.push_scope("api_call", attributes=attributes):
+                response = self._send_request(request)
+        else:
+            response = self._send_request(request)
         response_data = self._get_response_json(response, request_id=request_id)
         cached_parser = self._get_cached_parser(return_type)
         try:
@@ -260,7 +265,7 @@ class Client(ClientMethods):
     ) -> HTTPRequest:
         headers = self._build_auth_headers()
         headers[UNC_REQUEST_ID_HEADER] = request_id
-        headers[UNC_SDK_VERSION_HEADER] = __version__
+        headers[UNC_SDK_VERSION_HEADER] = get_version()
         method = api_request.method.lower()
         data = {"data": json.dumps(serialize_for_api(api_request.args))}
         match method:

uncountable/core/version.py

@@ -0,0 +1,11 @@
+import functools
+from importlib.metadata import PackageNotFoundError, version
+
+
+@functools.cache
+def get_version() -> str:
+    try:
+        version_str = version("UncountablePythonSDK")
+    except PackageNotFoundError:
+        version_str = "unknown"
+    return version_str

uncountable/integration/construct_client.py

@@ -3,6 +3,7 @@ import os
 from uncountable.core import AuthDetailsApiKey, Client
 from uncountable.core.client import ClientConfig
 from uncountable.core.types import AuthDetailsAll
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import (
     AuthRetrievalEnv,
     ProfileMetadata,
@@ -30,18 +31,23 @@ def _construct_auth_details(profile_meta: ProfileMetadata) -> AuthDetailsAll:
             return AuthDetailsApiKey(api_id=api_id, api_secret_key=api_secret_key)
 
 
-def _construct_client_config(profile_meta: ProfileMetadata) -> ClientConfig | None:
+def _construct_client_config(
+    profile_meta: ProfileMetadata, job_logger: JobLogger
+) -> ClientConfig | None:
     if profile_meta.client_options is None:
         return None
     return ClientConfig(
         allow_insecure_tls=profile_meta.client_options.allow_insecure_tls,
         extra_headers=profile_meta.client_options.extra_headers,
+        job_logger=job_logger,
     )
 
 
-def construct_uncountable_client(profile_meta: ProfileMetadata) -> Client:
+def construct_uncountable_client(
+    profile_meta: ProfileMetadata, job_logger: JobLogger
+) -> Client:
     return Client(
         base_url=profile_meta.base_url,
         auth_details=_construct_auth_details(profile_meta),
-        config=_construct_client_config(profile_meta),
+        config=_construct_client_config(profile_meta, job_logger),
     )

uncountable/integration/cron.py

@@ -4,7 +4,8 @@ from pkgs.argument_parser import CachedParser
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.integration.construct_client import construct_uncountable_client
 from uncountable.integration.executors.executors import resolve_executor
-from uncountable.integration.job import CronJobArguments, JobLogger
+from uncountable.integration.job import CronJobArguments
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import JobDefinition, ProfileMetadata
 
 
@@ -19,29 +20,36 @@ cron_args_parser = CachedParser(CronJobArgs)
 
 def cron_job_executor(**kwargs: dict) -> None:
     args_passed = cron_args_parser.parse_storage(kwargs)
-    client = construct_uncountable_client(profile_meta=args_passed.profile_metadata)
+    job_logger = JobLogger(
+        profile_metadata=args_passed.profile_metadata,
+        job_definition=args_passed.definition,
+    )
+    client = construct_uncountable_client(
+        profile_meta=args_passed.profile_metadata, job_logger=job_logger
+    )
     batch_processor = AsyncBatchProcessor(client=client)
     args = CronJobArguments(
         job_definition=args_passed.definition,
         client=client,
         batch_processor=batch_processor,
         profile_metadata=args_passed.profile_metadata,
-        logger=JobLogger(
-            profile_metadata=args_passed.profile_metadata,
-            job_definition=args_passed.definition,
-        ),
+        logger=job_logger,
     )
 
-    job = resolve_executor(args_passed.definition.executor, args_passed.profile_metadata)
+    with job_logger.push_scope(args_passed.definition.name) as job_logger:
+        job = resolve_executor(
+            args_passed.definition.executor, args_passed.profile_metadata
+        )
 
-    print(f"running job {args_passed.definition.name}")
+        job_logger.log_info("running job")
 
-    job.run(args=args)
+        job.run(args=args)
 
-    if batch_processor.current_queue_size() != 0:
-        batch_processor.send()
+        if batch_processor.current_queue_size() != 0:
+            batch_processor.send()
 
-    print(f"completed job {args_passed.definition.name}")
-    submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
-    if len(submitted_batch_job_ids) != 0:
-        print("submitted batch jobs", submitted_batch_job_ids)
+        submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
+        job_logger.log_info(
+            "completed job",
+            attributes={"submitted_batch_job_ids": submitted_batch_job_ids},
+        )

uncountable/integration/executors/generic_upload_executor.py

@@ -9,22 +9,27 @@ from pkgs.filesystem_utils import (
     FileObjectData,
     FileSystemFileReference,
     FileSystemObject,
+    FileSystemS3Config,
     FileSystemSFTPConfig,
     FileTransfer,
+    S3Session,
     SFTPSession,
 )
 from pkgs.filesystem_utils.filesystem_session import FileSystemSession
 from uncountable.core.file_upload import DataFileUpload, FileUpload
-from uncountable.integration.job import Job, JobArguments, JobLogger
+from uncountable.integration.job import Job, JobArguments
 from uncountable.integration.secret_retrieval import retrieve_secret
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.generic_upload_t import (
     GenericRemoteDirectoryScope,
     GenericUploadStrategy,
 )
 from uncountable.types.job_definition_t import (
     GenericUploadDataSource,
+    GenericUploadDataSourceS3,
     GenericUploadDataSourceSFTP,
     JobResult,
+    S3CloudProvider,
 )
 
 
@@ -185,6 +190,42 @@ class GenericUploadJob(Job):
                     pem_key=pem_key,
                 )
                 return SFTPSession(sftp_config=sftp_config)
+            case GenericUploadDataSourceS3():
+                if self.data_source.access_key_secret is not None:
+                    secret_access_key = retrieve_secret(
+                        self.data_source.access_key_secret,
+                        profile_metadata=args.profile_metadata,
+                    )
+                else:
+                    secret_access_key = None
+
+                if self.data_source.endpoint_url is None:
+                    assert (
+                        self.data_source.cloud_provider is not None
+                    ), "either cloud_provider or endpoint_url must be specified"
+                    match self.data_source.cloud_provider:
+                        case S3CloudProvider.AWS:
+                            endpoint_url = "https://s3.amazonaws.com"
+                        case S3CloudProvider.OVH:
+                            assert (
+                                self.data_source.region_name is not None
+                            ), "region_name must be specified for cloud_provider OVH"
+                            endpoint_url = (
+                                f"https://s3.{self.data_source.region_name}.cloud.ovh.net"
+                            )
+                else:
+                    endpoint_url = self.data_source.endpoint_url
+
+                s3_config = FileSystemS3Config(
+                    endpoint_url=endpoint_url,
+                    bucket_name=self.data_source.bucket_name,
+                    region_name=self.data_source.region_name,
+                    access_key_id=self.data_source.access_key_id,
+                    secret_access_key=secret_access_key,
+                    session_token=None,
+                )
+
+                return S3Session(s3_config=s3_config)
 
     def run(self, args: JobArguments) -> JobResult:
         client = args.client

uncountable/integration/job.py

@@ -3,24 +3,10 @@ from dataclasses import dataclass
 
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.client import Client
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import JobDefinition, JobResult, ProfileMetadata
 
 
-class JobLogger:
-    def __init__(
-        self, *, profile_metadata: ProfileMetadata, job_definition: JobDefinition
-    ) -> None:
-        self.profile_metadata = profile_metadata
-        self.job_definition = job_definition
-
-    def log_info(self, *log_objects: object) -> None:
-        # IMPROVE: log a json message with context that can be parsed by OT
-        print(
-            f"[{self.job_definition.id}] in profile ({self.profile_metadata.name}): ",
-            *log_objects,
-        )
-
-
 @dataclass
 class JobArgumentsBase:
     job_definition: JobDefinition

uncountable/integration/server.py

@@ -11,6 +11,7 @@ from apscheduler.triggers.cron import CronTrigger
 from sqlalchemy.engine.base import Engine
 
 from uncountable.integration.cron import CronJobArgs, cron_job_executor
+from uncountable.types import base_t
 from uncountable.types.client_config_t import ClientConfigOptions
 from uncountable.types.job_definition_t import (
     AuthRetrieval,
@@ -67,7 +68,14 @@ class IntegrationServer:
                         existing_job.reschedule(
                             CronTrigger.from_crontab(job_defn.cron_spec)
                         )
+                        if not job_defn.enabled:
+                            existing_job.pause()
+                        else:
+                            existing_job.resume()
                     else:
+                        job_opts: dict[str, base_t.JsonValue] = {}
+                        if not job_defn.enabled:
+                            job_opts["next_run_time"] = None
                         self._scheduler.add_job(
                             cron_job_executor,
                             # IMPROVE: reconsider these defaults
@@ -77,6 +85,7 @@ class IntegrationServer:
                             name=job_defn.name,
                             id=job_defn.id,
                             kwargs=job_kwargs,
+                            **job_opts,
                         )
                 case _:
                     assert_never(job_defn.trigger)

uncountable/integration/telemetry.py

@@ -0,0 +1,130 @@
+import functools
+import os
+import sys
+import time
+from contextlib import contextmanager
+from enum import StrEnum
+from typing import Generator, TextIO, assert_never, cast
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk._logs import LogRecord
+from opentelemetry.sdk.resources import Attributes, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    SimpleSpanProcessor,
+)
+from opentelemetry.trace import Tracer
+
+from uncountable.core.version import get_version
+from uncountable.types import base_t, job_definition_t
+
+
+def _cast_attributes(attributes: dict[str, base_t.JsonValue]) -> Attributes:
+    return cast(Attributes, attributes)
+
+
+@functools.cache
+def get_tracer() -> Tracer:
+    attributes: dict[str, base_t.JsonValue] = {
+        "service.name": "integration-server",
+        "sdk.version": get_version(),
+    }
+    unc_version = os.environ.get("UNC_VERSION")
+    if unc_version is not None:
+        attributes["service.version"] = unc_version
+    unc_env = os.environ.get("UNC_INTEGRATION_ENV")
+    if unc_env is not None:
+        attributes["deployment.environment"] = unc_env
+    resource = Resource.create(attributes=_cast_attributes(attributes))
+    provider = TracerProvider(resource=resource)
+    provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter()))
+    trace.set_tracer_provider(provider)
+    return provider.get_tracer("integration.telemetry")
+
+
+class LogSeverity(StrEnum):
+    INFO = "Info"
+    WARN = "Warn"
+    ERROR = "Error"
+
+
+class JobLogger:
+    current_span_id: int | None = None
+    current_trace_id: int | None = None
+
+    def __init__(
+        self,
+        *,
+        profile_metadata: job_definition_t.ProfileMetadata,
+        job_definition: job_definition_t.JobDefinition,
+    ) -> None:
+        self.profile_metadata = profile_metadata
+        self.job_definition = job_definition
+
+    def _patch_attributes(self, attributes: Attributes | None) -> Attributes:
+        patched_attributes: dict[str, base_t.JsonValue] = {
+            **(attributes if attributes is not None else {})
+        }
+        patched_attributes["profile.name"] = self.profile_metadata.name
+        patched_attributes["profile.base_url"] = self.profile_metadata.base_url
+        patched_attributes["job.name"] = self.job_definition.name
+        patched_attributes["job.id"] = self.job_definition.id
+        patched_attributes["job.definition_type"] = self.job_definition.type
+        match self.job_definition:
+            case job_definition_t.CronJobDefinition():
+                patched_attributes["job.definition.cron_spec"] = (
+                    self.job_definition.cron_spec
+                )
+            case _:
+                assert_never(self.job_definition)
+        patched_attributes["job.definition.executor.type"] = (
+            self.job_definition.executor.type
+        )
+        match self.job_definition.executor:
+            case job_definition_t.JobExecutorScript():
+                patched_attributes["job.definition.executor.import_path"] = (
+                    self.job_definition.executor.import_path
+                )
+            case job_definition_t.JobExecutorGenericUpload():
+                patched_attributes["job.definition.executor.data_source.type"] = (
+                    self.job_definition.executor.data_source.type
+                )
+            case _:
+                assert_never(self.job_definition.executor)
+        return _cast_attributes(patched_attributes)
+
+    @contextmanager
+    def push_scope(
+        self, scope_name: str, *, attributes: Attributes | None = None
+    ) -> Generator["JobLogger", None, None]:
+        with get_tracer().start_as_current_span(
+            scope_name, attributes=self._patch_attributes(attributes)
+        ) as span:
+            self.current_span_id = span.get_span_context().span_id
+            self.current_trace_id = span.get_span_context().trace_id
+            yield self
+
+    def _emit_log(
+        self, message: str, *, severity: LogSeverity, attributes: Attributes | None
+    ) -> None:
+        log_record = LogRecord(
+            body=message,
+            severity_text=severity,
+            timestamp=time.time_ns(),
+            attributes=self._patch_attributes(attributes),
+            span_id=self.current_span_id,
+            trace_id=self.current_trace_id,
+        )
+        log_file: TextIO = sys.stderr if severity == LogSeverity.ERROR else sys.stdout
+        log_file.write(log_record.to_json())
+        log_file.flush()
+
+    def log_info(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.INFO, attributes=attributes)
+
+    def log_warning(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.WARN, attributes=attributes)
+
+    def log_error(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.ERROR, attributes=attributes)

uncountable/types/api/uploader/invoke_uploader.py

@@ -29,6 +29,7 @@ class Arguments:
     file_id: base_t.ObjectId
     uploader_key: identifier_t.IdentifierKey
     material_family_key: identifier_t.IdentifierKey
+    recipe_key: typing.Optional[identifier_t.IdentifierKey] = None
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/async_batch_processor.py

@@ -163,6 +163,7 @@ class AsyncBatchProcessorBase(ABC):
         file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
         material_family_key: identifier_t.IdentifierKey,
+        recipe_key: typing.Optional[identifier_t.IdentifierKey] = None,
         depends_on: typing.Optional[list[str]] = None,
     ) -> async_batch_t.QueuedAsyncBatchRequest:
         """Runs a file through an uploader.
@@ -173,6 +174,7 @@ class AsyncBatchProcessorBase(ABC):
             file_id=file_id,
             uploader_key=uploader_key,
             material_family_key=material_family_key,
+            recipe_key=recipe_key,
         )
         json_data = serialize_for_api(args)
 

uncountable/types/client_base.py

@@ -857,6 +857,7 @@ class ClientMethods(ABC):
         file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
         material_family_key: identifier_t.IdentifierKey,
+        recipe_key: typing.Optional[identifier_t.IdentifierKey] = None,
     ) -> invoke_uploader_t.Data:
         """Runs a file through an uploader.
 
@@ -865,6 +866,7 @@ class ClientMethods(ABC):
             file_id=file_id,
             uploader_key=uploader_key,
             material_family_key=material_family_key,
+            recipe_key=recipe_key,
         )
         api_request = APIRequest(
             method=invoke_uploader_t.ENDPOINT_METHOD,

uncountable/types/job_definition.py

@@ -12,6 +12,8 @@ from .job_definition_t import JobExecutorBase as JobExecutorBase
 from .job_definition_t import JobExecutorScript as JobExecutorScript
 from .job_definition_t import GenericUploadDataSourceBase as GenericUploadDataSourceBase
 from .job_definition_t import GenericUploadDataSourceSFTP as GenericUploadDataSourceSFTP
+from .job_definition_t import S3CloudProvider as S3CloudProvider
+from .job_definition_t import GenericUploadDataSourceS3 as GenericUploadDataSourceS3
 from .job_definition_t import GenericUploadDataSource as GenericUploadDataSource
 from .job_definition_t import JobExecutorGenericUpload as JobExecutorGenericUpload
 from .job_definition_t import JobExecutor as JobExecutor

uncountable/types/job_definition_t.py

@@ -23,6 +23,7 @@ __all__: list[str] = [
     "CronJobDefinition",
     "GenericUploadDataSource",
     "GenericUploadDataSourceBase",
+    "GenericUploadDataSourceS3",
     "GenericUploadDataSourceSFTP",
     "GenericUploadDataSourceType",
     "JobDefinition",
@@ -36,6 +37,7 @@ __all__: list[str] = [
     "JobResult",
     "ProfileDefinition",
     "ProfileMetadata",
+    "S3CloudProvider",
 ]
 
 
@@ -58,6 +60,7 @@ class AuthRetrievalType(StrEnum):
 # DO NOT MODIFY -- This file is generated by type_spec
 class GenericUploadDataSourceType(StrEnum):
     SFTP = "sftp"
+    S3 = "s3"
 
 
 # DO NOT MODIFY -- This file is generated by type_spec
@@ -94,13 +97,35 @@ class GenericUploadDataSourceSFTP(GenericUploadDataSourceBase):
     pem_secret: secret_retrieval_t.SecretRetrieval
 
 
+# DO NOT MODIFY -- This file is generated by type_spec
+class S3CloudProvider(StrEnum):
+    OVH = "ovh"
+    AWS = "aws"
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    parse_require={"type"},
+)
+@dataclasses.dataclass(kw_only=True)
+class GenericUploadDataSourceS3(GenericUploadDataSourceBase):
+    type: typing.Literal[GenericUploadDataSourceType.S3] = GenericUploadDataSourceType.S3
+    bucket_name: str
+    cloud_provider: typing.Optional[S3CloudProvider] = None
+    endpoint_url: typing.Optional[str] = None
+    region_name: typing.Optional[str] = None
+    access_key_id: typing.Optional[str] = None
+    access_key_secret: typing.Optional[secret_retrieval_t.SecretRetrieval] = None
+
+
 # DO NOT MODIFY -- This file is generated by type_spec
 GenericUploadDataSource = typing.Annotated[
-    typing.Union[GenericUploadDataSourceSFTP],
+    typing.Union[GenericUploadDataSourceSFTP, GenericUploadDataSourceS3],
     serial_union_annotation(
         discriminator="type",
         discriminator_map={
             "sftp": GenericUploadDataSourceSFTP,
+            "s3": GenericUploadDataSourceS3,
         },
     ),
 ]
@@ -137,6 +162,7 @@ class JobDefinitionBase:
     id: str
     name: str
     executor: JobExecutor
+    enabled: bool = False
 
 
 # DO NOT MODIFY -- This file is generated by type_spec