UncountablePythonSDK 0.0.49__tar.gz → 0.0.50__tar.gz

{uncountablepythonsdk-0.0.49/UncountablePythonSDK.egg-info → uncountablepythonsdk-0.0.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.49
+Version: 0.0.50
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -27,6 +27,10 @@ Requires-Dist: PyYAML==6.*
 Requires-Dist: google-api-python-client==2.*
 Requires-Dist: tqdm==4.*
 Requires-Dist: pysftp==0.*
+Requires-Dist: opentelemetry-api==1.*
+Requires-Dist: opentelemetry-exporter-otlp-proto-common==1.*
+Requires-Dist: opentelemetry-exporter-otlp-proto-http==1.*
+Requires-Dist: opentelemetry-sdk==1.*
 Requires-Dist: paramiko==3.*
 Requires-Dist: boto3==1.*
 Provides-Extra: test

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50/UncountablePythonSDK.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.49
+Version: 0.0.50
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -27,6 +27,10 @@ Requires-Dist: PyYAML==6.*
 Requires-Dist: google-api-python-client==2.*
 Requires-Dist: tqdm==4.*
 Requires-Dist: pysftp==0.*
+Requires-Dist: opentelemetry-api==1.*
+Requires-Dist: opentelemetry-exporter-otlp-proto-common==1.*
+Requires-Dist: opentelemetry-exporter-otlp-proto-http==1.*
+Requires-Dist: opentelemetry-sdk==1.*
 Requires-Dist: paramiko==3.*
 Requires-Dist: boto3==1.*
 Provides-Extra: test

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/UncountablePythonSDK.egg-info/SOURCES.txt

@@ -87,12 +87,14 @@ uncountable/core/async_batch.py
 uncountable/core/client.py
 uncountable/core/file_upload.py
 uncountable/core/types.py
+uncountable/core/version.py
 uncountable/integration/__init__.py
 uncountable/integration/construct_client.py
 uncountable/integration/cron.py
 uncountable/integration/entrypoint.py
 uncountable/integration/job.py
 uncountable/integration/server.py
+uncountable/integration/telemetry.py
 uncountable/integration/db/__init__.py
 uncountable/integration/db/connect.py
 uncountable/integration/executors/__init__.py

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/UncountablePythonSDK.egg-info/requires.txt

@@ -9,6 +9,10 @@ PyYAML==6.*
 google-api-python-client==2.*
 tqdm==4.*
 pysftp==0.*
+opentelemetry-api==1.*
+opentelemetry-exporter-otlp-proto-common==1.*
+opentelemetry-exporter-otlp-proto-http==1.*
+opentelemetry-sdk==1.*
 paramiko==3.*
 boto3==1.*
 

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/docs/requirements.txt

@@ -2,6 +2,6 @@ furo==2024.7.18
 myst-parser==3.0.1
 sphinx-autoapi==3.2.0
 sphinx-copybutton==0.5.2
-Sphinx==7.4.4
+Sphinx==8.0.0
 sphinx_design==0.6.0
 sphinx-favicon==1.0.1

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/pyproject.toml

@@ -59,6 +59,10 @@ dependencies = [
     "google-api-python-client == 2.*",
     "tqdm == 4.*",
     "pysftp == 0.*",
+    "opentelemetry-api==1.*",
+    "opentelemetry-exporter-otlp-proto-common==1.*", 
+    "opentelemetry-exporter-otlp-proto-http==1.*", 
+    "opentelemetry-sdk==1.*",
     "paramiko == 3.*",
     "boto3 == 1.*"
 ]

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/core/client.py

@@ -4,16 +4,18 @@ import typing
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import StrEnum
-from importlib.metadata import PackageNotFoundError, version
 from urllib.parse import urljoin
 from uuid import uuid4
 
 import requests
+from opentelemetry.sdk.resources import Attributes
 from requests.exceptions import JSONDecodeError
 
 from pkgs.argument_parser import CachedParser
 from pkgs.serialization_util import serialize_for_api
 from pkgs.serialization_util.serialization_helpers import JsonValue
+from uncountable.core.version import get_version
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.client_base import APIRequest, ClientMethods
 from uncountable.types.client_config import ClientConfigOptions
 
@@ -25,12 +27,6 @@ UNC_REQUEST_ID_HEADER = "X-UNC-REQUEST-ID"
 UNC_SDK_VERSION_HEADER = "X-UNC-SDK-VERSION"
 
 
-try:
-    __version__ = version("UncountablePythonSDK")
-except PackageNotFoundError:
-    __version__ = "unknown"
-
-
 class EndpointMethod(StrEnum):
     POST = "POST"
     GET = "GET"
@@ -61,6 +57,7 @@ HTTPRequest = HTTPPostRequest | HTTPGetRequest
 @dataclass(kw_only=True)
 class ClientConfig(ClientConfigOptions):
     transform_request: typing.Callable[[requests.Request], requests.Request] | None = None
+    job_logger: typing.Optional[JobLogger] = None
 
 
 OAUTH_REFRESH_WINDOW_SECONDS = 60 * 5
@@ -204,7 +201,15 @@ class Client(ClientMethods):
             case _:
                 typing.assert_never(http_request)
         request.headers = http_request.headers
-        response = self._send_request(request)
+        if self._cfg.job_logger is not None:
+            attributes: Attributes = {
+                "method": http_request.method,
+                "endpoint": api_request.endpoint,
+            }
+            with self._cfg.job_logger.push_scope("api_call", attributes=attributes):
+                response = self._send_request(request)
+        else:
+            response = self._send_request(request)
         response_data = self._get_response_json(response, request_id=request_id)
         cached_parser = self._get_cached_parser(return_type)
         try:
@@ -260,7 +265,7 @@ class Client(ClientMethods):
     ) -> HTTPRequest:
         headers = self._build_auth_headers()
         headers[UNC_REQUEST_ID_HEADER] = request_id
-        headers[UNC_SDK_VERSION_HEADER] = __version__
+        headers[UNC_SDK_VERSION_HEADER] = get_version()
         method = api_request.method.lower()
         data = {"data": json.dumps(serialize_for_api(api_request.args))}
         match method:

uncountablepythonsdk-0.0.50/uncountable/core/version.py

@@ -0,0 +1,11 @@
+import functools
+from importlib.metadata import PackageNotFoundError, version
+
+
+@functools.cache
+def get_version() -> str:
+    try:
+        version_str = version("UncountablePythonSDK")
+    except PackageNotFoundError:
+        version_str = "unknown"
+    return version_str

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/integration/construct_client.py

@@ -3,6 +3,7 @@ import os
 from uncountable.core import AuthDetailsApiKey, Client
 from uncountable.core.client import ClientConfig
 from uncountable.core.types import AuthDetailsAll
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import (
     AuthRetrievalEnv,
     ProfileMetadata,
@@ -30,18 +31,23 @@ def _construct_auth_details(profile_meta: ProfileMetadata) -> AuthDetailsAll:
             return AuthDetailsApiKey(api_id=api_id, api_secret_key=api_secret_key)
 
 
-def _construct_client_config(profile_meta: ProfileMetadata) -> ClientConfig | None:
+def _construct_client_config(
+    profile_meta: ProfileMetadata, job_logger: JobLogger
+) -> ClientConfig | None:
     if profile_meta.client_options is None:
         return None
     return ClientConfig(
         allow_insecure_tls=profile_meta.client_options.allow_insecure_tls,
         extra_headers=profile_meta.client_options.extra_headers,
+        job_logger=job_logger,
     )
 
 
-def construct_uncountable_client(profile_meta: ProfileMetadata) -> Client:
+def construct_uncountable_client(
+    profile_meta: ProfileMetadata, job_logger: JobLogger
+) -> Client:
     return Client(
         base_url=profile_meta.base_url,
         auth_details=_construct_auth_details(profile_meta),
-        config=_construct_client_config(profile_meta),
+        config=_construct_client_config(profile_meta, job_logger),
     )

uncountablepythonsdk-0.0.50/uncountable/integration/cron.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+
+from pkgs.argument_parser import CachedParser
+from uncountable.core.async_batch import AsyncBatchProcessor
+from uncountable.integration.construct_client import construct_uncountable_client
+from uncountable.integration.executors.executors import resolve_executor
+from uncountable.integration.job import CronJobArguments
+from uncountable.integration.telemetry import JobLogger
+from uncountable.types.job_definition_t import JobDefinition, ProfileMetadata
+
+
+@dataclass
+class CronJobArgs:
+    definition: JobDefinition
+    profile_metadata: ProfileMetadata
+
+
+cron_args_parser = CachedParser(CronJobArgs)
+
+
+def cron_job_executor(**kwargs: dict) -> None:
+    args_passed = cron_args_parser.parse_storage(kwargs)
+    job_logger = JobLogger(
+        profile_metadata=args_passed.profile_metadata,
+        job_definition=args_passed.definition,
+    )
+    client = construct_uncountable_client(
+        profile_meta=args_passed.profile_metadata, job_logger=job_logger
+    )
+    batch_processor = AsyncBatchProcessor(client=client)
+    args = CronJobArguments(
+        job_definition=args_passed.definition,
+        client=client,
+        batch_processor=batch_processor,
+        profile_metadata=args_passed.profile_metadata,
+        logger=job_logger,
+    )
+
+    with job_logger.push_scope(args_passed.definition.name) as job_logger:
+        job = resolve_executor(
+            args_passed.definition.executor, args_passed.profile_metadata
+        )
+
+        job_logger.log_info("running job")
+
+        job.run(args=args)
+
+        if batch_processor.current_queue_size() != 0:
+            batch_processor.send()
+
+        submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
+        job_logger.log_info(
+            "completed job",
+            attributes={"submitted_batch_job_ids": submitted_batch_job_ids},
+        )

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/integration/executors/generic_upload_executor.py

@@ -17,8 +17,9 @@ from pkgs.filesystem_utils import (
 )
 from pkgs.filesystem_utils.filesystem_session import FileSystemSession
 from uncountable.core.file_upload import DataFileUpload, FileUpload
-from uncountable.integration.job import Job, JobArguments, JobLogger
+from uncountable.integration.job import Job, JobArguments
 from uncountable.integration.secret_retrieval import retrieve_secret
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.generic_upload_t import (
     GenericRemoteDirectoryScope,
     GenericUploadStrategy,

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/integration/job.py

@@ -3,24 +3,10 @@ from dataclasses import dataclass
 
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.client import Client
+from uncountable.integration.telemetry import JobLogger
 from uncountable.types.job_definition_t import JobDefinition, JobResult, ProfileMetadata
 
 
-class JobLogger:
-    def __init__(
-        self, *, profile_metadata: ProfileMetadata, job_definition: JobDefinition
-    ) -> None:
-        self.profile_metadata = profile_metadata
-        self.job_definition = job_definition
-
-    def log_info(self, *log_objects: object) -> None:
-        # IMPROVE: log a json message with context that can be parsed by OT
-        print(
-            f"[{self.job_definition.id}] in profile ({self.profile_metadata.name}): ",
-            *log_objects,
-        )
-
-
 @dataclass
 class JobArgumentsBase:
     job_definition: JobDefinition

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/integration/server.py

@@ -11,6 +11,7 @@ from apscheduler.triggers.cron import CronTrigger
 from sqlalchemy.engine.base import Engine
 
 from uncountable.integration.cron import CronJobArgs, cron_job_executor
+from uncountable.types import base_t
 from uncountable.types.client_config_t import ClientConfigOptions
 from uncountable.types.job_definition_t import (
     AuthRetrieval,
@@ -67,7 +68,14 @@ class IntegrationServer:
                         existing_job.reschedule(
                             CronTrigger.from_crontab(job_defn.cron_spec)
                         )
+                        if not job_defn.enabled:
+                            existing_job.pause()
+                        else:
+                            existing_job.resume()
                     else:
+                        job_opts: dict[str, base_t.JsonValue] = {}
+                        if not job_defn.enabled:
+                            job_opts["next_run_time"] = None
                         self._scheduler.add_job(
                             cron_job_executor,
                             # IMPROVE: reconsider these defaults
@@ -77,6 +85,7 @@ class IntegrationServer:
                             name=job_defn.name,
                             id=job_defn.id,
                             kwargs=job_kwargs,
+                            **job_opts,
                         )
                 case _:
                     assert_never(job_defn.trigger)

uncountablepythonsdk-0.0.50/uncountable/integration/telemetry.py

@@ -0,0 +1,130 @@
+import functools
+import os
+import sys
+import time
+from contextlib import contextmanager
+from enum import StrEnum
+from typing import Generator, TextIO, assert_never, cast
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk._logs import LogRecord
+from opentelemetry.sdk.resources import Attributes, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    SimpleSpanProcessor,
+)
+from opentelemetry.trace import Tracer
+
+from uncountable.core.version import get_version
+from uncountable.types import base_t, job_definition_t
+
+
+def _cast_attributes(attributes: dict[str, base_t.JsonValue]) -> Attributes:
+    return cast(Attributes, attributes)
+
+
+@functools.cache
+def get_tracer() -> Tracer:
+    attributes: dict[str, base_t.JsonValue] = {
+        "service.name": "integration-server",
+        "sdk.version": get_version(),
+    }
+    unc_version = os.environ.get("UNC_VERSION")
+    if unc_version is not None:
+        attributes["service.version"] = unc_version
+    unc_env = os.environ.get("UNC_INTEGRATION_ENV")
+    if unc_env is not None:
+        attributes["deployment.environment"] = unc_env
+    resource = Resource.create(attributes=_cast_attributes(attributes))
+    provider = TracerProvider(resource=resource)
+    provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter()))
+    trace.set_tracer_provider(provider)
+    return provider.get_tracer("integration.telemetry")
+
+
+class LogSeverity(StrEnum):
+    INFO = "Info"
+    WARN = "Warn"
+    ERROR = "Error"
+
+
+class JobLogger:
+    current_span_id: int | None = None
+    current_trace_id: int | None = None
+
+    def __init__(
+        self,
+        *,
+        profile_metadata: job_definition_t.ProfileMetadata,
+        job_definition: job_definition_t.JobDefinition,
+    ) -> None:
+        self.profile_metadata = profile_metadata
+        self.job_definition = job_definition
+
+    def _patch_attributes(self, attributes: Attributes | None) -> Attributes:
+        patched_attributes: dict[str, base_t.JsonValue] = {
+            **(attributes if attributes is not None else {})
+        }
+        patched_attributes["profile.name"] = self.profile_metadata.name
+        patched_attributes["profile.base_url"] = self.profile_metadata.base_url
+        patched_attributes["job.name"] = self.job_definition.name
+        patched_attributes["job.id"] = self.job_definition.id
+        patched_attributes["job.definition_type"] = self.job_definition.type
+        match self.job_definition:
+            case job_definition_t.CronJobDefinition():
+                patched_attributes["job.definition.cron_spec"] = (
+                    self.job_definition.cron_spec
+                )
+            case _:
+                assert_never(self.job_definition)
+        patched_attributes["job.definition.executor.type"] = (
+            self.job_definition.executor.type
+        )
+        match self.job_definition.executor:
+            case job_definition_t.JobExecutorScript():
+                patched_attributes["job.definition.executor.import_path"] = (
+                    self.job_definition.executor.import_path
+                )
+            case job_definition_t.JobExecutorGenericUpload():
+                patched_attributes["job.definition.executor.data_source.type"] = (
+                    self.job_definition.executor.data_source.type
+                )
+            case _:
+                assert_never(self.job_definition.executor)
+        return _cast_attributes(patched_attributes)
+
+    @contextmanager
+    def push_scope(
+        self, scope_name: str, *, attributes: Attributes | None = None
+    ) -> Generator["JobLogger", None, None]:
+        with get_tracer().start_as_current_span(
+            scope_name, attributes=self._patch_attributes(attributes)
+        ) as span:
+            self.current_span_id = span.get_span_context().span_id
+            self.current_trace_id = span.get_span_context().trace_id
+            yield self
+
+    def _emit_log(
+        self, message: str, *, severity: LogSeverity, attributes: Attributes | None
+    ) -> None:
+        log_record = LogRecord(
+            body=message,
+            severity_text=severity,
+            timestamp=time.time_ns(),
+            attributes=self._patch_attributes(attributes),
+            span_id=self.current_span_id,
+            trace_id=self.current_trace_id,
+        )
+        log_file: TextIO = sys.stderr if severity == LogSeverity.ERROR else sys.stdout
+        log_file.write(log_record.to_json())
+        log_file.flush()
+
+    def log_info(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.INFO, attributes=attributes)
+
+    def log_warning(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.WARN, attributes=attributes)
+
+    def log_error(self, message: str, *, attributes: Attributes | None = None) -> None:
+        self._emit_log(message=message, severity=LogSeverity.ERROR, attributes=attributes)

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/types/api/uploader/invoke_uploader.py

@@ -29,6 +29,7 @@ class Arguments:
     file_id: base_t.ObjectId
     uploader_key: identifier_t.IdentifierKey
     material_family_key: identifier_t.IdentifierKey
+    recipe_key: typing.Optional[identifier_t.IdentifierKey] = None
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/types/async_batch_processor.py

@@ -163,6 +163,7 @@ class AsyncBatchProcessorBase(ABC):
         file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
         material_family_key: identifier_t.IdentifierKey,
+        recipe_key: typing.Optional[identifier_t.IdentifierKey] = None,
         depends_on: typing.Optional[list[str]] = None,
     ) -> async_batch_t.QueuedAsyncBatchRequest:
         """Runs a file through an uploader.
@@ -173,6 +174,7 @@ class AsyncBatchProcessorBase(ABC):
             file_id=file_id,
             uploader_key=uploader_key,
             material_family_key=material_family_key,
+            recipe_key=recipe_key,
         )
         json_data = serialize_for_api(args)
 

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/types/client_base.py

@@ -857,6 +857,7 @@ class ClientMethods(ABC):
         file_id: base_t.ObjectId,
         uploader_key: identifier_t.IdentifierKey,
         material_family_key: identifier_t.IdentifierKey,
+        recipe_key: typing.Optional[identifier_t.IdentifierKey] = None,
     ) -> invoke_uploader_t.Data:
         """Runs a file through an uploader.
 
@@ -865,6 +866,7 @@ class ClientMethods(ABC):
             file_id=file_id,
             uploader_key=uploader_key,
             material_family_key=material_family_key,
+            recipe_key=recipe_key,
         )
         api_request = APIRequest(
             method=invoke_uploader_t.ENDPOINT_METHOD,

{uncountablepythonsdk-0.0.49 → uncountablepythonsdk-0.0.50}/uncountable/types/job_definition_t.py

@@ -162,6 +162,7 @@ class JobDefinitionBase:
     id: str
     name: str
     executor: JobExecutor
+    enabled: bool = False
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

uncountablepythonsdk-0.0.49/uncountable/integration/cron.py

@@ -1,47 +0,0 @@
-from dataclasses import dataclass
-
-from pkgs.argument_parser import CachedParser
-from uncountable.core.async_batch import AsyncBatchProcessor
-from uncountable.integration.construct_client import construct_uncountable_client
-from uncountable.integration.executors.executors import resolve_executor
-from uncountable.integration.job import CronJobArguments, JobLogger
-from uncountable.types.job_definition_t import JobDefinition, ProfileMetadata
-
-
-@dataclass
-class CronJobArgs:
-    definition: JobDefinition
-    profile_metadata: ProfileMetadata
-
-
-cron_args_parser = CachedParser(CronJobArgs)
-
-
-def cron_job_executor(**kwargs: dict) -> None:
-    args_passed = cron_args_parser.parse_storage(kwargs)
-    client = construct_uncountable_client(profile_meta=args_passed.profile_metadata)
-    batch_processor = AsyncBatchProcessor(client=client)
-    args = CronJobArguments(
-        job_definition=args_passed.definition,
-        client=client,
-        batch_processor=batch_processor,
-        profile_metadata=args_passed.profile_metadata,
-        logger=JobLogger(
-            profile_metadata=args_passed.profile_metadata,
-            job_definition=args_passed.definition,
-        ),
-    )
-
-    job = resolve_executor(args_passed.definition.executor, args_passed.profile_metadata)
-
-    print(f"running job {args_passed.definition.name}")
-
-    job.run(args=args)
-
-    if batch_processor.current_queue_size() != 0:
-        batch_processor.send()
-
-    print(f"completed job {args_passed.definition.name}")
-    submitted_batch_job_ids = batch_processor.get_submitted_job_ids()
-    if len(submitted_batch_job_ids) != 0:
-        print("submitted batch jobs", submitted_batch_job_ids)