UncountablePythonSDK 0.0.192__tar.gz → 0.0.193__tar.gz

{uncountablepythonsdk-0.0.192/UncountablePythonSDK.egg-info → uncountablepythonsdk-0.0.193}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: UncountablePythonSDK
-Version: 0.0.192
+Version: 0.0.193
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{uncountablepythonsdk-0.0.192 → uncountablepythonsdk-0.0.193/UncountablePythonSDK.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: UncountablePythonSDK
-Version: 0.0.192
+Version: 0.0.193
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{uncountablepythonsdk-0.0.192 → uncountablepythonsdk-0.0.193}/UncountablePythonSDK.egg-info/SOURCES.txt

@@ -57,6 +57,7 @@ examples/integration-server/jobs/materials_auto/example_scale_connection.py
 examples/integration-server/jobs/materials_auto/example_wh.py
 examples/integration-server/jobs/materials_auto/profile.yaml
 pkgs/__init__.py
+pkgs/codeowners.py
 pkgs/py.typed
 pkgs/argument_parser/__init__.py
 pkgs/argument_parser/cached_parser.py
@@ -179,6 +180,8 @@ uncountable/integration/executors/script_executor.py
 uncountable/integration/http_server/__init__.py
 uncountable/integration/http_server/types.py
 uncountable/integration/queue_runner/__init__.py
+uncountable/integration/queue_runner/command_handlers.py
+uncountable/integration/queue_runner/job_dispatcher.py
 uncountable/integration/queue_runner/job_process.py
 uncountable/integration/queue_runner/job_runner.py
 uncountable/integration/queue_runner/job_scheduler.py

{uncountablepythonsdk-0.0.192 → uncountablepythonsdk-0.0.193}/UncountablePythonSDK.egg-info/scm_file_list.json

@@ -65,6 +65,7 @@
     "pkgs/argument_parser/parser_inner.py",
     "pkgs/argument_parser/parser_options.py",
     "pkgs/argument_parser/type_predicates.py",
+    "pkgs/codeowners.py",
     "pkgs/filesystem_utils/__init__.py",
     "pkgs/filesystem_utils/_blob_session.py",
     "pkgs/filesystem_utils/_file_share_session.py",
@@ -165,6 +166,7 @@
     "uncountable/integration/job_queue.py",
     "uncountable/integration/ownership_lock.py",
     "uncountable/integration/queue_runner/__init__.py",
+    "uncountable/integration/queue_runner/command_handlers.py",
     "uncountable/integration/queue_runner/command_server/__init__.py",
     "uncountable/integration/queue_runner/command_server/command_client.py",
     "uncountable/integration/queue_runner/command_server/command_server.py",
@@ -183,6 +185,7 @@
     "uncountable/integration/queue_runner/datastore/factory.py",
     "uncountable/integration/queue_runner/datastore/interface.py",
     "uncountable/integration/queue_runner/datastore/model.py",
+    "uncountable/integration/queue_runner/job_dispatcher.py",
     "uncountable/integration/queue_runner/job_process.py",
     "uncountable/integration/queue_runner/job_runner.py",
     "uncountable/integration/queue_runner/job_scheduler.py",

uncountablepythonsdk-0.0.193/pkgs/codeowners.py

@@ -0,0 +1,157 @@
+import re
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from functools import cached_property
+from itertools import chain
+
+GLOB_CHARS = "*?["
+TEAM_OWNER_PREFIX = "@uncountableinc/"
+
+
+def generate_team_username(team_slug: str) -> str:
+    return f"{TEAM_OWNER_PREFIX}{team_slug}"
+
+
+def _glob_regex(pattern: str) -> re.Pattern[str]:
+    directory_pattern = pattern.endswith("/**")
+    normalized_pattern = pattern[:-3] if directory_pattern else pattern
+    regex_parts: list[str] = []
+    index = 0
+    while index < len(normalized_pattern):
+        dotfile_guard = (
+            "(?!\\.)" if index == 0 or normalized_pattern[index - 1] == "/" else ""
+        )
+        if normalized_pattern.startswith("**/", index):
+            regex_parts.append("(?:(?!\\.)[^/]+/)*")
+            index += 3
+        elif normalized_pattern.startswith("**", index):
+            regex_parts.append(".*")
+            index += 2
+        elif normalized_pattern[index] == "*":
+            regex_parts.append(f"{dotfile_guard}[^/]*")
+            index += 1
+        elif normalized_pattern[index] == "?":
+            regex_parts.append(f"{dotfile_guard}[^/]")
+            index += 1
+        elif normalized_pattern[index] == "[":
+            closing_index = normalized_pattern.find("]", index + 1)
+            if closing_index == -1:
+                regex_parts.append(r"\[")
+                index += 1
+            else:
+                character_class = normalized_pattern[index + 1 : closing_index]
+                if character_class.startswith("!"):
+                    character_class = f"^{character_class[1:]}"
+                regex_parts.append(f"{dotfile_guard}[{character_class}]")
+                index = closing_index + 1
+        else:
+            regex_parts.append(re.escape(normalized_pattern[index]))
+            index += 1
+    suffix = "(?:/(?!\\.)[^/]+)*" if directory_pattern else ""
+    return re.compile(f"^{''.join(regex_parts)}{suffix}$")
+
+
+@dataclass(kw_only=True)
+class CodeownerRule:
+    owners: list[str]
+    glob: str
+
+    @cached_property
+    def normalized_globs(self) -> list[str]:
+        normalized_glob = self.glob.strip("/")
+
+        # /path/to/file should the file /path/to/file
+        globs = [normalized_glob]
+
+        # /path/to/file should also match files in subdirectory
+        # /path/to/file/foo
+        # heuristic: treat as directory if no wildcard and last path part has no dot
+        last_part = normalized_glob.split("/")[-1]
+        if (
+            "*" not in normalized_glob
+            and "?" not in normalized_glob
+            and "." not in last_part
+        ):
+            globs.append(f"{normalized_glob}/**")
+
+        return globs
+
+    @cached_property
+    def matches_everything(self) -> bool:
+        return "*" in self.normalized_globs
+
+    @cached_property
+    def _compiled_matchers(self) -> list[re.Pattern[str]]:
+        return [_glob_regex(pattern) for pattern in self.normalized_globs]
+
+    def check_path(self, path: str) -> bool:
+        if self.matches_everything:
+            return True
+        return any(pattern.match(path) for pattern in self._compiled_matchers)
+
+
+def clean_line(line: str) -> str:
+    return line.strip().split("#", 1)[0]
+
+
+def _rules_from_lines(lines: Iterable[str]) -> list[CodeownerRule]:
+    rules: list[CodeownerRule] = []
+    for raw_line in lines:
+        line = clean_line(raw_line)
+        parts = line.split()
+        if len(parts) > 1:
+            rules.append(CodeownerRule(glob=parts[0], owners=parts[1:]))
+    return rules
+
+
+def get_codeowner_rules(codeowners_path: str) -> list[CodeownerRule]:
+    with open(codeowners_path, encoding="utf-8") as codeowners_file:
+        return _rules_from_lines(codeowners_file)
+
+
+def get_codeowner_rules_from_text(text: str) -> list[CodeownerRule]:
+    return _rules_from_lines(text.splitlines())
+
+
+def last_matching_rule(rules: list[CodeownerRule], path: str) -> CodeownerRule | None:
+    """The rule that owns `path` under CODEOWNERS last-match-wins precedence."""
+    matching_rules = [rule for rule in rules if rule.check_path(path)]
+    return matching_rules[-1] if matching_rules else None
+
+
+def get_owners_for_file(rules: list[CodeownerRule], path: str) -> list[str]:
+    rule = last_matching_rule(rules, path)
+    return rule.owners if rule is not None else []
+
+
+def evaluate_codeowner_rules(rules: list[CodeownerRule], files: list[str]) -> set[str]:
+    return set(chain.from_iterable(get_owners_for_file(rules, file) for file in files))
+
+
+def get_owners(codeowners_path: str, files: list[str]) -> set[str]:
+    return evaluate_codeowner_rules(get_codeowner_rules(codeowners_path), files)
+
+
+def get_codeowner_team_slugs(*, rules: list[CodeownerRule]) -> set[str]:
+    return {
+        owner.removeprefix(TEAM_OWNER_PREFIX)
+        for rule in rules
+        for owner in rule.owners
+        if owner.startswith(TEAM_OWNER_PREFIX)
+    }
+
+
+def get_codeowner_team_slugs_for_files(
+    *,
+    files: list[str],
+    codeowner_rules: list[CodeownerRule],
+    all_teams_lookup: Mapping[str, object],
+) -> list[str]:
+    if len(files) == 0:
+        return []
+    codeowners = evaluate_codeowner_rules(codeowner_rules, files)
+    return [
+        slug
+        for slug in all_teams_lookup
+        if generate_team_username(team_slug=slug) in codeowners
+    ]

{uncountablepythonsdk-0.0.192 → uncountablepythonsdk-0.0.193}/pkgs/type_spec/builder.py

@@ -103,6 +103,11 @@ class PropertyNameMode(StrEnum):
     deprecated_non_snake_case = "deprecated_non_snake_case"
 
 
+class StringEnumEmitMode(StrEnum):
+    STANDARD = "standard"
+    SOURCE_ENUM_ALIAS = "source_enum_alias"
+
+
 class BaseTypeName(StrEnum):
     """
     Base types that are supported.
@@ -912,6 +917,8 @@ class SpecTypeDefnStringEnum(SpecTypeDefn):
         self.desc: str | None = None
         self.sql_type_name: str | None = None
         self.emit_id_source = False
+        self.emit_mode = StringEnumEmitMode.STANDARD
+        self.alias_source_enum: SpecTypeDefnStringEnum | None = None
         self.source_enums: list[SpecType] = []
         self.property_name_mode = PropertyNameMode.strict
 
@@ -1036,12 +1043,15 @@ class SpecTypeDefnStringEnum(SpecTypeDefn):
 
         emit_data = data.get("emit")
         if emit_data is not None:
-            util.check_fields(emit_data, ["id_source"])
+            util.check_fields(emit_data, ["id_source", "mode"])
             emit_id_source = emit_data.get("id_source", False)
             assert isinstance(emit_id_source, bool)
             self.emit_id_source = emit_id_source
             if emit_id_source:
                 builder.emit_id_source_enums.add(self)
+            self.emit_mode = StringEnumEmitMode(
+                emit_data.get("mode", StringEnumEmitMode.STANDARD.value)
+            )
 
         if self.emit_id_source:
             assert len(self.namespace.path) == 1
@@ -1062,6 +1072,27 @@ class SpecTypeDefnStringEnum(SpecTypeDefn):
         for name in self.values:
             _validate_enum_name(name)
 
+        if self.emit_mode is StringEnumEmitMode.SOURCE_ENUM_ALIAS:
+            builder.ensure(
+                len(self.source_enums) == 1,
+                "source enum alias must reference one StringEnum",
+            )
+            source_enum = self.source_enums[0]
+            assert isinstance(source_enum, SpecTypeDefnStringEnum)
+            self.alias_source_enum = source_enum
+            builder.ensure(
+                self.values == source_enum.values,
+                "source enum alias must preserve all source values",
+            )
+            builder.ensure(
+                data_values is None,
+                "source enum alias cannot define local values",
+            )
+            builder.ensure(
+                self.sql_type_name is None,
+                "source enum alias cannot define a SQL type",
+            )
+
     def get_referenced_types(self) -> list[SpecType]:
         return self.source_enums
 

{uncountablepythonsdk-0.0.192 → uncountablepythonsdk-0.0.193}/pkgs/type_spec/emit_python.py

@@ -762,6 +762,16 @@ def _emit_endpoint_invocation_function(
 
 
 def _emit_string_enum(ctx: Context, stype: builder.SpecTypeDefnStringEnum) -> None:
+    if stype.alias_source_enum is not None:
+        if stype.emit_mode is not builder.StringEnumEmitMode.SOURCE_ENUM_ALIAS:
+            raise ValueError("source enum alias has an invalid emission mode")
+        ctx.out.write(
+            f"{python_class_name(stype.name)} = {refer_to(ctx, stype.alias_source_enum)}\n"
+        )
+        return
+    if stype.emit_mode is not builder.StringEnumEmitMode.STANDARD:
+        raise ValueError("string enum has an unsupported emission mode")
+
     ctx.use_enum = True
 
     # Detect if we need serial_string_enum. Ideally we can just stick this on all enums

{uncountablepythonsdk-0.0.192 → uncountablepythonsdk-0.0.193}/pkgs/type_spec/emit_typescript_util.py

@@ -270,6 +270,14 @@ def emit_type_ts(ctx: EmitTypescriptContext, stype: builder.SpecType) -> None:
         return
 
     if isinstance(stype, builder.SpecTypeDefnStringEnum):
+        if stype.alias_source_enum is not None:
+            if stype.emit_mode is not builder.StringEnumEmitMode.SOURCE_ENUM_ALIAS:
+                raise ValueError("source enum alias has an invalid emission mode")
+            source_enum_ref = refer_to(ctx, stype.alias_source_enum)
+            ctx.out.write(f"export import {stype.name} = {source_enum_ref}\n")
+            return
+        if stype.emit_mode is not builder.StringEnumEmitMode.STANDARD:
+            raise ValueError("string enum has an unsupported emission mode")
         ctx.out.write(f"export enum {stype.name} {{\n")
         assert stype.values
         for entry in stype.values.values():

{uncountablepythonsdk-0.0.192 → uncountablepythonsdk-0.0.193}/uncountable/core/client.py

@@ -4,7 +4,7 @@ import pathlib
 import re
 import typing
 import zipfile
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import UTC, timedelta
 from enum import StrEnum
 from io import BytesIO
@@ -181,13 +181,13 @@ class SDKError(Exception):
 
 @dataclass(kw_only=True)
 class OAuthBearerTokenCache:
-    token: str
+    token: str = field(repr=False)
     expires_at: datetime.datetime
 
 
 @dataclass(kw_only=True)
 class GetOauthBearerTokenData:
-    access_token: str
+    access_token: str = field(repr=False)
     expires_in: int
     token_type: str
     scope: str

{uncountablepythonsdk-0.0.192 → uncountablepythonsdk-0.0.193}/uncountable/core/types.py

@@ -1,21 +1,21 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 
 @dataclass(kw_only=True)
 class AuthDetailsApiKey:
     api_id: str
-    api_secret_key: str
+    api_secret_key: str = field(repr=False)
 
 
 @dataclass(kw_only=True)
 class AuthDetailsOAuth:
-    refresh_token: str
+    refresh_token: str = field(repr=False)
     scope: str = "unc.rnd"
 
 
 @dataclass(kw_only=True)
 class AuthDetailsBearerToken:
-    token: str
+    token: str = field(repr=False)
 
 
 AuthDetails = AuthDetailsApiKey  # Legacy Mapping

uncountablepythonsdk-0.0.193/uncountable/integration/queue_runner/command_handlers.py

@@ -0,0 +1,144 @@
+import os
+import threading
+import typing
+
+from uncountable.integration.queue_runner.command_server import (
+    CommandEnqueueJob,
+    CommandEnqueueJobResponse,
+    CommandRetryJob,
+    CommandRetryJobResponse,
+)
+from uncountable.integration.queue_runner.command_server.types import (
+    CommandCancelJob,
+    CommandCancelJobResponse,
+    CommandCancelJobStatus,
+)
+from uncountable.integration.queue_runner.datastore import Datastore
+from uncountable.integration.queue_runner.job_dispatcher import JobDispatcher
+from uncountable.integration.queue_runner.types import RESTART_EXIT_CODE
+from uncountable.integration.telemetry import Logger
+from uncountable.types import queued_job_t
+
+RESTART_DELAY_SECS = 5
+
+
+def _should_deduplicate_job(
+    invocation_context: queued_job_t.InvocationContext,
+) -> bool:
+    match invocation_context:
+        case (
+            queued_job_t.InvocationContextCron()
+            | queued_job_t.InvocationContextManual()
+        ):
+            return True
+        case queued_job_t.InvocationContextWebhook():
+            return False
+        case _:
+            typing.assert_never(invocation_context)
+
+
+async def _enqueue_or_deduplicate_job(
+    *,
+    job_ref_name: str,
+    payload: queued_job_t.QueuedJobPayload,
+    job_dispatcher: JobDispatcher,
+) -> str:
+    if _should_deduplicate_job(payload.invocation_context):
+        existing_queued_job = job_dispatcher.datastore.get_next_queued_job_for_ref_name(
+            job_ref_name=job_ref_name
+        )
+        if existing_queued_job is not None:
+            return existing_queued_job.queued_job_uuid
+    queued_job = job_dispatcher.datastore.add_job_to_queue(
+        job_payload=payload,
+        job_ref_name=job_ref_name,
+    )
+    job_dispatcher.logger.log_info(
+        "Job submitted successfully.",
+        attributes={
+            "job.id": queued_job.job_ref_name,
+            "job.queued_job_uuid": queued_job.queued_job_uuid,
+        },
+    )
+    await job_dispatcher.enqueue_queued_job(queued_job=queued_job)
+    return queued_job.queued_job_uuid
+
+
+async def handle_enqueue_job_command(
+    *, command: CommandEnqueueJob, job_dispatcher: JobDispatcher
+) -> None:
+    queued_job_uuid = await _enqueue_or_deduplicate_job(
+        job_ref_name=command.job_ref_name,
+        payload=command.payload,
+        job_dispatcher=job_dispatcher,
+    )
+    await command.response_queue.put(
+        CommandEnqueueJobResponse(queued_job_uuid=queued_job_uuid)
+    )
+
+
+async def handle_cancel_job_command(
+    *,
+    command: CommandCancelJob,
+    job_dispatcher: JobDispatcher,
+    running_job_uuids: set[str],
+) -> None:
+    queued_job = job_dispatcher.datastore.get_queued_job(uuid=command.queued_job_uuid)
+    if queued_job is None:
+        await command.response_queue.put(
+            CommandCancelJobResponse(status=CommandCancelJobStatus.NO_JOB_FOUND)
+        )
+        return
+
+    if queued_job.status != queued_job_t.JobStatus.QUEUED:
+        await command.response_queue.put(
+            CommandCancelJobResponse(
+                status=CommandCancelJobStatus.JOB_ALREADY_COMPLETED
+            )
+        )
+        return
+
+    job_dispatcher.datastore.remove_job_from_queue(command.queued_job_uuid)
+    if command.queued_job_uuid in running_job_uuids:
+        cancel_status = CommandCancelJobStatus.CANCELLED_WITH_RESTART
+        threading.Timer(
+            interval=RESTART_DELAY_SECS,
+            function=os._exit,
+            args=[RESTART_EXIT_CODE],
+        ).start()
+    else:
+        job_dispatcher.dispatched_job_uuids.discard(command.queued_job_uuid)
+        cancel_status = CommandCancelJobStatus.CANCELLED_QUEUED
+
+    await command.response_queue.put(CommandCancelJobResponse(status=cancel_status))
+    job_dispatcher.logger.log_debug(
+        "Job cancelled successfully.",
+        attributes={
+            "job.queued_job_uuid": command.queued_job_uuid,
+            "job.cancel_status": cancel_status,
+        },
+    )
+
+
+async def handle_retry_job_command(
+    *, command: CommandRetryJob, job_dispatcher: JobDispatcher
+) -> None:
+    queued_job = job_dispatcher.datastore.retry_job(command.queued_job_uuid)
+    if queued_job is None:
+        await command.response_queue.put(CommandRetryJobResponse(queued_job_uuid=None))
+        return
+
+    await job_dispatcher.enqueue_queued_job(queued_job=queued_job)
+    await command.response_queue.put(
+        CommandRetryJobResponse(queued_job_uuid=queued_job.queued_job_uuid)
+    )
+
+
+def handle_vacuum_queued_jobs_command(*, datastore: Datastore, logger: Logger) -> None:
+    logger.log_debug("Vaccuuming queued jobs...")
+    deleted_uuids = datastore.vaccuum_queued_jobs()
+
+    logger.log_debug(
+        "Queued jobs vacuumed successfully.",
+        attributes={"job.queued_job_uuids": deleted_uuids},
+    )

uncountablepythonsdk-0.0.193/uncountable/integration/queue_runner/job_dispatcher.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass, field
+
+from uncountable.integration.queue_runner.datastore import Datastore
+from uncountable.integration.queue_runner.worker import Worker
+from uncountable.integration.telemetry import Logger
+from uncountable.types import queued_job_t
+
+
+@dataclass(kw_only=True, frozen=True)
+class JobDispatcher:
+    datastore: Datastore
+    job_worker_lookup: dict[str, Worker]
+    logger: Logger
+    dispatched_job_uuids: set[str] = field(default_factory=set, init=False)
+
+    async def enqueue_queued_job(self, *, queued_job: queued_job_t.QueuedJob) -> None:
+        worker = self.job_worker_lookup.get(queued_job.job_ref_name)
+        if worker is None:
+            self.logger.log_error(
+                "No worker found for job.",
+                attributes={
+                    "job.id": queued_job.job_ref_name,
+                    "job.queued_job_uuid": queued_job.queued_job_uuid,
+                },
+            )
+            self.datastore.update_job_status(
+                queued_job.queued_job_uuid, queued_job_t.JobStatus.FAILED
+            )
+            return
+        self.dispatched_job_uuids.add(queued_job.queued_job_uuid)
+        await worker.listen_queue.put(queued_job)
+
+    async def dispatch_undispatched_queued_jobs(self) -> None:
+        undispatched_job_uuids = [
+            queued_job_uuid
+            for queued_job_uuid in self.datastore.list_queued_job_uuids()
+            if queued_job_uuid not in self.dispatched_job_uuids
+        ]
+        if len(undispatched_job_uuids) == 0:
+            return
+        for queued_job in self.datastore.load_queued_jobs(
+            queued_job_uuids=undispatched_job_uuids
+        ):
+            await self.enqueue_queued_job(queued_job=queued_job)