UncountablePythonSDK 0.0.79__tar.gz → 0.0.81__tar.gz

{uncountablepythonsdk-0.0.79/UncountablePythonSDK.egg-info → uncountablepythonsdk-0.0.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.79
+Version: 0.0.81
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81/UncountablePythonSDK.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.79
+Version: 0.0.81
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81}/UncountablePythonSDK.egg-info/SOURCES.txt

@@ -33,6 +33,7 @@ examples/set_recipe_output_file_sdk.py
 examples/upload_files.py
 examples/integration-server/pyproject.toml
 examples/integration-server/jobs/materials_auto/example_cron.py
+examples/integration-server/jobs/materials_auto/example_wh.py
 examples/integration-server/jobs/materials_auto/profile.yaml
 pkgs/__init__.py
 pkgs/py.typed

uncountablepythonsdk-0.0.81/examples/integration-server/jobs/materials_auto/example_wh.py

@@ -0,0 +1,15 @@
+import typing
+
+from uncountable.integration.job import JobArguments, WebhookJob, register_job
+from uncountable.types.job_definition_t import JobResult
+
+
+@register_job
+class WebhookExample(WebhookJob):
+    @property
+    def payload_type(self) -> typing.Any:
+        return super().payload_type
+
+    def run(self, args: JobArguments, payload: typing.Any) -> JobResult:
+        args.logger.log_info(f"webhook invoked with payload: {payload}")
+        return JobResult(success=True)

uncountablepythonsdk-0.0.81/examples/integration-server/jobs/materials_auto/profile.yaml

@@ -0,0 +1,43 @@
+base_url: http://host.docker.internal:5000
+auth_retrieval:
+  type: basic
+  api_id_secret:
+    type: env
+    env_key: API_ID
+  api_key_secret:
+    type: env
+    env_key: API_SECRET_KEY
+jobs:
+  - id: example_cron
+    enabled: true
+    type: cron
+    name: MyCron - Example
+    cron_spec: "* * * * *"
+    executor:
+      type: script
+      import_path: example_cron
+    logging_settings:
+      enabled: true
+      share_with_user_groups:
+        - admin:
+          type: ref_name
+          ref_name: admin
+
+  - id: example_wh1
+    type: webhook
+    name: Webhook 1
+    signature_key_secret:
+      type: env
+      env_key: WH1_SIGNATURE_KEY
+    executor:
+      type: script
+      import_path: example_wh
+  - id: example_wh2
+    type: webhook
+    name: Webhook 2
+    signature_key_secret:
+      type: env
+      env_key: WH2_SIGNATURE_KEY
+    executor:
+      type: script
+      import_path: example_wh

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81}/pkgs/serialization/serial_class.py

@@ -96,6 +96,10 @@ class SerialClassDataInspector:
     def named_type_path(self) -> Optional[str]:
         return self.current.named_type_path
 
+    @property
+    def is_field_proper(self) -> bool:
+        return self.current.from_decorator and self.current.named_type_path is not None
+
 
 def _get_merged_serial_class_data(type_class: type[Any]) -> _SerialClassData | None:
     base_class_data = (

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81}/pkgs/type_spec/parts/base.py.prepart

@@ -22,8 +22,8 @@ PureJsonScalar = Union[str, float, bool, None]
 # Regular expressions for identifying ref names and IDs. Ref names should be
 # using this regular expression as a constriant in the database.
 REF_NAME_REGEX = r"^[a-zA-Z0-9_/-]+$"
-# Does not support nonpositive integers at the moment.
-ID_REGEX = r"[1-9][0-9]{0,20}"
+# Ids matching a strict integer number are converted to integers
+ID_REGEX = r"-?[1-9][0-9]{0,20}"
 
 
 if TYPE_CHECKING:

uncountablepythonsdk-0.0.81/uncountable/integration/executors/executors.py

@@ -0,0 +1,137 @@
+from typing import assert_never
+
+from uncountable.core.client import Client
+from uncountable.integration.executors.generic_upload_executor import GenericUploadJob
+from uncountable.integration.executors.script_executor import resolve_script_executor
+from uncountable.integration.job import Job, JobArguments
+from uncountable.types import (
+    async_jobs_t,
+    entity_t,
+    field_values_t,
+    identifier_t,
+    job_definition_t,
+    transition_entity_phase_t,
+)
+
+
+def resolve_executor(
+    job_executor: job_definition_t.JobExecutor,
+    profile_metadata: job_definition_t.ProfileMetadata,
+) -> Job:
+    match job_executor:
+        case job_definition_t.JobExecutorScript():
+            return resolve_script_executor(
+                job_executor, profile_metadata=profile_metadata
+            )
+        case job_definition_t.JobExecutorGenericUpload():
+            return GenericUploadJob(
+                remote_directories=job_executor.remote_directories,
+                upload_strategy=job_executor.upload_strategy,
+                data_source=job_executor.data_source,
+            )
+    assert_never(job_executor)
+
+
+def _create_run_entity(
+    *, client: Client, logging_settings: job_definition_t.JobLoggingSettings
+) -> entity_t.Entity:
+    run_entity = client.create_entity(
+        entity_type=entity_t.EntityType.ASYNC_JOB,
+        definition_key=identifier_t.IdentifierKeyRefName(
+            ref_name="unc_integration_server_run_definition"
+        ),
+        field_values=[
+            field_values_t.FieldRefNameValue(
+                field_ref_name=async_jobs_t.ASYNC_JOB_TYPE_FIELD_REF_NAME,
+                value=async_jobs_t.AsyncJobType.INTEGRATION_SERVER_RUN,
+            ),
+            field_values_t.FieldRefNameValue(
+                field_ref_name=async_jobs_t.ASYNC_JOB_STATUS_FIELD_REF_NAME,
+                value=async_jobs_t.AsyncJobStatus.IN_PROGRESS,
+            ),
+        ],
+    ).entity
+    client.transition_entity_phase(
+        entity=run_entity,
+        transition=transition_entity_phase_t.TransitionIdentifierPhases(
+            phase_from_key=identifier_t.IdentifierKeyRefName(
+                ref_name="unc_integration_server_run__queued"
+            ),
+            phase_to_key=identifier_t.IdentifierKeyRefName(
+                ref_name="unc_integration_server_run__started"
+            ),
+        ),
+    )
+    if logging_settings.share_with_user_groups is not None:
+        client.grant_entity_permissions(
+            entity_type=entity_t.EntityType.ASYNC_JOB,
+            entity_key=identifier_t.IdentifierKeyId(id=run_entity.id),
+            permission_types=[
+                entity_t.EntityPermissionType.READ,
+                entity_t.EntityPermissionType.WRITE,
+            ],
+            user_group_keys=logging_settings.share_with_user_groups,
+        )
+    return run_entity
+
+
+def execute_job(
+    *,
+    job_definition: job_definition_t.JobDefinition,
+    profile_metadata: job_definition_t.ProfileMetadata,
+    args: JobArguments,
+) -> job_definition_t.JobResult:
+    with args.logger.push_scope(job_definition.name) as job_logger:
+        job = resolve_executor(job_definition.executor, profile_metadata)
+
+        job_logger.log_info("running job")
+
+        run_entity: entity_t.Entity | None = None
+        try:
+            if (
+                job_definition.logging_settings is not None
+                and job_definition.logging_settings.enabled
+            ):
+                run_entity = _create_run_entity(
+                    client=args.client, logging_settings=job_definition.logging_settings
+                )
+            result = job.run_outer(args=args)
+        except Exception as e:
+            job_logger.log_exception(e)
+            if run_entity is not None:
+                args.client.set_values(
+                    entity=run_entity,
+                    values=[
+                        field_values_t.ArgumentValueRefName(
+                            field_ref_name=async_jobs_t.ASYNC_JOB_STATUS_FIELD_REF_NAME,
+                            value=async_jobs_t.AsyncJobStatus.ERROR,
+                        ),
+                    ],
+                )
+            return job_definition_t.JobResult(success=False)
+
+        if args.batch_processor.current_queue_size() != 0:
+            args.batch_processor.send()
+
+        submitted_batch_job_ids = args.batch_processor.get_submitted_job_ids()
+        job_logger.log_info(
+            "completed job",
+            attributes={
+                "submitted_batch_job_ids": submitted_batch_job_ids,
+                "success": result.success,
+            },
+        )
+        if run_entity is not None:
+            args.client.set_values(
+                entity=run_entity,
+                values=[
+                    field_values_t.ArgumentValueRefName(
+                        field_ref_name=async_jobs_t.ASYNC_JOB_STATUS_FIELD_REF_NAME,
+                        value=async_jobs_t.AsyncJobStatus.COMPLETED
+                        if result.success
+                        else async_jobs_t.AsyncJobStatus.ERROR,
+                    ),
+                ],
+            )
+
+        return result

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81}/uncountable/integration/queue_runner/job_scheduler.py

@@ -8,6 +8,7 @@ from opentelemetry.trace import get_current_span
 from uncountable.integration.db.connect import IntegrationDBService, create_db_engine
 from uncountable.integration.db.session import get_session_maker
 from uncountable.integration.queue_runner.command_server import (
+    CommandEnqueueJob,
     CommandEnqueueJobResponse,
     CommandQueue,
     CommandTask,
@@ -99,6 +100,41 @@ async def start_scheduler(command_queue: CommandQueue) -> None:
                 return
             await worker.listen_queue.put(queued_job)
 
+        async def _enqueue_or_deduplicate_job(
+            job_ref_name: str,
+            payload: queued_job_t.QueuedJobPayload,
+        ) -> str:
+            if isinstance(
+                payload.invocation_context,
+                queued_job_t.InvocationContextCron,
+            ):
+                existing_queued_jobs = datastore.load_job_queue()
+                duplicate_job = next(
+                    (
+                        job
+                        for job in existing_queued_jobs
+                        if job.job_ref_name == job_ref_name
+                    ),
+                    None,
+                )
+                if duplicate_job is not None:
+                    return duplicate_job.queued_job_uuid
+            queued_job = datastore.add_job_to_queue(
+                job_payload=payload,
+                job_ref_name=job_ref_name,
+            )
+            await enqueue_queued_job(queued_job)
+            return queued_job.queued_job_uuid
+
+        async def _handle_enqueue_job_command(command: CommandEnqueueJob) -> None:
+            queued_job_uuid = await _enqueue_or_deduplicate_job(
+                job_ref_name=command.job_ref_name,
+                payload=command.payload,
+            )
+            await command.response_queue.put(
+                CommandEnqueueJobResponse(queued_job_uuid=queued_job_uuid)
+            )
+
         for queued_job in queued_jobs:
             await enqueue_queued_job(queued_job)
 
@@ -112,15 +148,11 @@ async def start_scheduler(command_queue: CommandQueue) -> None:
             for task in finished:
                 if task == command_task:
                     command = command_task.result()
-                    queued_job = datastore.add_job_to_queue(
-                        job_payload=command.payload, job_ref_name=command.job_ref_name
-                    )
-                    await command.response_queue.put(
-                        CommandEnqueueJobResponse(
-                            queued_job_uuid=queued_job.queued_job_uuid
-                        )
-                    )
-                    await enqueue_queued_job(queued_job)
+                    match command:
+                        case CommandEnqueueJob():
+                            await _handle_enqueue_job_command(command=command)
+                        case _:
+                            typing.assert_never(command)
                     command_task = asyncio.create_task(command_queue.get())
                 elif task == result_task:
                     queued_job_result = result_task.result()

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81}/uncountable/integration/queue_runner/worker.py

@@ -106,3 +106,6 @@ def run_queued_job(
         except Exception as e:
             job_logger.log_exception(e)
             return job_definition_t.JobResult(success=False)
+        except BaseException as e:
+            job_logger.log_exception(e)
+            raise e

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81}/uncountable/types/base_t.py

@@ -39,8 +39,8 @@ PureJsonScalar = Union[str, float, bool, None]
 # Regular expressions for identifying ref names and IDs. Ref names should be
 # using this regular expression as a constriant in the database.
 REF_NAME_REGEX = r"^[a-zA-Z0-9_/-]+$"
-# Does not support nonpositive integers at the moment.
-ID_REGEX = r"[1-9][0-9]{0,20}"
+# Ids matching a strict integer number are converted to integers
+ID_REGEX = r"-?[1-9][0-9]{0,20}"
 
 
 if TYPE_CHECKING:

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81}/uncountable/types/entity_t.py

@@ -70,10 +70,12 @@ __all__: list[str] = [
         "ingredient": "Ingredient",
         "ingredient_attribute": "Ingredient Attribute",
         "ingredient_attribute_category": "Ingredient Attribute Category",
+        "ingredient_attribute_value": "Ingredient Attribute Value",
         "ingredient_category": "Ingredient Category (Project)",
         "ingredient_category_all": "Ingredient Category",
         "ingredient_category_all_material_family": "Ingredient Category Material Family",
         "ingredient_lot": "Ingredient Lot",
+        "ingredient_mat_family": "Ingredient Material Family",
         "ingredient_role": "Ingredient Role",
         "ingredient_tag": "Ingredient Subcategory",
         "ingredient_tag_map": "Ingredient Tag Map",
@@ -225,10 +227,12 @@ class EntityType(StrEnum):
     INGREDIENT = "ingredient"
     INGREDIENT_ATTRIBUTE = "ingredient_attribute"
     INGREDIENT_ATTRIBUTE_CATEGORY = "ingredient_attribute_category"
+    INGREDIENT_ATTRIBUTE_VALUE = "ingredient_attribute_value"
     INGREDIENT_CATEGORY = "ingredient_category"
     INGREDIENT_CATEGORY_ALL = "ingredient_category_all"
     INGREDIENT_CATEGORY_ALL_MATERIAL_FAMILY = "ingredient_category_all_material_family"
     INGREDIENT_LOT = "ingredient_lot"
+    INGREDIENT_MAT_FAMILY = "ingredient_mat_family"
     INGREDIENT_ROLE = "ingredient_role"
     INGREDIENT_TAG = "ingredient_tag"
     INGREDIENT_TAG_MAP = "ingredient_tag_map"

{uncountablepythonsdk-0.0.79 → uncountablepythonsdk-0.0.81}/uncountable/types/job_definition_t.py

@@ -12,9 +12,9 @@ import dataclasses
 from pkgs.serialization import serial_class
 from pkgs.serialization import serial_union_annotation
 from . import auth_retrieval_t
-from . import base_t
 from . import client_config_t
 from . import generic_upload_t
+from . import identifier_t
 from . import integration_server_t
 from . import secret_retrieval_t
 
@@ -172,7 +172,7 @@ JobExecutor = typing.Annotated[
 @dataclasses.dataclass(kw_only=True)
 class JobLoggingSettings:
     enabled: bool = False
-    share_with_user_groups: typing.Optional[list[base_t.ObjectId]] = None
+    share_with_user_groups: typing.Optional[list[identifier_t.IdentifierKey]] = None
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

uncountablepythonsdk-0.0.79/examples/integration-server/jobs/materials_auto/profile.yaml

@@ -1,19 +0,0 @@
-base_url: http://host.docker.internal:5000
-auth_retrieval:
-  type: basic
-  api_id_secret:
-    type: env
-    env_key: API_ID
-  api_key_secret:
-    type: env
-    env_key: API_SECRET_KEY
-jobs:
-  - id: example_cron
-    enabled: true
-    type: cron
-    name: MyCron - Example
-    cron_spec: "* * * * *"
-    executor: 
-      type: script
-      import_path: example_cron
-

uncountablepythonsdk-0.0.79/uncountable/integration/executors/executors.py

@@ -1,56 +0,0 @@
-from typing import assert_never
-
-from uncountable.integration.executors.generic_upload_executor import GenericUploadJob
-from uncountable.integration.executors.script_executor import resolve_script_executor
-from uncountable.integration.job import Job, JobArguments
-from uncountable.types import job_definition_t
-
-
-def resolve_executor(
-    job_executor: job_definition_t.JobExecutor,
-    profile_metadata: job_definition_t.ProfileMetadata,
-) -> Job:
-    match job_executor:
-        case job_definition_t.JobExecutorScript():
-            return resolve_script_executor(
-                job_executor, profile_metadata=profile_metadata
-            )
-        case job_definition_t.JobExecutorGenericUpload():
-            return GenericUploadJob(
-                remote_directories=job_executor.remote_directories,
-                upload_strategy=job_executor.upload_strategy,
-                data_source=job_executor.data_source,
-            )
-    assert_never(job_executor)
-
-
-def execute_job(
-    *,
-    job_definition: job_definition_t.JobDefinition,
-    profile_metadata: job_definition_t.ProfileMetadata,
-    args: JobArguments,
-) -> job_definition_t.JobResult:
-    with args.logger.push_scope(job_definition.name) as job_logger:
-        job = resolve_executor(job_definition.executor, profile_metadata)
-
-        job_logger.log_info("running job")
-
-        try:
-            result = job.run_outer(args=args)
-        except Exception as e:
-            job_logger.log_exception(e)
-            return job_definition_t.JobResult(success=False)
-
-        if args.batch_processor.current_queue_size() != 0:
-            args.batch_processor.send()
-
-        submitted_batch_job_ids = args.batch_processor.get_submitted_job_ids()
-        job_logger.log_info(
-            "completed job",
-            attributes={
-                "submitted_batch_job_ids": submitted_batch_job_ids,
-                "success": result.success,
-            },
-        )
-
-        return result