UncountablePythonSDK 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl

type_spec/external/api/recipes/get_recipe_output_metadata.yaml

@@ -8,17 +8,17 @@ Arguments:
     type: Object
     properties:
         recipe_output_ids:
-            type: List<Integer>
+            type: List<ObjectId>
             desc: "Ids of the Recipe Outputs to retrieve metadata for"
 
 RecipeOutputMetadata:
     type: Object
     properties:
         recipe_output_id:
-            type: Integer
+            type: ObjectId
             desc: "Identifier for the recipe output"
         recipe_output_metadata_field_id:
-            type: Integer
+            type: ObjectId
             desc: "Identifier for the metadata field id"
         quantity_dec:
             type: Decimal
@@ -27,23 +27,10 @@ RecipeOutputMetadata:
             type: JsonValue
             desc: "The value for the metadata, if it is not of numeric type"
 
-RecipeOutputMetadataField:
-    type: Object
-    properties:
-        id:
-            type: Integer
-            desc: "Identifier for the recipe output metadata field"
-        name:
-            type: String
-            desc: "Name of the recipe output metadata field"
-        quantity_type:
-            type: String
-            desc: "The quantity type of the metadata, such as numeric or text"
-
 Data:
     type: Object
     properties:
         recipe_output_metadata:
             type: List<RecipeOutputMetadata>
         recipe_output_metadata_fields:
-            type: List<RecipeOutputMetadataField>
+            type: List<recipe_output_metadata.RecipeOutputMetadataField>

type_spec/external/api/recipes/get_recipes_data.yaml

@@ -7,89 +7,28 @@ $endpoint:
 Arguments:
     type: Object
     properties:
-        recipe_ids:
-            type: Optional<List<Integer>>
+        recipe_ids?:
+            type: Optional<List<ObjectId>>
             desc: "The recipes to get the data from. Either these or project_id must be filled in"
-        project_id:
-            type: Optional<Integer>
+        project_id?:
+            type: Optional<ObjectId>
             desc: "The projects to get the data from. Either these or recipe_ids must be filled in"
-        offset:
+        offset?:
             type: Optional<Integer>
             desc: "Used for pagination. All pagination is done in order of Recipe ID"
-        limit:
+        limit?:
             type: Optional<Integer>
             desc: "The number of data points to return. If not filled in, it will be set to 100, and cannot be set higher than 100"
 
-SimpleRecipeMetadataField:
-    type: Object
-    desc: "A descriptor of metadata. Metadata includes values that are neither ingredients nor process parameters, such as a location of an experiment"
-    properties:
-        metadata_id:
-            type: Integer
-            desc: "A unique ID for the metadata, referenceable elsewhere"
-        name:
-            type: String
-            desc: "The name of the metadata"
-        quantity_type:
-            type: String
-            desc: "The quantity type of the metadata, such as numeric or text"
-
-RecipeMetadata:
-    type: Object
-    desc: "The specific value of a metadata for a recipe. Recipe may only have 1 instance of each metadata"
-    properties:
-        metadata_id:
-            type: Integer
-            desc: "The unique ID for the metadata"
-        quantity_dec:
-            type: Decimal
-            desc: "The value for the metadata, if it is of numeric type"
-        quantity_json:
-            type: JsonValue
-            desc: "The value for the metadata, if it is not of numeric type"
-
-SimpleRecipeTag:
-    type: Object
-    desc: "A representation of recipe tags within Uncountable system"
-    properties:
-        recipe_tag_id:
-            type: Integer
-            desc: "A unique ID for the tag, referenceable elsewhere"
-        name:
-            type: String
-            desc: "The name of the tag"
-
-SimpleExperimentGroup:
-    type: Object
-    desc: "A representation of experiment groups within Uncountable system"
-    properties:
-        experiment_group_id:
-            type: Integer
-            desc: "A unique ID for the group, referenceable elsewhere"
-        name:
-            type: String
-            desc: "The name of the group"
-
-SimpleUnit:
-    type: Object
-    desc: "A representation of units within Uncountable system"
-    properties:
-        unit_id:
-            type: Integer
-            desc: "A unique ID for the unit, referenceable elsewhere"
-        name:
-            type: String
-            desc: "The name of the unit"
-
 RecipeOutput:
     type: Object
     desc: "A representation of a single measurement associated with a recipe / experiment within Uncountable system"
     properties:
         id:
-            type: Integer
+            type: ObjectId
             desc: "The global unique identifier for the recipe output. For cross-reference elsewhere."
         output_id:
-            type: Integer
+            type: ObjectId
             desc: "The output specifically measured in this case"
         replicate_num:
             type: Integer
@@ -101,35 +40,18 @@ RecipeOutput:
             type: JsonValue
             desc: "The quantity of the output, if not numeric"
         curve_id:
-            type: Integer
+            type: ObjectId
             desc: "The id of the curve associated with the output if the output is of curve type."
         output_condition_id:
-            type: Integer
+            type: ObjectId
             desc: "A reference to the condition ID for the output (null if there are no conditions). Used to encode complicated test settings such as test standard, aging time, etc."
 
-SimpleOutput:
-    type: Object
-    desc: "A condensed object representing an output / measurement within Uncountables system"
-    properties:
-        output_id:
-            type: Integer
-            desc: "The globally unique identifier for the output"
-        name:
-            type: String
-            desc: "The name of the output"
-        unit:
-            type: Optional<SimpleUnit>
-            desc: "The units for the output, if filled in"
-        quantity_type:
-            type: String
-            desc: "The quantity type of the output, with common types including numeric, text, categorical, curve"
-
 ConditionParameterValue:
     type: Object
     desc: "A condition parameter value represents a single association of value with a parameter. For instance, Test Method: ISO 23"
     properties:
         condition_parameter_id:
-            type: Integer
+            type: ObjectId
             desc: "The globally unique identifier for the condition parameter, i.e. Test Method or Aging Time"
         condition_parameter_name:
             type: String
@@ -146,47 +68,23 @@ SimpleOutputCondition:
     desc: "Within Uncountable, Output conditions represent how an output is measured. Common use cases include the test temperature, test standard, aging time, etc. An outputcondition is the full description of these parameters"
     properties:
         output_condition_id:
-            type: Integer
+            type: ObjectId
             desc: "The globally unique ID for the Output Condition, used for cross reference elsewhere"
         condition_parameter_values:
             type: List<ConditionParameterValue>
             desc: "The list of parameters and values that comprise this condition"
-SimpleWorkflowStep:
-    type: Object
-    desc: "A step associated with a workflow, representing where inputs are placed in the context of a recipe / experiment."
-    properties:
-        workflow_step_id:
-            type: Integer
-            desc: "The globally unique ID for the workflow step, used for cross reference elsewhere"
-        name:
-            type: String
-            desc: "The name of the workflow step"
-
-SimpleWorkflow:
-    type: Object
-    desc: "A workflow object. Workflows represent broad flowcharts for how experimental creation is done, including a graph of steps that lead into each other."
-    properties:
-        workflow_id:
-            type: Integer
-            desc: "The globally unique ID for the workflow, used for cross reference elsewhere"
-        name:
-            type: String
-            desc: "The name of the workflow"
-        workflow_steps:
-            type: List<SimpleWorkflowStep>
-            desc: "The steps associated with the workflow, representing where inputs are placed in the context of a recipe / experiment."
 
 RecipeInput:
     type: Object
     properties:
         input_id:
-            type: Integer
+            type: ObjectId
             desc: "The globally unique ID for the input, allowing for cross reference elsewhere"
         input_lot_recipe_id:
-            type: Optional<Integer>
+            type: Optional<ObjectId>
             desc: "The recipe id of the lot associated with the input"
         recipe_step_id:
-            type: Integer
+            type: ObjectId
             desc: "The unique Recipe Step ID for the input, allowing for placement within the workflow"
         quantity_dec:
             type: Decimal
@@ -207,25 +105,6 @@ RecipeInput:
             type: String
             desc: "The behavior of the input relative to the experiment. For example 'inclusion' or 'use quantity'."
 
-SimpleInput:
-    type: Object
-    properties:
-        input_id:
-            type: Integer
-            desc: "The globally unique ID for the input, allowing for cross reference elsewhere"
-        quantity_type:
-            type: String
-            desc: "The quantity type of the input in string form. Common types are numeric, recipe (representing a nested input), text, categorical"
-        name:
-            type: String
-            desc: "The name of the input"
-        is_parameter:
-            type: Boolean
-            desc: "Whether the input is a process parameter (true) or an ingredient (false)"
-        intermediate_recipe_id:
-            type: Optional<Integer>
-            desc: "The recipe ID of the input if the input is a a recipe (intermediate)."
-
 RecipeStep:
     type: Object
     properties:
@@ -233,7 +112,7 @@ RecipeStep:
             type: Optional<String>
             desc: "An optional name for the recipe step"
         recipe_step_id:
-            type: Integer
+            type: ObjectId
             desc: "An ID value for the recipe step, used elsewhere to reference the part of the recipe / experiment that the input was placed in"
         recipe_step_number:
             type: Integer
@@ -257,28 +136,15 @@ RecipeWorkflowStep:
     desc: "A workflow step associated with a recipe / experiment. Workflow steps are comprised of recipe step groups, in turn comprised of recipe steps, where inputs are located. These groupings exists to give customers the ability to flexibly place inputs into the correct process ordering"
     properties:
         recipe_workflow_step_id:
-            type: Integer
+            type: ObjectId
             desc: "A unique reference for the workflow step within the recipe."
         workflow_step_id:
-            type: Integer
+            type: ObjectId
             desc: "A reference to the workflow_step_id for the workflow. This will be shared across recipes / experiments, and can be used to do analysis about what part of a generic process inputs are located in."
         recipe_step_groups:
             type: List<RecipeStepGroup>
             desc: "A listing of recipe step groups within the recipe / experiment"
 
-SimpleUser:
-    type: Object
-    properties:
-        user_id:
-            type: Integer
-            desc: "The globally unique ID for the user, allowing for cross reference elsewhere"
-        display_name:
-            type: String
-            desc: "The display name of the user within Uncountable."
-        email:
-            type: String
-            desc: "The email address of the user in Uncountable's system."
-
 RecipeStepRelationship:
     type: Object
     properties:
@@ -305,10 +171,10 @@ Recipe:
     type: Object
     properties:
         recipe_id:
-            type: Integer
+            type: ObjectId
             desc: "A unique identifier for the recipe / experiment. Used for joining elsewhere"
         creating_user_id?:
-            type: Integer
+            type: ObjectId
             desc: "The user ID who originally created the recipe / experiment. Null when created from automated scripts, such as data transfers, or from the Uncountable implementation team."
         create_datetime:
             type: String
@@ -326,10 +192,10 @@ Recipe:
             type: String
             desc: "The value used in the barcoding system to lookup this experiment."
         workflow_id:
-            type: Integer
+            type: ObjectId
             desc: "The workflow ID associated with the recipe / experiment. Workflows correspond to a set of experimental steps performed, and are referenced in the workflows return object"
         metadata:
-            type: List<RecipeMetadata>
+            type: List<recipe_metadata.RecipeMetadata>
             desc: "Metadata associated with a recipe / experimen. Metadata includes values that are neither ingredients nor process parameters, such as a location of an experiment"
         inputs:
             type: List<RecipeInput>
@@ -341,10 +207,10 @@ Recipe:
             type: List<RecipeWorkflowStep>
             desc: "A reference of workflow steps in the recipe / experimen. This is used to reference input values to where they occurred in the experimental process."
         tag_ids:
-            type: List<Integer>
+            type: List<ObjectId>
             desc: A list of Tag IDs associated with the recipe / experiment
         experiment_group_ids:
-            type: List<Integer>
+            type: List<ObjectId>
             desc: A list of experiment group IDs associated with the recipe / experiment
         step_relationships:
             type: List<RecipeStepRelationship>
@@ -355,18 +221,18 @@ Data:
         recipes:
             type: List<Recipe>
         workflows:
-            type: List<SimpleWorkflow>
+            type: List<workflows.SimpleWorkflow>
         metadata:
-            type: List<SimpleRecipeMetadataField>
+            type: List<recipe_metadata.SimpleRecipeMetadataField>
         inputs:
-            type: List<SimpleInput>
+            type: List<inputs.SimpleInput>
         outputs:
-            type: List<SimpleOutput>
+            type: List<outputs.SimpleOutput>
         output_conditions:
             type: List<SimpleOutputCondition>
         users:
-            type: List<SimpleUser>
+            type: List<users.SimpleUser>
         recipe_tags:
-            type: List<SimpleRecipeTag>
+            type: List<recipe_tags.SimpleRecipeTag>
         experiment_groups:
-            type: List<SimpleExperimentGroup>
+            type: List<experiment_groups.SimpleExperimentGroup>

type_spec/external/api/recipes/set_recipe_inputs.yaml

@@ -9,13 +9,13 @@ RecipeInputValue:
     type: Object
     properties:
         recipe_id:
-            type: Integer
+            type: ObjectId
             desc: "A unique identifier for the recipe / experiment. Used for joining elsewhere"
         recipe_step_id?:
-            type: Optional<Integer>
+            type: Optional<ObjectId>
             desc: "Optionally the step id for the step of the recipe to be written to. If none is passed, the input will be written to first recipe workflow step"
         input_id:
-            type: Integer
+            type: ObjectId
             desc: "A unique identifier for the input. Used for joining and identification"
         value_numeric?:
             type: Decimal

type_spec/external/api/recipes/set_recipe_outputs.yaml

@@ -19,13 +19,13 @@ RecipeOutputValue:
     type: Object
     properties:
         recipe_id:
-            type: Integer
+            type: ObjectId
             desc: "A unique identifier for the recipe / experiment. Used for joining elsewhere"
         output_id:
-            type: Integer
+            type: ObjectId
             desc: "A unique identifier for the output. Used for joining and identification"
         condition_id?:
-            type: Integer
+            type: ObjectId
             desc: "A unique identifier the set of condition parameters associated with the output."
         experiment_num:
             type: Integer
@@ -48,9 +48,5 @@ Arguments:
             desc: "The outputs to set. Must be at most 100 entries long"
 
 Data:
-    type: Object
+    type: response.Response
     properties:
-        status:
-            type: Literal<'ok'>
-            default: "ok"
-            desc: "'ok' indicates the post request was successful"

uncountable/integration/construct_client.py

@@ -0,0 +1,30 @@
+import os
+from typing import assert_never
+
+from uncountable.core.client import AuthDetailsApiKey, Client
+from uncountable.integration.types import (
+    AuthRetrievalEnv,
+    ProfileMetadata,
+)
+
+
+def construct_uncountable_client(
+    profile_meta: ProfileMetadata
+) -> Client:
+    match profile_meta.auth_retrieval:
+        case AuthRetrievalEnv():
+            api_id = os.getenv(f"UNC_PROFILE_{profile_meta.name}_API_ID")
+            api_secret_key = os.getenv(
+                f"UNC_PROFILE_{profile_meta.name}_API_SECRET_KEY"
+            )
+
+            assert api_id is not None
+            assert api_secret_key is not None
+
+            return Client(
+                base_url=profile_meta.base_url,
+                auth_details=AuthDetailsApiKey(
+                    api_id=api_id, api_secret_key=api_secret_key
+                ),
+            )
+    assert_never(profile_meta.auth_retrieval)

uncountable/integration/cron.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+
+from pkgs.argument_parser import CachedParser
+from uncountable.integration.construct_client import construct_uncountable_client
+from uncountable.integration.executors.script_executor import resolve_script_executor
+from uncountable.integration.job import CronJobArguments
+from uncountable.integration.types import JobDefinition, ProfileMetadata
+
+
+@dataclass
+class CronJobArgs:
+    definition: JobDefinition
+    profile_metadata: ProfileMetadata
+
+
+cron_args_parser = CachedParser(CronJobArgs)
+
+
+def cron_job_executor(**kwargs: dict) -> None:
+    args_passed = cron_args_parser.parse_storage(kwargs)
+    args = CronJobArguments(
+        job_definition=args_passed.definition,
+        client=construct_uncountable_client(profile_meta=args_passed.profile_metadata),
+    )
+
+    job_class = resolve_script_executor(args_passed.definition.executor)
+
+    job = job_class()
+    job.run(args)

uncountable/integration/db/connect.py

@@ -0,0 +1,8 @@
+import os
+from sqlalchemy import create_engine
+from sqlalchemy.engine.base import Engine
+
+
+def create_db_engine() -> Engine:
+    return create_engine(os.environ["UNC_SQLITE_URI"])
+

uncountable/integration/entrypoint.py

@@ -0,0 +1,41 @@
+import os
+from importlib import resources
+
+from uncountable.integration.server import IntegrationServer
+from uncountable.integration.types import ProfileDefinition
+from pkgs.argument_parser import CachedParser
+from uncountable.integration.db.connect import create_db_engine
+
+
+profile_parser = CachedParser(ProfileDefinition)
+
+
+def main() -> None:
+    profiles_module = os.environ["UNC_PROFILES_MODULE"]
+    with IntegrationServer(create_db_engine()) as server:
+        # TODO: Loop through all job spec yaml files and call server.add_job
+        profiles = [
+            entry
+            for entry in resources.files(profiles_module).iterdir()
+            if entry.is_dir()
+        ]
+        for profile in profiles:
+            profile_name = profile.name
+            try:
+                profile = profile_parser.parse_yaml_resource(
+                    package=".".join([profiles_module, profile_name]),
+                    resource="profile.yaml",
+                )
+            except FileNotFoundError as e:
+                print("WARN: profile.yaml not found", e)
+                continue
+            server.register_profile(
+                profile_name=profile_name,
+                base_url=profile.base_url,
+                auth_retrieval=profile.auth_retrieval,
+                jobs=profile.jobs,
+            )
+
+
+if __name__ == "__main__":
+    main()

uncountable/integration/executors/script_executor.py

@@ -0,0 +1,18 @@
+
+
+import importlib
+import inspect
+from uncountable.integration.job import Job
+from uncountable.integration.types import JobExecutorScript
+
+
+def resolve_script_executor(executor: JobExecutorScript) -> type[Job]:
+    job_module = importlib.import_module(executor.import_path)
+    found_jobs: list[type[Job]] = []
+    for _, job_class in inspect.getmembers(job_module, inspect.isclass):
+        if Job in job_class.__bases__:
+            found_jobs.append(job_class())
+    assert (
+        len(found_jobs) == 1
+    ), f"expected exactly one job class in {executor.import_path}"
+    return found_jobs[0]

uncountable/integration/job.py

@@ -0,0 +1,39 @@
+from dataclasses import dataclass
+from uncountable.core.client import Client
+from uncountable.integration.types import JobDefinition
+
+from abc import ABC, abstractmethod
+
+
+@dataclass
+class JobArgumentsBase:
+    job_definition: JobDefinition
+    client: Client
+
+
+@dataclass
+class CronJobArguments(JobArgumentsBase):
+    # can imagine passing additional data such as in the sftp or webhook cases
+    pass
+
+
+JobArguments = CronJobArguments
+
+
+@dataclass
+class JobResult:
+    success: bool
+
+
+class Job(ABC):
+
+    @abstractmethod
+    def run(self, args: JobArguments) -> JobResult:
+        ...
+
+
+class CronJob(Job):
+
+    @abstractmethod
+    def run(self, args: CronJobArguments) -> JobResult:
+        ...

uncountable/integration/server.py

@@ -0,0 +1,86 @@
+from dataclasses import asdict
+from typing import assert_never
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.executors.pool import ThreadPoolExecutor
+from uncountable.integration.cron import CronJobArgs, cron_job_executor
+from apscheduler.triggers.cron import CronTrigger
+from sqlalchemy.engine.base import Engine
+
+from uncountable.integration.types import (
+    AuthRetrieval,
+    CronJobDefinition,
+    JobDefinition,
+    ProfileMetadata,
+)
+
+
+_MAX_APSCHEDULER_CONCURRENT_JOBS = 1
+
+
+class IntegrationServer:
+    _scheduler: BaseScheduler
+    _engine: Engine
+
+    def __init__(self, engine: Engine) -> None:
+        self._engine = engine
+        self._scheduler = BackgroundScheduler(
+            timezone="UTC",
+            jobstores={"default": SQLAlchemyJobStore(engine=engine)},
+            executors={"default": ThreadPoolExecutor(_MAX_APSCHEDULER_CONCURRENT_JOBS)},
+        )
+
+    def register_profile(
+        self,
+        *,
+        profile_name: str,
+        base_url: str,
+        auth_retrieval: AuthRetrieval,
+        jobs: list[JobDefinition],
+    ) -> None:
+        for job_defn in jobs:
+            profile_metadata = ProfileMetadata(
+                name=profile_name, auth_retrieval=auth_retrieval, base_url=base_url
+            )
+            match job_defn:
+                case CronJobDefinition():
+                    # Add to ap scheduler
+                    job_kwargs = asdict(
+                        CronJobArgs(
+                            definition=job_defn, profile_metadata=profile_metadata
+                        )
+                    )
+                    existing_job = self._scheduler.get_job(job_defn.id)
+                    if existing_job is not None:
+                        existing_job.modify(
+                            name=job_defn.name,
+                            kwargs=job_kwargs,
+                        )
+                        existing_job.reschedule(job_defn.cron_spec)
+                    else:
+                        self._scheduler.add_job(
+                            cron_job_executor,
+                            # IMPROVE: reconsider these defaults
+                            max_instances=1,
+                            coalesce=True,
+                            trigger=CronTrigger.from_crontab(job_defn.cron_spec),
+                            name=job_defn.name,
+                            id=job_defn.id,
+                            kwargs=job_kwargs,
+                        )
+                case _:
+                    assert_never(job_defn.trigger)
+
+    def _start_apscheduler(self) -> None:
+        self._scheduler.start()
+
+    def _stop_apscheduler(self) -> None:
+        self._scheduler.shutdown()
+
+    def __enter__(self) -> "IntegrationServer":
+        self._start_apscheduler()
+        return self
+
+    def __exit__(self) -> None:
+        self._stop_apscheduler()