UncountablePythonSDK 0.0.8__tar.gz → 0.0.10__tar.gz

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.8
+Version: 0.0.10
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -16,7 +16,11 @@ Classifier: Topic :: Utilities
 Classifier: Typing :: Typed
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: requests>=2.31.0
+Requires-Dist: requests==2.31.0
+Requires-Dist: SQLAlchemy==2.0.29
+Requires-Dist: APScheduler==3.10.4
+Requires-Dist: dateutil==2.9.0
+Requires-Dist: shelljob==0.6.3
 Provides-Extra: test
 Requires-Dist: mypy>=1.8.1; extra == "test"
 Requires-Dist: ruff>=0.2.1; extra == "test"

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/UncountablePythonSDK.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.8
+Version: 0.0.10
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -16,7 +16,11 @@ Classifier: Topic :: Utilities
 Classifier: Typing :: Typed
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: requests>=2.31.0
+Requires-Dist: requests==2.31.0
+Requires-Dist: SQLAlchemy==2.0.29
+Requires-Dist: APScheduler==3.10.4
+Requires-Dist: dateutil==2.9.0
+Requires-Dist: shelljob==0.6.3
 Provides-Extra: test
 Requires-Dist: mypy>=1.8.1; extra == "test"
 Requires-Dist: ruff>=0.2.1; extra == "test"

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/UncountablePythonSDK.egg-info/SOURCES.txt

@@ -79,6 +79,17 @@ uncountable/__init__.py
 uncountable/py.typed
 uncountable/core/__init__.py
 uncountable/core/client.py
+uncountable/integration/__init__.py
+uncountable/integration/construct_client.py
+uncountable/integration/cron.py
+uncountable/integration/entrypoint.py
+uncountable/integration/job.py
+uncountable/integration/server.py
+uncountable/integration/types.py
+uncountable/integration/db/__init__.py
+uncountable/integration/db/connect.py
+uncountable/integration/executors/__init__.py
+uncountable/integration/executors/script_executor.py
 uncountable/types/__init__.py
 uncountable/types/base.py
 uncountable/types/calculations.py

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/UncountablePythonSDK.egg-info/requires.txt

@@ -1,4 +1,8 @@
-requests>=2.31.0
+requests==2.31.0
+SQLAlchemy==2.0.29
+APScheduler==3.10.4
+dateutil==2.9.0
+shelljob==0.6.3
 
 [test]
 mypy>=1.8.1

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/pkgs/argument_parser/argument_parser.py

@@ -407,5 +407,5 @@ class CachedParser(typing.Generic[T]):
             return self.parse_storage(yaml.safe_load(data_in))
 
     def parse_yaml_resource(self, package: resources.Package, resource: str) -> T:
-        raw_data = resources.read_text(package, resource)
-        return self.parse_storage(yaml.safe_load(raw_data))
+        with resources.open_text(package, resource) as fp:
+            return self.parse_storage(yaml.safe_load(fp))

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/pkgs/type_spec/emit_open_api.py

@@ -141,6 +141,18 @@ def _serialize_global_context(ctx: EmitOpenAPIGlobalContext) -> str:
     return yaml.dump(oa_root, sort_keys=False)
 
 
+def _emit_endpoint_parameters(typ: OpenAPIType | None) -> dict[str, list[dict[str, str]]]:
+    if not isinstance(typ, OpenAPIObjectType):
+        return {}
+
+    return {
+        "parameters": [
+            {"$ref": f"#/components/schema/Arguments/{prop_name}"}
+            for prop_name in typ.properties
+        ]
+    }
+
+
 def _emit_namespace(
     gctx: EmitOpenAPIGlobalContext,
     ctx: EmitOpenAPIContext,
@@ -159,23 +171,28 @@ def _emit_namespace(
     if ctx.endpoint is not None:
         endpoint = ctx.endpoint
 
+        argument_type = ctx.types.get("Arguments")
         oa_endpoint = dict()
-        oa_endpoint[endpoint.method] = {
-            "tags": endpoint.tags,
-            "summary": endpoint.summary,
-            "description": endpoint.description,
-            "parameters": {"$ref": "#/components/schema/Arguments"},
-            "responses": {
-                "200": {
-                    "description": "OK",
-                    "content": {
-                        "application/json": {
-                            "schema": {"$ref": "#/components/schema/Data"}
-                        }
-                    },
-                }
-            },
-        }
+        oa_endpoint[endpoint.method] = (
+            {
+                "tags": endpoint.tags,
+                "summary": endpoint.summary,
+                "description": endpoint.description,
+            }
+            | _emit_endpoint_parameters(argument_type)
+            | {
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "content": {
+                            "application/json": {
+                                "schema": {"$ref": "#/components/schema/Data"}
+                            }
+                        },
+                    }
+                },
+            }
+        )
         oa_components["endpoint"] = oa_endpoint
 
     types = ctx.types
@@ -460,5 +477,5 @@ def open_api_type(
     ctx.namespaces.add(stype.namespace)
     # external namespace resolution
     return OpenAPIRefType(
-        source=f"{resolve_namespace_ref(ctx, stype.namespace, config=config)}/{stype.name}"
+        source=f"{resolve_namespace_ref(source_path=ctx.namespace.path, ref_path=stype.namespace.path, ref='/components/schema')}/{stype.name}"
     )

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/pkgs/type_spec/emit_open_api_util.py

@@ -9,7 +9,6 @@ from dataclasses import dataclass, field
 from typing import TypeAlias
 
 from . import builder
-from .config import OpenAPIConfig
 from .open_api_util import OpenAPIType
 
 MODIFY_NOTICE = "# DO NOT MODIFY -- This file is generated by type_spec"
@@ -75,12 +74,8 @@ class EmitOpenAPIContext:
 
 
 def resolve_namespace_ref(
-    ctx: EmitOpenAPIContext,
-    namespace: builder.SpecNamespace,
-    *,
-    config: OpenAPIConfig,
+    *, source_path: list[str], ref_path: list[str], ref: str
 ) -> str:
-    # TODO: Handle namespaces not in root directory
-    if len(ctx.namespace.path) == 1:
-        return f"{namespace.name}.yaml#/components/schema"
-    return f"{config.static_url_path}/common/{namespace.name}.yaml#/components/schema"
+    to_root = "/".join(".." for _ in source_path)
+    location = "/".join(ref_path)
+    return f"{to_root}/common/{location}.yaml#{ref}"

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/pkgs/type_spec/open_api_util.py

@@ -13,9 +13,10 @@ class OpenAPIType(ABC):
         self.nullable = nullable
 
     @abstractmethod
-    def asdict(self) -> dict[str, object]: ...
+    def asdict(self) -> dict[str, object]:
+        pass
 
-    def asarguments(self) -> list[dict[str, object]]:
+    def asarguments(self) -> dict[str, dict[str, object]]:
         raise UnsupportedOperation
 
     def add_addl_info(self, emitted: dict[str, object]) -> dict[str, object]:
@@ -147,8 +148,8 @@ class OpenAPIFreeFormObjectType(OpenAPIType):
     def asdict(self) -> dict[str, object]:
         return self.add_addl_info({"type": "object"})
 
-    def asarguments(self) -> list[dict[str, object]]:
-        return []
+    def asarguments(self) -> dict[str, dict[str, object]]:
+        return {}
 
 
 class OpenAPIObjectType(OpenAPIType):
@@ -185,17 +186,17 @@ class OpenAPIObjectType(OpenAPIType):
             },
         })
 
-    def asarguments(self) -> list[dict[str, object]]:
-        argument_types: list[dict[str, object]] = []
+    def asarguments(self) -> dict[str, dict[str, object]]:
+        argument_types: dict[str, dict[str, object]] = {}
         for property_name, property_type in self.properties.items():
             desc = self.property_desc.get(property_name)
-            argument_types.append({
+            argument_types[property_name] = {
                 "name": property_name,
                 "in": "query",
                 "schema": property_type.asdict(),
                 "required": not property_type.nullable,
                 "description": desc or "",
-            })
+            }
         return argument_types
 
 
@@ -219,11 +220,11 @@ class OpenAPIUnionType(OpenAPIType):
         # TODO: use parents description and nullable
         return {"oneOf": [base_type.asdict() for base_type in self.base_types]}
 
-    def asarguments(self) -> list[dict[str, object]]:
+    def asarguments(self) -> dict[str, dict[str, object]]:
         # TODO handle inheritence (allOf and refs); need to inline here...
         # for now skip this endpoint
 
-        return []
+        return {}
 
 
 class OpenAPIIntersectionType(OpenAPIType):
@@ -246,8 +247,8 @@ class OpenAPIIntersectionType(OpenAPIType):
         # TODO: use parents description and nullable
         return {"allOf": [base_type.asdict() for base_type in self.base_types]}
 
-    def asarguments(self) -> list[dict[str, object]]:
+    def asarguments(self) -> dict[str, dict[str, object]]:
         # TODO handle inheritence (allOf and refs); need to inline here...
         # for now skip this endpoint
 
-        return []
+        return {}

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/pyproject.toml

@@ -48,7 +48,11 @@ keywords = [ "uncountable", "sdk", "api", "uncountable-sdk" ]
 requires-python = ">=3.11"
 # May be compatible with older versions
 dependencies = [
-    "requests >= 2.31.0",
+    "requests == 2.31.0",
+    "SQLAlchemy == 2.0.29",
+    "APScheduler == 3.10.4",
+    "dateutil == 2.9.0",
+    "shelljob == 0.6.3"
 ]
 dynamic = ["version"]
 

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/type_spec/external/api/inputs/get_input_data.yaml

@@ -76,7 +76,7 @@ FullInput:
             type: List<input_attributes.InputAttributeValue>
             desc: "Attributes associated with an input, such as CAS number, density, etc."
         global_category_id:
-            type: ObjectId
+            type: Optional<ObjectId>
             desc: "The global category ID associated with the input"
         subcategory_ids:
             type: List<ObjectId>

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/type_spec/external/api/outputs/get_output_data.yaml

@@ -73,10 +73,10 @@ FullOutput:
             type: List<OutputAttrVal>
             desc: "Attributes associated with an output, such as a test documentation"
         global_category_id:
-            type: ObjectId
+            type: Optional<ObjectId>
             desc: "The global category ID associated with the output"
         unit_id:
-            type: ObjectId
+            type: Optional<ObjectId>
             desc: "The unit for the output, which all data is stored in. This may be different than the users default displayed information for the output"
 
 Data:

UncountablePythonSDK-0.0.10/uncountable/integration/construct_client.py

@@ -0,0 +1,30 @@
+import os
+from typing import assert_never
+
+from uncountable.core.client import AuthDetailsApiKey, Client
+from uncountable.integration.types import (
+    AuthRetrievalEnv,
+    ProfileMetadata,
+)
+
+
+def construct_uncountable_client(
+    profile_meta: ProfileMetadata
+) -> Client:
+    match profile_meta.auth_retrieval:
+        case AuthRetrievalEnv():
+            api_id = os.getenv(f"UNC_PROFILE_{profile_meta.name}_API_ID")
+            api_secret_key = os.getenv(
+                f"UNC_PROFILE_{profile_meta.name}_API_SECRET_KEY"
+            )
+
+            assert api_id is not None
+            assert api_secret_key is not None
+
+            return Client(
+                base_url=profile_meta.base_url,
+                auth_details=AuthDetailsApiKey(
+                    api_id=api_id, api_secret_key=api_secret_key
+                ),
+            )
+    assert_never(profile_meta.auth_retrieval)

UncountablePythonSDK-0.0.10/uncountable/integration/cron.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+
+from pkgs.argument_parser import CachedParser
+from uncountable.integration.construct_client import construct_uncountable_client
+from uncountable.integration.executors.script_executor import resolve_script_executor
+from uncountable.integration.job import CronJobArguments
+from uncountable.integration.types import JobDefinition, ProfileMetadata
+
+
+@dataclass
+class CronJobArgs:
+    definition: JobDefinition
+    profile_metadata: ProfileMetadata
+
+
+cron_args_parser = CachedParser(CronJobArgs)
+
+
+def cron_job_executor(**kwargs: dict) -> None:
+    args_passed = cron_args_parser.parse_storage(kwargs)
+    args = CronJobArguments(
+        job_definition=args_passed.definition,
+        client=construct_uncountable_client(profile_meta=args_passed.profile_metadata),
+    )
+
+    job_class = resolve_script_executor(args_passed.definition.executor)
+
+    job = job_class()
+    job.run(args)

UncountablePythonSDK-0.0.10/uncountable/integration/db/connect.py

@@ -0,0 +1,8 @@
+import os
+from sqlalchemy import create_engine
+from sqlalchemy.engine.base import Engine
+
+
+def create_db_engine() -> Engine:
+    return create_engine(os.environ["UNC_SQLITE_URI"])
+

UncountablePythonSDK-0.0.10/uncountable/integration/entrypoint.py

@@ -0,0 +1,41 @@
+import os
+from importlib import resources
+
+from uncountable.integration.server import IntegrationServer
+from uncountable.integration.types import ProfileDefinition
+from pkgs.argument_parser import CachedParser
+from uncountable.integration.db.connect import create_db_engine
+
+
+profile_parser = CachedParser(ProfileDefinition)
+
+
+def main() -> None:
+    profiles_module = os.environ["UNC_PROFILES_MODULE"]
+    with IntegrationServer(create_db_engine()) as server:
+        # TODO: Loop through all job spec yaml files and call server.add_job
+        profiles = [
+            entry
+            for entry in resources.files(profiles_module).iterdir()
+            if entry.is_dir()
+        ]
+        for profile in profiles:
+            profile_name = profile.name
+            try:
+                profile = profile_parser.parse_yaml_resource(
+                    package=".".join([profiles_module, profile_name]),
+                    resource="profile.yaml",
+                )
+            except FileNotFoundError as e:
+                print("WARN: profile.yaml not found", e)
+                continue
+            server.register_profile(
+                profile_name=profile_name,
+                base_url=profile.base_url,
+                auth_retrieval=profile.auth_retrieval,
+                jobs=profile.jobs,
+            )
+
+
+if __name__ == "__main__":
+    main()

UncountablePythonSDK-0.0.10/uncountable/integration/executors/script_executor.py

@@ -0,0 +1,18 @@
+
+
+import importlib
+import inspect
+from uncountable.integration.job import Job
+from uncountable.integration.types import JobExecutorScript
+
+
+def resolve_script_executor(executor: JobExecutorScript) -> type[Job]:
+    job_module = importlib.import_module(executor.import_path)
+    found_jobs: list[type[Job]] = []
+    for _, job_class in inspect.getmembers(job_module, inspect.isclass):
+        if Job in job_class.__bases__:
+            found_jobs.append(job_class())
+    assert (
+        len(found_jobs) == 1
+    ), f"expected exactly one job class in {executor.import_path}"
+    return found_jobs[0]

UncountablePythonSDK-0.0.10/uncountable/integration/job.py

@@ -0,0 +1,39 @@
+from dataclasses import dataclass
+from uncountable.core.client import Client
+from uncountable.integration.types import JobDefinition
+
+from abc import ABC, abstractmethod
+
+
+@dataclass
+class JobArgumentsBase:
+    job_definition: JobDefinition
+    client: Client
+
+
+@dataclass
+class CronJobArguments(JobArgumentsBase):
+    # can imagine passing additional data such as in the sftp or webhook cases
+    pass
+
+
+JobArguments = CronJobArguments
+
+
+@dataclass
+class JobResult:
+    success: bool
+
+
+class Job(ABC):
+
+    @abstractmethod
+    def run(self, args: JobArguments) -> JobResult:
+        ...
+
+
+class CronJob(Job):
+
+    @abstractmethod
+    def run(self, args: CronJobArguments) -> JobResult:
+        ...

UncountablePythonSDK-0.0.10/uncountable/integration/server.py

@@ -0,0 +1,86 @@
+from dataclasses import asdict
+from typing import assert_never
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.executors.pool import ThreadPoolExecutor
+from uncountable.integration.cron import CronJobArgs, cron_job_executor
+from apscheduler.triggers.cron import CronTrigger
+from sqlalchemy.engine.base import Engine
+
+from uncountable.integration.types import (
+    AuthRetrieval,
+    CronJobDefinition,
+    JobDefinition,
+    ProfileMetadata,
+)
+
+
+_MAX_APSCHEDULER_CONCURRENT_JOBS = 1
+
+
+class IntegrationServer:
+    _scheduler: BaseScheduler
+    _engine: Engine
+
+    def __init__(self, engine: Engine) -> None:
+        self._engine = engine
+        self._scheduler = BackgroundScheduler(
+            timezone="UTC",
+            jobstores={"default": SQLAlchemyJobStore(engine=engine)},
+            executors={"default": ThreadPoolExecutor(_MAX_APSCHEDULER_CONCURRENT_JOBS)},
+        )
+
+    def register_profile(
+        self,
+        *,
+        profile_name: str,
+        base_url: str,
+        auth_retrieval: AuthRetrieval,
+        jobs: list[JobDefinition],
+    ) -> None:
+        for job_defn in jobs:
+            profile_metadata = ProfileMetadata(
+                name=profile_name, auth_retrieval=auth_retrieval, base_url=base_url
+            )
+            match job_defn:
+                case CronJobDefinition():
+                    # Add to ap scheduler
+                    job_kwargs = asdict(
+                        CronJobArgs(
+                            definition=job_defn, profile_metadata=profile_metadata
+                        )
+                    )
+                    existing_job = self._scheduler.get_job(job_defn.id)
+                    if existing_job is not None:
+                        existing_job.modify(
+                            name=job_defn.name,
+                            kwargs=job_kwargs,
+                        )
+                        existing_job.reschedule(job_defn.cron_spec)
+                    else:
+                        self._scheduler.add_job(
+                            cron_job_executor,
+                            # IMPROVE: reconsider these defaults
+                            max_instances=1,
+                            coalesce=True,
+                            trigger=CronTrigger.from_crontab(job_defn.cron_spec),
+                            name=job_defn.name,
+                            id=job_defn.id,
+                            kwargs=job_kwargs,
+                        )
+                case _:
+                    assert_never(job_defn.trigger)
+
+    def _start_apscheduler(self) -> None:
+        self._scheduler.start()
+
+    def _stop_apscheduler(self) -> None:
+        self._scheduler.shutdown()
+
+    def __enter__(self) -> "IntegrationServer":
+        self._start_apscheduler()
+        return self
+
+    def __exit__(self) -> None:
+        self._stop_apscheduler()

UncountablePythonSDK-0.0.10/uncountable/integration/types.py

@@ -0,0 +1,89 @@
+# TODO: move to type spec
+
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Literal
+
+
+class JobDefinitionType(StrEnum):
+    CRON = "cron"
+    # also imagine other job types like webhook-triggered or sftp-triggered,
+    # manual, etc
+
+
+class JobExecutorType(StrEnum):
+    SCRIPT = "script"
+    # also imagine builtin executors like 'sftp_sync'
+
+
+class AuthRetrievalType(StrEnum):
+    ENV = "env"
+    # also imagine secrets manager, keyvault, etc
+
+
+@dataclass
+class JobExecutorBase:
+    type: JobExecutorType
+
+
+@dataclass
+class JobExecutorScript(JobExecutorBase):
+    type: Literal[JobExecutorType.SCRIPT]
+    import_path: str
+
+
+JobExecutor = JobExecutorScript
+
+
+@dataclass
+class JobDefinitionBase:
+    id: str
+    name: str
+
+
+@dataclass
+class CronJobDefinition(JobDefinitionBase):
+    type: Literal[JobDefinitionType.CRON]
+    cron_spec: str
+    # Here we assert that the executor has to be a script, but we could add
+    # other builtin executor types that cron jobs can support later
+    executor: JobExecutorScript
+
+
+JobDefinition = CronJobDefinition
+
+
+@dataclass
+class AuthRetrievalBase:
+    type: AuthRetrievalType
+
+
+@dataclass
+class AuthRetrievalEnv:
+    # We don't really need any extra info here, we can enforce that the auth
+    # keys are named like UNC_PROFILE_{profile name}_API_SECRET_KEY. For
+    # supporting pulling secrets from secrets manager etc it will be nice to
+    # use dataclass properties to get the secret name, region etc.
+    type: Literal[AuthRetrievalType.ENV]
+
+
+AuthRetrieval = AuthRetrievalEnv
+
+
+@dataclass
+class ProfileDefinition:
+    # profile name (expected to be something like customer_name) will be
+    # obtained from the folder name instead of specified here. Forces jobs to
+    # be organized nicely in folders that separate their identities.
+    auth_retrieval: AuthRetrieval
+    base_url: str
+    jobs: list[JobDefinition]
+
+
+@dataclass
+class ProfileMetadata:
+    # supplied by inspecting the folder name that the profile is in
+    name: str
+    base_url: str
+    auth_retrieval: AuthRetrieval

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/uncountable/types/api/inputs/get_input_data.py

@@ -66,7 +66,7 @@ class FullInput:
     quantity_type: str
     is_parameter: bool
     attributes: list[input_attributes_t.InputAttributeValue]
-    global_category_id: base_t.ObjectId
+    global_category_id: typing.Optional[base_t.ObjectId]
     subcategory_ids: list[base_t.ObjectId]
 
 

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/uncountable/types/api/outputs/get_output_data.py

@@ -70,8 +70,8 @@ class FullOutput:
     name: str
     quantity_type: str
     attributes: list[OutputAttrVal]
-    global_category_id: base_t.ObjectId
-    unit_id: base_t.ObjectId
+    global_category_id: typing.Optional[base_t.ObjectId]
+    unit_id: typing.Optional[base_t.ObjectId]
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

{UncountablePythonSDK-0.0.8 → UncountablePythonSDK-0.0.10}/uncountable/types/workflows.py

@@ -20,7 +20,7 @@ __all__: list[str] = [
 @dataclass(kw_only=True)
 class SimpleWorkflowStep:
     workflow_step_id: base_t.ObjectId
-    name: str
+    name: typing.Optional[str]
 
 
 # DO NOT MODIFY -- This file is generated by type_spec