UncountablePythonSDK 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

{UncountablePythonSDK-0.0.8.dist-info → UncountablePythonSDK-0.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UncountablePythonSDK
-Version: 0.0.8
+Version: 0.0.10
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
@@ -16,7 +16,11 @@ Classifier: Topic :: Utilities
 Classifier: Typing :: Typed
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: requests >=2.31.0
+Requires-Dist: requests ==2.31.0
+Requires-Dist: SQLAlchemy ==2.0.29
+Requires-Dist: APScheduler ==3.10.4
+Requires-Dist: dateutil ==2.9.0
+Requires-Dist: shelljob ==0.6.3
 Provides-Extra: test
 Requires-Dist: mypy >=1.8.1 ; extra == 'test'
 Requires-Dist: ruff >=0.2.1 ; extra == 'test'

{UncountablePythonSDK-0.0.8.dist-info → UncountablePythonSDK-0.0.10.dist-info}/RECORD

@@ -4,7 +4,7 @@ pkgs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pkgs/argument_parser/__init__.py,sha256=CsQ6QoPKSLLRVl-z6URAmPkiUL9ZPZoV4rJHgy_-RjA,385
 pkgs/argument_parser/_is_enum.py,sha256=JQPlVIW0z-N9Qux61TQXaqP3lIHEhOOGwe9--JcM4Ro,393
 pkgs/argument_parser/_is_namedtuple.py,sha256=Rjc1bKanIPPogl3qG5JPBxglG1TqWYOo1nxxhBASQWY,265
-pkgs/argument_parser/argument_parser.py,sha256=0jXJ_gUNYj9dxpuyFl64o-zRVDwWBiSC9mXKdP53i6I,14446
+pkgs/argument_parser/argument_parser.py,sha256=TC6cP6OQeLyzYU0m56f9BXSuCcR67FgyRlOa6OZyjgk,14445
 pkgs/argument_parser/case_convert.py,sha256=J9wahIE-P95LvTqn4M4gDUx_RXeiW2SRo9i_1bz1E6A,558
 pkgs/serialization/__init__.py,sha256=quvXMSl1szddLTr4Yxo9KA9oBMoeX7qGpFkkAplFBbY,603
 pkgs/serialization/missing_sentry.py,sha256=aM_9KxbCk9dVvXvcOKgkIQBqFWvLhv8QlIUCiuFEXMo,806
@@ -20,13 +20,13 @@ pkgs/type_spec/__main__.py,sha256=5bJaX9Y_-FavP0qwzhk-z-V97UY7uaezJTa1zhO_HHQ,10
 pkgs/type_spec/builder.py,sha256=Pl3UzHYgci3XRYsJ-pEsrdKu26mrJeLQv66OS2KePGc,38993
 pkgs/type_spec/config.py,sha256=wXALehQr_CcteOY_KbeI3rtmGgqVzHcveAhgKCDfQoQ,4934
 pkgs/type_spec/emit_io_ts.py,sha256=gCEfS81w_ifqjLVJ3_cpy9Gq03o6H5nEsh35WAkqGGE,5606
-pkgs/type_spec/emit_open_api.py,sha256=8tidMRWubBmiiq-XCu_2732IvwTc6R3miurdlBCv3-w,16939
-pkgs/type_spec/emit_open_api_util.py,sha256=S4eHeryxiGUNujypmwn8UZQSlaCCWn7uMpiZzNJJqCg,2054
+pkgs/type_spec/emit_open_api.py,sha256=dnGazdVBjdEWQ4WK8Ujq966quRZY2TyQ05PGR48Md2I,17461
+pkgs/type_spec/emit_open_api_util.py,sha256=xnzPOuPO8Pbhf6xf1tkHUKgXs1n-jH-lWFac8pOtd4I,1882
 pkgs/type_spec/emit_python.py,sha256=ixJ5aXzrpeGuNyuX3pfGRSWgUWgRNegX3ltgwqrmQJA,35580
 pkgs/type_spec/emit_typescript.py,sha256=6tlzvyHc7iIEic01AtjYWuzy7Ac3KijtXRQ3IQSN7JE,18670
 pkgs/type_spec/emit_typescript_util.py,sha256=MZMigIsYIYOXLjJqp-DSRYsztWbIKd9QQEmYRRqnGQ4,894
 pkgs/type_spec/load_types.py,sha256=LMIkrk0n-Kh4I4CUOBzuboYtsAHBYuMrL9d2hkU83f4,2184
-pkgs/type_spec/open_api_util.py,sha256=eJBYgKmu6twqParqWUIIywzx8RM-2dOIyoBLRMHMmqo,7414
+pkgs/type_spec/open_api_util.py,sha256=kOxDc1P4dgjWRxOQNNRtayUEHiHcxbc4OGzBsCWsKsM,7462
 pkgs/type_spec/test.py,sha256=4ueujBq-pEgnX3Z69HyPmD-bullFXmpixcpVzfOkhP4,489
 pkgs/type_spec/util.py,sha256=pTV9EB4_M0tucWwape9_1m5maIi3UKPw44SA_Mc8KNA,4856
 pkgs/type_spec/parts/base.py.prepart,sha256=2RsjF9nTwERb7M0LKsPjGbN5cL2QMwMVt4QHpUPPM70,2147
@@ -47,11 +47,11 @@ type_spec/external/api/entity/resolve_entity_ids.yaml,sha256=Zf3OhAohwLJO7wWj0e-
 type_spec/external/api/entity/set_values.yaml,sha256=_UDxSk09Ke0KDABt8ldOScC2CLrbAhBPArZvno5iD4I,422
 type_spec/external/api/input_groups/get_input_group_names.yaml,sha256=LYgnm2Or7ZWCzDGjl53Y1PcHvej8G7Mpp5H96Z4UZCg,913
 type_spec/external/api/inputs/create_inputs.yaml,sha256=pX8jatpve4iJxK7kYw7SBlQXlVHJVfLDCOrDdpn8Gsw,1561
-type_spec/external/api/inputs/get_input_data.yaml,sha256=-C4-9MbQjwKlWnst2RCA8yXAOVzHssXKfGDx-219pMA,3815
+type_spec/external/api/inputs/get_input_data.yaml,sha256=kjX3VE0y2DxxfUk0J2kjnufSFVAbk_3v28EPuHBhvRE,3825
 type_spec/external/api/inputs/get_input_names.yaml,sha256=AgGcenh6wEbmhmx8i5CY-0hY0q5geZ04ECayE9Hr0Xw,1522
 type_spec/external/api/inputs/get_inputs_data.yaml,sha256=9wKwnOXC_KokThWnz5Me5RzIcGlxffwcKJtgFow-kvU,2368
 type_spec/external/api/inputs/set_input_attribute_values.yaml,sha256=RpOtVIWBlOp_Nd5Xp67IaQmCzGnLkNwFKQDi4WD8oX4,1021
-type_spec/external/api/outputs/get_output_data.yaml,sha256=78ywbknD-j8TNUabSCGv4JNE1FIehhvRQtrB0wQ7krY,3465
+type_spec/external/api/outputs/get_output_data.yaml,sha256=ABaKme64kFCZ-A11AUM5ARQs0Pb9f71atWx-QMYFNZs,3485
 type_spec/external/api/outputs/get_output_names.yaml,sha256=IqEYCOHeyHIu45Lf37sfzn-o_0VFk0u_Bqsb0NKHPzw,1334
 type_spec/external/api/outputs/resolve_output_conditions.yaml,sha256=lGrG6XPGiQtHx24GccPcO18xxAoMynTSC_2WwxCjFrc,1640
 type_spec/external/api/project/get_projects.yaml,sha256=pQXGGPHenmFWYUuzEmZ-ORf0_d6lpQkTrXXMRrEqvDI,1676
@@ -70,6 +70,17 @@ uncountable/__init__.py,sha256=281cC2hs8pbrD0jVKMol-tbWSh7Zcsc8oRT42dKteyE,102
 uncountable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/core/__init__.py,sha256=GCXRRhhYv-yKzqKjAij7GcNiOr90BBbgdhhpusWtmUc,100
 uncountable/core/client.py,sha256=jcJBGwe31fOqfGiUJzW_KMcfE0hax55jgVVliALBNLQ,4160
+uncountable/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+uncountable/integration/construct_client.py,sha256=Ojdj6Mh5orAjXZLxb6SzvH4zRmkvqp6nHF_8Ph7h1ho,896
+uncountable/integration/cron.py,sha256=QZP4yvxdGUZsOH5wtxZzqja_0UBHL4lwERzEHGTR-E0,917
+uncountable/integration/entrypoint.py,sha256=31QJO6pFDvPR8k5Bi7RSm-IUI5kXj_VHdz4XZW3j5DI,1363
+uncountable/integration/job.py,sha256=orPmhLWs5mB4U8XweIb1Pa3fUKKE89yUnHXLkHy7KOs,699
+uncountable/integration/server.py,sha256=VfbJLFqHl6cQAL8e2EOmrtSRfruskQ89h_2Mu3EMeOI,2998
+uncountable/integration/types.py,sha256=n9idu2_qHOA5CQdE6NK8HS6aZ8ugTZKTfTTnm4sy8tI,2066
+uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+uncountable/integration/db/connect.py,sha256=iI9e8a2hfbFP-dvH0MGLsrG-RpM0dHKCL-oCLkah9hs,181
+uncountable/integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+uncountable/integration/executors/script_executor.py,sha256=l_mniGlS9Ni6FdmspN7bvMWVocOaXKCo8FM9SFW8n8c,600
 uncountable/types/__init__.py,sha256=SbBDFhHu5iqnvnYjWCBOkjg3_mXuaBHEGfnq021wfHw,3873
 uncountable/types/base.py,sha256=7yMC3sCQ13wd92896Ga2XbZffCqq98SKv09rFGb1ulg,2682
 uncountable/types/calculations.py,sha256=16J-KKMp-I8ZQUkYNmKCHfAn6DGb99cFinALcDIdGHY,562
@@ -90,7 +101,7 @@ uncountable/types/recipe_tags.py,sha256=lYpksHAxXCcIjZKR7JoZOTH2cBSovwxZaHwjZy_y
 uncountable/types/response.py,sha256=ZI0CG7ZxBM2k5_W-6mNMU3UlB0p1i-0nrwOvsMaS-vU,620
 uncountable/types/units.py,sha256=_kZ7KkXIbRiY2fOdkTsbJBpWRah5TCC2WWiG05e-1DA,565
 uncountable/types/users.py,sha256=SUjNHBDcImKnnE7IN096Wfr1fmjNjCkQ7yQgKUPffz8,588
-uncountable/types/workflows.py,sha256=bj2xaH6mFAPLNeFDepGw1TvuBbsyP43wTx8_LY0nwcM,808
+uncountable/types/workflows.py,sha256=JJKM0SeDqAMA2oxwzlG1c1FxeSBI7E49duoITcUrEZo,825
 uncountable/types/api/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/batch/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/batch/execute_batch.py,sha256=cCZo_akwJR4_ET6oOBChdlrCQlxUw8V9i8tDpv7JTIc,1696
@@ -105,12 +116,12 @@ uncountable/types/api/input_groups/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHk
 uncountable/types/api/input_groups/get_input_group_names.py,sha256=LdHWWEfVNGys6Tudienjich56Zz4bj7uXznpyYitCYA,1033
 uncountable/types/api/inputs/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/inputs/create_inputs.py,sha256=ykXjqKoE8WOy5N47jcjDVzSdiZx9GlB0FcTyuJdn8FU,1675
-uncountable/types/api/inputs/get_input_data.py,sha256=X30bb3eJEl7RweXMsSGlEpAZbF4rZvr8Mse5IMNAao4,2161
+uncountable/types/api/inputs/get_input_data.py,sha256=RrgmS4QORMrm1HTNey-A3zzCdYdkmTbokVYPTI_bjPo,2178
 uncountable/types/api/inputs/get_input_names.py,sha256=8vUA9maZdogngeYbr-DpifNrokJ16BgDN5LQvTrEvSc,1153
 uncountable/types/api/inputs/get_inputs_data.py,sha256=sqZ6xEjzQqRQYMaKXLGWR8WPhWbO6J6zZT8zSTEer84,1841
 uncountable/types/api/inputs/set_input_attribute_values.py,sha256=yvWgIVzl818ewhUA4v3ldSBfmZKngdWTtFn7fqk-dOE,1341
 uncountable/types/api/outputs/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
-uncountable/types/api/outputs/get_output_data.py,sha256=Zs1-8jJEPNltMq6DQisSGSItqgTpRD9WFi0wMfDscf8,2182
+uncountable/types/api/outputs/get_output_data.py,sha256=CpYlXNGWWNTiMgDPBRwucSseSrX4gnLuLIp2ljiONrI,2216
 uncountable/types/api/outputs/get_output_names.py,sha256=Id_ApombSzzFdq5rD4uOfWIKzthic6KBAeyqcrhsx18,1086
 uncountable/types/api/outputs/resolve_output_conditions.py,sha256=XZqUseXcGhApHmPm-2u7a37Y4blLRxoPoLhvYlBfCpI,1799
 uncountable/types/api/project/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
@@ -128,7 +139,7 @@ uncountable/types/api/recipes/get_recipe_output_metadata.py,sha256=L9s2ykPP4pd02
 uncountable/types/api/recipes/get_recipes_data.py,sha256=dOKokz6rJp3AiqNrF8rAZFlmJSs3ejdNIJhwKw0Utr0,5317
 uncountable/types/api/recipes/set_recipe_inputs.py,sha256=sHEwPocBucWRnnoX7nbNaFqdflxFkqdjuVydNezqluY,1326
 uncountable/types/api/recipes/set_recipe_outputs.py,sha256=QYq39TNchQ80ET1C77OE9fwhbu_HmIoEDmrQJHkkCu0,1609
-UncountablePythonSDK-0.0.8.dist-info/METADATA,sha256=Tbt6PxvCFuyUjcs2twY0ze73oOxiNWsPV8DgdLu6EmI,1125
-UncountablePythonSDK-0.0.8.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-UncountablePythonSDK-0.0.8.dist-info/top_level.txt,sha256=8ga1DCSWt4Uc_XJ5cR0QrDQuyoeo2uoSzaAJdQT2KBo,36
-UncountablePythonSDK-0.0.8.dist-info/RECORD,,
+UncountablePythonSDK-0.0.10.dist-info/METADATA,sha256=-k3bLSnSPbkgbodLSCUIvBzwdRtpcfIcBKEPHtmxEnY,1261
+UncountablePythonSDK-0.0.10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+UncountablePythonSDK-0.0.10.dist-info/top_level.txt,sha256=8ga1DCSWt4Uc_XJ5cR0QrDQuyoeo2uoSzaAJdQT2KBo,36
+UncountablePythonSDK-0.0.10.dist-info/RECORD,,

pkgs/argument_parser/argument_parser.py

@@ -407,5 +407,5 @@ class CachedParser(typing.Generic[T]):
             return self.parse_storage(yaml.safe_load(data_in))
 
     def parse_yaml_resource(self, package: resources.Package, resource: str) -> T:
-        raw_data = resources.read_text(package, resource)
-        return self.parse_storage(yaml.safe_load(raw_data))
+        with resources.open_text(package, resource) as fp:
+            return self.parse_storage(yaml.safe_load(fp))

pkgs/type_spec/emit_open_api.py

@@ -141,6 +141,18 @@ def _serialize_global_context(ctx: EmitOpenAPIGlobalContext) -> str:
     return yaml.dump(oa_root, sort_keys=False)
 
 
+def _emit_endpoint_parameters(typ: OpenAPIType | None) -> dict[str, list[dict[str, str]]]:
+    if not isinstance(typ, OpenAPIObjectType):
+        return {}
+
+    return {
+        "parameters": [
+            {"$ref": f"#/components/schema/Arguments/{prop_name}"}
+            for prop_name in typ.properties
+        ]
+    }
+
+
 def _emit_namespace(
     gctx: EmitOpenAPIGlobalContext,
     ctx: EmitOpenAPIContext,
@@ -159,23 +171,28 @@ def _emit_namespace(
     if ctx.endpoint is not None:
         endpoint = ctx.endpoint
 
+        argument_type = ctx.types.get("Arguments")
         oa_endpoint = dict()
-        oa_endpoint[endpoint.method] = {
-            "tags": endpoint.tags,
-            "summary": endpoint.summary,
-            "description": endpoint.description,
-            "parameters": {"$ref": "#/components/schema/Arguments"},
-            "responses": {
-                "200": {
-                    "description": "OK",
-                    "content": {
-                        "application/json": {
-                            "schema": {"$ref": "#/components/schema/Data"}
-                        }
-                    },
-                }
-            },
-        }
+        oa_endpoint[endpoint.method] = (
+            {
+                "tags": endpoint.tags,
+                "summary": endpoint.summary,
+                "description": endpoint.description,
+            }
+            | _emit_endpoint_parameters(argument_type)
+            | {
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "content": {
+                            "application/json": {
+                                "schema": {"$ref": "#/components/schema/Data"}
+                            }
+                        },
+                    }
+                },
+            }
+        )
         oa_components["endpoint"] = oa_endpoint
 
     types = ctx.types
@@ -460,5 +477,5 @@ def open_api_type(
     ctx.namespaces.add(stype.namespace)
     # external namespace resolution
     return OpenAPIRefType(
-        source=f"{resolve_namespace_ref(ctx, stype.namespace, config=config)}/{stype.name}"
+        source=f"{resolve_namespace_ref(source_path=ctx.namespace.path, ref_path=stype.namespace.path, ref='/components/schema')}/{stype.name}"
     )

pkgs/type_spec/emit_open_api_util.py

@@ -9,7 +9,6 @@ from dataclasses import dataclass, field
 from typing import TypeAlias
 
 from . import builder
-from .config import OpenAPIConfig
 from .open_api_util import OpenAPIType
 
 MODIFY_NOTICE = "# DO NOT MODIFY -- This file is generated by type_spec"
@@ -75,12 +74,8 @@ class EmitOpenAPIContext:
 
 
 def resolve_namespace_ref(
-    ctx: EmitOpenAPIContext,
-    namespace: builder.SpecNamespace,
-    *,
-    config: OpenAPIConfig,
+    *, source_path: list[str], ref_path: list[str], ref: str
 ) -> str:
-    # TODO: Handle namespaces not in root directory
-    if len(ctx.namespace.path) == 1:
-        return f"{namespace.name}.yaml#/components/schema"
-    return f"{config.static_url_path}/common/{namespace.name}.yaml#/components/schema"
+    to_root = "/".join(".." for _ in source_path)
+    location = "/".join(ref_path)
+    return f"{to_root}/common/{location}.yaml#{ref}"

pkgs/type_spec/open_api_util.py

@@ -13,9 +13,10 @@ class OpenAPIType(ABC):
         self.nullable = nullable
 
     @abstractmethod
-    def asdict(self) -> dict[str, object]: ...
+    def asdict(self) -> dict[str, object]:
+        pass
 
-    def asarguments(self) -> list[dict[str, object]]:
+    def asarguments(self) -> dict[str, dict[str, object]]:
         raise UnsupportedOperation
 
     def add_addl_info(self, emitted: dict[str, object]) -> dict[str, object]:
@@ -147,8 +148,8 @@ class OpenAPIFreeFormObjectType(OpenAPIType):
     def asdict(self) -> dict[str, object]:
         return self.add_addl_info({"type": "object"})
 
-    def asarguments(self) -> list[dict[str, object]]:
-        return []
+    def asarguments(self) -> dict[str, dict[str, object]]:
+        return {}
 
 
 class OpenAPIObjectType(OpenAPIType):
@@ -185,17 +186,17 @@ class OpenAPIObjectType(OpenAPIType):
             },
         })
 
-    def asarguments(self) -> list[dict[str, object]]:
-        argument_types: list[dict[str, object]] = []
+    def asarguments(self) -> dict[str, dict[str, object]]:
+        argument_types: dict[str, dict[str, object]] = {}
         for property_name, property_type in self.properties.items():
             desc = self.property_desc.get(property_name)
-            argument_types.append({
+            argument_types[property_name] = {
                 "name": property_name,
                 "in": "query",
                 "schema": property_type.asdict(),
                 "required": not property_type.nullable,
                 "description": desc or "",
-            })
+            }
         return argument_types
 
 
@@ -219,11 +220,11 @@ class OpenAPIUnionType(OpenAPIType):
         # TODO: use parents description and nullable
         return {"oneOf": [base_type.asdict() for base_type in self.base_types]}
 
-    def asarguments(self) -> list[dict[str, object]]:
+    def asarguments(self) -> dict[str, dict[str, object]]:
         # TODO handle inheritence (allOf and refs); need to inline here...
         # for now skip this endpoint
 
-        return []
+        return {}
 
 
 class OpenAPIIntersectionType(OpenAPIType):
@@ -246,8 +247,8 @@ class OpenAPIIntersectionType(OpenAPIType):
         # TODO: use parents description and nullable
         return {"allOf": [base_type.asdict() for base_type in self.base_types]}
 
-    def asarguments(self) -> list[dict[str, object]]:
+    def asarguments(self) -> dict[str, dict[str, object]]:
         # TODO handle inheritence (allOf and refs); need to inline here...
         # for now skip this endpoint
 
-        return []
+        return {}

type_spec/external/api/inputs/get_input_data.yaml

@@ -76,7 +76,7 @@ FullInput:
             type: List<input_attributes.InputAttributeValue>
             desc: "Attributes associated with an input, such as CAS number, density, etc."
         global_category_id:
-            type: ObjectId
+            type: Optional<ObjectId>
             desc: "The global category ID associated with the input"
         subcategory_ids:
             type: List<ObjectId>

type_spec/external/api/outputs/get_output_data.yaml

@@ -73,10 +73,10 @@ FullOutput:
             type: List<OutputAttrVal>
             desc: "Attributes associated with an output, such as a test documentation"
         global_category_id:
-            type: ObjectId
+            type: Optional<ObjectId>
             desc: "The global category ID associated with the output"
         unit_id:
-            type: ObjectId
+            type: Optional<ObjectId>
             desc: "The unit for the output, which all data is stored in. This may be different than the users default displayed information for the output"
 
 Data:

uncountable/integration/construct_client.py

@@ -0,0 +1,30 @@
+import os
+from typing import assert_never
+
+from uncountable.core.client import AuthDetailsApiKey, Client
+from uncountable.integration.types import (
+    AuthRetrievalEnv,
+    ProfileMetadata,
+)
+
+
+def construct_uncountable_client(
+    profile_meta: ProfileMetadata
+) -> Client:
+    match profile_meta.auth_retrieval:
+        case AuthRetrievalEnv():
+            api_id = os.getenv(f"UNC_PROFILE_{profile_meta.name}_API_ID")
+            api_secret_key = os.getenv(
+                f"UNC_PROFILE_{profile_meta.name}_API_SECRET_KEY"
+            )
+
+            assert api_id is not None
+            assert api_secret_key is not None
+
+            return Client(
+                base_url=profile_meta.base_url,
+                auth_details=AuthDetailsApiKey(
+                    api_id=api_id, api_secret_key=api_secret_key
+                ),
+            )
+    assert_never(profile_meta.auth_retrieval)

uncountable/integration/cron.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+
+from pkgs.argument_parser import CachedParser
+from uncountable.integration.construct_client import construct_uncountable_client
+from uncountable.integration.executors.script_executor import resolve_script_executor
+from uncountable.integration.job import CronJobArguments
+from uncountable.integration.types import JobDefinition, ProfileMetadata
+
+
+@dataclass
+class CronJobArgs:
+    definition: JobDefinition
+    profile_metadata: ProfileMetadata
+
+
+cron_args_parser = CachedParser(CronJobArgs)
+
+
+def cron_job_executor(**kwargs: dict) -> None:
+    args_passed = cron_args_parser.parse_storage(kwargs)
+    args = CronJobArguments(
+        job_definition=args_passed.definition,
+        client=construct_uncountable_client(profile_meta=args_passed.profile_metadata),
+    )
+
+    job_class = resolve_script_executor(args_passed.definition.executor)
+
+    job = job_class()
+    job.run(args)

uncountable/integration/db/connect.py

@@ -0,0 +1,8 @@
+import os
+from sqlalchemy import create_engine
+from sqlalchemy.engine.base import Engine
+
+
+def create_db_engine() -> Engine:
+    return create_engine(os.environ["UNC_SQLITE_URI"])
+

uncountable/integration/entrypoint.py

@@ -0,0 +1,41 @@
+import os
+from importlib import resources
+
+from uncountable.integration.server import IntegrationServer
+from uncountable.integration.types import ProfileDefinition
+from pkgs.argument_parser import CachedParser
+from uncountable.integration.db.connect import create_db_engine
+
+
+profile_parser = CachedParser(ProfileDefinition)
+
+
+def main() -> None:
+    profiles_module = os.environ["UNC_PROFILES_MODULE"]
+    with IntegrationServer(create_db_engine()) as server:
+        # TODO: Loop through all job spec yaml files and call server.add_job
+        profiles = [
+            entry
+            for entry in resources.files(profiles_module).iterdir()
+            if entry.is_dir()
+        ]
+        for profile in profiles:
+            profile_name = profile.name
+            try:
+                profile = profile_parser.parse_yaml_resource(
+                    package=".".join([profiles_module, profile_name]),
+                    resource="profile.yaml",
+                )
+            except FileNotFoundError as e:
+                print("WARN: profile.yaml not found", e)
+                continue
+            server.register_profile(
+                profile_name=profile_name,
+                base_url=profile.base_url,
+                auth_retrieval=profile.auth_retrieval,
+                jobs=profile.jobs,
+            )
+
+
+if __name__ == "__main__":
+    main()

uncountable/integration/executors/script_executor.py

@@ -0,0 +1,18 @@
+
+
+import importlib
+import inspect
+from uncountable.integration.job import Job
+from uncountable.integration.types import JobExecutorScript
+
+
+def resolve_script_executor(executor: JobExecutorScript) -> type[Job]:
+    job_module = importlib.import_module(executor.import_path)
+    found_jobs: list[type[Job]] = []
+    for _, job_class in inspect.getmembers(job_module, inspect.isclass):
+        if Job in job_class.__bases__:
+            found_jobs.append(job_class())
+    assert (
+        len(found_jobs) == 1
+    ), f"expected exactly one job class in {executor.import_path}"
+    return found_jobs[0]

uncountable/integration/job.py

@@ -0,0 +1,39 @@
+from dataclasses import dataclass
+from uncountable.core.client import Client
+from uncountable.integration.types import JobDefinition
+
+from abc import ABC, abstractmethod
+
+
+@dataclass
+class JobArgumentsBase:
+    job_definition: JobDefinition
+    client: Client
+
+
+@dataclass
+class CronJobArguments(JobArgumentsBase):
+    # can imagine passing additional data such as in the sftp or webhook cases
+    pass
+
+
+JobArguments = CronJobArguments
+
+
+@dataclass
+class JobResult:
+    success: bool
+
+
+class Job(ABC):
+
+    @abstractmethod
+    def run(self, args: JobArguments) -> JobResult:
+        ...
+
+
+class CronJob(Job):
+
+    @abstractmethod
+    def run(self, args: CronJobArguments) -> JobResult:
+        ...

uncountable/integration/server.py

@@ -0,0 +1,86 @@
+from dataclasses import asdict
+from typing import assert_never
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.executors.pool import ThreadPoolExecutor
+from uncountable.integration.cron import CronJobArgs, cron_job_executor
+from apscheduler.triggers.cron import CronTrigger
+from sqlalchemy.engine.base import Engine
+
+from uncountable.integration.types import (
+    AuthRetrieval,
+    CronJobDefinition,
+    JobDefinition,
+    ProfileMetadata,
+)
+
+
+_MAX_APSCHEDULER_CONCURRENT_JOBS = 1
+
+
+class IntegrationServer:
+    _scheduler: BaseScheduler
+    _engine: Engine
+
+    def __init__(self, engine: Engine) -> None:
+        self._engine = engine
+        self._scheduler = BackgroundScheduler(
+            timezone="UTC",
+            jobstores={"default": SQLAlchemyJobStore(engine=engine)},
+            executors={"default": ThreadPoolExecutor(_MAX_APSCHEDULER_CONCURRENT_JOBS)},
+        )
+
+    def register_profile(
+        self,
+        *,
+        profile_name: str,
+        base_url: str,
+        auth_retrieval: AuthRetrieval,
+        jobs: list[JobDefinition],
+    ) -> None:
+        for job_defn in jobs:
+            profile_metadata = ProfileMetadata(
+                name=profile_name, auth_retrieval=auth_retrieval, base_url=base_url
+            )
+            match job_defn:
+                case CronJobDefinition():
+                    # Add to ap scheduler
+                    job_kwargs = asdict(
+                        CronJobArgs(
+                            definition=job_defn, profile_metadata=profile_metadata
+                        )
+                    )
+                    existing_job = self._scheduler.get_job(job_defn.id)
+                    if existing_job is not None:
+                        existing_job.modify(
+                            name=job_defn.name,
+                            kwargs=job_kwargs,
+                        )
+                        existing_job.reschedule(job_defn.cron_spec)
+                    else:
+                        self._scheduler.add_job(
+                            cron_job_executor,
+                            # IMPROVE: reconsider these defaults
+                            max_instances=1,
+                            coalesce=True,
+                            trigger=CronTrigger.from_crontab(job_defn.cron_spec),
+                            name=job_defn.name,
+                            id=job_defn.id,
+                            kwargs=job_kwargs,
+                        )
+                case _:
+                    assert_never(job_defn.trigger)
+
+    def _start_apscheduler(self) -> None:
+        self._scheduler.start()
+
+    def _stop_apscheduler(self) -> None:
+        self._scheduler.shutdown()
+
+    def __enter__(self) -> "IntegrationServer":
+        self._start_apscheduler()
+        return self
+
+    def __exit__(self) -> None:
+        self._stop_apscheduler()

uncountable/integration/types.py

@@ -0,0 +1,89 @@
+# TODO: move to type spec
+
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Literal
+
+
+class JobDefinitionType(StrEnum):
+    CRON = "cron"
+    # also imagine other job types like webhook-triggered or sftp-triggered,
+    # manual, etc
+
+
+class JobExecutorType(StrEnum):
+    SCRIPT = "script"
+    # also imagine builtin executors like 'sftp_sync'
+
+
+class AuthRetrievalType(StrEnum):
+    ENV = "env"
+    # also imagine secrets manager, keyvault, etc
+
+
+@dataclass
+class JobExecutorBase:
+    type: JobExecutorType
+
+
+@dataclass
+class JobExecutorScript(JobExecutorBase):
+    type: Literal[JobExecutorType.SCRIPT]
+    import_path: str
+
+
+JobExecutor = JobExecutorScript
+
+
+@dataclass
+class JobDefinitionBase:
+    id: str
+    name: str
+
+
+@dataclass
+class CronJobDefinition(JobDefinitionBase):
+    type: Literal[JobDefinitionType.CRON]
+    cron_spec: str
+    # Here we assert that the executor has to be a script, but we could add
+    # other builtin executor types that cron jobs can support later
+    executor: JobExecutorScript
+
+
+JobDefinition = CronJobDefinition
+
+
+@dataclass
+class AuthRetrievalBase:
+    type: AuthRetrievalType
+
+
+@dataclass
+class AuthRetrievalEnv:
+    # We don't really need any extra info here, we can enforce that the auth
+    # keys are named like UNC_PROFILE_{profile name}_API_SECRET_KEY. For
+    # supporting pulling secrets from secrets manager etc it will be nice to
+    # use dataclass properties to get the secret name, region etc.
+    type: Literal[AuthRetrievalType.ENV]
+
+
+AuthRetrieval = AuthRetrievalEnv
+
+
+@dataclass
+class ProfileDefinition:
+    # profile name (expected to be something like customer_name) will be
+    # obtained from the folder name instead of specified here. Forces jobs to
+    # be organized nicely in folders that separate their identities.
+    auth_retrieval: AuthRetrieval
+    base_url: str
+    jobs: list[JobDefinition]
+
+
+@dataclass
+class ProfileMetadata:
+    # supplied by inspecting the folder name that the profile is in
+    name: str
+    base_url: str
+    auth_retrieval: AuthRetrieval

uncountable/types/api/inputs/get_input_data.py

@@ -66,7 +66,7 @@ class FullInput:
     quantity_type: str
     is_parameter: bool
     attributes: list[input_attributes_t.InputAttributeValue]
-    global_category_id: base_t.ObjectId
+    global_category_id: typing.Optional[base_t.ObjectId]
     subcategory_ids: list[base_t.ObjectId]
 
 

uncountable/types/api/outputs/get_output_data.py

@@ -70,8 +70,8 @@ class FullOutput:
     name: str
     quantity_type: str
     attributes: list[OutputAttrVal]
-    global_category_id: base_t.ObjectId
-    unit_id: base_t.ObjectId
+    global_category_id: typing.Optional[base_t.ObjectId]
+    unit_id: typing.Optional[base_t.ObjectId]
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/workflows.py

@@ -20,7 +20,7 @@ __all__: list[str] = [
 @dataclass(kw_only=True)
 class SimpleWorkflowStep:
     workflow_step_id: base_t.ObjectId
-    name: str
+    name: typing.Optional[str]
 
 
 # DO NOT MODIFY -- This file is generated by type_spec