tracdap-runtime 0.8.0rc2__py3-none-any.whl → 0.9.0b2__py3-none-any.whl

tracdap/rt/_impl/exec/dev_mode.py

@@ -58,11 +58,6 @@ class DevModeTranslator:
 
         cls._log.info(f"Applying dev mode config translation to system config")
 
-        # TODO: In code gen, default object types to a new object unless the field is marked as optional
-        # This would match the general semantics of protobuf
-        if sys_config.storage is None:
-            sys_config.storage = _cfg.StorageConfig()
-
         sys_config = cls._add_integrated_repo(sys_config)
         sys_config = cls._process_storage(sys_config, config_mgr)
 
@@ -73,11 +68,12 @@ class DevModeTranslator:
 
         # Add the integrated model repo trac_integrated
 
-        integrated_repo_config = _cfg.PluginConfig(
+        integrated_repo_config = _meta.ResourceDefinition(
+            resourceType=_meta.ResourceType.MODEL_REPOSITORY,
             protocol="integrated",
             properties={})
 
-        sys_config.repositories["trac_integrated"] = integrated_repo_config
+        sys_config.resources["trac_integrated"] = integrated_repo_config
 
         return sys_config
 
@@ -86,23 +82,17 @@ class DevModeTranslator:
             cls, sys_config: _cfg.RuntimeConfig,
             config_mgr: _cfg_p.ConfigManager):
 
-        storage_config = copy.deepcopy(sys_config.storage)
-
-        for bucket_key, bucket_config in storage_config.buckets.items():
-            storage_config.buckets[bucket_key] = cls._resolve_storage_location(
-                bucket_key, bucket_config, config_mgr)
-
-        for bucket_key, bucket_config in storage_config.external.items():
-            storage_config.external[bucket_key] = cls._resolve_storage_location(
-                bucket_key, bucket_config, config_mgr)
+        sys_config.properties[_cfg_p.ConfigKeys.STORAGE_DEFAULT_LAYOUT] = _meta.StorageLayout.DEVELOPER_LAYOUT.name
 
-        sys_config = copy.copy(sys_config)
-        sys_config.storage = storage_config
+        for resource_key, resource in sys_config.resources.items():
+            if resource.resourceType in [_meta.ResourceType.INTERNAL_STORAGE, _meta.ResourceType.EXTERNAL_STORAGE]:
+                sys_config.resources[resource_key] = cls._resolve_storage_location(
+                resource_key, resource, config_mgr)
 
         return sys_config
 
     @classmethod
-    def _resolve_storage_location(cls, bucket_key, bucket_config, config_mgr: _cfg_p.ConfigManager):
+    def _resolve_storage_location(cls, bucket_key, bucket_config: _meta.ResourceDefinition, config_mgr: _cfg_p.ConfigManager):
 
         if bucket_config.protocol != "LOCAL":
             return bucket_config
@@ -166,6 +156,9 @@ class DevModeTranslator:
             job_config, job_def = self.translate_job_def(job_config, job_def, model_class)
             job_config.job = job_def
 
+            # Include some basic tags and attributes in the generated metadata
+            job_config = self.generate_dev_mode_tags(job_config)
+
             return job_config
 
         finally:
@@ -218,27 +211,31 @@ class DevModeTranslator:
         return job_config, job_def
 
     @classmethod
-    def _add_job_resource(
+    def _add_job_metadata(
             cls, job_config: _cfg.JobConfig,
             obj_id: _meta.TagHeader, obj: _meta.ObjectDefinition) \
             -> _cfg.JobConfig:
 
         obj_key = _util.object_key(obj_id)
-        job_config.resources[obj_key] = obj
+
+        job_config.objectMapping[obj_key] = obj_id
+        job_config.objects[obj_key] = obj
 
         return job_config
 
     @classmethod
-    def _process_job_id(cls, job_config: _cfg.JobConfig):
+    def _process_job_id(cls, job_config: _cfg.JobConfig) -> _cfg.JobConfig:
 
         job_id = _util.new_object_id(_meta.ObjectType.JOB)
+        result_id = _util.new_object_id(_meta.ObjectType.RESULT)
 
         cls._log.info(f"Assigning job ID = [{_util.object_key(job_id)}]")
+        cls._log.info(f"Assigning result ID = [{_util.object_key(result_id)}]")
 
-        translated_config = copy.copy(job_config)
-        translated_config.jobId = job_id
+        job_config.jobId = job_id
+        job_config.resultId = result_id
 
-        return translated_config
+        return job_config
 
     @classmethod
     def _process_job_type(cls, job_def: _meta.JobDefinition):
@@ -346,7 +343,7 @@ class DevModeTranslator:
 
             model_id, model_obj = self._generate_model_for_class(model_class)
             job_detail.model = _util.selector_for(model_id)
-            job_config = self._add_job_resource(job_config, model_id, model_obj)
+            job_config = self._add_job_metadata(job_config, model_id, model_obj)
 
         # Otherwise look for models specified as a single string, and take that as the entry point
         else:
@@ -355,7 +352,7 @@ class DevModeTranslator:
             if hasattr(job_detail, "model") and isinstance(job_detail.model, str):
                 model_id, model_obj = self._generate_model_for_entry_point(job_detail.model)  # noqa
                 job_detail.model = _util.selector_for(model_id)
-                job_config = self._add_job_resource(job_config, model_id, model_obj)
+                job_config = self._add_job_metadata(job_config, model_id, model_obj)
 
             elif hasattr(job_detail, "model") and isinstance(job_detail.model, _meta.TagSelector):
                 if job_detail.model.objectType == _meta.ObjectType.OBJECT_TYPE_NOT_SET:
@@ -369,7 +366,7 @@ class DevModeTranslator:
                     if isinstance(model_detail, str):
                         model_id, model_obj = self._generate_model_for_entry_point(model_detail)
                         job_detail.models[model_key] = _util.selector_for(model_id)
-                        job_config = self._add_job_resource(job_config, model_id, model_obj)
+                        job_config = self._add_job_metadata(job_config, model_id, model_obj)
 
         return job_config, job_def
 
@@ -446,8 +443,8 @@ class DevModeTranslator:
         job_def.runFlow.flow = _util.selector_for(flow_id)
 
         job_config = copy.copy(job_config)
-        job_config.resources = copy.copy(job_config.resources)
-        job_config = self._add_job_resource(job_config, flow_id, flow_obj)
+        job_config.objects = copy.copy(job_config.objects)
+        job_config = self._add_job_metadata(job_config, flow_id, flow_obj)
 
         return job_config, job_def
 
@@ -472,7 +469,7 @@ class DevModeTranslator:
         for model_name, model_node in model_nodes.items():
 
             model_selector = job_def.runFlow.models[model_name]
-            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_obj = _util.get_job_metadata(model_selector, job_config)
 
             model_inputs = set(model_obj.model.inputs.keys())
             model_outputs = set(model_obj.model.outputs.keys())
@@ -540,7 +537,7 @@ class DevModeTranslator:
                 # Generate node param sockets needed by the model
                 if node_name in job.models:
                     model_selector = job.models[node_name]
-                    model_obj = _util.get_job_resource(model_selector, job_config)
+                    model_obj = _util.get_job_metadata(model_selector, job_config)
                     for param_name in model_obj.model.parameters:
                         add_param_to_flow(node_name, param_name)
                         if param_name not in node.parameters:
@@ -622,7 +619,7 @@ class DevModeTranslator:
         for target in targets:
 
             model_selector = job_def.runFlow.models.get(target.node)
-            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_obj = _util.get_job_metadata(model_selector, job_config)
             model_param = model_obj.model.parameters.get(target.socket)
             model_params.append(model_param)
 
@@ -659,7 +656,7 @@ class DevModeTranslator:
         for target in targets:
 
             model_selector = job_def.runFlow.models.get(target.node)
-            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_obj = _util.get_job_metadata(model_selector, job_config)
             model_input = model_obj.model.inputs.get(target.socket)
             model_inputs.append(model_input)
 
@@ -694,7 +691,7 @@ class DevModeTranslator:
         for source in sources:
 
             model_selector = job_def.runFlow.models.get(source.node)
-            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_obj = _util.get_job_metadata(model_selector, job_config)
             model_input = model_obj.model.outputs.get(source.socket)
             model_outputs.append(model_input)
 
@@ -727,10 +724,10 @@ class DevModeTranslator:
 
         if hasattr(job_detail, "model"):
             model_key = _util.object_key(job_detail.model)
-            model_or_flow = job_config.resources[model_key].model
+            model_or_flow = job_config.objects[model_key].model
         elif hasattr(job_detail, "flow"):
             flow_key = _util.object_key(job_detail.flow)
-            model_or_flow = job_config.resources[flow_key].flow
+            model_or_flow = job_config.objects[flow_key].flow
         else:
             model_or_flow = None
 
@@ -784,71 +781,67 @@ class DevModeTranslator:
         job_detail = self._get_job_detail(job_def)
 
         if hasattr(job_detail, "model"):
-            model_obj = _util.get_job_resource(job_detail.model, job_config)
+            model_obj = _util.get_job_metadata(job_detail.model, job_config)
             required_inputs = model_obj.model.inputs
-            required_outputs = model_obj.model.outputs
+            expected_outputs = model_obj.model.outputs
 
         elif hasattr(job_detail, "flow"):
-            flow_obj = _util.get_job_resource(job_detail.flow, job_config)
+            flow_obj = _util.get_job_metadata(job_detail.flow, job_config)
             required_inputs = flow_obj.flow.inputs
-            required_outputs = flow_obj.flow.outputs
+            expected_outputs = flow_obj.flow.outputs
 
         else:
             return job_config, job_def
 
         job_inputs = job_detail.inputs
         job_outputs = job_detail.outputs
-        job_resources = job_config.resources
-
-        for input_key, input_value in job_inputs.items():
-            if not (isinstance(input_value, str) and input_value in job_resources):
-
-                model_input = required_inputs[input_key]
-
-                if model_input.objectType == _meta.ObjectType.DATA:
-                    schema = model_input.schema if model_input and not model_input.dynamic else None
-                    input_id = self._process_data_socket(input_key, input_value, schema, job_resources, new_unique_file=False)
-                elif model_input.objectType == _meta.ObjectType.FILE:
-                    file_type = model_input.fileType
-                    input_id = self._process_file_socket(input_key, input_value, file_type, job_resources, new_unique_file=False)
-                else:
-                    raise _ex.EUnexpected()
-
-                job_inputs[input_key] = _util.selector_for(input_id)
+        job_prior_outputs = job_detail.priorOutputs
+
+        for key, schema in required_inputs.items():
+            if key not in job_inputs:
+                if not schema.optional:
+                    raise _ex.EJobValidation(f"Missing required input [{key}]")
+                continue
+            supplied_input = job_inputs.pop(key) if key in job_inputs else None
+            input_selector = self._process_socket(key, schema, supplied_input, job_config, is_output=False)
+            if input_selector is not None:
+                job_inputs[key] = input_selector
+
+        for key, schema in expected_outputs.items():
+            if key not in job_outputs:
+                raise _ex.EJobValidation(f"Missing required output [{key}]")
+            supplied_output = job_outputs.pop(key)
+            output_selector = self._process_socket(key, schema, supplied_output, job_config, is_output=True)
+            if output_selector is not None:
+                job_prior_outputs[key] = output_selector
 
-        for output_key, output_value in job_outputs.items():
-            if not (isinstance(output_value, str) and output_value in job_resources):
+        return job_config, job_def
 
-                model_output = required_outputs[output_key]
+    def _process_socket(self, key, socket, supplied_value, job_config, is_output) -> _meta.TagSelector:
 
-                if model_output.objectType == _meta.ObjectType.DATA:
-                    schema = model_output.schema if model_output and not model_output.dynamic else None
-                    output_id = self._process_data_socket(output_key, output_value, schema, job_resources, new_unique_file=True)
-                elif model_output.objectType == _meta.ObjectType.FILE:
-                    file_type = model_output.fileType
-                    output_id = self._process_file_socket(output_key, output_value, file_type, job_resources, new_unique_file=True)
-                else:
-                    raise _ex.EUnexpected()
+        if socket.objectType == _meta.ObjectType.DATA:
+            schema = socket.schema if socket and not socket.dynamic else None
+            return self._process_data_socket(key, supplied_value, schema, job_config, is_output)
 
-                job_outputs[output_key] = _util.selector_for(output_id)
+        elif socket.objectType == _meta.ObjectType.FILE:
+            file_type = socket.fileType
+            return self._process_file_socket(key, supplied_value, file_type, job_config, is_output)
 
-        return job_config, job_def
+        else:
+            raise _ex.EUnexpected()
 
     def _process_data_socket(
             self, data_key, data_value, schema: tp.Optional[_meta.SchemaDefinition],
-            resources: tp.Dict[str, _meta.ObjectDefinition], new_unique_file=False) \
-            -> _meta.TagHeader:
+            job_config: _cfg.JobConfig, is_output: bool)\
+            -> _meta.TagSelector:
 
         data_id = _util.new_object_id(_meta.ObjectType.DATA)
         storage_id = _util.new_object_id(_meta.ObjectType.STORAGE)
 
-        self._log.info(f"Generating data definition for [{data_key}] with ID = [{_util.object_key(data_id)}]")
-
         if isinstance(data_value, str):
             storage_path = data_value
-            storage_key = self._sys_config.storage.defaultBucket
-            storage_format = self.infer_format(storage_path, self._sys_config.storage, schema)
-            snap_version = 1
+            storage_key = _util.read_property(self._sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
+            storage_format = self.infer_format(storage_path, self._sys_config, schema)
 
         elif isinstance(data_value, dict):
 
@@ -857,58 +850,65 @@ class DevModeTranslator:
             if not storage_path:
                 raise _ex.EConfigParse(f"Invalid configuration for input [{data_key}] (missing required value 'path'")
 
-            storage_key = data_value.get("storageKey") or self._sys_config.storage.defaultBucket
-            storage_format = data_value.get("format") or self.infer_format(storage_path, self._sys_config.storage, schema)
-            snap_version = 1
+            storage_key = data_value.get("storageKey") or _util.read_property(self._sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
+            storage_format = data_value.get("format") or self.infer_format(storage_path, self._sys_config, schema)
 
         else:
             raise _ex.EConfigParse(f"Invalid configuration for input '{data_key}'")
 
-        # For unique outputs, increment the snap number to find a new unique snap
-        # These are not incarnations, bc likely in dev mode model code and inputs are changing
-        # Incarnations are for recreation of a dataset using the exact same code path and inputs
+        # Scan for existing versions using hte DEVELOPER storage layout
+
+        self._log.info(f"Looking for {'output' if is_output else 'input'} [{data_key}]...")
 
-        if new_unique_file:
-            storage_path, snap_version = self._new_unique_file(data_key, storage_key, storage_path, snap_version)
+        storage_path, version = self._find_latest_version(storage_key, storage_path)
+        data_id.objectVersion = version
+
+        if version > 0:
+            self._log.info(f"Found {'output' if is_output else 'input'} [{data_key}] version {version}")
+            self._log.info(f"Generating {'prior' if is_output else 'data'} definition for [{data_key}] with ID = [{_util.object_key(data_id)}]")
+        elif is_output:
+            self._log.info(f"No prior data for output [{data_key}]")
+        else:
+            # This is allowed for some scenarios, e.g. inside a job group
+            self._log.warning(f"No data found for input [{data_key}]")
 
         part_key = _meta.PartKey(opaqueKey="part-root", partType=_meta.PartType.PART_ROOT)
-        delta_index = 1
-        incarnation_index = 1
+        snap_index = version - 1 if version > 0 else 0
+        delta_index = 0
+        incarnation_index = 0
 
         # This is also defined in functions.DynamicDataSpecFunc, maybe centralize?
-        data_item = f"data/table/{data_id.objectId}/{part_key.opaqueKey}/snap-{snap_version}/delta-{delta_index}"
+        data_item = f"data/table/{data_id.objectId}/{part_key.opaqueKey}/snap-{snap_index}/delta-{delta_index}"
 
         data_obj = self._generate_data_definition(
-            part_key, snap_version, delta_index, data_item,
+            part_key, snap_index, delta_index, data_item,
             schema, storage_id)
 
         storage_obj = self._generate_storage_definition(
             storage_id, storage_key, storage_path, storage_format,
             data_item, incarnation_index)
 
-        resources[_util.object_key(data_id)] = data_obj
-        resources[_util.object_key(storage_id)] = storage_obj
+        self._add_job_metadata(job_config, data_id, data_obj)
+        self._add_job_metadata(job_config, storage_id, storage_obj)
 
-        return data_id
+        return _util.selector_for(data_id)
 
     def _process_file_socket(
             self, file_key, file_value, file_type: _meta.FileType,
-            resources: tp.Dict[str, _meta.ObjectDefinition], new_unique_file=False) \
-            -> _meta.TagHeader:
+            job_config: _cfg.JobConfig, is_output: bool) \
+            -> tp.Optional[_meta.TagSelector]:
 
         file_id = _util.new_object_id(_meta.ObjectType.FILE)
         storage_id = _util.new_object_id(_meta.ObjectType.STORAGE)
 
-        self._log.info(f"Generating file definition for [{file_key}] with ID = [{_util.object_key(file_id)}]")
-
         if isinstance(file_value, str):
 
-            storage_key = self._sys_config.storage.defaultBucket
+            storage_key = _util.read_property(self._sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
             storage_path = file_value
 
         elif isinstance(file_value, dict):
 
-            storage_key = file_value.get("storageKey") or self._sys_config.storage.defaultBucket
+            storage_key = file_value.get("storageKey") or _util.read_property(self._sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
             storage_path = file_value.get("path")
 
             if not storage_path:
@@ -917,17 +917,28 @@ class DevModeTranslator:
         else:
             raise _ex.EConfigParse(f"Invalid configuration for input '{file_key}'")
 
-        storage_format = "application/x-binary"
-        file_version = 1
+        # Scan for existing versions using hte DEVELOPER storage layout
+
+        self._log.info(f"Looking for {'output' if is_output else 'input'} [{file_key}]...")
 
-        if new_unique_file:
-            storage_path, file_version = self._new_unique_file(file_key, storage_key, storage_path, file_version)
-            file_size = 0
+        storage_path, version = self._find_latest_version(storage_key, storage_path)
+        file_id.objectVersion = version
+
+        if version > 0:
+            self._log.info(f"Found {'output' if is_output else 'input'} [{file_key}] version {version}")
+            self._log.info(f"Generating {'prior' if is_output else 'file'} definition for [{file_key}] with ID = [{_util.object_key(file_id)}]")
+        elif is_output:
+            self._log.info(f"No prior data for output [{file_key}]")
         else:
-            storage = self._storage_manager.get_file_storage(storage_key)
-            file_size = storage.size(storage_path)
+            # This is allowed for some scenarios, e.g. inside a job group
+            self._log.warning(f"No data found for input [{file_key}]")
+
+        storage = self._storage_manager.get_file_storage(storage_key)
+        file_size = storage.size(storage_path) if storage.exists(storage_path) else 0
 
-        data_item = f"file/{file_id.objectId}/version-{file_version}"
+        storage_format = "application/x-binary"
+
+        data_item = f"file/{file_id.objectId}/version-{version}"
         file_name = f"{file_key}.{file_type.extension}"
 
         file_obj = self._generate_file_definition(
@@ -936,15 +947,15 @@ class DevModeTranslator:
 
         storage_obj = self._generate_storage_definition(
             storage_id, storage_key, storage_path, storage_format,
-            data_item, incarnation_index=1)
+            data_item, incarnation_index=0)
 
-        resources[_util.object_key(file_id)] = file_obj
-        resources[_util.object_key(storage_id)] = storage_obj
+        self._add_job_metadata(job_config, file_id, file_obj)
+        self._add_job_metadata(job_config, storage_id, storage_obj)
 
-        return file_id
+        return _util.selector_for(file_id)
 
     @staticmethod
-    def infer_format(storage_path: str, storage_config: _cfg.StorageConfig, schema: tp.Optional[_meta.SchemaDefinition]):
+    def infer_format(storage_path: str, sys_config: _cfg.RuntimeConfig, schema: tp.Optional[_meta.SchemaDefinition]):
 
         schema_type = schema.schemaType if schema and schema.schemaType else _meta.SchemaType.TABLE
 
@@ -958,27 +969,30 @@ class DevModeTranslator:
                 return extension[1:] if extension.startswith(".") else extension
 
         else:
-            return storage_config.defaultFormat
+            return _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_FORMAT, "CSV")
 
-    def _new_unique_file(self, socket_name, storage_key, storage_path, version):
+    def _find_latest_version(self, storage_key, storage_path):
 
-        x_storage = self._storage_manager.get_file_storage(storage_key)
-        x_orig_path = pathlib.PurePath(storage_path)
-        x_name = x_orig_path.name
+        storage = self._storage_manager.get_file_storage(storage_key)
+        orig_path = pathlib.PurePath(storage_path)
+        version = 0
 
-        if x_storage.exists(str(x_orig_path.parent)):
-            listing = x_storage.ls(str(x_orig_path.parent))
-            existing_files = list(map(lambda stat: stat.file_name, listing))
-        else:
-            existing_files = []
+        if not storage.exists(str(orig_path.parent)):
+            return storage_path, version
 
-        while x_name in existing_files:
+        listing = storage.ls(str(orig_path.parent))
+        existing_files = list(map(lambda stat: stat.file_name, listing))
 
-            version += 1
-            x_name = f"{x_orig_path.stem}-{version}{x_orig_path.suffix}"
-            storage_path = str(x_orig_path.parent.joinpath(x_name))
+        next_version = version + 1
+        next_name = f"{orig_path.stem}{orig_path.suffix}"
 
-        self._log.info(f"Output for [{socket_name}] will be version {version}")
+        while next_name in existing_files:
+
+            storage_path = str(orig_path.parent.joinpath(next_name))
+            version = next_version
+
+            next_version = version + 1
+            next_name = f"{orig_path.stem}-{next_version}{orig_path.suffix}"
 
         return storage_path, version
 
@@ -1043,12 +1057,40 @@ class DevModeTranslator:
 
         storage_def = _meta.StorageDefinition()
         storage_def.dataItems[data_item] = storage_item
+        storage_def.layout = _meta.StorageLayout.DEVELOPER_LAYOUT
 
         if storage_format.lower() == "csv":
             storage_def.storageOptions["lenient_csv_parser"] = _types.MetadataCodec.encode_value(True)
 
         return _meta.ObjectDefinition(objectType=_meta.ObjectType.STORAGE, storage=storage_def)
 
+    @classmethod
+    def generate_dev_mode_tags(cls, job_config: _cfg.JobConfig) -> _cfg.JobConfig:
+
+        job_config = copy.copy(job_config)
+
+        for key, object_id in job_config.objectMapping.items():
+            if key not in job_config.tags:
+
+                tag = _meta.Tag(header=object_id)
+
+                timestamp = _types.MetadataCodec.convert_datetime_value(object_id.objectTimestamp.isoDatetime)
+                user_id = _types.MetadataCodec.encode_value("local_user")
+                user_name = _types.MetadataCodec.encode_value("Local User")
+
+                tag.attrs["trac_dev_mode"] = _types.MetadataCodec.encode_value(True)
+
+                tag.attrs["trac_create_time"] = timestamp
+                tag.attrs["trac_create_user_id"] = user_id
+                tag.attrs["trac_create_user_name"] = user_name
+
+                tag.attrs["trac_update_time"] = timestamp
+                tag.attrs["trac_update_user_id"] = user_id
+                tag.attrs["trac_update_user_name"] = user_name
+
+                job_config.tags[key] = tag
+
+        return job_config
 
 
 DevModeTranslator._log = _logging.logger_for_class(DevModeTranslator)