tracdap-runtime 0.8.0rc2__py3-none-any.whl → 0.9.0b1__py3-none-any.whl

tracdap/rt/_impl/core/repos.py

@@ -38,6 +38,13 @@ class RepositoryManager:
 
             try:
 
+                # Add global properties related to the repo protocol
+                related_props = {
+                    k: v for (k, v) in sys_config.properties.items()
+                    if k.startswith(f"{repo_config.protocol}.")}
+
+                repo_config.properties.update(related_props)
+
                 self._repos[repo_name] = plugins.PluginManager.load_plugin(IModelRepository, repo_config)
 
             except ex.EPluginNotAvailable as e:

tracdap/rt/_impl/core/storage.py

@@ -81,7 +81,7 @@ class StorageManager:
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
         self.__external: tp.List[str] = list()
-        self.__settings = sys_config.storage
+        self.__sys_config = sys_config
 
         for storage_key, storage_config in sys_config.storage.buckets.items():
             self.create_storage(storage_key, storage_config)
@@ -93,13 +93,20 @@ class StorageManager:
             self.create_storage(storage_key, storage_config)
 
     def default_storage_key(self):
-        return self.__settings.defaultBucket
+        return self.__sys_config.storage.defaultBucket
 
     def default_storage_format(self):
-        return self.__settings.defaultFormat
+        return self.__sys_config.storage.defaultFormat
 
     def create_storage(self, storage_key: str, storage_config: _cfg.PluginConfig):
 
+        # Add global properties related to the storage protocol
+        related_props = {
+            k: v for (k, v) in self.__sys_config.properties.items()
+            if k.startswith(f"{storage_config.protocol}.")}
+
+        storage_config.properties.update(related_props)
+
         if plugins.PluginManager.is_plugin_available(IStorageProvider, storage_config.protocol):
             self._create_storage_from_provider(storage_key, storage_config)
         else:

tracdap/rt/_impl/core/util.py

@@ -16,6 +16,7 @@
 import datetime as dt
 import pathlib
 import platform
+import re
 
 import typing as tp
 import uuid
@@ -30,6 +31,7 @@ import traceback as tb
 __IS_WINDOWS = platform.system() == "Windows"
 __FIRST_MODEL_FRAME_NAME = "run_model"
 __FIRST_MODEL_FRAME_TEST_NAME = "_callTestMethod"
+__OBJ_KEY_PATTERN = re.compile(r"([A-Z]+)-(.*)-v(\d+)")
 
 
 def is_windows():
@@ -60,7 +62,7 @@ def format_file_size(size: int) -> str:
 
 def new_object_id(object_type: meta.ObjectType) -> meta.TagHeader:
 
-    timestamp = dt.datetime.utcnow()
+    timestamp = dt.datetime.now(dt.timezone.utc)
 
     return meta.TagHeader(
         objectType=object_type,
@@ -71,6 +73,19 @@ def new_object_id(object_type: meta.ObjectType) -> meta.TagHeader:
         tagTimestamp=meta.DatetimeValue(timestamp.isoformat()))
 
 
+def new_object_version(prior_id: meta.TagHeader) -> meta.TagHeader:
+
+    timestamp = dt.datetime.now(dt.timezone.utc)
+
+    return meta.TagHeader(
+        objectType=prior_id.objectType,
+        objectId=prior_id.objectId,
+        objectVersion=prior_id.objectVersion + 1,
+        objectTimestamp=meta.DatetimeValue(timestamp.isoformat()),
+        tagVersion=1,
+        tagTimestamp=meta.DatetimeValue(timestamp.isoformat()))
+
+
 def object_key(object_id: tp.Union[meta.TagHeader, meta.TagSelector]) -> str:
 
     if isinstance(object_id, meta.TagHeader):
@@ -106,29 +121,57 @@ def selector_for_latest(object_id: meta.TagHeader) -> meta.TagSelector:
         latestTag=True)
 
 
-def get_job_resource(
+def get_job_metadata(
         selector: tp.Union[meta.TagHeader, meta.TagSelector],
         job_config: cfg.JobConfig,
-        optional: bool = False):
+        optional: bool = False) \
+        -> tp.Optional[meta.ObjectDefinition]:
 
-    resource_key = object_key(selector)
-    resource_id = job_config.resourceMapping.get(resource_key)
+    obj_key = object_key(selector)
+    obj_id = job_config.objectMapping.get(obj_key)
 
-    if resource_id is not None:
-        resource_key = object_key(resource_id)
+    if obj_id is not None:
+        obj_key = object_key(obj_id)
 
-    resource = job_config.resources.get(resource_key)
+    obj = job_config.objects.get(obj_key)
 
-    if resource is not None:
-        return resource
+    if obj is not None:
+        return obj
 
     if optional:
         return None
 
-    err = f"Missing required {selector.objectType.name} resource [{object_key(selector)}]"
+    err = f"Missing required {selector.objectType.name} object for [{object_key(selector)}]"
     raise ex.ERuntimeValidation(err)
 
 
+def get_job_mapping(
+        selector: tp.Union[meta.TagHeader, meta.TagSelector],
+        job_config: cfg.JobConfig) \
+        -> meta.TagHeader:
+
+    obj_key = object_key(selector)
+    obj_id = job_config.objectMapping.get(obj_key)
+
+    if obj_id is not None:
+        return obj_id
+
+    obj_key_match = __OBJ_KEY_PATTERN.match(obj_key)
+
+    if not obj_key_match:
+        err = f"Missing required {selector.objectType.name} ID for [{object_key(selector)}]"
+        raise ex.ERuntimeValidation(err)
+
+    obj_type = obj_key_match.group(1)
+    obj_id = obj_key_match.group(2)
+    obj_ver = obj_key_match.group(3)
+    obj_ts = job_config.jobId.objectTimestamp
+
+    return meta.TagHeader(
+        meta.ObjectType.__members__[obj_type], obj_id,
+        int(obj_ver), obj_ts, 1, obj_ts)
+
+
 def get_origin(metaclass: type):
 
     # Minimum supported Python is 3.7, which does not provide get_origin and get_args

tracdap/rt/_impl/exec/dev_mode.py

@@ -87,6 +87,7 @@ class DevModeTranslator:
             config_mgr: _cfg_p.ConfigManager):
 
         storage_config = copy.deepcopy(sys_config.storage)
+        storage_config.defaultLayout = _meta.StorageLayout.DEVELOPER_LAYOUT
 
         for bucket_key, bucket_config in storage_config.buckets.items():
             storage_config.buckets[bucket_key] = cls._resolve_storage_location(
@@ -218,27 +219,29 @@ class DevModeTranslator:
         return job_config, job_def
 
     @classmethod
-    def _add_job_resource(
+    def _add_job_metadata(
             cls, job_config: _cfg.JobConfig,
             obj_id: _meta.TagHeader, obj: _meta.ObjectDefinition) \
             -> _cfg.JobConfig:
 
         obj_key = _util.object_key(obj_id)
-        job_config.resources[obj_key] = obj
+        job_config.objects[obj_key] = obj
 
         return job_config
 
     @classmethod
-    def _process_job_id(cls, job_config: _cfg.JobConfig):
+    def _process_job_id(cls, job_config: _cfg.JobConfig) -> _cfg.JobConfig:
 
         job_id = _util.new_object_id(_meta.ObjectType.JOB)
+        result_id = _util.new_object_id(_meta.ObjectType.RESULT)
 
         cls._log.info(f"Assigning job ID = [{_util.object_key(job_id)}]")
+        cls._log.info(f"Assigning result ID = [{_util.object_key(result_id)}]")
 
-        translated_config = copy.copy(job_config)
-        translated_config.jobId = job_id
+        job_config.jobId = job_id
+        job_config.resultId = result_id
 
-        return translated_config
+        return job_config
 
     @classmethod
     def _process_job_type(cls, job_def: _meta.JobDefinition):
@@ -346,7 +349,7 @@ class DevModeTranslator:
 
             model_id, model_obj = self._generate_model_for_class(model_class)
             job_detail.model = _util.selector_for(model_id)
-            job_config = self._add_job_resource(job_config, model_id, model_obj)
+            job_config = self._add_job_metadata(job_config, model_id, model_obj)
 
         # Otherwise look for models specified as a single string, and take that as the entry point
         else:
@@ -355,7 +358,7 @@ class DevModeTranslator:
             if hasattr(job_detail, "model") and isinstance(job_detail.model, str):
                 model_id, model_obj = self._generate_model_for_entry_point(job_detail.model)  # noqa
                 job_detail.model = _util.selector_for(model_id)
-                job_config = self._add_job_resource(job_config, model_id, model_obj)
+                job_config = self._add_job_metadata(job_config, model_id, model_obj)
 
             elif hasattr(job_detail, "model") and isinstance(job_detail.model, _meta.TagSelector):
                 if job_detail.model.objectType == _meta.ObjectType.OBJECT_TYPE_NOT_SET:
@@ -369,7 +372,7 @@ class DevModeTranslator:
                     if isinstance(model_detail, str):
                         model_id, model_obj = self._generate_model_for_entry_point(model_detail)
                         job_detail.models[model_key] = _util.selector_for(model_id)
-                        job_config = self._add_job_resource(job_config, model_id, model_obj)
+                        job_config = self._add_job_metadata(job_config, model_id, model_obj)
 
         return job_config, job_def
 
@@ -446,8 +449,8 @@ class DevModeTranslator:
         job_def.runFlow.flow = _util.selector_for(flow_id)
 
         job_config = copy.copy(job_config)
-        job_config.resources = copy.copy(job_config.resources)
-        job_config = self._add_job_resource(job_config, flow_id, flow_obj)
+        job_config.objects = copy.copy(job_config.objects)
+        job_config = self._add_job_metadata(job_config, flow_id, flow_obj)
 
         return job_config, job_def
 
@@ -472,7 +475,7 @@ class DevModeTranslator:
         for model_name, model_node in model_nodes.items():
 
             model_selector = job_def.runFlow.models[model_name]
-            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_obj = _util.get_job_metadata(model_selector, job_config)
 
             model_inputs = set(model_obj.model.inputs.keys())
             model_outputs = set(model_obj.model.outputs.keys())
@@ -540,7 +543,7 @@ class DevModeTranslator:
                 # Generate node param sockets needed by the model
                 if node_name in job.models:
                     model_selector = job.models[node_name]
-                    model_obj = _util.get_job_resource(model_selector, job_config)
+                    model_obj = _util.get_job_metadata(model_selector, job_config)
                     for param_name in model_obj.model.parameters:
                         add_param_to_flow(node_name, param_name)
                         if param_name not in node.parameters:
@@ -622,7 +625,7 @@ class DevModeTranslator:
         for target in targets:
 
             model_selector = job_def.runFlow.models.get(target.node)
-            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_obj = _util.get_job_metadata(model_selector, job_config)
             model_param = model_obj.model.parameters.get(target.socket)
             model_params.append(model_param)
 
@@ -659,7 +662,7 @@ class DevModeTranslator:
         for target in targets:
 
             model_selector = job_def.runFlow.models.get(target.node)
-            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_obj = _util.get_job_metadata(model_selector, job_config)
             model_input = model_obj.model.inputs.get(target.socket)
             model_inputs.append(model_input)
 
@@ -694,7 +697,7 @@ class DevModeTranslator:
         for source in sources:
 
             model_selector = job_def.runFlow.models.get(source.node)
-            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_obj = _util.get_job_metadata(model_selector, job_config)
             model_input = model_obj.model.outputs.get(source.socket)
             model_outputs.append(model_input)
 
@@ -727,10 +730,10 @@ class DevModeTranslator:
 
         if hasattr(job_detail, "model"):
             model_key = _util.object_key(job_detail.model)
-            model_or_flow = job_config.resources[model_key].model
+            model_or_flow = job_config.objects[model_key].model
         elif hasattr(job_detail, "flow"):
             flow_key = _util.object_key(job_detail.flow)
-            model_or_flow = job_config.resources[flow_key].flow
+            model_or_flow = job_config.objects[flow_key].flow
         else:
             model_or_flow = None
 
@@ -784,71 +787,68 @@ class DevModeTranslator:
         job_detail = self._get_job_detail(job_def)
 
         if hasattr(job_detail, "model"):
-            model_obj = _util.get_job_resource(job_detail.model, job_config)
+            model_obj = _util.get_job_metadata(job_detail.model, job_config)
             required_inputs = model_obj.model.inputs
-            required_outputs = model_obj.model.outputs
+            expected_outputs = model_obj.model.outputs
 
         elif hasattr(job_detail, "flow"):
-            flow_obj = _util.get_job_resource(job_detail.flow, job_config)
+            flow_obj = _util.get_job_metadata(job_detail.flow, job_config)
             required_inputs = flow_obj.flow.inputs
-            required_outputs = flow_obj.flow.outputs
+            expected_outputs = flow_obj.flow.outputs
 
         else:
             return job_config, job_def
 
+        job_metadata = job_config.objects
         job_inputs = job_detail.inputs
         job_outputs = job_detail.outputs
-        job_resources = job_config.resources
-
-        for input_key, input_value in job_inputs.items():
-            if not (isinstance(input_value, str) and input_value in job_resources):
-
-                model_input = required_inputs[input_key]
+        job_prior_outputs = job_detail.priorOutputs
+
+        for key, schema in required_inputs.items():
+            if key not in job_inputs:
+                if not schema.optional:
+                    raise _ex.EJobValidation(f"Missing required input [{key}]")
+                continue
+            supplied_input = job_inputs.pop(key) if key in job_inputs else None
+            input_selector = self._process_socket(key, schema, supplied_input, job_metadata, is_output=False)
+            if input_selector is not None:
+                job_inputs[key] = input_selector
+
+        for key, schema in expected_outputs.items():
+            if key not in job_outputs:
+                raise _ex.EJobValidation(f"Missing required output [{key}]")
+            supplied_output = job_outputs.pop(key)
+            output_selector = self._process_socket(key, schema, supplied_output, job_metadata, is_output=True)
+            if output_selector is not None:
+                job_prior_outputs[key] = output_selector
 
-                if model_input.objectType == _meta.ObjectType.DATA:
-                    schema = model_input.schema if model_input and not model_input.dynamic else None
-                    input_id = self._process_data_socket(input_key, input_value, schema, job_resources, new_unique_file=False)
-                elif model_input.objectType == _meta.ObjectType.FILE:
-                    file_type = model_input.fileType
-                    input_id = self._process_file_socket(input_key, input_value, file_type, job_resources, new_unique_file=False)
-                else:
-                    raise _ex.EUnexpected()
-
-                job_inputs[input_key] = _util.selector_for(input_id)
-
-        for output_key, output_value in job_outputs.items():
-            if not (isinstance(output_value, str) and output_value in job_resources):
+        return job_config, job_def
 
-                model_output = required_outputs[output_key]
+    def _process_socket(self, key, socket, supplied_value, job_metadata, is_output) -> _meta.TagSelector:
 
-                if model_output.objectType == _meta.ObjectType.DATA:
-                    schema = model_output.schema if model_output and not model_output.dynamic else None
-                    output_id = self._process_data_socket(output_key, output_value, schema, job_resources, new_unique_file=True)
-                elif model_output.objectType == _meta.ObjectType.FILE:
-                    file_type = model_output.fileType
-                    output_id = self._process_file_socket(output_key, output_value, file_type, job_resources, new_unique_file=True)
-                else:
-                    raise _ex.EUnexpected()
+        if socket.objectType == _meta.ObjectType.DATA:
+            schema = socket.schema if socket and not socket.dynamic else None
+            return self._process_data_socket(key, supplied_value, schema, job_metadata, is_output)
 
-                job_outputs[output_key] = _util.selector_for(output_id)
+        elif socket.objectType == _meta.ObjectType.FILE:
+            file_type = socket.fileType
+            return self._process_file_socket(key, supplied_value, file_type, job_metadata, is_output)
 
-        return job_config, job_def
+        else:
+            raise _ex.EUnexpected()
 
     def _process_data_socket(
             self, data_key, data_value, schema: tp.Optional[_meta.SchemaDefinition],
-            resources: tp.Dict[str, _meta.ObjectDefinition], new_unique_file=False) \
-            -> _meta.TagHeader:
+            job_metadata: tp.Dict[str, _meta.ObjectDefinition], is_output: bool)\
+            -> _meta.TagSelector:
 
         data_id = _util.new_object_id(_meta.ObjectType.DATA)
         storage_id = _util.new_object_id(_meta.ObjectType.STORAGE)
 
-        self._log.info(f"Generating data definition for [{data_key}] with ID = [{_util.object_key(data_id)}]")
-
         if isinstance(data_value, str):
             storage_path = data_value
             storage_key = self._sys_config.storage.defaultBucket
             storage_format = self.infer_format(storage_path, self._sys_config.storage, schema)
-            snap_version = 1
 
         elif isinstance(data_value, dict):
 
@@ -859,48 +859,55 @@ class DevModeTranslator:
 
             storage_key = data_value.get("storageKey") or self._sys_config.storage.defaultBucket
             storage_format = data_value.get("format") or self.infer_format(storage_path, self._sys_config.storage, schema)
-            snap_version = 1
 
         else:
             raise _ex.EConfigParse(f"Invalid configuration for input '{data_key}'")
 
-        # For unique outputs, increment the snap number to find a new unique snap
-        # These are not incarnations, bc likely in dev mode model code and inputs are changing
-        # Incarnations are for recreation of a dataset using the exact same code path and inputs
+        # Scan for existing versions using hte DEVELOPER storage layout
+
+        self._log.info(f"Looking for {'output' if is_output else 'input'} [{data_key}]...")
+
+        storage_path, version = self._find_latest_version(storage_key, storage_path)
+        data_id.objectVersion = version
 
-        if new_unique_file:
-            storage_path, snap_version = self._new_unique_file(data_key, storage_key, storage_path, snap_version)
+        if version > 0:
+            self._log.info(f"Found {'output' if is_output else 'input'} [{data_key}] version {version}")
+            self._log.info(f"Generating {'prior' if is_output else 'data'} definition for [{data_key}] with ID = [{_util.object_key(data_id)}]")
+        elif is_output:
+            self._log.info(f"No prior data for output [{data_key}]")
+        else:
+            # This is allowed for some scenarios, e.g. inside a job group
+            self._log.warning(f"No data found for input [{data_key}]")
 
         part_key = _meta.PartKey(opaqueKey="part-root", partType=_meta.PartType.PART_ROOT)
-        delta_index = 1
-        incarnation_index = 1
+        snap_index = version - 1 if version > 0 else 0
+        delta_index = 0
+        incarnation_index = 0
 
         # This is also defined in functions.DynamicDataSpecFunc, maybe centralize?
-        data_item = f"data/table/{data_id.objectId}/{part_key.opaqueKey}/snap-{snap_version}/delta-{delta_index}"
+        data_item = f"data/table/{data_id.objectId}/{part_key.opaqueKey}/snap-{snap_index}/delta-{delta_index}"
 
         data_obj = self._generate_data_definition(
-            part_key, snap_version, delta_index, data_item,
+            part_key, snap_index, delta_index, data_item,
             schema, storage_id)
 
         storage_obj = self._generate_storage_definition(
             storage_id, storage_key, storage_path, storage_format,
             data_item, incarnation_index)
 
-        resources[_util.object_key(data_id)] = data_obj
-        resources[_util.object_key(storage_id)] = storage_obj
+        job_metadata[_util.object_key(data_id)] = data_obj
+        job_metadata[_util.object_key(storage_id)] = storage_obj
 
-        return data_id
+        return _util.selector_for(data_id)
 
     def _process_file_socket(
             self, file_key, file_value, file_type: _meta.FileType,
-            resources: tp.Dict[str, _meta.ObjectDefinition], new_unique_file=False) \
-            -> _meta.TagHeader:
+            job_metadata: tp.Dict[str, _meta.ObjectDefinition], is_output: bool) \
+            -> tp.Optional[_meta.TagSelector]:
 
         file_id = _util.new_object_id(_meta.ObjectType.FILE)
         storage_id = _util.new_object_id(_meta.ObjectType.STORAGE)
 
-        self._log.info(f"Generating file definition for [{file_key}] with ID = [{_util.object_key(file_id)}]")
-
         if isinstance(file_value, str):
 
             storage_key = self._sys_config.storage.defaultBucket
@@ -917,17 +924,28 @@ class DevModeTranslator:
         else:
             raise _ex.EConfigParse(f"Invalid configuration for input '{file_key}'")
 
-        storage_format = "application/x-binary"
-        file_version = 1
+        # Scan for existing versions using hte DEVELOPER storage layout
 
-        if new_unique_file:
-            storage_path, file_version = self._new_unique_file(file_key, storage_key, storage_path, file_version)
-            file_size = 0
+        self._log.info(f"Looking for {'output' if is_output else 'input'} [{file_key}]...")
+
+        storage_path, version = self._find_latest_version(storage_key, storage_path)
+        file_id.objectVersion = version
+
+        if version > 0:
+            self._log.info(f"Found {'output' if is_output else 'input'} [{file_key}] version {version}")
+            self._log.info(f"Generating {'prior' if is_output else 'file'} definition for [{file_key}] with ID = [{_util.object_key(file_id)}]")
+        elif is_output:
+            self._log.info(f"No prior data for output [{file_key}]")
         else:
-            storage = self._storage_manager.get_file_storage(storage_key)
-            file_size = storage.size(storage_path)
+            # This is allowed for some scenarios, e.g. inside a job group
+            self._log.warning(f"No data found for input [{file_key}]")
+
+        storage = self._storage_manager.get_file_storage(storage_key)
+        file_size = storage.size(storage_path) if storage.exists(storage_path) else 0
 
-        data_item = f"file/{file_id.objectId}/version-{file_version}"
+        storage_format = "application/x-binary"
+
+        data_item = f"file/{file_id.objectId}/version-{version}"
         file_name = f"{file_key}.{file_type.extension}"
 
         file_obj = self._generate_file_definition(
@@ -936,12 +954,12 @@ class DevModeTranslator:
 
         storage_obj = self._generate_storage_definition(
             storage_id, storage_key, storage_path, storage_format,
-            data_item, incarnation_index=1)
+            data_item, incarnation_index=0)
 
-        resources[_util.object_key(file_id)] = file_obj
-        resources[_util.object_key(storage_id)] = storage_obj
+        job_metadata[_util.object_key(file_id)] = file_obj
+        job_metadata[_util.object_key(storage_id)] = storage_obj
 
-        return file_id
+        return _util.selector_for(file_id)
 
     @staticmethod
     def infer_format(storage_path: str, storage_config: _cfg.StorageConfig, schema: tp.Optional[_meta.SchemaDefinition]):
@@ -960,25 +978,28 @@ class DevModeTranslator:
         else:
             return storage_config.defaultFormat
 
-    def _new_unique_file(self, socket_name, storage_key, storage_path, version):
+    def _find_latest_version(self, storage_key, storage_path):
 
-        x_storage = self._storage_manager.get_file_storage(storage_key)
-        x_orig_path = pathlib.PurePath(storage_path)
-        x_name = x_orig_path.name
+        storage = self._storage_manager.get_file_storage(storage_key)
+        orig_path = pathlib.PurePath(storage_path)
+        version = 0
 
-        if x_storage.exists(str(x_orig_path.parent)):
-            listing = x_storage.ls(str(x_orig_path.parent))
-            existing_files = list(map(lambda stat: stat.file_name, listing))
-        else:
-            existing_files = []
+        if not storage.exists(str(orig_path.parent)):
+            return storage_path, version
+
+        listing = storage.ls(str(orig_path.parent))
+        existing_files = list(map(lambda stat: stat.file_name, listing))
+
+        next_version = version + 1
+        next_name = f"{orig_path.stem}{orig_path.suffix}"
 
-        while x_name in existing_files:
+        while next_name in existing_files:
 
-            version += 1
-            x_name = f"{x_orig_path.stem}-{version}{x_orig_path.suffix}"
-            storage_path = str(x_orig_path.parent.joinpath(x_name))
+            storage_path = str(orig_path.parent.joinpath(next_name))
+            version = next_version
 
-        self._log.info(f"Output for [{socket_name}] will be version {version}")
+            next_version = version + 1
+            next_name = f"{orig_path.stem}-{next_version}{orig_path.suffix}"
 
         return storage_path, version
 
@@ -1043,6 +1064,7 @@ class DevModeTranslator:
 
         storage_def = _meta.StorageDefinition()
         storage_def.dataItems[data_item] = storage_item
+        storage_def.layout = _meta.StorageLayout.DEVELOPER_LAYOUT
 
         if storage_format.lower() == "csv":
             storage_def.storageOptions["lenient_csv_parser"] = _types.MetadataCodec.encode_value(True)