metaflow 2.13.3__tar.gz → 2.13.5__tar.gz

{metaflow-2.13.3/metaflow.egg-info → metaflow-2.13.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: metaflow
-Version: 2.13.3
+Version: 2.13.5
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.3; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.5; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/cli.py

@@ -504,6 +504,11 @@ def start(
     # *after* the run decospecs so that they don't take precedence. In other
     # words, for the same decorator, we want `myflow.py run --with foo` to
     # take precedence over any other `foo` decospec
+
+    # Note that top-level decospecs are used primarily with non run/resume
+    # options as well as with the airflow/argo/sfn integrations which pass
+    # all the decospecs (the ones from top-level but also the ones from the
+    # run/resume level) through the tl decospecs.
     ctx.obj.tl_decospecs = list(decospecs or [])
 
     # initialize current and parameter context for deploy-time parameters

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/cli_components/run_cmds.py

@@ -39,6 +39,8 @@ def before_run(obj, tags, decospecs):
         + list(obj.environment.decospecs() or [])
     )
     if all_decospecs:
+        # These decospecs are the ones from run/resume PLUS the ones from the
+        # environment (for example the @conda)
         decorators._attach_decorators(obj.flow, all_decospecs)
         decorators._init(obj.flow)
         # Regenerate graph if we attached more decorators

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/cli_components/step_cmd.py

@@ -77,14 +77,6 @@ from ..util import decompress_list
     default=None,
     help="Run id of the origin flow, if this task is part of a flow being resumed.",
 )
-@click.option(
-    "--with",
-    "decospecs",
-    multiple=True,
-    help="Add a decorator to this task. You can specify this "
-    "option multiple times to attach multiple decorators "
-    "to this task.",
-)
 @click.option(
     "--ubf-context",
     default="none",
@@ -112,7 +104,6 @@ def step(
     max_user_code_retries=None,
     clone_only=None,
     clone_run_id=None,
-    decospecs=None,
     ubf_context="none",
     num_parallel=None,
 ):
@@ -136,10 +127,6 @@ def step(
         raise CommandException("Function *%s* is not a step." % step_name)
     echo("Executing a step, *%s*" % step_name, fg="magenta", bold=False)
 
-    if decospecs:
-        decorators._attach_decorators_to_step(func, decospecs)
-        decorators._init(ctx.obj.flow)
-
     step_kwargs = ctx.params
     # Remove argument `step_name` from `step_kwargs`.
     step_kwargs.pop("step_name", None)

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/decorators.py

@@ -150,8 +150,8 @@ class Decorator(object):
             return
 
         # Note that by design, later values override previous ones.
-        self.attributes = unpack_delayed_evaluator(self.attributes)
-        self._user_defined_attributes.update(self.attributes.keys())
+        self.attributes, new_user_attributes = unpack_delayed_evaluator(self.attributes)
+        self._user_defined_attributes.update(new_user_attributes)
         self.attributes = resolve_delayed_evaluator(self.attributes)
 
         self._ran_init = True

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/parameters.py

@@ -367,7 +367,9 @@ class Parameter(object):
         )
 
         # Resolve any value from configurations
-        self.kwargs = unpack_delayed_evaluator(self.kwargs, ignore_errors=ignore_errors)
+        self.kwargs, _ = unpack_delayed_evaluator(
+            self.kwargs, ignore_errors=ignore_errors
+        )
         # Do it one item at a time so errors are ignored at that level (as opposed to
         # at the entire kwargs level)
         self.kwargs = {

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/plugins/__init__.py

@@ -16,6 +16,7 @@ CLIS_DESC = [
     ("argo-workflows", ".argo.argo_workflows_cli.cli"),
     ("card", ".cards.card_cli.cli"),
     ("tag", ".tag_cli.cli"),
+    ("spot-metadata", ".kubernetes.spot_metadata_cli.cli"),
     ("logs", ".logs_cli.cli"),
 ]
 
@@ -104,6 +105,10 @@ SIDECARS_DESC = [
         "save_logs_periodically",
         "..mflog.save_logs_periodically.SaveLogsPeriodicallySidecar",
     ),
+    (
+        "spot_termination_monitor",
+        ".kubernetes.spot_monitor_sidecar.SpotTerminationMonitorSidecar",
+    ),
     ("heartbeat", "metaflow.metadata_provider.heartbeat.MetadataHeartBeat"),
 ]
 

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/plugins/argo/argo_workflows.py

@@ -1705,6 +1705,7 @@ class ArgoWorkflows(object):
                     },
                     **{
                         # Some optional values for bookkeeping
+                        "METAFLOW_FLOW_FILENAME": os.path.basename(sys.argv[0]),
                         "METAFLOW_FLOW_NAME": self.flow.name,
                         "METAFLOW_STEP_NAME": node.name,
                         "METAFLOW_RUN_ID": run_id,

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/plugins/cards/card_decorator.py

@@ -76,6 +76,12 @@ class CardDecorator(StepDecorator):
 
     card_creator = None
 
+    _config_values = None
+
+    _config_file_name = None
+
+    task_finished_decos = 0
+
     def __init__(self, *args, **kwargs):
         super(CardDecorator, self).__init__(*args, **kwargs)
         self._task_datastore = None
@@ -106,6 +112,25 @@ class CardDecorator(StepDecorator):
     def _increment_step_counter(cls):
         cls.step_counter += 1
 
+    @classmethod
+    def _increment_completed_counter(cls):
+        cls.task_finished_decos += 1
+
+    @classmethod
+    def _set_config_values(cls, config_values):
+        cls._config_values = config_values
+
+    @classmethod
+    def _set_config_file_name(cls, flow):
+        # Only create a config file from the very first card decorator.
+        if cls._config_values and not cls._config_file_name:
+            with tempfile.NamedTemporaryFile(
+                mode="w", encoding="utf-8", delete=False
+            ) as config_file:
+                config_value = dump_config_values(flow)
+                json.dump(config_value, config_file)
+                cls._config_file_name = config_file.name
+
     def step_init(
         self, flow, graph, step_name, decorators, environment, flow_datastore, logger
     ):
@@ -116,11 +141,13 @@ class CardDecorator(StepDecorator):
 
         # We check for configuration options. We do this here before they are
         # converted to properties.
-        self._config_values = [
-            (config.name, ConfigInput.make_key_name(config.name))
-            for _, config in flow._get_parameters()
-            if config.IS_CONFIG_PARAMETER
-        ]
+        self._set_config_values(
+            [
+                (config.name, ConfigInput.make_key_name(config.name))
+                for _, config in flow._get_parameters()
+                if config.IS_CONFIG_PARAMETER
+            ]
+        )
 
         self.card_options = self.attributes["options"]
 
@@ -159,15 +186,11 @@ class CardDecorator(StepDecorator):
 
         # If we have configs, we need to dump them to a file so we can re-use them
         # when calling the card creation subprocess.
-        if self._config_values:
-            with tempfile.NamedTemporaryFile(
-                mode="w", encoding="utf-8", delete=False
-            ) as config_file:
-                config_value = dump_config_values(flow)
-                json.dump(config_value, config_file)
-                self._config_file_name = config_file.name
-        else:
-            self._config_file_name = None
+        # Since a step can contain multiple card decorators, and all the card creation processes
+        # will reference the same config file (because of how the CardCreator is created (only single class instance)),
+        # we need to ensure that a single config file is being referenced for all card create commands.
+        # This config file will be removed when the last card decorator has finished creating its card.
+        self._set_config_file_name(flow)
 
         card_type = self.attributes["type"]
         card_class = get_card_class(card_type)
@@ -246,12 +269,7 @@ class CardDecorator(StepDecorator):
             self.card_creator.create(mode="render", final=True, **create_options)
             self.card_creator.create(mode="refresh", final=True, **create_options)
 
-        # Unlink the config file if it exists
-        if self._config_file_name:
-            try:
-                os.unlink(self._config_file_name)
-            except Exception as e:
-                pass
+        self._cleanup(step_name)
 
     @staticmethod
     def _options(mapping):
@@ -286,3 +304,18 @@ class CardDecorator(StepDecorator):
             top_level_options["local-config-file"] = self._config_file_name
 
         return list(self._options(top_level_options))
+
+    def task_exception(
+        self, exception, step_name, flow, graph, retry_count, max_user_code_retries
+    ):
+        self._cleanup(step_name)
+
+    def _cleanup(self, step_name):
+        self._increment_completed_counter()
+        if self.task_finished_decos == self.total_decos_on_step[step_name]:
+            # Unlink the config file if it exists
+            if self._config_file_name:
+                try:
+                    os.unlink(self._config_file_name)
+                except Exception as e:
+                    pass

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/plugins/events_decorator.py

@@ -362,11 +362,7 @@ class TriggerOnFinishDecorator(FlowDecorator):
     """
 
     name = "trigger_on_finish"
-    defaults = {
-        "flow": None,  # flow_name or project_flow_name
-        "flows": [],  # flow_names or project_flow_names
-        "options": {},
-    }
+
     options = {
         "trigger": dict(
             multiple=True,
@@ -374,6 +370,14 @@ class TriggerOnFinishDecorator(FlowDecorator):
             help="Specify run pathspec for testing @trigger_on_finish locally.",
         ),
     }
+    defaults = {
+        "flow": None,  # flow_name or project_flow_name
+        "flows": [],  # flow_names or project_flow_names
+        "options": {},
+        # Re-enable if you want to support TL options directly in the decorator like
+        # for @project decorator
+        #    **{k: v["default"] for k, v in options.items()},
+    }
 
     def flow_init(
         self,
@@ -539,13 +543,43 @@ class TriggerOnFinishDecorator(FlowDecorator):
         self.options = self.attributes["options"]
 
         # Handle scenario for local testing using --trigger.
+
+        # Re-enable this code if you want to support passing trigger directly in the
+        # decorator in a way similar to how production and branch are passed in the
+        # project decorator.
+
+        # # This is overkill since default is None for all options but adding this code
+        # # to make it safe if other non None-default options are added in the future.
+        # for op in options:
+        #     if (
+        #         op in self._user_defined_attributes
+        #         and options[op] != self.defaults[op]
+        #         and self.attributes[op] != options[op]
+        #     ):
+        #         # Exception if:
+        #         #  - the user provides a value in the attributes field
+        #         #  - AND the user provided a value in the command line (non default)
+        #         #  - AND the values are different
+        #         # Note that this won't raise an error if the user provided the default
+        #         # value in the command line and provided one in attribute but although
+        #         # slightly inconsistent, it is not incorrect.
+        #         raise MetaflowException(
+        #             "You cannot pass %s as both a command-line argument and an attribute "
+        #             "of the @trigger_on_finish decorator." % op
+        #         )
+
+        # if "trigger" in self._user_defined_attributes:
+        #    trigger_option = self.attributes["trigger"]
+        # else:
+        trigger_option = options["trigger"]
+
         self._option_values = options
-        if options["trigger"]:
+        if trigger_option:
             from metaflow import Run
             from metaflow.events import Trigger
 
             run_objs = []
-            for run_pathspec in options["trigger"]:
+            for run_pathspec in trigger_option:
                 if len(run_pathspec.split("/")) != 2:
                     raise MetaflowException(
                         "Incorrect format for run pathspec for *--trigger*. "

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -190,7 +190,7 @@ def step(
     executable = ctx.obj.environment.executable(step_name, executable)
 
     # Set environment
-    env = {}
+    env = {"METAFLOW_FLOW_FILENAME": os.path.basename(sys.argv[0])}
     env_deco = [deco for deco in node.decorators if deco.name == "environment"]
     if env_deco:
         env = env_deco[0].attributes["vars"]

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -547,6 +547,13 @@ class KubernetesDecorator(StepDecorator):
             self._save_logs_sidecar = Sidecar("save_logs_periodically")
             self._save_logs_sidecar.start()
 
+            # Start spot termination monitor sidecar.
+            current._update_env(
+                {"spot_termination_notice": "/tmp/spot_termination_notice"}
+            )
+            self._spot_monitor_sidecar = Sidecar("spot_termination_monitor")
+            self._spot_monitor_sidecar.start()
+
         num_parallel = None
         if hasattr(flow, "_parallel_ubf_iter"):
             num_parallel = flow._parallel_ubf_iter.num_parallel
@@ -605,6 +612,7 @@ class KubernetesDecorator(StepDecorator):
 
         try:
             self._save_logs_sidecar.terminate()
+            self._spot_monitor_sidecar.terminate()
         except:
             # Best effort kill
             pass

metaflow-2.13.5/metaflow/plugins/kubernetes/spot_metadata_cli.py

@@ -0,0 +1,69 @@
+from metaflow._vendor import click
+from datetime import datetime, timezone
+from metaflow.tagging_util import validate_tags
+from metaflow.metadata_provider import MetaDatum
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.group(help="Commands related to spot metadata.")
+def spot_metadata():
+    pass
+
+
+@spot_metadata.command(help="Record spot termination metadata for a task.")
+@click.option(
+    "--run-id",
+    required=True,
+    help="Run ID for which metadata is to be recorded.",
+)
+@click.option(
+    "--step-name",
+    required=True,
+    help="Step Name for which metadata is to be recorded.",
+)
+@click.option(
+    "--task-id",
+    required=True,
+    help="Task ID for which metadata is to be recorded.",
+)
+@click.option(
+    "--termination-notice-time",
+    required=True,
+    help="Spot termination notice time.",
+)
+@click.option(
+    "--tag",
+    "tags",
+    multiple=True,
+    required=False,
+    default=None,
+    help="List of tags.",
+)
+@click.pass_obj
+def record(obj, run_id, step_name, task_id, termination_notice_time, tags=None):
+    validate_tags(tags)
+
+    tag_list = list(tags) if tags else []
+
+    entries = [
+        MetaDatum(
+            field="spot-termination-received-at",
+            value=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            type="spot-termination-received-at",
+            tags=tag_list,
+        ),
+        MetaDatum(
+            field="spot-termination-time",
+            value=termination_notice_time,
+            type="spot-termination-time",
+            tags=tag_list,
+        ),
+    ]
+
+    obj.metadata.register_metadata(
+        run_id=run_id, step_name=step_name, task_id=task_id, metadata=entries
+    )

metaflow-2.13.5/metaflow/plugins/kubernetes/spot_monitor_sidecar.py

@@ -0,0 +1,109 @@
+import os
+import sys
+import time
+import signal
+import requests
+import subprocess
+from multiprocessing import Process
+from datetime import datetime, timezone
+from metaflow.sidecar import MessageTypes
+
+
+class SpotTerminationMonitorSidecar(object):
+    EC2_TYPE_URL = "http://169.254.169.254/latest/meta-data/instance-life-cycle"
+    METADATA_URL = "http://169.254.169.254/latest/meta-data/spot/termination-time"
+    TOKEN_URL = "http://169.254.169.254/latest/api/token"
+    POLL_INTERVAL = 5  # seconds
+
+    def __init__(self):
+        self.is_alive = True
+        self._process = None
+        self._token = None
+        self._token_expiry = 0
+
+        if self._is_aws_spot_instance():
+            self._process = Process(target=self._monitor_loop)
+            self._process.start()
+
+    def process_message(self, msg):
+        if msg.msg_type == MessageTypes.SHUTDOWN:
+            self.is_alive = False
+            if self._process:
+                self._process.terminate()
+
+    @classmethod
+    def get_worker(cls):
+        return cls
+
+    def _get_imds_token(self):
+        current_time = time.time()
+        if current_time >= self._token_expiry - 60:  # Refresh 60s before expiry
+            try:
+                response = requests.put(
+                    url=self.TOKEN_URL,
+                    headers={"X-aws-ec2-metadata-token-ttl-seconds": "300"},
+                    timeout=1,
+                )
+                if response.status_code == 200:
+                    self._token = response.text
+                    self._token_expiry = current_time + 240  # Slightly less than TTL
+            except requests.exceptions.RequestException:
+                pass
+        return self._token
+
+    def _make_ec2_request(self, url, timeout):
+        token = self._get_imds_token()
+        headers = {"X-aws-ec2-metadata-token": token} if token else {}
+        response = requests.get(url=url, headers=headers, timeout=timeout)
+        return response
+
+    def _is_aws_spot_instance(self):
+        try:
+            response = self._make_ec2_request(url=self.EC2_TYPE_URL, timeout=1)
+            return response.status_code == 200 and response.text == "spot"
+        except (requests.exceptions.RequestException, requests.exceptions.Timeout):
+            return False
+
+    def _monitor_loop(self):
+        while self.is_alive:
+            try:
+                response = self._make_ec2_request(url=self.METADATA_URL, timeout=1)
+                if response.status_code == 200:
+                    termination_time = response.text
+                    self._emit_termination_metadata(termination_time)
+                    os.kill(os.getppid(), signal.SIGTERM)
+                    break
+            except (requests.exceptions.RequestException, requests.exceptions.Timeout):
+                pass
+            time.sleep(self.POLL_INTERVAL)
+
+    def _emit_termination_metadata(self, termination_time):
+        flow_filename = os.getenv("METAFLOW_FLOW_FILENAME")
+        pathspec = os.getenv("MF_PATHSPEC")
+        _, run_id, step_name, task_id = pathspec.split("/")
+        retry_count = os.getenv("MF_ATTEMPT")
+
+        with open("/tmp/spot_termination_notice", "w") as fp:
+            fp.write(termination_time)
+
+        command = [
+            sys.executable,
+            f"/metaflow/{flow_filename}",
+            "spot-metadata",
+            "record",
+            "--run-id",
+            run_id,
+            "--step-name",
+            step_name,
+            "--task-id",
+            task_id,
+            "--termination-notice-time",
+            termination_time,
+            "--tag",
+            "attempt_id:{}".format(retry_count),
+        ]
+
+        result = subprocess.run(command, capture_output=True, text=True)
+
+        if result.returncode != 0:
+            print(f"Failed to record spot termination metadata: {result.stderr}")

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/plugins/project_decorator.py

@@ -72,7 +72,6 @@ class ProjectDecorator(FlowDecorator):
     """
 
     name = "project"
-    defaults = {"name": None}
 
     options = {
         "production": dict(
@@ -91,19 +90,48 @@ class ProjectDecorator(FlowDecorator):
         ),
     }
 
+    defaults = {"name": None, **{k: v["default"] for k, v in options.items()}}
+
     def flow_init(
         self, flow, graph, environment, flow_datastore, metadata, logger, echo, options
     ):
         self._option_values = options
         project_name = self.attributes.get("name")
+        for op in options:
+            if (
+                op in self._user_defined_attributes
+                and options[op] != self.defaults[op]
+                and self.attributes[op] != options[op]
+            ):
+                # Exception if:
+                #  - the user provides a value in the attributes field
+                #  - AND the user provided a value in the command line (non default)
+                #  - AND the values are different
+                # Note that this won't raise an error if the user provided the default
+                # value in the command line and provided one in attribute but although
+                # slightly inconsistent, it is not incorrect.
+                raise MetaflowException(
+                    "You cannot pass %s as both a command-line argument and an attribute "
+                    "of the @project decorator." % op
+                )
+        if "branch" in self._user_defined_attributes:
+            project_branch = self.attributes["branch"]
+        else:
+            project_branch = options["branch"]
+
+        if "production" in self._user_defined_attributes:
+            project_production = self.attributes["production"]
+        else:
+            project_production = options["production"]
+
         project_flow_name, branch_name = format_name(
             flow.name,
             project_name,
-            options["production"],
-            options["branch"],
+            project_production,
+            project_branch,
             get_username(),
         )
-        is_user_branch = options["branch"] is None and not options["production"]
+        is_user_branch = project_branch is None and not project_production
         echo(
             "Project: *%s*, Branch: *%s*" % (project_name, branch_name),
             fg="magenta",
@@ -114,7 +142,7 @@ class ProjectDecorator(FlowDecorator):
                 "project_name": project_name,
                 "branch_name": branch_name,
                 "is_user_branch": is_user_branch,
-                "is_production": options["production"],
+                "is_production": project_production,
                 "project_flow_name": project_flow_name,
             }
         )

{metaflow-2.13.3 → metaflow-2.13.5}/metaflow/user_configs/config_parameters.py

@@ -3,7 +3,7 @@ import json
 import os
 import re
 
-from typing import Any, Callable, Dict, Optional, TYPE_CHECKING, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING, Union
 
 
 from ..exception import MetaflowException
@@ -171,7 +171,7 @@ class DelayEvaluator(collections.abc.Mapping):
         yield "%s%d" % (UNPACK_KEY, id(self))
 
     def __getitem__(self, key):
-        if key == "%s%d" % (UNPACK_KEY, id(self)):
+        if isinstance(key, str) and key == "%s%d" % (UNPACK_KEY, id(self)):
             return self
         if self._access is None:
             raise KeyError(key)
@@ -196,12 +196,23 @@ class DelayEvaluator(collections.abc.Mapping):
         if flow_cls is None:
             # We are not executing inside a flow (ie: not the CLI)
             raise MetaflowException(
-                "Config object can only be used directly in the FlowSpec defining them. "
-                "If using outside of the FlowSpec, please use ConfigEval"
+                "Config object can only be used directly in the FlowSpec defining them "
+                "(or their flow decorators)."
             )
         if self._access is not None:
             # Build the final expression by adding all the fields in access as . fields
-            self._config_expr = ".".join([self._config_expr] + self._access)
+            access_list = [self._config_expr]
+            for a in self._access:
+                if isinstance(a, str):
+                    access_list.append(a)
+                elif isinstance(a, DelayEvaluator):
+                    # Supports things like config[other_config.selector].var
+                    access_list.append(a())
+                else:
+                    raise MetaflowException(
+                        "Field '%s' of type '%s' is not supported" % (str(a), type(a))
+                    )
+            self._config_expr = ".".join(access_list)
         # Evaluate the expression setting the config values as local variables
         try:
             return eval(
@@ -369,7 +380,7 @@ class Config(Parameter, collections.abc.Mapping):
 
     def __getitem__(self, key):
         self._init_delayed_evaluator()
-        if key.startswith(UNPACK_KEY):
+        if isinstance(key, str) and key.startswith(UNPACK_KEY):
             return self._delayed_evaluator[key]
         return DelayEvaluator(self.name.lower())[key]
 
@@ -406,17 +417,20 @@ def resolve_delayed_evaluator(v: Any, ignore_errors: bool = False) -> Any:
 
 def unpack_delayed_evaluator(
     to_unpack: Dict[str, Any], ignore_errors: bool = False
-) -> Dict[str, Any]:
+) -> Tuple[Dict[str, Any], List[str]]:
     result = {}
+    new_keys = []
     for k, v in to_unpack.items():
         if not isinstance(k, str) or not k.startswith(UNPACK_KEY):
             result[k] = v
         else:
             # k.startswith(UNPACK_KEY)
             try:
-                result.update(resolve_delayed_evaluator(v))
+                new_vals = resolve_delayed_evaluator(v)
+                new_keys.extend(new_vals.keys())
+                result.update(new_vals)
             except Exception as e:
                 if ignore_errors:
                     continue
                 raise e
-    return result
+    return result, new_keys

metaflow-2.13.5/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.13.5"