octavia 15.0.0.0rc1__py3-none-any.whl → 16.0.0.0rc1__py3-none-any.whl

octavia/controller/worker/v2/tasks/network_tasks.py

@@ -17,16 +17,19 @@ import time
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import excutils
+from sqlalchemy.orm import exc as sa_exception
 from taskflow import task
 from taskflow.types import failure
 import tenacity
 
 from octavia.common import constants
 from octavia.common import data_models
+from octavia.common import exceptions
 from octavia.common import utils
 from octavia.controller.worker import task_utils
 from octavia.db import api as db_apis
 from octavia.db import repositories as repo
+from octavia.i18n import _
 from octavia.network import base
 from octavia.network import data_models as n_data_models
 
@@ -43,6 +46,7 @@ class BaseNetworkTask(task.Task):
         self.task_utils = task_utils.TaskUtils()
         self.loadbalancer_repo = repo.LoadBalancerRepository()
         self.amphora_repo = repo.AmphoraRepository()
+        self.amphora_member_port_repo = repo.AmphoraMemberPortRepository()
 
     @property
     def network_driver(self):
@@ -78,6 +82,7 @@ class CalculateAmphoraDelta(BaseNetworkTask):
             loadbalancer[constants.VIP_NETWORK_ID]
         }
 
+        net_vnic_type_map = {}
         for pool in db_lb.pools:
             for member in pool.members:
                 if (member.subnet_id and
@@ -85,6 +90,8 @@ class CalculateAmphoraDelta(BaseNetworkTask):
                         constants.PENDING_DELETE):
                     member_network = self.network_driver.get_subnet(
                         member.subnet_id).network_id
+                    net_vnic_type_map[member_network] = getattr(
+                        member, 'vnic_type', constants.VNIC_TYPE_NORMAL)
                     desired_subnet_to_net_map[member.subnet_id] = (
                         member_network)
 
@@ -117,7 +124,8 @@ class CalculateAmphoraDelta(BaseNetworkTask):
                 n_data_models.FixedIP(
                     subnet_id=subnet_id)
                 for subnet_id, net_id in desired_subnet_to_net_map.items()
-                if net_id == add_net_id])
+                if net_id == add_net_id],
+            vnic_type=net_vnic_type_map[add_net_id])
             for add_net_id in add_ids]
 
         # Calculate member Subnet deltas
@@ -215,45 +223,6 @@ class GetPlumbedNetworks(BaseNetworkTask):
             amphora[constants.COMPUTE_ID])
 
 
-class PlugNetworks(BaseNetworkTask):
-    """Task to plug the networks.
-
-    This uses the delta to add all missing networks/nics
-    """
-
-    def execute(self, amphora, delta):
-        """Update the amphora networks for the delta."""
-
-        LOG.debug("Plug or unplug networks for amphora id: %s",
-                  amphora[constants.ID])
-
-        if not delta:
-            LOG.debug("No network deltas for amphora id: %s",
-                      amphora[constants.ID])
-            return
-
-        # add nics
-        for nic in delta[constants.ADD_NICS]:
-            self.network_driver.plug_network(amphora[constants.COMPUTE_ID],
-                                             nic[constants.NETWORK_ID])
-
-    def revert(self, amphora, delta, *args, **kwargs):
-        """Handle a failed network plug by removing all nics added."""
-
-        LOG.warning("Unable to plug networks for amp id %s",
-                    amphora[constants.ID])
-        if not delta:
-            return
-
-        for nic in delta[constants.ADD_NICS]:
-            try:
-                self.network_driver.unplug_network(
-                    amphora[constants.COMPUTE_ID],
-                    nic[constants.NETWORK_ID])
-            except base.NetworkNotFound:
-                pass
-
-
 class UnPlugNetworks(BaseNetworkTask):
     """Task to unplug the networks
 
@@ -316,6 +285,14 @@ class HandleNetworkDelta(BaseNetworkTask):
             fixed_ip.subnet = self.network_driver.get_subnet(
                 fixed_ip.subnet_id)
 
+    def _cleanup_port(self, port_id, compute_id):
+        try:
+            self.network_driver.delete_port(port_id)
+        except Exception:
+            LOG.error(f'Unable to delete port {port_id} after failing to plug '
+                      f'the port into compute {compute_id}. This port '
+                      f'may now be abandoned in neutron.')
+
     def execute(self, amphora, delta):
         """Handle network plugging based off deltas."""
         session = db_apis.get_session()
@@ -324,20 +301,43 @@ class HandleNetworkDelta(BaseNetworkTask):
                                            id=amphora.get(constants.ID))
         updated_ports = {}
         for nic in delta[constants.ADD_NICS]:
+            network_id = nic[constants.NETWORK_ID]
             subnet_id = nic[constants.FIXED_IPS][0][constants.SUBNET_ID]
-            interface = self.network_driver.plug_network(
-                db_amp.compute_id, nic[constants.NETWORK_ID])
-            port = self.network_driver.get_port(interface.port_id)
-            # nova may plugged undesired subnets (it plugs one of the subnets
-            # of the network), we can safely unplug the subnets we don't need,
-            # the desired subnet will be added in the 'ADD_SUBNETS' loop.
-            extra_subnets = [
-                fixed_ip.subnet_id
-                for fixed_ip in port.fixed_ips
-                if fixed_ip.subnet_id != subnet_id]
-            for subnet_id in extra_subnets:
-                port = self.network_driver.unplug_fixed_ip(
-                    port_id=interface.port_id, subnet_id=subnet_id)
+
+            try:
+                port = self.network_driver.create_port(
+                    network_id,
+                    name=f'octavia-lb-member-{amphora.get(constants.ID)}',
+                    vnic_type=nic[constants.VNIC_TYPE])
+            except exceptions.NotFound as e:
+                if 'Network' in str(e):
+                    raise base.NetworkNotFound(str(e))
+                raise base.CreatePortException(str(e))
+            except Exception as e:
+                message = _(f'Error creating a port on network {network_id}. ')
+                LOG.exception(message)
+                raise base.CreatePortException(message) from e
+
+            try:
+                self.network_driver.plug_port(db_amp, port)
+            except exceptions.NotFound as e:
+                self._cleanup_port(port.id, db_amp.compute_id)
+                if 'Instance' in str(e):
+                    raise base.AmphoraNotFound(str(e))
+                raise base.PlugNetworkException(str(e))
+            except Exception as e:
+                self._cleanup_port(port.id, db_amp.compute_id)
+                message = _('Error plugging amphora (compute_id: '
+                            '{compute_id}) into network {network_id}.').format(
+                    compute_id=db_amp.compute_id, network_id=network_id)
+                LOG.exception(message)
+                raise base.PlugNetworkException(message) from e
+            with session.begin():
+                self.amphora_member_port_repo.create(
+                    session, port_id=port.id,
+                    amphora_id=amphora.get(constants.ID),
+                    network_id=network_id)
+
             self._fill_port_info(port)
             updated_ports[port.network_id] = port.to_dict(recurse=True)
 
@@ -395,6 +395,14 @@ class HandleNetworkDelta(BaseNetworkTask):
                 self.network_driver.delete_port(port_id)
             except Exception:
                 LOG.exception("Unable to delete the port")
+            try:
+                with session.begin():
+                    self.amphora_member_port_repo.delete(session,
+                                                         port_id=port_id)
+            except sa_exception.NoResultFound:
+                # Passively fail here for upgrade compatibility
+                LOG.warning("No Amphora member port records found for "
+                            "port_id: %s", port_id)
 
             updated_ports.pop(network_id, None)
         return {amphora[constants.ID]: list(updated_ports.values())}
@@ -497,6 +505,20 @@ class UpdateVIPSecurityGroup(BaseNetworkTask):
         return sg_id
 
 
+class UpdateAmphoraSecurityGroup(BaseNetworkTask):
+    """Task to update SGs for an Amphora."""
+
+    def execute(self, loadbalancer_id: str):
+        session = db_apis.get_session()
+        with session.begin():
+            db_lb = self.loadbalancer_repo.get(
+                session, id=loadbalancer_id)
+        for amp in db_lb.amphorae:
+            self.network_driver.update_aap_port_sg(db_lb,
+                                                   amp,
+                                                   db_lb.vip)
+
+
 class GetSubnetFromVIP(BaseNetworkTask):
     """Task to plumb a VIP."""
 
@@ -536,11 +558,10 @@ class PlugVIPAmphora(BaseNetworkTask):
         """Handle a failure to plumb a vip."""
         if isinstance(result, failure.Failure):
             return
+        lb_id = loadbalancer[constants.LOADBALANCER_ID]
         LOG.warning("Unable to plug VIP for amphora id %s "
                     "load balancer id %s",
-                    amphora.get(constants.ID),
-                    loadbalancer[constants.LOADBALANCER_ID])
-
+                    amphora.get(constants.ID), lb_id)
         try:
             session = db_apis.get_session()
             with session.begin():
@@ -550,15 +571,16 @@ class PlugVIPAmphora(BaseNetworkTask):
                 db_amp.ha_port_id = result[constants.HA_PORT_ID]
                 db_subnet = self.network_driver.get_subnet(
                     subnet[constants.ID])
-                db_lb = self.loadbalancer_repo.get(
-                    session,
-                    id=loadbalancer[constants.LOADBALANCER_ID])
-
+                db_lb = self.loadbalancer_repo.get(session, id=lb_id)
             self.network_driver.unplug_aap_port(db_lb.vip,
                                                 db_amp, db_subnet)
         except Exception as e:
-            LOG.error('Failed to unplug AAP port. Resources may still be in '
-                      'use for VIP: %s due to error: %s', db_lb.vip, str(e))
+            LOG.error(
+                'Failed to unplug AAP port for load balancer: %s. '
+                'Resources may still be in use for VRRP port: %s. '
+                'Due to error: %s',
+                lb_id, result[constants.VRRP_PORT_ID], str(e)
+            )
 
 
 class UnplugVIP(BaseNetworkTask):
@@ -906,24 +928,29 @@ class DeletePort(BaseNetworkTask):
         """Delete the network port."""
         if port_id is None:
             return
-        if self.execute.retry.statistics.get(constants.ATTEMPT_NUMBER, 1) == 1:
+        # tenacity 8.5.0 moves statistics from the retry object to the function
+        try:
+            retry_statistics = self.execute.statistics
+        except AttributeError:
+            retry_statistics = self.execute.retry.statistics
+
+        if retry_statistics.get(constants.ATTEMPT_NUMBER, 1) == 1:
             LOG.debug("Deleting network port %s", port_id)
         else:
             LOG.warning('Retrying network port %s delete attempt %s of %s.',
                         port_id,
-                        self.execute.retry.statistics[
-                            constants.ATTEMPT_NUMBER],
+                        retry_statistics[constants.ATTEMPT_NUMBER],
                         self.execute.retry.stop.max_attempt_number)
         # Let the Taskflow engine know we are working and alive
         # Don't use get with a default for 'attempt_number', we need to fail
         # if that number is missing.
         self.update_progress(
-            self.execute.retry.statistics[constants.ATTEMPT_NUMBER] /
+            retry_statistics[constants.ATTEMPT_NUMBER] /
             self.execute.retry.stop.max_attempt_number)
         try:
             self.network_driver.delete_port(port_id)
         except Exception:
-            if (self.execute.retry.statistics[constants.ATTEMPT_NUMBER] !=
+            if (retry_statistics[constants.ATTEMPT_NUMBER] !=
                     self.execute.retry.stop.max_attempt_number):
                 LOG.warning('Network port delete for port id: %s failed. '
                             'Retrying.', port_id)
@@ -950,6 +977,22 @@ class DeletePort(BaseNetworkTask):
                 raise
 
 
+class DeleteAmphoraMemberPorts(BaseNetworkTask):
+    """Task to delete all of the member ports on an Amphora."""
+
+    def execute(self, amphora_id, passive_failure=False):
+        delete_port = DeletePort()
+        session = db_apis.get_session()
+
+        with session.begin():
+            ports = self.amphora_member_port_repo.get_port_ids(
+                session, amphora_id)
+        for port in ports:
+            delete_port.execute(port, passive_failure)
+            with session.begin():
+                self.amphora_member_port_repo.delete(session, port_id=port)
+
+
 class CreateVIPBasePort(BaseNetworkTask):
     """Task to create the VIP base port for an amphora."""
 
@@ -963,16 +1006,21 @@ class CreateVIPBasePort(BaseNetworkTask):
     def execute(self, vip, vip_sg_id, amphora_id, additional_vips):
         port_name = constants.AMP_BASE_PORT_PREFIX + amphora_id
         fixed_ips = [{constants.SUBNET_ID: vip[constants.SUBNET_ID]}]
-        sg_id = []
+        sg_ids = []
+        # NOTE(gthiemonge) clarification:
+        # - vip_sg_id is the ID of the SG created and managed by Octavia.
+        # - vip['sg_ids'] are the IDs of the SGs provided by the user.
         if vip_sg_id:
-            sg_id = [vip_sg_id]
+            sg_ids = [vip_sg_id]
+        if vip["sg_ids"]:
+            sg_ids += vip["sg_ids"]
         secondary_ips = [vip[constants.IP_ADDRESS]]
         for add_vip in additional_vips:
             secondary_ips.append(add_vip[constants.IP_ADDRESS])
         port = self.network_driver.create_port(
             vip[constants.NETWORK_ID], name=port_name, fixed_ips=fixed_ips,
             secondary_ips=secondary_ips,
-            security_group_ids=sg_id,
+            security_group_ids=sg_ids,
             qos_policy_id=vip[constants.QOS_POLICY_ID])
         LOG.info('Created port %s with ID %s for amphora %s',
                  port_name, port.id, amphora_id)

octavia/db/base_models.py

@@ -11,6 +11,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+from typing import Optional
 
 from wsme import types as wtypes
 
@@ -58,9 +59,14 @@ class OctaviaBase(models.ModelBase):
         if obj.__class__.__name__ in ['AdditionalVip']:
             return (obj.__class__.__name__ +
                     obj.load_balancer_id + obj.subnet_id)
+        if obj.__class__.__name__ in ['VipSecurityGroup']:
+            return obj.__class__.__name__ + obj.load_balancer_id + obj.sg_id
+        if obj.__class__.__name__ in ['AmphoraMemberPort']:
+            return obj.__class__.__name__ + obj.port_id
         raise NotImplementedError
 
-    def to_data_model(self, _graph_nodes=None):
+    def to_data_model(
+            self, _graph_nodes=None, recursion_depth: Optional[int] = None):
         """Converts to a data model graph.
 
         In order to make the resulting data model graph usable no matter how
@@ -71,6 +77,13 @@ class OctaviaBase(models.ModelBase):
                              method. Should not be called from the outside.
                              Contains a dictionary of all OctaviaBase type
                              objects in the generated graph
+        :param recursion_depth: Used only for configuring recursion.
+                                This option allows to limit recursion depth.
+                                It could be used when we need only main node
+                                and its first level relationships.
+                                It allows to save time on recursion calls for
+                                huge graphs, when only main object is
+                                necessary.
         """
         _graph_nodes = _graph_nodes or {}
         if not self.__data_model__:
@@ -88,9 +101,16 @@ class OctaviaBase(models.ModelBase):
         dm_self = self.__data_model__(**dm_kwargs)
         dm_key = self._get_unique_key(dm_self)
         _graph_nodes.update({dm_key: dm_self})
+        new_depth = recursion_depth
+        need_recursion = recursion_depth is None or recursion_depth > 0
+        # decrease depth of recursion on new recursion call
+        if new_depth:
+            new_depth -= 1
         for attr_name in attr_names:
             attr = getattr(self, attr_name)
-            if isinstance(attr, OctaviaBase) and attr.__class__:
+            if (need_recursion and
+                    isinstance(attr, OctaviaBase) and
+                    attr.__class__):
                 # If this attr is already in the graph node list, just
                 # reference it there and don't recurse.
                 ukey = self._get_unique_key(attr)
@@ -98,18 +118,22 @@ class OctaviaBase(models.ModelBase):
                     setattr(dm_self, attr_name, _graph_nodes[ukey])
                 else:
                     setattr(dm_self, attr_name, attr.to_data_model(
-                        _graph_nodes=_graph_nodes))
+                        _graph_nodes=_graph_nodes,
+                        recursion_depth=new_depth))
             elif isinstance(attr, (collections.InstrumentedList, list)):
                 setattr(dm_self, attr_name, [])
                 listref = getattr(dm_self, attr_name)
                 for item in attr:
-                    if isinstance(item, OctaviaBase) and item.__class__:
+                    if (need_recursion and
+                            isinstance(item, OctaviaBase) and
+                            item.__class__):
                         ukey = self._get_unique_key(item)
                         if ukey in _graph_nodes.keys():
                             listref.append(_graph_nodes[ukey])
                         else:
                             listref.append(
-                                item.to_data_model(_graph_nodes=_graph_nodes))
+                                item.to_data_model(_graph_nodes=_graph_nodes,
+                                                   recursion_depth=new_depth))
                     elif not isinstance(item, OctaviaBase):
                         listref.append(item)
         return dm_self

octavia/db/migration/alembic_migrations/versions/3097e55493ae_add_sg_id_to_vip_table.py

@@ -0,0 +1,39 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""add sg_id to vip table
+
+Revision ID: 3097e55493ae
+Revises: db2a73e82626
+Create Date: 2024-04-05 10:04:32.015445
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3097e55493ae'
+down_revision = 'db2a73e82626'
+
+
+def upgrade():
+    op.create_table(
+        "vip_security_group",
+        sa.Column("load_balancer_id", sa.String(36), nullable=False),
+        sa.Column("sg_id", sa.String(36), nullable=False),
+        sa.ForeignKeyConstraint(["load_balancer_id"],
+                                ["vip.load_balancer_id"],
+                                name="fk_vip_sg_vip_lb_id"),
+        sa.PrimaryKeyConstraint("load_balancer_id", "sg_id")
+    )

octavia/db/migration/alembic_migrations/versions/8db7a6443785_add_member_vnic_type.py

@@ -0,0 +1,36 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Add member vnic_type
+
+Revision ID: 8db7a6443785
+Revises: 3097e55493ae
+Create Date: 2024-03-29 20:34:37.263847
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+from octavia.common import constants
+
+# revision identifiers, used by Alembic.
+revision = '8db7a6443785'
+down_revision = '3097e55493ae'
+
+
+def upgrade():
+    op.add_column(
+        u'member',
+        sa.Column(u'vnic_type', sa.String(64), nullable=False,
+                  server_default=constants.VNIC_TYPE_NORMAL)
+    )

octavia/db/migration/alembic_migrations/versions/fabf4983846b_add_member_port_table.py

@@ -0,0 +1,40 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""add_member_port_table
+
+Revision ID: fabf4983846b
+Revises: 8db7a6443785
+Create Date: 2024-08-30 23:12:01.713217
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'fabf4983846b'
+down_revision = '8db7a6443785'
+
+
+def upgrade():
+    op.create_table(
+        'amphora_member_port',
+        sa.Column('port_id', sa.String(36), primary_key=True),
+        sa.Column('amphora_id', sa.String(36), nullable=False, index=True),
+        sa.Column('network_id', sa.String(36)),
+        sa.Column('created_at', sa.DateTime()),
+        sa.Column('updated_at', sa.DateTime())
+    )
+    op.create_foreign_key(
+        'fk_member_port_amphora_id', 'amphora_member_port',
+        'amphora', ['amphora_id'], ['id']
+    )

octavia/db/models.py

@@ -229,6 +229,7 @@ class Member(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
         nullable=False)
     enabled = sa.Column(sa.Boolean(), nullable=False)
     pool = orm.relationship("Pool", back_populates="members")
+    vnic_type = sa.Column(sa.String(64), nullable=True)
 
     _tags = orm.relationship(
         'Tags',
@@ -246,7 +247,7 @@ class Member(base_models.BASE, base_models.IdMixin, base_models.ProjectMixin,
                 f"ip_address={self.ip_address!r}, "
                 f"protocol_port={self.protocol_port!r}, "
                 f"operating_status={self.operating_status!r}, "
-                f"weight={self.weight!r})")
+                f"weight={self.weight!r}, vnic_type={self.vnic_type!r})")
 
 
 class HealthMonitor(base_models.BASE, base_models.IdMixin,
@@ -508,6 +509,14 @@ class Vip(base_models.BASE):
     octavia_owned = sa.Column(sa.Boolean(), nullable=True)
     vnic_type = sa.Column(sa.String(64), nullable=True)
 
+    sgs = orm.relationship(
+        "VipSecurityGroup", cascade="all,delete-orphan",
+        uselist=True, backref=orm.backref("vip", uselist=False))
+
+    @property
+    def sg_ids(self) -> list[str]:
+        return [sg.sg_id for sg in self.sgs]
+
 
 class AdditionalVip(base_models.BASE):
 
@@ -976,3 +985,36 @@ class ListenerCidr(base_models.BASE):
         sa.ForeignKey("listener.id", name="fk_listener_cidr_listener_id"),
         nullable=False)
     cidr = sa.Column(sa.String(64), nullable=False)
+
+
+class VipSecurityGroup(base_models.BASE):
+
+    __data_model__ = data_models.VipSecurityGroup
+
+    __tablename__ = "vip_security_group"
+    __table_args__ = (
+        sa.PrimaryKeyConstraint('load_balancer_id', 'sg_id'),
+    )
+
+    load_balancer_id = sa.Column(
+        sa.String(36),
+        sa.ForeignKey("vip.load_balancer_id", name="fk_vip_sg_vip_lb_id"),
+        nullable=False)
+    sg_id = sa.Column(sa.String(64), nullable=False)
+
+
+class AmphoraMemberPort(base_models.BASE, models.TimestampMixin):
+
+    __data_model__ = data_models.AmphoraMemberPort
+
+    __tablename__ = "amphora_member_port"
+
+    port_id = sa.Column(
+        sa.String(36),
+        primary_key=True)
+    amphora_id = sa.Column(
+        sa.String(36),
+        sa.ForeignKey("amphora.id", name="fk_member_port_amphora_id"),
+        nullable=False)
+    network_id = sa.Column(
+        sa.String(36))