octavia 13.0.0__py3-none-any.whl → 13.0.1__py3-none-any.whl

octavia/controller/worker/v2/flows/load_balancer_flows.py

@@ -637,6 +637,14 @@ class LoadBalancerFlows(object):
                 requires=constants.LOADBALANCER_ID,
                 provides=constants.AMPHORAE))
 
+            failover_LB_flow.add(
+                amphora_driver_tasks.AmphoraeGetConnectivityStatus(
+                    name=(new_amp_role + '-' +
+                          constants.AMPHORAE_GET_CONNECTIVITY_STATUS),
+                    requires=constants.AMPHORAE,
+                    rebind={constants.NEW_AMPHORA_ID: constants.AMPHORA_ID},
+                    provides=constants.AMPHORAE_STATUS))
+
             # Listeners update needs to be run on all amphora to update
             # their peer configurations. So parallelize this with an
             # unordered subflow.
@@ -651,14 +659,18 @@ class LoadBalancerFlows(object):
                 amphora_driver_tasks.AmphoraIndexListenerUpdate(
                     name=(constants.AMPHORA + '-0-' +
                           constants.AMP_LISTENER_UPDATE),
-                    requires=(constants.LOADBALANCER, constants.AMPHORAE),
+                    requires=(constants.LOADBALANCER, constants.AMPHORAE,
+                              constants.AMPHORAE_STATUS),
+                    rebind={constants.NEW_AMPHORA_ID: constants.AMPHORA_ID},
                     inject={constants.AMPHORA_INDEX: 0,
                             constants.TIMEOUT_DICT: timeout_dict}))
             update_amps_subflow.add(
                 amphora_driver_tasks.AmphoraIndexListenerUpdate(
                     name=(constants.AMPHORA + '-1-' +
                           constants.AMP_LISTENER_UPDATE),
-                    requires=(constants.LOADBALANCER, constants.AMPHORAE),
+                    requires=(constants.LOADBALANCER, constants.AMPHORAE,
+                              constants.AMPHORAE_STATUS),
+                    rebind={constants.NEW_AMPHORA_ID: constants.AMPHORA_ID},
                     inject={constants.AMPHORA_INDEX: 1,
                             constants.TIMEOUT_DICT: timeout_dict}))
 
@@ -667,7 +679,8 @@ class LoadBalancerFlows(object):
             # Configure and enable keepalived in the amphora
             failover_LB_flow.add(self.amp_flows.get_vrrp_subflow(
                 new_amp_role + '-' + constants.GET_VRRP_SUBFLOW,
-                timeout_dict, create_vrrp_group=False))
+                timeout_dict, create_vrrp_group=False,
+                get_amphorae_status=False))
 
             # #### End of standby ####
 
@@ -682,6 +695,7 @@ class LoadBalancerFlows(object):
                     name=(new_amp_role + '-' +
                           constants.AMPHORA_RELOAD_LISTENER),
                     requires=(constants.LOADBALANCER, constants.AMPHORAE),
+                    rebind={constants.NEW_AMPHORA_ID: constants.AMPHORA_ID},
                     inject={constants.AMPHORA_INDEX: 1,
                             constants.TIMEOUT_DICT: timeout_dict}))
 

octavia/controller/worker/v2/tasks/amphora_driver_tasks.py

@@ -14,6 +14,9 @@
 #
 
 import copy
+from typing import List
+from typing import Optional
+
 from cryptography import fernet
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -102,10 +105,19 @@ class AmpListenersUpdate(BaseAmphoraTask):
 class AmphoraIndexListenerUpdate(BaseAmphoraTask):
     """Task to update the listeners on one amphora."""
 
-    def execute(self, loadbalancer, amphora_index, amphorae, timeout_dict=()):
+    def execute(self, loadbalancer, amphora_index, amphorae,
+                amphorae_status: dict, new_amphora_id: str, timeout_dict=()):
         # Note, we don't want this to cause a revert as it may be used
         # in a failover flow with both amps failing. Skip it and let
         # health manager fix it.
+
+        amphora_id = amphorae[amphora_index].get(constants.ID)
+        amphora_status = amphorae_status.get(amphora_id, {})
+        if amphora_status.get(constants.UNREACHABLE):
+            LOG.warning("Skipping listener update because amphora %s "
+                        "is not reachable.", amphora_id)
+            return
+
         try:
             # TODO(johnsom) Optimize this to use the dicts and not need the
             #               DB lookups
@@ -120,14 +132,16 @@ class AmphoraIndexListenerUpdate(BaseAmphoraTask):
             self.amphora_driver.update_amphora_listeners(
                 db_lb, db_amp, timeout_dict)
         except Exception as e:
-            amphora_id = amphorae[amphora_index].get(constants.ID)
             LOG.error('Failed to update listeners on amphora %s. Skipping '
                       'this amphora as it is failing to update due to: %s',
                       amphora_id, str(e))
-            session = db_apis.get_session()
-            with session.begin():
-                self.amphora_repo.update(session, amphora_id,
-                                         status=constants.ERROR)
+            # Update only the status of the newly created amphora during the
+            # failover
+            if amphora_id == new_amphora_id:
+                session = db_apis.get_session()
+                with session.begin():
+                    self.amphora_repo.update(session, amphora_id,
+                                             status=constants.ERROR)
 
 
 class ListenersUpdate(BaseAmphoraTask):
@@ -193,10 +207,18 @@ class AmphoraIndexListenersReload(BaseAmphoraTask):
     """Task to reload all listeners on an amphora."""
 
     def execute(self, loadbalancer, amphora_index, amphorae,
-                timeout_dict=None):
+                amphorae_status: dict, new_amphora_id: str, timeout_dict=None):
         """Execute listener reload routines for listeners on an amphora."""
         if amphorae is None:
             return
+
+        amphora_id = amphorae[amphora_index].get(constants.ID)
+        amphora_status = amphorae_status.get(amphora_id, {})
+        if amphora_status.get(constants.UNREACHABLE):
+            LOG.warning("Skipping listener reload because amphora %s "
+                        "is not reachable.", amphora_id)
+            return
+
         # TODO(johnsom) Optimize this to use the dicts and not need the
         #               DB lookups
         session = db_apis.get_session()
@@ -210,13 +232,15 @@ class AmphoraIndexListenersReload(BaseAmphoraTask):
             try:
                 self.amphora_driver.reload(db_lb, db_amp, timeout_dict)
             except Exception as e:
-                amphora_id = amphorae[amphora_index][constants.ID]
                 LOG.warning('Failed to reload listeners on amphora %s. '
                             'Skipping this amphora as it is failing to '
                             'reload due to: %s', amphora_id, str(e))
-                with session.begin():
-                    self.amphora_repo.update(session, amphora_id,
-                                             status=constants.ERROR)
+                # Update only the status of the newly created amphora during
+                # the failover
+                if amphora_id == new_amphora_id:
+                    with session.begin():
+                        self.amphora_repo.update(session, amphora_id,
+                                                 status=constants.ERROR)
 
 
 class ListenerDelete(BaseAmphoraTask):
@@ -478,8 +502,15 @@ class AmphoraUpdateVRRPInterface(BaseAmphoraTask):
 class AmphoraIndexUpdateVRRPInterface(BaseAmphoraTask):
     """Task to get and update the VRRP interface device name from amphora."""
 
-    def execute(self, amphora_index, amphorae, timeout_dict=None):
+    def execute(self, amphora_index, amphorae, amphorae_status: dict,
+                new_amphora_id: str, timeout_dict=None):
         amphora_id = amphorae[amphora_index][constants.ID]
+        amphora_status = amphorae_status.get(amphora_id, {})
+        if amphora_status.get(constants.UNREACHABLE):
+            LOG.warning("Skipping VRRP interface update because amphora %s "
+                        "is not reachable.", amphora_id)
+            return None
+
         try:
             # TODO(johnsom) Optimize this to use the dicts and not need the
             #               DB lookups
@@ -494,9 +525,12 @@ class AmphoraIndexUpdateVRRPInterface(BaseAmphoraTask):
             LOG.error('Failed to get amphora VRRP interface on amphora '
                       '%s. Skipping this amphora as it is failing due to: '
                       '%s', amphora_id, str(e))
-            with session.begin():
-                self.amphora_repo.update(session, amphora_id,
-                                         status=constants.ERROR)
+            # Update only the status of the newly created amphora during the
+            # failover
+            if amphora_id == new_amphora_id:
+                with session.begin():
+                    self.amphora_repo.update(session, amphora_id,
+                                             status=constants.ERROR)
             return None
 
         with session.begin():
@@ -542,12 +576,19 @@ class AmphoraIndexVRRPUpdate(BaseAmphoraTask):
     """Task to update the VRRP configuration of an amphora."""
 
     def execute(self, loadbalancer_id, amphorae_network_config, amphora_index,
-                amphorae, amp_vrrp_int, timeout_dict=None):
+                amphorae, amphorae_status: dict, amp_vrrp_int: Optional[str],
+                new_amphora_id: str, timeout_dict=None):
         """Execute update_vrrp_conf."""
         # Note, we don't want this to cause a revert as it may be used
         # in a failover flow with both amps failing. Skip it and let
         # health manager fix it.
         amphora_id = amphorae[amphora_index][constants.ID]
+        amphora_status = amphorae_status.get(amphora_id, {})
+        if amphora_status.get(constants.UNREACHABLE):
+            LOG.warning("Skipping VRRP configuration because amphora %s "
+                        "is not reachable.", amphora_id)
+            return
+
         try:
             # TODO(johnsom) Optimize this to use the dicts and not need the
             #               DB lookups
@@ -564,9 +605,12 @@ class AmphoraIndexVRRPUpdate(BaseAmphoraTask):
             LOG.error('Failed to update VRRP configuration amphora %s. '
                       'Skipping this amphora as it is failing to update due '
                       'to: %s', amphora_id, str(e))
-            with session.begin():
-                self.amphora_repo.update(session, amphora_id,
-                                         status=constants.ERROR)
+            # Update only the status of the newly created amphora during the
+            # failover
+            if amphora_id == new_amphora_id:
+                with session.begin():
+                    self.amphora_repo.update(session, amphora_id,
+                                             status=constants.ERROR)
             return
         LOG.debug("Uploaded VRRP configuration of amphora %s.", amphora_id)
 
@@ -594,10 +638,17 @@ class AmphoraIndexVRRPStart(BaseAmphoraTask):
     This will reload keepalived if it is already running.
     """
 
-    def execute(self, amphora_index, amphorae, timeout_dict=None):
+    def execute(self, amphora_index, amphorae, amphorae_status: dict,
+                new_amphora_id: str, timeout_dict=None):
         # TODO(johnsom) Optimize this to use the dicts and not need the
         #               DB lookups
         amphora_id = amphorae[amphora_index][constants.ID]
+        amphora_status = amphorae_status.get(amphora_id, {})
+        if amphora_status.get(constants.UNREACHABLE):
+            LOG.warning("Skipping VRRP start because amphora %s "
+                        "is not reachable.", amphora_id)
+            return
+
         session = db_apis.get_session()
         with session.begin():
             db_amp = self.amphora_repo.get(session, id=amphora_id)
@@ -607,9 +658,12 @@ class AmphoraIndexVRRPStart(BaseAmphoraTask):
             LOG.error('Failed to start VRRP on amphora %s. '
                       'Skipping this amphora as it is failing to start due '
                       'to: %s', amphora_id, str(e))
-            with session.begin():
-                self.amphora_repo.update(session, amphora_id,
-                                         status=constants.ERROR)
+            # Update only the status of the newly created amphora during the
+            # failover
+            if amphora_id == new_amphora_id:
+                with session.begin():
+                    self.amphora_repo.update(session, amphora_id,
+                                             status=constants.ERROR)
             return
         LOG.debug("Started VRRP on amphora %s.",
                   amphorae[amphora_index][constants.ID])
@@ -669,3 +723,40 @@ class AmphoraConfigUpdate(BaseAmphoraTask):
                       'update. Please update the amphora image for this '
                       'amphora. Skipping.'.
                       format(amphora.get(constants.ID)))
+
+
+class AmphoraeGetConnectivityStatus(BaseAmphoraTask):
+    """Task that checks amphorae connectivity status.
+
+    Check and return the connectivity status of both amphorae in ACTIVE STANDBY
+    load balancers
+    """
+
+    def execute(self, amphorae: List[dict], new_amphora_id: str,
+                timeout_dict=None):
+        amphorae_status = {}
+
+        for amphora in amphorae:
+            amphora_id = amphora[constants.ID]
+            amphorae_status[amphora_id] = {}
+
+            session = db_apis.get_session()
+            with session.begin():
+                db_amp = self.amphora_repo.get(session, id=amphora_id)
+
+            try:
+                # Verify if the amphora is reachable
+                self.amphora_driver.check(db_amp, timeout_dict=timeout_dict)
+            except Exception as e:
+                LOG.exception("Cannot get status for amphora %s",
+                              amphora_id)
+                # In case it fails and the tested amphora is the newly created
+                # amphora, it's not a normal error handling, re-raise the
+                # exception
+                if amphora_id == new_amphora_id:
+                    raise e
+                amphorae_status[amphora_id][constants.UNREACHABLE] = True
+            else:
+                amphorae_status[amphora_id][constants.UNREACHABLE] = False
+
+        return amphorae_status

octavia/controller/worker/v2/tasks/database_tasks.py

@@ -130,13 +130,17 @@ class CreateAmphoraInDB(BaseDatabaseTask):
         LOG.warning("Reverting create amphora in DB for amp id %s ", result)
 
         # Delete the amphora for now. May want to just update status later
-        try:
-            with db_apis.session().begin() as session:
+        with db_apis.session().begin() as session:
+            try:
                 self.amphora_repo.delete(session, id=result)
-        except Exception as e:
-            LOG.error("Failed to delete amphora %(amp)s "
-                      "in the database due to: "
-                      "%(except)s", {'amp': result, 'except': str(e)})
+            except Exception as e:
+                LOG.error("Failed to delete amphora %(amp)s "
+                          "in the database due to: "
+                          "%(except)s", {'amp': result, 'except': str(e)})
+            try:
+                self.amp_health_repo.delete(session, amphora_id=result)
+            except Exception:
+                pass
 
 
 class MarkLBAmphoraeDeletedInDB(BaseDatabaseTask):
@@ -2621,15 +2625,13 @@ class DecrementHealthMonitorQuota(BaseDatabaseTask):
 
             try:
                 session = db_apis.get_session()
-                lock_session = db_apis.get_session()
                 try:
                     self.repos.check_quota_met(session,
-                                               lock_session,
                                                data_models.HealthMonitor,
                                                project_id)
-                    lock_session.commit()
+                    session.commit()
                 except Exception:
-                    lock_session.rollback()
+                    session.rollback()
             except Exception:
                 # Don't fail the revert flow
                 pass
@@ -2679,15 +2681,13 @@ class DecrementListenerQuota(BaseDatabaseTask):
 
             try:
                 session = db_apis.get_session()
-                lock_session = db_apis.get_session()
                 try:
                     self.repos.check_quota_met(session,
-                                               lock_session,
                                                data_models.Listener,
                                                project_id)
-                    lock_session.commit()
+                    session.commit()
                 except Exception:
-                    lock_session.rollback()
+                    session.rollback()
             except Exception:
                 # Don't fail the revert flow
                 pass
@@ -2739,15 +2739,13 @@ class DecrementLoadBalancerQuota(BaseDatabaseTask):
 
             try:
                 session = db_apis.get_session()
-                lock_session = db_apis.get_session()
                 try:
                     self.repos.check_quota_met(session,
-                                               lock_session,
                                                data_models.LoadBalancer,
                                                project_id)
-                    lock_session.commit()
+                    session.commit()
                 except Exception:
-                    lock_session.rollback()
+                    session.rollback()
             except Exception:
                 # Don't fail the revert flow
                 pass
@@ -2798,15 +2796,13 @@ class DecrementMemberQuota(BaseDatabaseTask):
 
             try:
                 session = db_apis.get_session()
-                lock_session = db_apis.get_session()
                 try:
                     self.repos.check_quota_met(session,
-                                               lock_session,
                                                data_models.Member,
                                                project_id)
-                    lock_session.commit()
+                    session.commit()
                 except Exception:
-                    lock_session.rollback()
+                    session.rollback()
             except Exception:
                 # Don't fail the revert flow
                 pass
@@ -2871,42 +2867,38 @@ class DecrementPoolQuota(BaseDatabaseTask):
             # in case other quota actions have occurred
             try:
                 session = db_apis.get_session()
-                lock_session = db_apis.get_session()
                 try:
                     self.repos.check_quota_met(session,
-                                               lock_session,
                                                data_models.Pool,
                                                project_id)
-                    lock_session.commit()
+                    session.commit()
                 except Exception:
-                    lock_session.rollback()
+                    session.rollback()
 
                 # Attempt to increment back the health monitor quota
                 if pool_child_count['HM'] > 0:
-                    lock_session = db_apis.get_session()
+                    session = db_apis.get_session()
                     try:
                         self.repos.check_quota_met(session,
-                                                   lock_session,
                                                    data_models.HealthMonitor,
                                                    project_id)
-                        lock_session.commit()
+                        session.commit()
                     except Exception:
-                        lock_session.rollback()
+                        session.rollback()
 
                 # Attempt to increment back the member quota
                 # This is separate calls to maximize the correction
                 # should other factors have increased the in use quota
                 # before this point in the revert flow
                 for i in range(pool_child_count['member']):
-                    lock_session = db_apis.get_session()
+                    session = db_apis.get_session()
                     try:
                         self.repos.check_quota_met(session,
-                                                   lock_session,
                                                    data_models.Member,
                                                    project_id)
-                        lock_session.commit()
+                        session.commit()
                     except Exception:
-                        lock_session.rollback()
+                        session.rollback()
             except Exception:
                 # Don't fail the revert flow
                 pass
@@ -2954,7 +2946,8 @@ class DecrementL7policyQuota(BaseDatabaseTask):
                                        data_models.L7Policy,
                                        l7policy[constants.PROJECT_ID])
             db_l7policy = self.l7policy_repo.get(
-                db_apis.get_session(), id=l7policy[constants.L7POLICY_ID])
+                lock_session,
+                id=l7policy[constants.L7POLICY_ID])
 
             if db_l7policy and db_l7policy.l7rules:
                 self.repos.decrement_quota(lock_session,
@@ -2982,28 +2975,26 @@ class DecrementL7policyQuota(BaseDatabaseTask):
         if not isinstance(result, failure.Failure):
             try:
                 session = db_apis.get_session()
-                lock_session = db_apis.get_session()
                 try:
                     self.repos.check_quota_met(session,
-                                               lock_session,
                                                data_models.L7Policy,
                                                l7policy[constants.PROJECT_ID])
-                    lock_session.commit()
+                    session.commit()
                 except Exception:
-                    lock_session.rollback()
+                    session.rollback()
                 db_l7policy = self.l7policy_repo.get(
                     session, id=l7policy[constants.L7POLICY_ID])
                 if db_l7policy:
                     # Attempt to increment back the L7Rule quota
                     for i in range(len(db_l7policy.l7rules)):
-                        lock_session = db_apis.get_session()
+                        session = db_apis.get_session()
                         try:
                             self.repos.check_quota_met(
-                                session, lock_session, data_models.L7Rule,
+                                session, data_models.L7Rule,
                                 db_l7policy.project_id)
-                            lock_session.commit()
+                            session.commit()
                         except Exception:
-                            lock_session.rollback()
+                            session.rollback()
             except Exception:
                 # Don't fail the revert flow
                 pass
@@ -3054,15 +3045,13 @@ class DecrementL7ruleQuota(BaseDatabaseTask):
 
             try:
                 session = db_apis.get_session()
-                lock_session = db_apis.get_session()
                 try:
                     self.repos.check_quota_met(session,
-                                               lock_session,
                                                data_models.L7Rule,
                                                l7rule[constants.PROJECT_ID])
-                    lock_session.commit()
+                    session.commit()
                 except Exception:
-                    lock_session.rollback()
+                    session.rollback()
             except Exception:
                 # Don't fail the revert flow
                 pass