fix: Avoid race when removing interfaces via NNCP (#2347)

* Avoid race when removing interfaces via NNCP

Remoivng an interface that was created using an NNCP, is done by editing the same
NNCP. This sometimes resulted in a race, in which the NNCP success status actually
presented the prvious status, leading to deleting the NNCP before the configuration
was completed, leaving hanging interfaces in the cluster nodes, with
node native interfaces occupied as the ports of these tests-created interfaces.
A recent PR made this failed flow to always occur.
This PR aims to assure that the timestamp of the AVAIALBLE status is updated for
the recent change (the interface removal) and not the previous change (setup or
modification).
This PR is based on the fix that was presented in
RedHatQE/openshift-virtualization-tests#430.

* Store fixed initial time outside loop

* Handle CR comments

* More CR fixes

Verify orderly teardown of a failed NNCP.

* More CR fixes

Remove redundant overriding update().

* Fix CI failure

Add datetime to flake8.

* Fix more CR comments

Clarify comment
Rename args for clarification
Add typing for new functions

* Fix CI failure

* Add clarifying comment

* Apply code-reabbit suggestions

* Fix comments

Author: yossisegev

.flake8

@@ -58,6 +58,7 @@ fcn_exclude_functions =
     click,
     ast,
     filecmp,
+    datetime,
 
 enable-extensions =
     FCN,

ocp_resources/node_network_configuration_policy.py

@@ -1,16 +1,17 @@
 import re
+from datetime import datetime
 
 from kubernetes.dynamic.exceptions import ConflictError
 
-from ocp_resources.utils.constants import TIMEOUT_4MINUTES
+from ocp_resources.utils.constants import TIMEOUT_1MINUTE, TIMEOUT_4MINUTES, TIMEOUT_5SEC
 from ocp_resources.exceptions import NNCPConfigurationFailed
 from ocp_resources.node import Node
 from ocp_resources.node_network_configuration_enactment import (
     NodeNetworkConfigurationEnactment,
 )
 from ocp_resources.node_network_state import NodeNetworkState
 from ocp_resources.resource import Resource, ResourceEditor
-from timeout_sampler import TimeoutExpiredError, TimeoutSampler, TimeoutWatch
+from timeout_sampler import TimeoutExpiredError, TimeoutSampler, TimeoutWatch, retry
 
 IPV4_STR = "ipv4"
 IPV6_STR = "ipv6"
@@ -322,10 +323,43 @@ def _absent_interface(self):
         if self.ports:
             self.add_ports()
 
+        # The current time-stamp of the NNCP's available status will change after the NNCP is updated, therefore
+        # it must be fetched and stored before the update, and compared with the new time-stamp after.
+        initial_success_status_time = self._get_last_successful_transition_time()
         ResourceEditor(
             patches={self: {"spec": {"desiredState": {"interfaces": self.desired_state["interfaces"]}}}}
         ).update()
 
+        # If the NNCP failed on setup, then its tear-down AVAIALBLE status will necessarily be the first.
+        if initial_success_status_time:
+            self._wait_for_nncp_status_update(initial_transition_time=initial_success_status_time)
+
+    def _get_last_successful_transition_time(self) -> str | None:
+        for condition in self.instance.status.conditions:
+            if (
+                condition["type"] == self.Conditions.Type.AVAILABLE
+                and condition["status"] == Resource.Condition.Status.TRUE
+                and condition["reason"] == self.Conditions.Reason.SUCCESSFULLY_CONFIGURED
+            ):
+                return condition["lastTransitionTime"]
+        return None
+
+    @retry(
+        wait_timeout=TIMEOUT_1MINUTE,
+        sleep=TIMEOUT_5SEC,
+    )
+    def _wait_for_nncp_status_update(self, initial_transition_time: str) -> bool:
+        date_format = "%Y-%m-%dT%H:%M:%SZ"
+        formatted_initial_transition_time = datetime.strptime(initial_transition_time, date_format)
+        for condition in self.instance.get("status", {}).get("conditions", []):
+            if (
+                condition["type"] == self.Conditions.Type.AVAILABLE
+                and condition["status"] == Resource.Condition.Status.TRUE
+                and datetime.strptime(condition["lastTransitionTime"], date_format) > formatted_initial_transition_time
+            ):
+                return True
+        return False
+
     @property
     def status(self):
         for condition in self.instance.status.conditions: