chore: Debug rest apis

Signed-off-by: ntkathole <nikhilkathole2683@gmail.com>

Author: ntkathole

.github/workflows/registry-rest-api-tests.yml

@@ -160,7 +160,46 @@ jobs:
       - name: Debug KIND Cluster when there is a failure
         if: failure()
         run: |
-          kubectl get pods --all-namespaces
+          echo "=== All pods ==="
+          kubectl get pods --all-namespaces -o wide
+
+          echo ""
+          echo "=== Test namespace pods ==="
+          kubectl get pods -n test-ns-feast-rest -o wide 2>/dev/null || echo "Namespace not found"
+
+          echo ""
+          echo "=== Services & Endpoints ==="
+          kubectl get svc,endpoints -n test-ns-feast-rest 2>/dev/null || true
+
+          echo ""
+          echo "=== Ingress ==="
+          kubectl describe ingress -n test-ns-feast-rest 2>/dev/null || true
+
+          echo ""
+          echo "=== FeatureStore CRs ==="
+          kubectl get feast -n test-ns-feast-rest -o wide 2>/dev/null || true
+
+          echo ""
+          echo "=== Non-normal events ==="
+          kubectl get events -n test-ns-feast-rest --sort-by=.lastTimestamp --field-selector=type!=Normal 2>/dev/null || true
+
+          echo ""
+          echo "=== Pod logs (last 80 lines each) ==="
+          for pod in $(kubectl get pods -n test-ns-feast-rest --no-headers -o custom-columns=:metadata.name 2>/dev/null); do
+            echo "--- Logs: $pod ---"
+            kubectl logs "$pod" -n test-ns-feast-rest --tail=80 --all-containers 2>/dev/null || true
+          done
+
+          echo ""
+          echo "=== Ingress controller logs ==="
+          kubectl logs -n ingress-nginx -l app.kubernetes.io/component=controller --tail=40 2>/dev/null || true
+
+          echo ""
+          echo "=== Feast operator logs ==="
+          kubectl logs -n feast-operator-system -l control-plane=controller-manager --tail=40 2>/dev/null || true
+
+          echo ""
+          echo "=== Node status ==="
           kubectl describe nodes
 
       - name: Clean up

sdk/python/feast/infra/registry/sql.py

@@ -23,6 +23,7 @@
     update,
 )
 from sqlalchemy.engine import Engine
+from sqlalchemy.exc import IntegrityError
 
 from feast import utils
 from feast.base_feature_view import BaseFeatureView
@@ -1028,8 +1029,15 @@ def _maybe_init_project_metadata(self, project):
                     "last_updated_timestamp": update_time,
                     "project_id": project,
                 }
-                insert_stmt = insert(feast_metadata).values(values)
-                conn.execute(insert_stmt)
+                try:
+                    with conn.begin_nested():
+                        conn.execute(insert(feast_metadata).values(values))
+                except IntegrityError:
+                    logger.info(
+                        "Project metadata for %s already initialized by "
+                        "another process.",
+                        project,
+                    )
 
     def _delete_object(
         self,

sdk/python/tests/integration/rest_api/conftest.py

@@ -13,6 +13,7 @@
     create_route,
     delete_namespace,
     deploy_and_validate_pod,
+    dump_kubernetes_diagnostics,
     execPodCommand,
     get_pod_name_by_prefix,
     run_kubectl_apply_with_sed,
@@ -38,7 +39,12 @@ def get(self, endpoint, params=None):
         return requests.get(url, params=params, verify=False)
 
 
-def _wait_for_http_ready(route_url: str, timeout: int = 180, interval: int = 5) -> None:
+def _wait_for_http_ready(
+    route_url: str,
+    namespace: str = "test-ns-feast-rest",
+    timeout: int = 300,
+    interval: int = 5,
+) -> None:
     """
     Poll the HTTP endpoint until it returns a non-502 response.
 
@@ -49,6 +55,8 @@ def _wait_for_http_ready(route_url: str, timeout: int = 180, interval: int = 5)
     health_url = f"{route_url}/api/v1/projects"
     deadline = time.time() + timeout
     last_status = None
+    last_body = None
+    consecutive_errors = 0
 
     print(
         f"\n Waiting for HTTP endpoint to become ready (timeout={timeout}s): {health_url}"
@@ -58,18 +66,37 @@ def _wait_for_http_ready(route_url: str, timeout: int = 180, interval: int = 5)
         try:
             resp = requests.get(health_url, timeout=10, verify=False)
             last_status = resp.status_code
+            last_body = resp.text[:500] if resp.text else ""
             if resp.status_code != 502:
                 print(f" HTTP endpoint is ready (status={resp.status_code})")
                 return
+            consecutive_errors += 1
             print(
                 f" HTTP endpoint returned {resp.status_code}, retrying in {interval}s..."
             )
+            if consecutive_errors % 6 == 0:
+                elapsed = timeout - (deadline - time.time())
+                print(
+                    f" Still waiting after ~{int(elapsed)}s. "
+                    f"Response body: {last_body[:200]}"
+                )
         except requests.exceptions.RequestException as exc:
             last_status = str(exc)
+            last_body = None
+            consecutive_errors += 1
             print(f" HTTP request failed ({exc}), retrying in {interval}s...")
 
         time.sleep(interval)
 
+    print(
+        f"\n HTTP endpoint did not become ready within {timeout}s. "
+        f"Last status: {last_status}"
+    )
+    if last_body:
+        print(f" Last response body: {last_body[:500]}")
+
+    dump_kubernetes_diagnostics(namespace)
+
     raise RuntimeError(
         f"HTTP endpoint {health_url} did not become ready within {timeout}s "
         f"(last status: {last_status})"
@@ -184,7 +211,7 @@ def feast_rest_client():
         # Wait for the HTTP endpoint to become ready before running tests.
         # Pod/CR readiness does not guarantee the backend is serving traffic;
         # the ingress may return 502 while the Feast server is still starting.
-        _wait_for_http_ready(route_url)
+        _wait_for_http_ready(route_url, namespace=namespace)
 
         print(f"\n Connected to Feast REST at: {route_url}")
         yield FeastRestClient(route_url)

sdk/python/tests/integration/rest_api/support.py

@@ -4,10 +4,14 @@
 
 from kubernetes import client
 
+DEFAULT_EXEC_TIMEOUT = 300
 
-def run_command(cmd, cwd=None, check=True):
+
+def run_command(cmd, cwd=None, check=True, timeout=None):
     print(f"Running command: {' '.join(cmd)}")
-    result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, check=True)
+    result = subprocess.run(
+        cmd, cwd=cwd, capture_output=True, text=True, check=True, timeout=timeout
+    )
     if check and result.returncode != 0:
         print(result.stdout)
         print(result.stderr)
@@ -30,14 +34,23 @@ def delete_namespace(api_instance, namespace_name):
     api_instance.delete_namespace(namespace_name)
 
 
-def run_kubectl_command(args):
+def run_kubectl_command(args, timeout=None):
     try:
         result = subprocess.run(
-            ["kubectl"] + args, capture_output=True, text=True, check=True
+            ["kubectl"] + args,
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=timeout,
         )
         return result.stdout.strip()
+    except subprocess.TimeoutExpired:
+        print(f"TIMEOUT executing 'kubectl {' '.join(args)}' after {timeout}s")
+        return None
     except subprocess.CalledProcessError as e:
         print(f"Error executing 'kubectl {' '.join(args)}': {e}")
+        if e.stderr:
+            print(f"  stderr: {e.stderr.strip()}")
         return None
 
 
@@ -236,8 +249,116 @@ def applyFeastProject(namespace, feast_project):
     return apply_output
 
 
-def execPodCommand(namespace, podName, command_args):
+def execPodCommand(namespace, podName, command_args, timeout=DEFAULT_EXEC_TIMEOUT):
     apply_args = ["exec", podName, "-n", namespace, "--"] + command_args
-    apply_output = run_kubectl_command(apply_args)
+    apply_output = run_kubectl_command(apply_args, timeout=timeout)
     print("Output of args apply:\n", apply_output)
     return apply_output
+
+
+def _kubectl_print(args, timeout=30):
+    """Run a kubectl command and print its output (for diagnostics)."""
+    output = run_kubectl_command(args, timeout=timeout)
+    if output:
+        print(output)
+    return output
+
+
+def dump_kubernetes_diagnostics(namespace):
+    """Dump diagnostic info for debugging infrastructure failures."""
+    separator = "=" * 60
+    print(f"\n{separator}")
+    print(f"  KUBERNETES DIAGNOSTICS FOR NAMESPACE: {namespace}")
+    print(f"{separator}\n")
+
+    print("--- Pod status ---")
+    _kubectl_print(["get", "pods", "-n", namespace, "-o", "wide"])
+
+    print("\n--- Pod descriptions (non-Running) ---")
+    pods_output = run_kubectl_command(
+        [
+            "get",
+            "pods",
+            "-n",
+            namespace,
+            "--no-headers",
+            "-o",
+            "custom-columns=NAME:.metadata.name,STATUS:.status.phase",
+        ],
+        timeout=30,
+    )
+    if pods_output:
+        for line in pods_output.splitlines():
+            parts = line.split()
+            if len(parts) == 2 and parts[1] != "Running":
+                print(f"\n  Describing non-running pod: {parts[0]}")
+                _kubectl_print(["describe", "pod", parts[0], "-n", namespace])
+
+    print("\n--- Services and endpoints ---")
+    _kubectl_print(["get", "svc", "-n", namespace])
+    _kubectl_print(["get", "endpoints", "-n", namespace])
+
+    print("\n--- Ingress ---")
+    _kubectl_print(["get", "ingress", "-n", namespace, "-o", "wide"])
+    _kubectl_print(["describe", "ingress", "-n", namespace])
+
+    print("\n--- FeatureStore CRs ---")
+    _kubectl_print(["get", "feast", "-n", namespace, "-o", "wide"])
+
+    print("\n--- Warning/Error events ---")
+    _kubectl_print(
+        [
+            "get",
+            "events",
+            "-n",
+            namespace,
+            "--sort-by=.lastTimestamp",
+            "--field-selector=type!=Normal",
+        ]
+    )
+
+    print("\n--- Pod logs (last 50 lines each) ---")
+    pods_names = run_kubectl_command(
+        [
+            "get",
+            "pods",
+            "-n",
+            namespace,
+            "--no-headers",
+            "-o",
+            "custom-columns=:metadata.name",
+        ],
+        timeout=30,
+    )
+    if pods_names:
+        for pod_name in pods_names.splitlines():
+            pod_name = pod_name.strip()
+            if not pod_name:
+                continue
+            print(f"\n  --- Logs for pod: {pod_name} ---")
+            _kubectl_print(
+                [
+                    "logs",
+                    pod_name,
+                    "-n",
+                    namespace,
+                    "--tail=50",
+                    "--all-containers",
+                ]
+            )
+
+    print("\n--- Ingress controller logs (last 30 lines) ---")
+    _kubectl_print(
+        [
+            "logs",
+            "-n",
+            "ingress-nginx",
+            "-l",
+            "app.kubernetes.io/component=controller",
+            "--tail=30",
+        ]
+    )
+
+    print(f"\n{separator}")
+    print("  END DIAGNOSTICS")
+    print(f"{separator}\n")