Merge pull request #2175 from NVIDIA/dcgm-probes

[dcgm][dcgm-exporter] add liveness and readiness probes

Author: tariq1890

assets/state-dcgm-exporter/0800_daemonset.yaml

@@ -52,6 +52,17 @@ spec:
         ports:
         - name: "metrics"
           containerPort: 9400
+        livenessProbe:
+          httpGet:
+            port: 9400
+            path: /health
+          initialDelaySeconds: 45
+          periodSeconds: 5
+        readinessProbe:
+          httpGet:
+            port: 9400
+            path: /health
+          initialDelaySeconds: 45
         volumeMounts:
         - name: "pod-gpu-resources"
           readOnly: true

assets/state-dcgm/0400_dcgm.yml

@@ -43,6 +43,14 @@ spec:
         ports:
         - name: "dcgm"
           containerPort: 5555
+        livenessProbe:
+          tcpSocket:
+            port: 5555
+          initialDelaySeconds: 15
+        readinessProbe:
+          tcpSocket:
+            port: 5555
+          initialDelaySeconds: 15
       volumes:
         - name: run-nvidia
           hostPath: