MTV-3613 | fix transient ConnectionTestFailed error

pkg/controller/ova/builder.go

@@ -191,6 +191,19 @@ func (r *Builder) PodSpec(provider *api.Provider, pvc *core.PersistentVolumeClai
 					},
 				},
 				SecurityContext: r.securityContext(),
+				ReadinessProbe: &core.Probe{
+					ProbeHandler: core.ProbeHandler{
+						HTTPGet: &core.HTTPGetAction{
+							Path: "/test_connection",
+							Port: intstr.FromInt32(8080),
+						},
+					},
+					InitialDelaySeconds: 5,
+					PeriodSeconds:       3,
+					TimeoutSeconds:      2,
+					SuccessThreshold:    1,
+					FailureThreshold:    3,
+				},
 				VolumeMounts: []core.VolumeMount{
 					{
 						Name:      NFSVolumeMountName,

pkg/controller/provider/controller.go

@@ -215,6 +215,9 @@ func (r Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (r
 	provider.Status.BeginStagingConditions()
 
 	defer func() {
+		// This must be called before updateProviderStatus to clean up unstaged conditions
+		provider.Status.EndStagingConditions()
+
 		if err == nil {
 			if err = r.updateProviderStatus(provider); err != nil {
 				err = liberr.Wrap(err)
@@ -280,9 +283,6 @@ func (r Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (r
 			})
 	}
 
-	// End staging conditions.
-	provider.Status.EndStagingConditions()
-
 	// Update the DB.
 	err = r.updateProvider(provider)
 	if err != nil {

pkg/controller/provider/validation.go

@@ -8,19 +8,23 @@ import (
 	"net/url"
 	"regexp"
 	"strings"
+	"time"
 
 	api "github.com/kubev2v/forklift/pkg/apis/forklift/v1beta1"
 	"github.com/kubev2v/forklift/pkg/controller/base"
+	"github.com/kubev2v/forklift/pkg/controller/ova"
 	"github.com/kubev2v/forklift/pkg/controller/provider/container"
 	vsphere "github.com/kubev2v/forklift/pkg/controller/provider/model/vsphere"
 	libcnd "github.com/kubev2v/forklift/pkg/lib/condition"
 	liberr "github.com/kubev2v/forklift/pkg/lib/error"
 	"github.com/kubev2v/forklift/pkg/lib/inventory/model"
 	libref "github.com/kubev2v/forklift/pkg/lib/ref"
 	"github.com/kubev2v/forklift/pkg/lib/util"
+	appsv1 "k8s.io/api/apps/v1"
 	core "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -42,6 +46,7 @@ const (
 	SSHReady                = "SSHReady"
 	SSHNotReady             = "SSHNotReady"
 	SMBCSIDriverNotReady    = "SMBCSIDriverNotReady"
+	WaitingForService       = "WaitingForService"
 )
 
 // CSI driver names
@@ -370,6 +375,14 @@ func (r *Reconciler) testConnection(provider *api.Provider, secret *core.Secret)
 	if provider.Status.HasBlockerCondition() {
 		return nil
 	}
+
+	// For OVA providers, check if the inventory service pod is ready before testing connection
+	if provider.Type() == api.Ova {
+		if ready := r.checkOVAServiceReady(provider); !ready {
+			return nil
+		}
+	}
+
 	rl := container.Build(nil, provider, secret)
 	status, err := rl.Test()
 	if err == nil {
@@ -420,6 +433,139 @@ func (r *Reconciler) testConnection(provider *api.Provider, secret *core.Secret)
 	return nil
 }
 
+func (r *Reconciler) checkOVAServiceReady(provider *api.Provider) bool {
+	if provider.Status.Service == nil {
+		log.Info("Waiting for OVA inventory service to be created.")
+		provider.Status.SetCondition(
+			libcnd.Condition{
+				Type:     WaitingForService,
+				Status:   True,
+				Reason:   Started,
+				Category: Advisory,
+				Message:  "Waiting for OVA inventory service to be created.",
+			})
+		return false
+	}
+
+	// Service exists, check if the Deployment has ready replicas
+	labeler := ova.Labeler{}
+	deploymentList := &appsv1.DeploymentList{}
+	err := r.List(
+		context.TODO(),
+		deploymentList,
+		&client.ListOptions{
+			LabelSelector: labels.SelectorFromSet(labeler.ProviderLabels(provider)),
+			Namespace:     Settings.Namespace,
+		})
+
+	if err != nil {
+		// If we can't list deployments, log but don't fail - allow connection test to proceed
+		log.Error(err, "Failed to list deployments for OVA provider")
+		return true
+	}
+
+	if len(deploymentList.Items) == 0 {
+		// No deployment found yet, keep waiting
+		log.Info("Waiting for OVA inventory service deployment to be created.")
+		provider.Status.SetCondition(
+			libcnd.Condition{
+				Type:     WaitingForService,
+				Status:   True,
+				Reason:   Started,
+				Category: Advisory,
+				Message:  "Waiting for OVA inventory service deployment to be created.",
+			})
+		return false
+	}
+
+	deployment := &deploymentList.Items[0]
+	if deployment.Status.ReadyReplicas == 0 {
+		// Check if pods are in error state before just waiting
+		podList := &core.PodList{}
+		err := r.List(
+			context.TODO(),
+			podList,
+			&client.ListOptions{
+				LabelSelector: labels.SelectorFromSet(labeler.ProviderLabels(provider)),
+				Namespace:     Settings.Namespace,
+			})
+
+		if err != nil {
+			// If we can't list pods, log but continue waiting
+			log.Error(err, "Failed to list pods for OVA provider")
+		} else if len(podList.Items) > 0 {
+			pod := &podList.Items[0]
+
+			// Check for container errors
+			for _, containerStatus := range pod.Status.ContainerStatuses {
+				if waiting := containerStatus.State.Waiting; waiting != nil {
+					switch waiting.Reason {
+					case "ErrImagePull", "ImagePullBackOff", "InvalidImageName":
+						provider.Status.Phase = ValidationFailed
+						provider.Status.SetCondition(
+							libcnd.Condition{
+								Type:     ConnectionTestFailed,
+								Status:   True,
+								Reason:   Tested,
+								Category: Critical,
+								Message:  fmt.Sprintf("OVA inventory service has image error: %s - %s", waiting.Reason, waiting.Message),
+							})
+						return false
+					case "CrashLoopBackOff", "CreateContainerConfigError", "CreateContainerError":
+						provider.Status.Phase = ValidationFailed
+						provider.Status.SetCondition(
+							libcnd.Condition{
+								Type:     ConnectionTestFailed,
+								Status:   True,
+								Reason:   Tested,
+								Category: Critical,
+								Message:  fmt.Sprintf("OVA inventory service failed to start: %s - %s", waiting.Reason, waiting.Message),
+							})
+						return false
+					}
+				}
+			}
+
+			// Check for pods stuck in pending state (e.g., mount failures)
+			if pod.Status.Phase == core.PodPending {
+				if time.Since(pod.CreationTimestamp.Time) > 2*time.Minute {
+					provider.Status.Phase = ValidationFailed
+					provider.Status.SetCondition(
+						libcnd.Condition{
+							Type:     ConnectionTestFailed,
+							Status:   True,
+							Reason:   Tested,
+							Category: Critical,
+							Message:  fmt.Sprintf("OVA inventory service '%s' has been pending for over 2 minutes. Check pod events for mount or configuration errors.", pod.Name),
+						})
+					return false
+				}
+			}
+		}
+
+		// No errors found, continue waiting
+		log.Info("Waiting for OVA inventory service to become ready.",
+			"deployment", deployment.Name,
+			"replicas", deployment.Status.Replicas,
+			"readyReplicas", deployment.Status.ReadyReplicas)
+		provider.Status.SetCondition(
+			libcnd.Condition{
+				Type:     WaitingForService,
+				Status:   True,
+				Reason:   Started,
+				Category: Advisory,
+				Message:  "Waiting for OVA inventory service to become ready.",
+			})
+		return false
+	}
+
+	// Deployment has ready replicas, proceed with connection test
+	log.Info("OVA inventory service pod is ready, proceeding with connection test.",
+		"deployment", deployment.Name,
+		"readyReplicas", deployment.Status.ReadyReplicas)
+	return true
+}
+
 // Validate inventory created.
 func (r *Reconciler) inventoryCreated(provider *api.Provider) error {
 	if provider.Status.HasBlockerCondition() {