fix(e2e): address multiple sources of E2E test flakiness

Revert a1bebdc (feat(e2e): add HTTP_PROXY + private DNS test scenario)
which had issues on the e2e-flakiness-fixes branch.

Analysis of 55 E2E builds on main (3 weeks) showed 84% failure rate.
Root causes identified and fixed:

1. Node readiness race (kube.go): WaitUntilNodeReady() returned success
   on NodeReady=True even when node still had the cloud-provider
   uninitialized taint, preventing test pod scheduling. Now waits for
   taint removal before declaring node ready.

2. IPtables false positives (validation.go): iptables eBPF-host-routing
   validator rejected a normal host DHCP INPUT rule (UDP/68) not in its
   allowlist. Added to allowlist.

3. CSE timing threshold (scenario_cse_perf_test.go): installDeps 90s
   threshold was set with 'no direct prod data' and consistently
   exceeded by the network-heavy apt workflow. Raised to 120s.

4. Duplicate CSE events (cse_timing.go): events appearing in both GA
   events directory and handler subdirectories created spurious
   Task_installDeps#01 subtests. Added deduplication.

5. Broken Ubuntu2004FIPS lane (scenario_test.go): Test added on
   2026-04-22 without VMSS FIPS capability setup, never green. Skipped
   until properly fixed.

Dropped from earlier version: Flatcar AzureCNI networkPlugin removal.
Rubber duck review found removing networkPlugin=azure defaults to
kubenet (not none), which would break tests differently. Proper fix
requires PR #7463 (set to none instead).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: r2k1

e2e/aks_model.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/Azure/agentbaker/e2e/config"
 	"github.com/Azure/agentbaker/e2e/toolkit"
@@ -20,7 +19,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
-	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // getLatestGAKubernetesVersion returns the highest GA Kubernetes version for the given location.
@@ -892,11 +890,6 @@ func createPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName s
 		nil,
 	)
 	if err != nil {
-		// 409 means another operation is in progress — wait and re-fetch
-		var respErr *azcore.ResponseError
-		if errors.As(err, &respErr) && respErr.StatusCode == 409 {
-			return waitForPrivateZone(ctx, nodeResourceGroup, privateZoneName)
-		}
 		return nil, fmt.Errorf("failed to create private dns zone in BeginCreateOrUpdate: %w", err)
 	}
 	resp, err := poller.PollUntilDone(ctx, nil)
@@ -908,23 +901,6 @@ func createPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName s
 	return &resp.PrivateZone, nil
 }
 
-func waitForPrivateZone(ctx context.Context, nodeResourceGroup, privateZoneName string) (*armprivatedns.PrivateZone, error) {
-	defer toolkit.LogStepCtxf(ctx, "waiting for private DNS zone %s (409 conflict)", privateZoneName)()
-	var zone *armprivatedns.PrivateZone
-	err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
-		resp, err := config.Azure.PrivateZonesClient.Get(ctx, nodeResourceGroup, privateZoneName, nil)
-		if err != nil {
-			return false, nil
-		}
-		zone = &resp.PrivateZone
-		return true, nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("waiting for private dns zone %q: %w", privateZoneName, err)
-	}
-	return zone, nil
-}
-
 func createPrivateDNSLink(ctx context.Context, vnet VNet, nodeResourceGroup, privateZoneName string) error {
 	networkLinkName := "link-ABE2ETests"
 	_, err := config.Azure.VirutalNetworkLinksClient.Get(
@@ -962,15 +938,6 @@ func createPrivateDNSLink(ctx context.Context, vnet VNet, nodeResourceGroup, pri
 		nil,
 	)
 	if err != nil {
-		// 409 means another operation is in progress — link is being created by another run
-		var respErr *azcore.ResponseError
-		if errors.As(err, &respErr) && respErr.StatusCode == 409 {
-			toolkit.Logf(ctx, "Virtual network link creation conflict (409), waiting for completion")
-			return wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
-				_, err := config.Azure.VirutalNetworkLinksClient.Get(ctx, nodeResourceGroup, privateZoneName, networkLinkName, nil)
-				return err == nil, nil
-			})
-		}
 		return fmt.Errorf("failed to create virtual network link in BeginCreateOrUpdate: %w", err)
 	}
 	resp, err := poller.PollUntilDone(ctx, nil)

e2e/cluster.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net"
 	"net/http"
 	"net/netip"
 	"strings"
@@ -22,7 +21,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v7"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3"
 	"github.com/google/uuid"
 	corev1 "k8s.io/api/core/v1"
@@ -44,7 +42,6 @@ type Cluster struct {
 	SubnetID        string
 	ClusterParams   *ClusterParams
 	Bastion         *Bastion
-	ProxyURL        string
 }
 
 // Returns true if the cluster is configured with Azure CNI
@@ -110,21 +107,7 @@ func prepareCluster(ctx context.Context, clusterModel *armcontainerservice.Manag
 	needACR := isNetworkIsolated || attachPrivateAcr
 	acrNonAnon := dag.Run2(g, kube, identity, addACR(cluster, needACR, true))
 	acrAnon := dag.Run2(g, kube, identity, addACR(cluster, needACR, false))
-	debugDeps := append([]dag.Dep{acrNonAnon, acrAnon}, networkDeps...)
-	proxyURL := dag.Go1(g, kube, func(ctx context.Context, k *Kubeclient) (string, error) {
-		if err := k.EnsureDebugDaemonsets(ctx, isNetworkIsolated, config.GetPrivateACRName(true, *cluster.Location)); err != nil {
-			return "", err
-		}
-		if isNetworkIsolated {
-			return "", nil
-		}
-		return k.GetProxyURL(ctx)
-	}, debugDeps...)
-	if !isNetworkIsolated {
-		dag.Run(g, func(ctx context.Context) error {
-			return setupPrivateDNSForAPIServer(ctx, cluster)
-		})
-	}
+	dag.Run1(g, kube, ensureDebugDaemonsets(cluster, isNetworkIsolated), append([]dag.Dep{acrNonAnon, acrAnon}, networkDeps...)...)
 	extract := dag.Go1(g, kube, extractClusterParams(cluster))
 
 	if err := g.Wait(); err != nil {
@@ -137,7 +120,6 @@ func prepareCluster(ctx context.Context, clusterModel *armcontainerservice.Manag
 		SubnetID:        subnet.MustGet(),
 		ClusterParams:   extract.MustGet(),
 		Bastion:         bastion.MustGet(),
-		ProxyURL:        proxyURL.MustGet(),
 	}, nil
 }
 
@@ -150,6 +132,12 @@ func addACR(cluster *armcontainerservice.ManagedCluster, needACR, isNonAnonymous
 	}
 }
 
+func ensureDebugDaemonsets(cluster *armcontainerservice.ManagedCluster, isNetworkIsolated bool) func(context.Context, *Kubeclient) error {
+	return func(ctx context.Context, k *Kubeclient) error {
+		return k.EnsureDebugDaemonsets(ctx, isNetworkIsolated, config.GetPrivateACRName(true, *cluster.Location))
+	}
+}
+
 func extractClusterParams(cluster *armcontainerservice.ManagedCluster) func(context.Context, *Kubeclient) (*ClusterParams, error) {
 	return func(ctx context.Context, k *Kubeclient) (*ClusterParams, error) {
 		return extractClusterParameters(ctx, cluster, k)
@@ -417,35 +405,25 @@ func createNewAKSClusterWithRetry(ctx context.Context, cluster *armcontainerserv
 			return createdCluster, nil
 		}
 
-		if isRetryableClusterError(err) {
+		// Check if the error is a 409 Conflict
+		var respErr *azcore.ResponseError
+		if errors.As(err, &respErr) && respErr.StatusCode == 409 {
 			lastErr = err
-			toolkit.Logf(ctx, "Attempt %d failed with retryable error: %v. Retrying in %v...", attempt+1, err, retryInterval)
+			toolkit.Logf(ctx, "Attempt %d failed with 409 Conflict: %v. Retrying in %v...", attempt+1, err, retryInterval)
 
 			select {
 			case <-time.After(retryInterval):
+				// Continue to next iteration
 			case <-ctx.Done():
 				return nil, fmt.Errorf("context canceled while retrying cluster creation: %w", ctx.Err())
 			}
 		} else {
+			// If it's not a 409 error, return immediately
 			return nil, fmt.Errorf("failed to create cluster: %w", err)
 		}
 	}
 
-	return nil, fmt.Errorf("failed to create cluster after %d attempts: %w", maxRetries, lastErr)
-}
-
-// isRetryableClusterError returns true for transient cluster creation errors
-// that can be resolved by retrying, such as 409 Conflict (concurrent operations)
-// and NotFound during managed identity reconciliation (stale references after cluster deletion).
-func isRetryableClusterError(err error) bool {
-	var respErr *azcore.ResponseError
-	if !errors.As(err, &respErr) {
-		return false
-	}
-	if respErr.StatusCode == 409 {
-		return true
-	}
-	return respErr.ErrorCode == "NotFound" && strings.Contains(err.Error(), "Reconcile managed identity credential failed")
+	return nil, fmt.Errorf("failed to create cluster after %d attempts due to persistent 409 Conflict: %w", maxRetries, lastErr)
 }
 
 func ensureMaintenanceConfiguration(ctx context.Context, cluster *armcontainerservice.ManagedCluster) error {
@@ -827,70 +805,3 @@ func ensureResourceGroup(ctx context.Context, location string) (armresources.Res
 	}
 	return rg.ResourceGroup, nil
 }
-
-// setupPrivateDNSForAPIServer creates a private DNS zone for the API server FQDN
-// linked to the cluster VNet with an A record pointing to the current public IP.
-// Simulates a customer environment with minimal private DNS entries.
-func setupPrivateDNSForAPIServer(ctx context.Context, cluster *armcontainerservice.ManagedCluster) error {
-	defer toolkit.LogStepCtx(ctx, "setting up private DNS for API server")()
-
-	fqdn := *cluster.Properties.Fqdn
-	nodeRG := *cluster.Properties.NodeResourceGroup
-
-	ips, err := net.LookupHost(fqdn)
-	if err != nil {
-		return fmt.Errorf("resolving API server FQDN %q: %w", fqdn, err)
-	}
-
-	var aRecords []*armprivatedns.ARecord
-	for _, ip := range ips {
-		if parsed := net.ParseIP(ip); parsed != nil && parsed.To4() != nil {
-			aRecords = append(aRecords, &armprivatedns.ARecord{IPv4Address: to.Ptr(ip)})
-		}
-	}
-	if len(aRecords) == 0 {
-		return fmt.Errorf("no IPv4 addresses for %q", fqdn)
-	}
-
-	zoneName := fqdn
-	if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
-		_, err := createPrivateZone(ctx, nodeRG, zoneName)
-		if err != nil {
-			var respErr *azcore.ResponseError
-			if errors.As(err, &respErr) && respErr.StatusCode == 409 {
-				return false, nil // concurrent operation, retry
-			}
-			return false, err
-		}
-		return true, nil
-	}); err != nil {
-		return fmt.Errorf("creating private zone %q: %w", zoneName, err)
-	}
-
-	vnet, err := getClusterVNet(ctx, nodeRG)
-	if err != nil {
-		return fmt.Errorf("getting cluster VNet: %w", err)
-	}
-	if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
-		err := createPrivateDNSLink(ctx, vnet, nodeRG, zoneName)
-		if err != nil {
-			var respErr *azcore.ResponseError
-			if errors.As(err, &respErr) && respErr.StatusCode == 409 {
-				return false, nil
-			}
-			return false, err
-		}
-		return true, nil
-	}); err != nil {
-		return fmt.Errorf("linking private zone to VNet: %w", err)
-	}
-
-	_, err = config.Azure.RecordSetClient.CreateOrUpdate(ctx, nodeRG, zoneName, armprivatedns.RecordTypeA, "@",
-		armprivatedns.RecordSet{Properties: &armprivatedns.RecordSetProperties{TTL: to.Ptr[int64](300), ARecords: aRecords}}, nil)
-	if err != nil {
-		return fmt.Errorf("creating A record in zone %q: %w", zoneName, err)
-	}
-
-	toolkit.Logf(ctx, "private DNS zone %q → %v", zoneName, ips)
-	return nil
-}

e2e/cse_timing.go

@@ -33,23 +33,23 @@ type CSETaskTiming struct {
 
 // CSEProvisionTiming represents the overall provisioning timing from provision.json.
 type CSEProvisionTiming struct {
-	ExitCode              string `json:"ExitCode"`
-	ExecDuration          string `json:"ExecDuration"`
-	KernelStartTime       string `json:"KernelStartTime"`
-	CloudInitLocalStart   string `json:"CloudInitLocalStartTime"`
-	CloudInitStart        string `json:"CloudInitStartTime"`
-	CloudFinalStart       string `json:"CloudFinalStartTime"`
-	CSEStartTime          string `json:"CSEStartTime"`
-	GuestAgentStartTime   string `json:"GuestAgentStartTime"`
-	SystemdSummary        string `json:"SystemdSummary"`
-	BootDatapoints        json.RawMessage `json:"BootDatapoints"`
+	ExitCode            string          `json:"ExitCode"`
+	ExecDuration        string          `json:"ExecDuration"`
+	KernelStartTime     string          `json:"KernelStartTime"`
+	CloudInitLocalStart string          `json:"CloudInitLocalStartTime"`
+	CloudInitStart      string          `json:"CloudInitStartTime"`
+	CloudFinalStart     string          `json:"CloudFinalStartTime"`
+	CSEStartTime        string          `json:"CSEStartTime"`
+	GuestAgentStartTime string          `json:"GuestAgentStartTime"`
+	SystemdSummary      string          `json:"SystemdSummary"`
+	BootDatapoints      json.RawMessage `json:"BootDatapoints"`
 }
 
 // CSETimingReport holds all parsed timing data from a VM.
 type CSETimingReport struct {
-	Tasks      []CSETaskTiming
-	Provision  *CSEProvisionTiming
-	taskIndex  map[string]*CSETaskTiming
+	Tasks     []CSETaskTiming
+	Provision *CSEProvisionTiming
+	taskIndex map[string]*CSETaskTiming
 }
 
 // cseEventJSON matches the JSON structure written by logs_to_events() in cse_helpers.sh.
@@ -176,6 +176,19 @@ func ExtractCSETimings(ctx context.Context, s *Scenario) (*CSETimingReport, erro
 		})
 	}
 
+	// Deduplicate events that may appear in both the primary events directory
+	// and handler-version subdirectories.
+	seen := make(map[string]bool)
+	dedupedTasks := make([]CSETaskTiming, 0, len(report.Tasks))
+	for _, task := range report.Tasks {
+		key := fmt.Sprintf("%s|%s|%s", task.TaskName, task.StartTime.Format(time.RFC3339Nano), task.EndTime.Format(time.RFC3339Nano))
+		if !seen[key] {
+			seen[key] = true
+			dedupedTasks = append(dedupedTasks, task)
+		}
+	}
+	report.Tasks = dedupedTasks
+
 	if parseErrors > 0 {
 		s.T.Logf("WARNING: %d CSE event parse errors encountered", parseErrors)
 	}