Skip to content

Modify output data for ease of use in visualisations #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lwr20 merged 9 commits into from
Feb 10, 2026
Merged

Modify output data for ease of use in visualisations #37

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
762c84d
Add optional junit output
lwr20 Feb 6, 2026
32e3415
Add RELEASE_STREAM to results
lwr20 Feb 6, 2026
e7519d9
Fixes
lwr20 Feb 6, 2026
f0ae3cb
Better error handling
lwr20 Feb 6, 2026
4572d39
Add junit files to gitignore
lwr20 Feb 6, 2026
9587945
Add release stream (if set) to test name
lwr20 Feb 6, 2026
8f93a79
Don't error on tcpdump timeout
lwr20 Feb 6, 2026
73ac354
review markups
lwr20 Feb 6, 2026
e8dac76
Add kibana dashboard
lwr20 Feb 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,6 @@ kubeconfig*
results.json
TODO.md
venv
run_output*.log
results*
junit*.xml
107 changes: 87 additions & 20 deletions cmd/benchmark.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"path"
"runtime"
"strconv"
"time"

log "github.com/sirupsen/logrus"
ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
Expand All @@ -32,6 +33,7 @@ import (
"github.com/projectcalico/tiger-bench/pkg/dnsperf"
"github.com/projectcalico/tiger-bench/pkg/elasticsearch"
"github.com/projectcalico/tiger-bench/pkg/iperf"
"github.com/projectcalico/tiger-bench/pkg/junit"
"github.com/projectcalico/tiger-bench/pkg/policy"
"github.com/projectcalico/tiger-bench/pkg/qperf"
"github.com/projectcalico/tiger-bench/pkg/results"
Expand Down Expand Up @@ -84,33 +86,56 @@ func main() {
}

var benchmarkResults []results.Result
startTime := time.Now()
log.Debug("Starting tests")
log.Debug("Number of TestConfigs: ", len(cfg.TestConfigs))
for _, testConfig := range cfg.TestConfigs {
err = cluster.ConfigureCluster(ctx, cfg, clients, *testConfig)
// Clean up any leftover resources from previous runs
cleanupNamespace(ctx, clients, testConfig)

thisResult := results.Result{}
thisResult.Config = *testConfig
thisResult.ClusterDetails, _ = cluster.GetClusterDetails(ctx, clients)
thisResult.Status = "failed"

if err != nil {
log.WithError(err).Fatal("failed to configure cluster")
log.WithError(err).Error("failed to configure cluster")
Comment thread
lwr20 marked this conversation as resolved.
thisResult.Error = fmt.Sprintf("failed to configure cluster: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
defer cleanupNamespace(ctx, clients, testConfig)
// update cluster details after reconfig
thisResult.ClusterDetails, _ = cluster.GetClusterDetails(ctx, clients)

err = cluster.SetupStandingConfig(ctx, clients, *testConfig, testConfig.TestNamespace, cfg.WebServerImage)
if err != nil {
log.WithError(err).Fatal("failed to setup standing config on cluster")
log.WithError(err).Error("failed to setup standing config on cluster")
thisResult.Error = fmt.Sprintf("failed to setup standing config: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
thisResult := results.Result{}
thisResult.Config = *testConfig
switch testConfig.TestKind {
case config.TestKindNone:
// No test to run
case config.TestKindIperf:
var iperfResults []*iperf.Results
err = policy.CreateTestPolicy(ctx, clients, testPolicyName, testConfig.TestNamespace, []int{testConfig.Perf.TestPort})
if err != nil {
log.WithError(err).Fatal("failed to create iperf test policy")
log.WithError(err).Error("failed to create iperf test policy")
thisResult.Error = fmt.Sprintf("failed to create iperf test policy: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
err = iperf.DeployIperfPods(ctx, clients, testConfig.TestNamespace, testConfig.HostNetwork, cfg.PerfImage, testConfig.Perf.TestPort)
if err != nil {
log.WithError(err).Fatal("failed to deploy iperf pods")
log.WithError(err).Error("failed to deploy iperf pods")
thisResult.Error = fmt.Sprintf("failed to deploy iperf pods: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
log.Info("Running iperf tests, Iterations=", testConfig.Iterations)
for j := 0; j < testConfig.Iterations; j++ {
Expand All @@ -130,11 +155,19 @@ func main() {
var qperfResults []*qperf.Results
err = policy.CreateTestPolicy(ctx, clients, testPolicyName, testConfig.TestNamespace, []int{testConfig.Perf.ControlPort, testConfig.Perf.TestPort})
if err != nil {
log.WithError(err).Fatal("failed to create qperf test policy")
log.WithError(err).Error("failed to create qperf test policy")
thisResult.Error = fmt.Sprintf("failed to create qperf test policy: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
err = qperf.DeployQperfPods(ctx, clients, testConfig.TestNamespace, testConfig.HostNetwork, cfg.PerfImage, testConfig.Perf.ControlPort, testConfig.Perf.TestPort)
if err != nil {
log.WithError(err).Fatal("failed to deploy qperf pods")
log.WithError(err).Error("failed to deploy qperf pods")
thisResult.Error = fmt.Sprintf("failed to deploy qperf pods: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
for j := 0; j < testConfig.Iterations; j++ {
log.Debug("entering qperf loop")
Expand All @@ -155,11 +188,19 @@ func main() {
if testConfig.DNSPerf.TestDNSPolicy {
mypol, err := dnsperf.MakeDNSPolicy(testConfig.TestNamespace, testPolicyName, testConfig.DNSPerf.NumDomains, testConfig.DNSPerf.TargetURL)
if err != nil {
log.WithError(err).Fatal("failed to create dnsperf DNS policy object")
log.WithError(err).Error("failed to create dnsperf DNS policy object")
thisResult.Error = fmt.Sprintf("failed to create dnsperf DNS policy object: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
_, err = policy.GetOrCreateDNSPolicy(ctx, clients, mypol)
if err != nil {
log.WithError(err).Fatal("failed to create dnsperf DNS policy")
log.WithError(err).Error("failed to create dnsperf DNS policy")
thisResult.Error = fmt.Sprintf("failed to create dnsperf DNS policy: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
}
thisResult.DNSPerf, err = dnsperf.RunDNSPerfTests(ctx, clients, testConfig, cfg.WebServerImage, cfg.PerfImage)
Expand All @@ -172,7 +213,11 @@ func main() {
// Apply standing policy (that applies to both server and test pods)
err := policy.CreateTestPolicy(ctx, clients, testPolicyName, testConfig.TestNamespace, []int{8080})
if err != nil {
log.WithError(err).Fatal("failed to create ttfr test policy")
log.WithError(err).Error("failed to create ttfr test policy")
thisResult.Error = fmt.Sprintf("failed to create ttfr test policy: %v", err)
benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
log.Info("Running ttfr tests, Iterations=", testConfig.Iterations)
for j := 0; j < testConfig.Iterations; j++ {
Expand All @@ -190,19 +235,23 @@ func main() {
}
}
default:
log.Fatal("test type unknown")
log.Error("test type unknown")
thisResult.Error = fmt.Sprintf("unknown test type: %s", testConfig.TestKind)

benchmarkResults = append(benchmarkResults, thisResult)
cleanupNamespace(ctx, clients, testConfig)
continue
}
if thisResult.Error == "" {
thisResult.Status = "success"
}
// If we set the CPU limit, unset it again.
if testConfig.CalicoNodeCPULimit != "" {
err = cluster.SetCalicoNodeCPULimit(ctx, clients, "0")
if err != nil {
log.WithError(err).Fatal("failed to reset calico node CPU limit")
log.WithError(err).Error("failed to reset calico node CPU limit")
}
}
thisResult.ClusterDetails, err = cluster.GetClusterDetails(ctx, clients)
if err != nil {
log.WithError(err).Error("error getting cluster details")
}
log.Debugf("Result: %+v", thisResult)
err = elasticsearch.UploadResult(cfg, thisResult, false)
if err != nil {
Expand All @@ -212,7 +261,25 @@ func main() {
log.Infof("Results: %+v", benchmarkResults)
err = writeResultToFile(cfg.ResultsFile, benchmarkResults)
if err != nil {
log.WithError(err).Fatal("failed to write results to file")
log.WithError(err).Error("failed to write results to file")
}

// Clean up after test completes
cleanupNamespace(ctx, clients, testConfig)
}

// Generate and write JUnit report after all tests complete
if cfg.JUnitReportFile != "" {
junitReport, err := junit.GenerateJUnitReport(benchmarkResults, startTime)
if err != nil {
log.WithError(err).Error("failed to generate JUnit report")
} else {
err = junit.WriteJUnitReport(cfg.JUnitReportFile, junitReport)
if err != nil {
log.WithError(err).Error("failed to write JUnit report")
} else {
log.Infof("JUnit report written to %s", cfg.JUnitReportFile)
}
}
}
}
Expand Down
8 changes: 8 additions & 0 deletions pkg/cluster/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"fmt"
"net"
"os"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -318,6 +319,7 @@ type Details struct {
K8SVersion string
CRIVersion string
CNIOption string
ReleaseStream string
}

// GetClusterDetails gets details about the cluster
Expand Down Expand Up @@ -493,6 +495,12 @@ func GetClusterDetails(ctx context.Context, clients config.Clients) (Details, er

details.CNIOption = installation.Spec.CNI.Type.String()

// This grabs the RELEASE_STREAM environment variable if it exists, overnight runs use to describe the release chosen for the cluster.
// By adding it to the results, we can easily filter and compare overnight runs by release stream in visualisations like Kibana.
if releaseStream := os.Getenv("RELEASE_STREAM"); releaseStream != "" {
details.ReleaseStream = releaseStream
}

if strings.Contains(testnode.Name, "ip") && strings.Contains(testnode.Name, "compute.internal") {
details.Cloud = "aws"
} else { // TODO: Add more cloud providers
Expand Down
1 change: 1 addition & 0 deletions pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ type Config struct {
ESAPIKey string `envconfig:"ELASTICSEARCH_KEY" default:""`
TestTimeout int `default:"600"`
ResultsFile string `envconfig:"RESULTS_FILE" default:"/results/results.json"`
JUnitReportFile string `envconfig:"JUNIT_REPORT_FILE" default:""`
CalicoVersion string `default:""`
ProxyAddress string `envconfig:"HTTP_PROXY" default:""`
TestConfigFile string `envconfig:"TESTCONFIGFILE" required:"true"`
Expand Down
12 changes: 9 additions & 3 deletions pkg/dnsperf/dnsperf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,9 @@ func RunDNSPerfTests(ctx context.Context, clients config.Clients, testConfig *co
}
go func(tcpdumpPod corev1.Pod, testPod corev1.Pod) {
if e := runTCPDump(testctx, clients, &tcpdumpPod, testPod, targetPort, testConfig.Duration+60); e != nil {
log.WithError(e).Error("failed to run tcpdump")
if !strings.Contains(e.Error(), "context deadline exceeded") {
log.WithError(e).Error("failed to run tcpdump")
}
}
}(tcpdumpPod, testPod)
}
Expand Down Expand Up @@ -438,8 +440,12 @@ func runTCPDump(ctx context.Context, clients config.Clients, pod *corev1.Pod, te
var out string
out, _, err = utils.ExecCommandInPod(ctx, pod, cmd, timeout+30)
if err != nil {
log.WithError(err).Error("failed to run tcpdump command")
return err
if strings.Contains(err.Error(), "context deadline exceeded") {
log.WithError(err).Debug("tcpdump context deadline exceeded (expected - tcpdump runs for full test duration)")
} else {
log.WithError(err).Error("failed to run tcpdump command")
return err
}
}
log.Infof("stdout=%s", out)

Expand Down
4 changes: 2 additions & 2 deletions pkg/elasticsearch/elasticsearch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func TestCreateESDoc(t *testing.T) {
DNSPerf: &dnsperf,
}

expectedDoc := `{"config":{"TestKind":"dnsperf","Encap":"none","Dataplane":"bpf","NumPolicies":30,"NumIdlePolicies":0,"NumServices":20,"NumPods":10,"HostNetwork":false,"TestNamespace":"","Iterations":0,"Duration":0,"DNSPerf":{"NumDomains":0,"Mode":"Inline","RunStress":false,"TestDNSPolicy":false,"TargetURL":""},"Perf":null,"TTFRConfig":null,"CalicoNodeCPULimit":"40m","LeaveStandingConfig":false},"ClusterDetails":{"Cloud":"","Provisioner":"","NodeType":"","NodeOS":"","NodeKernel":"","NodeArch":"","NumNodes":0,"Dataplane":"","IPFamily":"","Encapsulation":"","WireguardEnabled":false,"Product":"","CalicoVersion":"","K8SVersion":"","CRIVersion":"","CNIOption":""},"dnsperf":{"LookupTime":{"min":0.001,"max":0.013641,"avg":0.004745,"P50":0.00364,"P75":0.004745,"P90":0.006914,"P99":0.013641,"datapoints":17},"ConnectTime":{"min":0.000239,"max":0.003917,"avg":0.000542,"P50":0.000439,"P75":0.000542,"P90":0.00088,"P99":0.003917,"datapoints":17},"DuplicateSYN":101,"DuplicateSYNACK":0,"FailedCurls":16,"SuccessfulCurls":1326}}`
expectedDoc := `{"config":{"TestKind":"dnsperf","Encap":"none","Dataplane":"bpf","NumPolicies":30,"NumIdlePolicies":0,"NumServices":20,"NumPods":10,"HostNetwork":false,"TestNamespace":"","Iterations":0,"Duration":0,"DNSPerf":{"NumDomains":0,"Mode":"Inline","RunStress":false,"TestDNSPolicy":false,"TargetURL":""},"Perf":null,"TTFRConfig":null,"CalicoNodeCPULimit":"40m","LeaveStandingConfig":false},"ClusterDetails":{"Cloud":"","Provisioner":"","NodeType":"","NodeOS":"","NodeKernel":"","NodeArch":"","NumNodes":0,"Dataplane":"","IPFamily":"","Encapsulation":"","WireguardEnabled":false,"Product":"","CalicoVersion":"","K8SVersion":"","CRIVersion":"","CNIOption":"","ReleaseStream":""},"dnsperf":{"LookupTime":{"min":0.001,"max":0.013641,"avg":0.004745,"P50":0.00364,"P75":0.004745,"P90":0.006914,"P99":0.013641,"datapoints":17},"ConnectTime":{"min":0.000239,"max":0.003917,"avg":0.000542,"P50":0.000439,"P75":0.000542,"P90":0.00088,"P99":0.003917,"datapoints":17},"DuplicateSYN":101,"DuplicateSYNACK":0,"FailedCurls":16,"SuccessfulCurls":1326}}`

doc, err := createESDoc(result)
require.NoError(t, err)
Expand All @@ -84,7 +84,7 @@ func TestCreateESDocBlank(t *testing.T) {
DNSPerf: &dnsperf,
}

expectedDoc := `{"config":{"TestKind":"","Encap":"","Dataplane":"","NumPolicies":0,"NumIdlePolicies":0,"NumServices":0,"NumPods":0,"HostNetwork":false,"TestNamespace":"","Iterations":0,"Duration":0,"DNSPerf":null,"Perf":null,"TTFRConfig":null,"CalicoNodeCPULimit":"","LeaveStandingConfig":false},"ClusterDetails":{"Cloud":"","Provisioner":"","NodeType":"","NodeOS":"","NodeKernel":"","NodeArch":"","NumNodes":0,"Dataplane":"","IPFamily":"","Encapsulation":"","WireguardEnabled":false,"Product":"","CalicoVersion":"","K8SVersion":"","CRIVersion":"","CNIOption":""},"dnsperf":{"LookupTime":{},"ConnectTime":{},"DuplicateSYN":0,"DuplicateSYNACK":0,"FailedCurls":0,"SuccessfulCurls":0}}`
expectedDoc := `{"config":{"TestKind":"","Encap":"","Dataplane":"","NumPolicies":0,"NumIdlePolicies":0,"NumServices":0,"NumPods":0,"HostNetwork":false,"TestNamespace":"","Iterations":0,"Duration":0,"DNSPerf":null,"Perf":null,"TTFRConfig":null,"CalicoNodeCPULimit":"","LeaveStandingConfig":false},"ClusterDetails":{"Cloud":"","Provisioner":"","NodeType":"","NodeOS":"","NodeKernel":"","NodeArch":"","NumNodes":0,"Dataplane":"","IPFamily":"","Encapsulation":"","WireguardEnabled":false,"Product":"","CalicoVersion":"","K8SVersion":"","CRIVersion":"","CNIOption":"","ReleaseStream":""},"dnsperf":{"LookupTime":{},"ConnectTime":{},"DuplicateSYN":0,"DuplicateSYNACK":0,"FailedCurls":0,"SuccessfulCurls":0}}`
doc, err := createESDoc(result)
require.NoError(t, err)

Expand Down
Loading
Loading