feat(2.53.5): opt-in fix for export.match from config

Implements step 1 from go/gmp:matchstuck

Signed-off-by: bwplotka <bwplotka@gmail.com>

Author: bwplotka

google/config/config.go

@@ -2,6 +2,9 @@ package config
 
 // GoogleCloudConfig represents a custom Prometheus config entry
 // required by the hot-reloading requirements in high security k8s setups.
+//
+// All configuration parameters overrides the corresponding EXTRA_ARGS environment
+// or CLI flags.
 type GoogleCloudConfig struct {
 	Export GoogleCloudExportConfig `yaml:"export,omitempty"`
 	Query  GoogleCloudQueryConfig  `yaml:"query,omitempty"`
@@ -11,10 +14,24 @@ type GoogleCloudConfig struct {
 type GoogleCloudExportConfig struct {
 	// Compression controls the ExporterOpts.Compression setting.
 	Compression string `yaml:"compression,omitempty"`
-	// CredentialsFile controls ExporterOpts.CredentialsFile
+	// CredentialsFile controls the ExporterOpts.CredentialsFile setting.
 	CredentialsFile string `yaml:"credentials,omitempty"`
 
-	// Deprecated: This option no longer works, see https://github.com/GoogleCloudPlatform/prometheus-engine/pull/1688
+	// EnableMatch allows additional control over matching filtering
+	// given the go/gmp:matchstuck context.
+	//
+	// Available settings:
+	// * not set: This config's Match settings are ignored. Custom export.match flags are used.
+	// * false: Filtering feature is explicitly disabled; export matches all series.
+	// * true: This config's Match settings are used. Overwrites all custom export.match flags.
+	EnableMatch *bool `yaml:"enable_match,omitempty"`
+	// Match, if EnableMatch is true, controls the ExporterOpts.Matchers setting.
+	// It offers runtime change-able control of the "matchOneOf" filtering.
+	// This filtering will skip certain series from entering the series cache (for export).
+	// This data will still be ingested into Prometheus local storage.
+	//
+	// IMPORTANT: This option is prone to misconfiguration, thus opt-in and removed from the public docs.
+	// See https://cloud.google.com/stackdriver/docs/managed-prometheus/setup-managed#filter-metrics.
 	Match []string `yaml:"match,omitempty"`
 }
 

google/export/export.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -149,6 +150,13 @@ type Exporter struct {
 	// Used to construct a new metric client when options change, or at initialization. It
 	// is exposed as a variable so that unit tests may change the constructor.
 	newMetricClient func(ctx context.Context, opts ExporterOpts) (metricServiceClient, error)
+
+	// Currently applied matchers in seriesCache.matchers. Avoids locking for reads.
+	matchers Matchers
+
+	// We allow runtime config path. For startup consistency make exporter noop until export
+	// will see the first ApplyConfig.
+	configured bool
 }
 
 const (
@@ -206,7 +214,8 @@ type ExporterOpts struct {
 	// A list of metric matchers. Only Prometheus time series satisfying at
 	// least one of the matchers are exported.
 	// This option matches the semantics of the Prometheus federation match[]
-	// parameter.
+	// parameter. This is a setting from flags only, see ApplyConfig on
+	// how it's applied dynamically in conjunction with Export.Match runtime option.
 	Matchers Matchers
 
 	// Prefix under which metrics are written to GCM.
@@ -367,6 +376,7 @@ func defaultNewMetricClient(ctx context.Context, opts ExporterOpts) (metricServi
 }
 
 // New returns a new Cloud Monitoring Exporter.
+// IMPORTANT: Created exporter is noop until ApplyConfig is called at least once. This is for configuration startup consistency.
 func New(ctx context.Context, logger log.Logger, reg prometheus.Registerer, opts ExporterOpts, lease Lease) (*Exporter, error) {
 	grpc_prometheus.EnableClientHandlingTimeHistogram(
 		grpc_prometheus.WithHistogramBuckets([]float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 40, 50, 60}),
@@ -409,7 +419,7 @@ func New(ctx context.Context, logger log.Logger, reg prometheus.Registerer, opts
 		ctx:                  ctx,
 		opts:                 opts,
 		metricClient:         metricClient,
-		seriesCache:          newSeriesCache(logger, reg, opts.MetricTypePrefix, opts.Matchers),
+		seriesCache:          newSeriesCache(logger, reg, opts.MetricTypePrefix),
 		externalLabels:       createLabelSet(&config.Config{}, &opts),
 		newMetricClient:      defaultNewMetricClient,
 		nextc:                make(chan struct{}, 1),
@@ -418,14 +428,17 @@ func New(ctx context.Context, logger log.Logger, reg prometheus.Registerer, opts
 		lease:                lease,
 	}
 
+	// Set initial matchers.
+	e.matchers = opts.Matchers
+	e.seriesCache.setMatchers(e.matchers)
+
 	// Whenever the lease is lost, clear the series cache so we don't start off of out-of-range
 	// reset timestamps when we gain the lease again.
 	lease.OnLeaderChange(e.seriesCache.clear)
 
 	for i := range e.shards {
 		e.shards[i] = newShard(opts.Efficiency.ShardBufferSize)
 	}
-
 	return e, nil
 }
 
@@ -442,10 +455,12 @@ const (
 // ApplyConfig updates the exporter state to the given configuration. For the
 // Prometheus fork config's google_cloud entry, ExporterOpts might be
 // changed and applied to the exporter, potentially recreating the metric client.
+// NOTE: Runtime configuration will not override disable/disableAuth options from flags.
 func (e *Exporter) ApplyConfig(cfg *config.Config) (err error) {
 	// Note: We don't expect the NopExporter to call this. Only the config reloader calls it.
 	e.mtx.Lock()
 	defer e.mtx.Unlock()
+	e.configured = true
 
 	// Don't recreate the metric client each time. If the metric client is recreated, it has to
 	// potentially redo the TCP handshake. With HTTP/2, TCP connections are kept alive for a small
@@ -463,7 +478,6 @@ func (e *Exporter) ApplyConfig(cfg *config.Config) (err error) {
 		e.opts.CredentialsFile = cfg.GoogleCloud.Export.CredentialsFile
 		recreateClient = true
 	}
-	// NOTE(bwplotka): Matchers are deprecated, see https://github.com/GoogleCloudPlatform/prometheus-engine/pull/1688
 
 	lset := createLabelSet(cfg, &e.opts)
 	labelsChanged := !labels.Equal(e.externalLabels, lset)
@@ -477,20 +491,41 @@ func (e *Exporter) ApplyConfig(cfg *config.Config) (err error) {
 		}
 	}
 
-	// If changed, or we're calling this for the first time, we need to recreate the client.
+	var toApply Matchers
+	switch {
+	case cfg.GoogleCloud.Export.EnableMatch == nil:
+		// Ignore Export.Match, whatever was in flag or default wins.
+		toApply = e.opts.Matchers
+	case !*cfg.GoogleCloud.Export.EnableMatch:
+		// Explicit apply all matching (disabling the filtering).
+		toApply = nil
+	case *cfg.GoogleCloud.Export.EnableMatch:
+		// Runtime Export.Match opted-in; override:
+		for _, match := range cfg.GoogleCloud.Export.Match {
+			if err := toApply.Set(match); err != nil {
+				return err
+			}
+		}
+	}
+
 	if recreateClient {
+		// If changed, or we're calling this for the first time, we need to recreate the client.
 		e.metricClient, err = e.newMetricClient(e.ctx, e.opts)
 		if err != nil {
 			return fmt.Errorf("create metric client: %w", err)
 		}
 	}
 
+	// Updates series cache settings; those require extra care on cache entries.
+	if !e.matchers.Equals(toApply) {
+		e.matchers = toApply
+		e.seriesCache.setMatchers(e.matchers)
+	}
 	if labelsChanged {
 		e.externalLabels = lset
 		// New external labels possibly invalidate the cached series conversions.
 		e.seriesCache.forceRefresh()
 	}
-
 	return nil
 }
 
@@ -557,13 +592,18 @@ func (e *Exporter) Export(metadata MetadataFunc, batch []record.RefSample, exemp
 		return
 	}
 
-	metadata = e.wrapMetadata(metadata)
-
 	e.mtx.Lock()
+	if !e.configured {
+		// ApplyConfig must be called at least once, otherwise we risk a race between flag and runtime config change on start.
+		e.mtx.Unlock()
+		return
+	}
 	externalLabels := e.externalLabels
 	start, end, ok := e.lease.Range()
 	e.mtx.Unlock()
 
+	metadata = e.wrapMetadata(metadata)
+
 	if !ok {
 		exemplarsDropped.WithLabelValues("not-in-ha-range").Add(float64(len(exemplarMap)))
 		samplesDropped.WithLabelValues("not-in-ha-range").Add(float64(batchSize))
@@ -1030,6 +1070,21 @@ func (m *Matchers) IsCumulative() bool {
 	return true
 }
 
+// Equals returns true if m is the same as other matchers. Order matters.
+//
+// NOTE: In practice order semi-matter, as the match result should be similar, but
+// things are faster or slower depending on the order. For efficiency of this function
+// and deterministic efficiency of matching, we assume order matter.
+func (m *Matchers) Equals(other Matchers) bool {
+	return slices.EqualFunc(*m, other, func(msel, osel labels.Selector) bool {
+		return slices.EqualFunc(msel, osel, func(matcher, otherMatcher *labels.Matcher) bool {
+			return matcher.Type == otherMatcher.Type &&
+					matcher.Name == otherMatcher.Name &&
+					matcher.Value == otherMatcher.Value
+		})
+	})
+}
+
 func (m *Matchers) Matches(lset labels.Labels) bool {
 	if len(*m) == 0 {
 		return true