Clean up legacy CDC logic (#11471)

Currently, there are two levels of CDC implementation. One is within the
pebble cache, acting underneath the storage layer to act as a more
efficient storage system, the other is remote cache application layer
logic in the CAS, built to be visible through the remote cache APIs.

Keeping this old implementation around has caused confusion, since the
flags are so similar.

The new implementation has been adjusted so that it adheres to:
-
bazelbuild/remote-apis@de5501d
-
bazelbuild/remote-apis@080cf12
-
bazelbuild/remote-apis@9ef19c6

Moving the CDC logic from pebble_cache -> CAS/ByteStream layer allows us
to make it distributed, so that if large files share chunks, we only
need to store the chunks once. It also means that clients can only
request specific chunks, rather than being required to also tell the
server which blob they are trying to reconstruct.

Author: tyler-french

enterprise/server/backends/cache_config/cache_config.go

@@ -61,7 +61,6 @@ type PebbleCacheConfig struct {
 	AtimeBufferSize             *int                    `yaml:"atime_buffer_size"`
 	MinEvictionAge              *time.Duration          `yaml:"min_eviction_age"`
 	MinBytesAutoZstdCompression *int64                  `yaml:"min_bytes_auto_zstd_compression"`
-	AverageChunkSizeBytes       int                     `yaml:"average_chunk_size_bytes"`
 	ClearCacheOnStartup         bool                    `yaml:"clear_cache_on_startup"`
 	ActiveKeyVersion            *int64                  `yaml:"active_key_version"`
 	GCSConfig                   GCSConfig               `yaml:"gcs"`

enterprise/server/backends/metacache/metacache.go

@@ -386,7 +386,7 @@ func (c *Cache) writerForRecord(ctx context.Context, fileRecord *sgpb.FileRecord
 	}
 }
 
-func (c *Cache) newWrappedWriter(ctx context.Context, fileRecord *sgpb.FileRecord, sizeHint int64, fileType sgpb.FileMetadata_FileType, fn writeMetadataFn) (interfaces.CommittedWriteCloser, error) {
+func (c *Cache) newWrappedWriter(ctx context.Context, fileRecord *sgpb.FileRecord, sizeHint int64, fn writeMetadataFn) (interfaces.CommittedWriteCloser, error) {
 	wcm, err := c.writerForRecord(ctx, fileRecord, sizeHint)
 	if err != nil {
 		return nil, err
@@ -403,7 +403,6 @@ func (c *Cache) newWrappedWriter(ctx context.Context, fileRecord *sgpb.FileRecor
 			StoredSizeBytes:    bytesWritten,
 			LastAccessUsec:     now,
 			LastModifyUsec:     now,
-			FileType:           fileType,
 		}
 		if bytesWritten == 0 {
 			log.Infof("Rejecting zero-length write. Metadata: %+v", md)
@@ -478,7 +477,7 @@ func (c *Cache) writer(ctx context.Context, r *rspb.ResourceName, sizeHint int64
 		return nil, err
 	}
 
-	wc, err := c.newWrappedWriter(ctx, fileRecord, sizeHint, sgpb.FileMetadata_COMPLETE_FILE_TYPE, fn)
+	wc, err := c.newWrappedWriter(ctx, fileRecord, sizeHint, fn)
 	if err != nil {
 		return nil, err
 	}
@@ -704,11 +703,9 @@ func (c *Cache) Delete(ctx context.Context, r *rspb.ResourceName) (resultErr err
 }
 
 func (c *Cache) reader(ctx context.Context, md *sgpb.FileMetadata, r *rspb.ResourceName, uncompressedOffset, uncompressedLimit int64) (io.ReadCloser, error) {
-	// If this object was not chunked, and is somehow stored as a zero-
-	// length file, pretend it does not exist.
-	storageMetadata := md.GetStorageMetadata()
-
-	if chunkedMD := storageMetadata.GetChunkedMetadata(); chunkedMD == nil && md.GetStoredSizeBytes() == 0 {
+	// If this object is somehow stored as a zero-length file, pretend it
+	// does not exist.
+	if md.GetStoredSizeBytes() == 0 {
 		log.CtxInfof(ctx, "Ignoring zero-length file. md: %+v, resource: %v", md, r)
 		return nil, status.NotFoundError("object not found (zero-length)")
 	}
@@ -769,9 +766,7 @@ func (c *Cache) reader(ctx context.Context, md *sgpb.FileMetadata, r *rspb.Resou
 			}
 			reader = d
 		}
-		if shouldDecompress && storageMetadata.GetChunkedMetadata() == nil {
-			// We don't need to decompress the chunked reader's content since
-			// it already returns decompressed content from its children.
+		if shouldDecompress {
 			dr, err := compression.NewZstdDecompressingReader(reader)
 			if err != nil {
 				return nil, err

enterprise/server/backends/metacache/metacache_test.go

@@ -256,7 +256,6 @@ func getCrypterEnv(t *testing.T) (*testenv.TestEnv, string) {
 func TestEncryption(t *testing.T) {
 	testCases := []struct {
 		desc                   string
-		averageChunkSizeBytes  int
 		maxInlineFileSizeBytes int64
 		digestSize             int64
 	}{

enterprise/server/backends/migration_cache/migration_cache.go

@@ -195,7 +195,6 @@ func pebbleCacheFromConfig(env environment.Env, cfg *cache_config.PebbleCacheCon
 		AtimeUpdateThreshold:        cfg.AtimeUpdateThreshold,
 		AtimeBufferSize:             cfg.AtimeBufferSize,
 		MinEvictionAge:              cfg.MinEvictionAge,
-		AverageChunkSizeBytes:       cfg.AverageChunkSizeBytes,
 		ClearCacheOnStartup:         cfg.ClearCacheOnStartup,
 		ActiveKeyVersion:            cfg.ActiveKeyVersion,
 		GCSBucket:                   cfg.GCSConfig.Bucket,

enterprise/server/backends/pebble_cache/BUILD

@@ -20,7 +20,6 @@ go_library(
         "//server/interfaces",
         "//server/metrics",
         "//server/real_environment",
-        "//server/remote_cache/chunking",
         "//server/remote_cache/digest",
         "//server/util/alert",
         "//server/util/approxlru",
@@ -52,7 +51,6 @@ go_test(
     name = "pebble_cache_test",
     size = "medium",
     srcs = ["pebble_cache_test.go"],
-    shard_count = 2,
     deps = [
         ":pebble_cache",
         "//enterprise/server/backends/kms",