Update S3-related tests to run in Testcontainers (#117)

Fix S3-related tests from #86:

- Add `MinIOContainer` and `S3Application` to manage MinIO
- Generate fixtures via `SmallDataFactory` (prefer composition over inheritance)
- Migrate `CloudAccessTest` and `S3SelectTest` to TestContainers suite.
- Pre-bake test `s3`/`s3-invalid` PXF server configs into image. 
- Move S3 tests from the singlecluster to the testcontainers pipeline (`pxf-s3`) and delete the unused legacy MinIO scripting.
- Replace s3a client with aws S3 client in tests data management.

Author: ostinru

.github/workflows/pxf-ci.yml

@@ -330,7 +330,6 @@ jobs:
           - jdbc
           - proxy
           - unused
-          - s3
           - features
           - gpdb
           - gpdb_fdw
@@ -501,7 +500,6 @@ jobs:
           - jdbc
           - proxy
           - unused
-          - s3
           - features
           - gpdb
           - gpdb_fdw
@@ -659,6 +657,7 @@ jobs:
       matrix:
         tc_group:
           - 'pxf-jdbc'
+          - 'pxf-s3'
         use_fdw:
           - 'false'
           - 'true'

automation/Makefile

@@ -35,11 +35,6 @@ endif
 # lowercase the protocol
 PROTOCOL := $(shell echo $(PROTOCOL) | tr A-Z a-z)
 
-ifeq "$(PROTOCOL)" "minio"
-	MINIO=true
-	PROTOCOL=s3
-endif
-
 PXF_BASE ?= $(PXF_HOME)
 TEMPLATES_DIR=$(PXF_HOME)/templates
 
@@ -193,21 +188,6 @@ ifneq "$(PROTOCOL)" ""
 			cp $(TEMPLATES_DIR)/pxf-site.xml $(PROTOCOL_HOME)/; \
 			sed $(SED_OPTS) 's|</configuration>|<property><name>pxf.fs.basePath</name><value>$(BASE_PATH)</value></property></configuration>|g' $(PROTOCOL_HOME)/pxf-site.xml; \
 		fi; \
-		if [ $(PROTOCOL) = s3 ]; then \
-			if [ "$(MINIO)" = "true" ]; then \
-				cp $(TEMPLATES_DIR)/minio-site.xml $(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \
-				sed $(SED_OPTS) "s|YOUR_MINIO_URL|http://localhost:9000|" $(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \
-			fi; \
-			mkdir -p $(PROTOCOL_HOME)-invalid; \
-			cp $(TEMPLATES_DIR)/$(PROTOCOL)-site.xml $(PROTOCOL_HOME)-invalid/; \
-			if [ -z "$(ACCESS_KEY_ID)" ] || [ -z "$(SECRET_ACCESS_KEY)" ]; then \
-				echo "AWS Keys (ACCESS_KEY_ID, SECRET_ACCESS_KEY) not set"; \
-				rm -rf $(PROTOCOL_HOME); \
-				exit 1; \
-			fi; \
-			sed $(SED_OPTS) "s|YOUR_AWS_ACCESS_KEY_ID|$(ACCESS_KEY_ID)|" $(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \
-			sed $(SED_OPTS) "s|YOUR_AWS_SECRET_ACCESS_KEY|$(SECRET_ACCESS_KEY)|" $(PROTOCOL_HOME)/$(PROTOCOL)-site.xml; \
-		fi; \
 		if [ $(PROTOCOL) = abfss ]; then \
 			if [ -z "$(ABFSS_ACCOUNT)" ] || [ -z "$(ABFSS_CLIENT_ENDPOINT)" ] || [ -z "$(ABFSS_CLIENT_ID)" ] || [ -z "$(ABFSS_CLIENT_SECRET)" ]; then \
 				echo "ADL Keys (ABFSS_ACCOUNT, ABFSS_CLIENT_ID, ABFSS_CLIENT_SECRET, ABFSS_CLIENT_ENDPOINT) not set"; \
@@ -255,6 +235,7 @@ endif
 # Usage:
 #   make test-tc                          => run all testcontainers tests (Ubuntu)
 #   make test-tc TC_GROUP=pxf-jdbc        => run only pxf-jdbc group
+#   make test-tc TC_GROUP=pxf-s3          => run S3 Select + CloudAccess tests (MinIO sidecar)
 #   make test-tc DISTRO=rocky9            => run with Rocky Linux 9 base image
 .PHONY: test-tc
 test-tc: check-env symlink_pxf_jars pxf_regress

automation/pom.xml

@@ -23,6 +23,7 @@
         <orc.version>1.6.13</orc.version>
         <snappy.java.version>1.1.10.4</snappy.java.version>
         <powermock.version>1.6.4</powermock.version>
+        <argLine.extra></argLine.extra>
     </properties>
 
     <repositories>
@@ -59,7 +60,7 @@
                 <version>2.15</version>
                 <configuration>
                     <testFailureIgnore>true</testFailureIgnore>
-                    <argLine>-Xmx4096m</argLine>
+                    <argLine>-Xmx4096m ${argLine.extra}</argLine>
                     <forkCount>1</forkCount>
                     <reuseForks>false</reuseForks>
                 </configuration>
@@ -535,4 +536,16 @@
             <version>1.9.5</version>
         </dependency>
     </dependencies>
+
+    <profiles>
+        <profile>
+            <id>java9-plus</id>
+            <activation>
+                <jdk>[9,)</jdk>
+            </activation>
+            <properties>
+                <argLine.extra>--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED</argLine.extra>
+            </properties>
+        </profile>
+    </profiles>
 </project>

automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java

@@ -0,0 +1,98 @@
+package org.apache.cloudberry.pxf.automation.applications;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import com.amazonaws.services.s3.model.ListObjectsV2Result;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * S3 API access wrapper used by automation tests to seed and clean fixtures
+ * in a MinIO bucket. Owns the AmazonS3 client; callers should call
+ * `shutdown()` when done (typically in afterClass before stopping the
+ * MinIO container).
+ */
+public class S3Application implements AutoCloseable {
+
+    private final AmazonS3 s3Client;
+
+    public S3Application(MinIOContainer minio) {
+        this.s3Client = buildS3Client(minio.getHostEndpoint(), minio.getAccessKey(), minio.getSecretKey());
+    }
+
+    public void createBucket(String bucket) {
+        if (!s3Client.doesBucketExistV2(bucket)) {
+            s3Client.createBucket(bucket);
+        }
+    }
+
+    public void putObject(String bucket, String key, Path localFile) throws IOException {
+        s3Client.putObject(new PutObjectRequest(bucket, key, localFile.toFile()));
+    }
+
+    public void deletePrefix(String bucket, String prefix) {
+        ListObjectsV2Request request = new ListObjectsV2Request()
+                .withBucketName(bucket)
+                .withPrefix(prefix);
+        ListObjectsV2Result listing;
+        do {
+            listing = s3Client.listObjectsV2(request);
+            List<String> keys = new ArrayList<>();
+            for (S3ObjectSummary summary : listing.getObjectSummaries()) {
+                keys.add(summary.getKey());
+            }
+            if (!keys.isEmpty()) {
+                s3Client.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys.toArray(new String[0])));
+            }
+            request.setContinuationToken(listing.getNextContinuationToken());
+        } while (listing.isTruncated());
+    }
+
+    public void shutdown() {
+        s3Client.shutdown();
+    }
+
+    @Override
+    public void close() {
+        shutdown();
+    }
+
+    private static AmazonS3 buildS3Client(String endpoint, String accessKey, String secretKey) {
+        return AmazonS3ClientBuilder.standard()
+                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "us-east-1"))
+                .withPathStyleAccessEnabled(true)
+                .withCredentials(new AWSStaticCredentialsProvider(
+                        new BasicAWSCredentials(accessKey, secretKey)))
+                .build();
+    }
+}

automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java

@@ -0,0 +1,78 @@
+package org.apache.cloudberry.pxf.automation.testcontainers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * TestContainers wrapper around MinIO for S3 / S3 Select automation tests.
+ * The container joins a shared Docker network with alias minio, so PXF inside the
+ * Cloudberry container can reach it at http://minio:9000.
+ *
+ * This class only manages the container lifecycle and exposes endpoint /
+ * credential accessors. S3 API access (buckets, objects) lives in
+ * {@link org.apache.cloudberry.pxf.automation.applications.S3Application}.
+ */
+public class MinIOContainer extends GenericContainer<MinIOContainer> {
+
+    private static final String DEFAULT_IMAGE = "minio/minio:RELEASE.2024-11-07T00-52-20Z";
+    private static final String NETWORK_ALIAS = "minio";
+
+    public static final int API_PORT = 9000;
+    public static final int CONSOLE_PORT = 9001;
+
+    public static final String ACCESS_KEY = "admin";
+    public static final String SECRET_KEY = "password";
+    public static final String DEFAULT_BUCKET = "gpdb-ud-scratch";
+
+    public MinIOContainer(Network network) {
+        super(DockerImageName.parse(DEFAULT_IMAGE));
+
+        withNetwork(network)
+                .withNetworkAliases(NETWORK_ALIAS)
+                .withExposedPorts(API_PORT, CONSOLE_PORT)
+                .withEnv("MINIO_ROOT_USER", ACCESS_KEY)
+                .withEnv("MINIO_ROOT_PASSWORD", SECRET_KEY)
+                .withEnv("MINIO_API_SELECT_PARQUET", "on")
+                .withCommand("server", "/data", "--console-address", ":" + CONSOLE_PORT)
+                .waitingFor(Wait.forHttp("/minio/health/live").forPort(API_PORT));
+    }
+
+    /** S3 API endpoint reachable from the test JVM (mapped port). */
+    public String getHostEndpoint() {
+        return "http://localhost:" + getMappedPort(API_PORT);
+    }
+
+    /** S3 API endpoint for PXF and other containers on the same Docker network. */
+    public String getInternalEndpoint() {
+        return "http://" + NETWORK_ALIAS + ":" + API_PORT;
+    }
+
+    public String getAccessKey() {
+        return ACCESS_KEY;
+    }
+
+    public String getSecretKey() {
+        return SECRET_KEY;
+    }
+}

automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/PXFCloudberryContainer.java

@@ -31,8 +31,8 @@
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.MountableFile;
+import org.apache.cloudberry.pxf.automation.utils.AutomationUtils;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -137,7 +137,7 @@ private static String resolveDistro() {
      */
     public static synchronized PXFCloudberryContainer getInstance() {
         if (instance == null) {
-            String repo = resolveProperty("pxf.test.repo.path", findRepoPath());
+            String repo = resolveProperty("pxf.test.repo.path", AutomationUtils.findRepoRoot().toString());
             String distro = resolveDistro();
             String imageName = "pxf/cbdb-testcontainer-" + distro + ":1";
             String baseImage = BASE_IMAGES.getOrDefault(distro, BASE_IMAGES.get("ubuntu"));
@@ -204,21 +204,6 @@ private static String resolveProperty(String key, String fallback) {
         return (value != null && !value.isEmpty()) ? value : fallback;
     }
 
-    private static String findRepoPath() {
-        File dir = new File(System.getProperty("user.dir"));
-        for (int i = 0; i < 5; i++) {
-            if (new File(dir, "automation/pom.xml").exists()) {
-                return dir.getAbsolutePath();
-            }
-            dir = dir.getParentFile();
-            if (dir == null)
-                break;
-        }
-        throw new IllegalStateException(
-                "Cannot auto-detect cloudberry-pxf repo root. Set -Dpxf.test.repo.path=...");
-    }
-
-
     public Network getSharedNetwork() {
         return network;
     }

automation/src/main/java/org/apache/cloudberry/pxf/automation/utils/AutomationUtils.java

@@ -0,0 +1,60 @@
+package org.apache.cloudberry.pxf.automation.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.nio.file.Path;
+
+/**
+ * Shared helpers for the cloudberry-pxf automation suite.
+ */
+public final class AutomationUtils {
+
+    private static final String REPO_MARKER = "automation/pom.xml";
+    private static final int MAX_PARENT_HOPS = 6;
+
+    private AutomationUtils() {
+    }
+
+    /**
+     * Walks upward from {@code user.dir} looking for the cloudberry-pxf repo
+     * root (identified by the {@code automation/pom.xml} marker). Used by code
+     * that needs to read resources from the working tree at runtime.
+     *
+     * @throws IllegalStateException if the marker is not found within a handful
+     *     of parent directories. Set {@code -Dpxf.test.repo.path=...} to bypass.
+     */
+    public static Path findRepoRoot() {
+        File dir = new File(System.getProperty("user.dir"));
+        for (int i = 0; i < MAX_PARENT_HOPS; i++) {
+            if (new File(dir, REPO_MARKER).exists()) {
+                return dir.toPath().toAbsolutePath().normalize();
+            }
+            dir = dir.getParentFile();
+            if (dir == null) {
+                break;
+            }
+        }
+        throw new IllegalStateException(
+                "Cannot auto-detect cloudberry-pxf repo root from user.dir="
+                        + System.getProperty("user.dir")
+                        + ". Set -Dpxf.test.repo.path=... to override.");
+    }
+}