add S3 dep to Play server and fix TPC-H generation script

Author: refset

deps.edn

@@ -6,6 +6,7 @@
   ;; XTDB dependencies
   com.xtdb/xtdb-api {:mvn/version "2.x-SNAPSHOT"}
   com.xtdb/xtdb-core {:mvn/version "2.x-SNAPSHOT"}
+  com.xtdb/xtdb-aws {:mvn/version "2.x-SNAPSHOT"}
 
   org.postgresql/postgresql {:mvn/version "42.7.4"}
   com.github.seancorfield/next.jdbc {:mvn/version "1.3.939"}

scripts/README.md

@@ -42,12 +42,13 @@ The action will generate the dataset and upload it to S3 automatically.
 To generate a TPC-H dataset locally at scale factor 0.01:
 
 ```bash
-clojure -M:generate-tpch xtdb-play-datasets 0.01
+clojure -M:generate-tpch xtdb-play-datasets 0.01 eu-west-1
 ```
 
 **Arguments:**
 - `bucket-name`: The S3 bucket to upload to (e.g., `xtdb-play-datasets`)
 - `scale-factor`: The TPC-H scale factor (0.01 = ~10MB, 1.0 = ~1GB, etc.)
+- `region`: AWS region where the bucket is located (e.g., `eu-west-1`)
 
 **Note:** Local generation requires AWS credentials configured in your environment (see Prerequisites).
 

scripts/generate_tpch.clj

@@ -8,63 +8,65 @@
   (:require [clojure.tools.logging :as log]
             [xtdb.api :as xt]
             [xtdb.datasets.tpch :as tpch]
-            [xtdb.node :as xtn]))
+            [xtdb.node :as xtn]
+            [xtdb.db-catalog :as db-catalog])
+  (:import [xtdb.aws.s3 S3Configurator]
+           [software.amazon.awssdk.regions Region]))
 
 (defn generate-dataset
   "Generates TPC-H dataset at given scale factor and stores in S3 bucket."
-  [bucket-name scale-factor]
-  (log/info "Generating TPC-H dataset at scale factor" scale-factor "to bucket" bucket-name)
+  [bucket-name scale-factor region]
+  (log/info "Generating TPC-H dataset at scale factor" scale-factor "to bucket" bucket-name "in region" region)
 
-  (let [node-config {:storage
-                     {:object-store
-                      {:module 'xtdb.s3/s3-object-store
-                       :bucket bucket-name
-                       :prefix (str "tpch-sf" scale-factor "/")
-                       ;; Public bucket - no credentials needed for read
-                       ;; Write requires AWS credentials from environment
-                       }}
-                     :server {:port 5432}}]
+  (let [node-config {:storage [:remote {:object-store [:s3 {:bucket bucket-name
+                                                            :prefix (str "tpch-sf" scale-factor "/")
+                                                            :configurator (reify S3Configurator
+                                                                            (configureClient [_ builder]
+                                                                              (.region builder (Region/of region))))}]}]
+                     :disk-cache {:path "/tmp/xtdb-tpch-cache"}}]
 
-    (log/info "Starting XTDB node with config:" node-config)
+    (log/info "Starting XTDB node with S3 storage in region:" region)
 
     (with-open [node (xtn/start-node node-config)]
       (log/info "Node started, generating TPC-H data...")
 
-      ;; Generate TPC-H dataset
-      (tpch/submit-tpch! node {:scale-factor scale-factor})
+      ;; Generate TPC-H dataset using DML (INSERT statements)
+      (tpch/submit-dml! node scale-factor)
 
       (log/info "TPC-H data submitted, waiting for ingest...")
 
       ;; Wait for data to be ingested
       (Thread/sleep 5000)
 
-      ;; Finish chunk to ensure all data is written
-      (log/info "Calling finish-chunk to ensure data is persisted...")
-      (.finishChunk node)
+;; Finish block to ensure all data is written
+      (log/info "Calling finish-block to ensure data is persisted...")
+      (.finishBlock (.getLogProcessor (db-catalog/primary-db node)))
 
       (log/info "Dataset generation complete!"))))
 
 (defn -main [& args]
-  (when (not= (count args) 2)
-    (println "Usage: clojure -M:generate-tpch <bucket-name> <scale-factor>")
-    (println "Example: clojure -M:generate-tpch xtdb-play-datasets 0.01")
+  (when (not= (count args) 3)
+    (println "Usage: clojure -M:generate-tpch <bucket-name> <scale-factor> <region>")
+    (println "Example: clojure -M:generate-tpch xtdb-play-datasets 0.01 eu-west-1")
     (System/exit 1))
 
   (let [bucket-name (first args)
-        scale-factor (Double/parseDouble (second args))]
+        scale-factor (Double/parseDouble (second args))
+        region (nth args 2)]
 
     (println "=" (repeat 60 "="))
     (println "TPC-H Dataset Generator")
     (println "=" (repeat 60 "="))
     (println "Bucket:" bucket-name)
     (println "Scale Factor:" scale-factor)
+    (println "Region:" region)
     (println)
 
     (try
-      (generate-dataset bucket-name scale-factor)
+      (generate-dataset bucket-name scale-factor region)
       (println)
       (println "✓ Success! Dataset is now available in S3:")
-      (println "  s3://" bucket-name "/tpch-sf" scale-factor "/")
+      (println (str "  s3://" bucket-name "/tpch-sf" scale-factor "/"))
       (catch Exception e
         (println "✗ Error generating dataset:")
         (println (.getMessage e))