Add batch serving functionality to docker compose setup

Author: Shu Heng

docs/getting-started/installing-feast.md

@@ -20,18 +20,32 @@ A docker compose file is provided to quickly test Feast with official docker ima
 * Retrieve features for online serving
 * Updating the feature set
 
+The docker compose setup uses Direct Runner for the Apache Beam jobs.
+
 ### 0. Requirements
 
 1. [Docker compose](https://docs.docker.com/compose/install/) should be installed.
-2. Port 6565, 6566 and 8888 are not in used. Otherwise, modify the port mappings in  `infra/docker-compose/docker-compose.yml` to use unoccupied ports.
+2. Port 6565, 6566 and 8888, 9094 are not in used. Otherwise, modify the port mappings in  `infra/docker-compose/docker-compose.yml` to use unoccupied ports.
+3. For batch serving, you will also need a service account key that has access to GCS and BigQuery. Port 6567 will be used for the batch serving endpoint.
 
-### 1. Step-by-step guide
+### 1. Step-by-step guide (Online serving)
 1. Navigate to `infra/docker-compose`.
 2. Copy `.env.sample` to `.env`.
 3. `docker-compose up -d`
 4. A jupyter notebook server should be accessible via `localhost:8888`
 5. Please wait a minute or two for the Feast services to be ready before running the notebook. You will know that the services are ready when port `6565` and `6566` starts listening.
 
+### 2. Step-by-step guide (Batch serving)
+1. Navigate to `infra/docker-compose`.
+2. Copy `.env.sample` to `.env`.
+3. Copy your GCP account service key(s) to `infra/docker-compose/gcp-service-accounts`.
+4. Modify the value of `FEAST_<SERVICE_NAME>_GCP_SERVICE_ACCOUNT_KEY` in your `.env` file. It should be the json file name without extension.
+5. Modify the value of `infra/docker-compose/serving/bq-store.yml`. Alternatively, you can also point to a different store configuration file by modifying `FEAST_BATCH_STORE_CONFIG` in your `.env` file.
+5. `docker-compose -f docker-compose.yml -f docker-compose.batch.yml up -d`
+6. A jupyter notebook server should be accessible via `localhost:8888`
+7. Please wait a minute or two for the Feast services to be ready before running the notebook. You will know that the services are ready when port `6565` and `6567` starts listening.
+8. When you are done, run `docker-compose -f docker-compose.yml -f docker-compose.batch.yml down` to shutdown the services.
+
 ## Minikube
 
 ### Overview

infra/docker-compose/.env.sample

@@ -1,16 +1,19 @@
 COMPOSE_PROJECT_NAME=feast
 
 FEAST_VERSION=latest
-FEAST_GCP_SERVICE_ACCOUNT_KEY_PATH=/etc/gcloud/service-accounts/key.json
 
 FEAST_CORE_IMAGE=gcr.io/kf-feast/feast-core
-FEAST_CORE_CONFIG=override
-FEAST_CORE_CONFIG_PATH=/etc/feast/feast-core/override.yaml
+FEAST_CORE_CONFIG=direct-runner
 FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY=placeholder
 
 FEAST_SERVING_IMAGE=gcr.io/kf-feast/feast-serving
-FEAST_SERVING_STORE_CONFIG=redis-store
-FEAST_SERVING_STORE_CONFIG_PATH=/etc/feast/feast-serving/store.yaml
-FEAST_SERVING_GCP_SERVICE_ACCOUNT_KEY=placeholder
+FEAST_ONLINE_SERVING_CONFIG=online-serving
+FEAST_ONLINE_STORE_CONFIG=redis-store
+FEAST_BATCH_SERVING_CONFIG=batch-serving
+FEAST_BATCH_STORE_CONFIG=bq-store
+FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY=placeholder
+FEAST_JOB_STAGING_LOCATION=gs://your-gcp-project/bucket
+
+FEAST_JUPYTER_IMAGE=gcr.io/kf-feast/feast-jupyter
+FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder
 
-FEAST_JUPYTER_IMAGE=gcr.io/kf-feast/feast-jupyter

infra/docker-compose/core/override.yml …ra/docker-compose/core/direct-runner.ymlinfra/docker-compose/core/override.yml renamed to infra/docker-compose/core/direct-runner.yml infra/docker-compose/core/override.yml renamed to infra/docker-compose/core/direct-runner.yml

@@ -0,0 +1,25 @@
+version: "3.7"
+
+services:
+  batch-serving:
+    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
+    volumes:
+      - ./serving/${FEAST_BATCH_SERVING_CONFIG}.yml:/etc/feast/application.yml
+      - ./serving/${FEAST_BATCH_STORE_CONFIG}.yml:/etc/feast/store.yml
+      - ./gcp-service-accounts/${FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
+    depends_on:
+      - core
+      - redis
+    ports:
+      - 6567:6567
+    restart: on-failure
+    environment:
+      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
+      FEAST_JOB_STAGING_LOCATION: ${FEAST_JOB_STAGING_LOCATION}
+    command:
+      - "java"
+      - "-Xms1024m"
+      - "-Xmx1024m"
+      - "-jar"
+      - "/opt/feast/feast-serving.jar"
+      - "--spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml"

infra/docker-compose/docker-compose.batch.yml

@@ -2,12 +2,14 @@ version: "3.7"
 
 services:
   core:
+    image: ${FEAST_CORE_IMAGE}:${FEAST_VERSION}
     volumes:
-      - ./core/${FEAST_CORE_CONFIG}.yml:${FEAST_CORE_CONFIG_PATH}
-      - ./gcp-service-accounts/${FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY}.json:${FEAST_GCP_SERVICE_ACCOUNT_KEY_PATH}
+      - ./core/${FEAST_CORE_CONFIG}.yml:/etc/feast/application.yml
+      - ./gcp-service-accounts/${FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
     environment:
       DB_HOST: db
-      GOOGLE_APPLICATION_CREDENTIALS: ${FEAST_GCP_SERVICE_ACCOUNT_KEY_PATH}
+      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
+    restart: on-failure
     depends_on:
       - db
       - kafka
@@ -17,34 +19,38 @@ services:
       - java
       - -jar
       - /opt/feast/feast-core.jar
-      - --spring.config.location=classpath:/application.yml,file:${FEAST_CORE_CONFIG_PATH}
-    restart: on-failure
+      - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yaml
 
-  serving:
+  online-serving:
+    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
     volumes:
-      - ./serving/${FEAST_SERVING_STORE_CONFIG}.yml:${FEAST_SERVING_STORE_CONFIG_PATH}
-      - ./gcp-service-accounts/${FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY}.json:${FEAST_GCP_SERVICE_ACCOUNT_KEY_PATH}
-    environment:
-      FEAST_CORE_HOST: core
-      FEAST_STORE_CONFIG_PATH: ${FEAST_SERVING_STORE_CONFIG_PATH}
-      GOOGLE_APPLICATION_CREDENTIALS: ${FEAST_GCP_SERVICE_ACCOUNT_KEY_PATH}
+      - ./serving/${FEAST_ONLINE_SERVING_CONFIG}.yml:/etc/feast/application.yml
+      - ./serving/${FEAST_ONLINE_STORE_CONFIG}.yml:/etc/feast/store.yml
     depends_on:
       - core
       - redis
     ports:
       - 6566:6566
     restart: on-failure
+    command:
+      - java
+      - -jar
+      - /opt/feast/feast-serving.jar
+      - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
 
   jupyter:
+    image: ${FEAST_JUPYTER_IMAGE}:${FEAST_VERSION}
     volumes:
       - ./jupyter/notebooks:/home/jovyan/feast-notebooks
       - ./jupyter/features:/home/jovyan/features
+      - ./gcp-service-accounts/${FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
     depends_on:
       - core
-      - serving
+      - online-serving
     environment:
-      FEAST_SERVING_URL: serving:6566
       FEAST_CORE_URL: core:6565
+      FEAST_SERVING_URL: online-serving:6566
+      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
     ports:
       - 8888:8888
     command:
@@ -56,11 +62,16 @@ services:
 
   kafka:
     image: confluentinc/cp-kafka:5.2.1
-    restart: on-failure
     environment:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9092,OUTSIDE://localhost:9094
+      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
+    ports:
+      - 9094:9092
+
     depends_on:
       - zookeeper
 
@@ -70,4 +81,4 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
 
   db:
-    image: postgres:12-alpine
+    image: postgres:12-alpine