diff --git a/.prow/config.yaml b/.prow/config.yaml index 225e89ee5b1..c1305bf6ec2 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -21,7 +21,7 @@ plank: deck: tide_update_period: 1s spyglass: - size_limit: 100e+6 # 100MB + size_limit: 50e+6 # 50MB viewers: "started.json|finished.json": ["metadata"] "build-log.txt": ["buildlog"] @@ -50,132 +50,56 @@ tide: # presubmits list Prow jobs that run on pull requests presubmits: gojek/feast: - - name: unit-test-core + - name: test-core-and-ingestion decorate: true always_run: true spec: - volumes: - - name: service-account - secret: - secretName: prow-service-account containers: - image: maven:3.6-jdk-8 - volumeMounts: - - name: service-account - mountPath: /etc/service-account - readOnly: true - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - command: [".prow/scripts/run_unit_test.sh", "--component", "core"] + command: [".prow/scripts/test-core-ingestion.sh"] - - name: unit-test-ingestion + - name: test-serving decorate: true always_run: true spec: - volumes: - - name: service-account - secret: - secretName: prow-service-account containers: - image: maven:3.6-jdk-8 - volumeMounts: - - name: service-account - mountPath: /etc/service-account - readOnly: true - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - command: [".prow/scripts/run_unit_test.sh", "--component", "ingestion"] + command: [".prow/scripts/test-serving.sh"] - - name: unit-test-serving + - name: test-java-sdk decorate: true always_run: true spec: containers: - image: maven:3.6-jdk-8 - command: [".prow/scripts/run_unit_test.sh", "--component", "serving"] + command: [".prow/scripts/test-java-sdk.sh"] - - name: unit-test-cli + - name: test-python-sdk decorate: true always_run: true spec: containers: - - image: golang:1.12 - env: - - name: GO111MODULE - value: "on" - command: [".prow/scripts/run_unit_test.sh", "--component", "cli"] + - image: python:3.7 + command: [".prow/scripts/test-python-sdk.sh"] - - name: unit-test-python-sdk + - name: test-golang-sdk decorate: true always_run: true spec: - volumes: - - name: service-account - secret: - secretName: prow-service-account containers: - - image: python:3.6 - volumeMounts: - - name: service-account - mountPath: /etc/service-account - readOnly: true - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/service-account/service-account.json - command: [".prow/scripts/run_unit_test.sh", "--component", "python-sdk"] + - image: golang:1.13 + command: [".prow/scripts/test-golang-sdk.sh"] - - name: integration-test + - name: test-end-to-end decorate: true always_run: true spec: - volumes: - - name: docker-socket-volume - hostPath: - path: /var/run/docker.sock - type: File - - name: service-account - secret: - secretName: prow-service-account - nodeSelector: - os: ubuntu containers: - - image: google/cloud-sdk - # securityContext and docker socket volume mounts are needed because we are building - # Docker images in this job - securityContext: - privileged: true - volumeMounts: - - name: docker-socket-volume - mountPath: /var/run/docker.sock - - name: service-account - mountPath: /etc/service-account - readOnly: true - command: - - bash - - -c - - | - export FEAST_HOME=${PWD} - export FEAST_IMAGE_REGISTRY=us.gcr.io - export FEAST_IMAGE_TAG=${PULL_PULL_SHA} - export FEAST_WAREHOUSE_DATASET=feast_build_${BUILD_ID} - export FEAST_CORE_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 - export FEAST_SERVING_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 - export FEAST_RELEASE_NAME=feast-${BUILD_ID:0:5} - export BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${BUILD_ID:0:5}/integration-tests/testdata/feature_values/ingestion_1.csv - export KAFKA_BROKERS=10.128.0.201:9092 - export KAFKA_TOPICS=feast_build_${BUILD_ID:0:5} - - . .prow/scripts/prepare_integration_test.sh - .prow/scripts/install_feast_and_run_e2e_test.sh - TEST_EXIT_CODE=$? - .prow/scripts/cleanup_feast_installation.sh - - exit ${TEST_EXIT_CODE} + - image: maven:3.6-jdk-8 + command: [".prow/scripts/test-end-to-end.sh"] # TODO: do a release when a git tag is pushed -# postsubmits list Prow jobs that run on every push # +# postsubmits list Prow jobs that run on every push # postsubmits: -# gojek/feast: +# gojek/feast: \ No newline at end of file diff --git a/.prow/scripts/cleanup_feast_installation.sh b/.prow/scripts/cleanup_feast_installation.sh deleted file mode 100755 index 78b7d83a327..00000000000 --- a/.prow/scripts/cleanup_feast_installation.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e - -bq -q rm -rf --dataset ${FEAST_WAREHOUSE_DATASET} -gsutil -q rm ${BATCH_IMPORT_DATA_GCS_PATH} -helm delete --purge $FEAST_RELEASE_NAME diff --git a/.prow/scripts/prepare_maven_cache.sh b/.prow/scripts/download-maven-cache.sh similarity index 79% rename from .prow/scripts/prepare_maven_cache.sh rename to .prow/scripts/download-maven-cache.sh index e5113389426..c9704878e0a 100755 --- a/.prow/scripts/prepare_maven_cache.sh +++ b/.prow/scripts/download-maven-cache.sh @@ -23,5 +23,11 @@ done if [[ ! ${ARCHIVE_URI} ]]; then usage; exit 1; fi if [[ ! ${OUTPUT_DIR} ]]; then usage; exit 1; fi +# Install Google Cloud SDK if gsutil command not exists +if [[ ! $(command -v gsutil) ]]; then + CURRENT_DIR=$(dirname "$BASH_SOURCE") + . "${CURRENT_DIR}"/install_google_cloud_sdk.sh +fi + gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar tar xf /tmp/.m2.tar -C ${OUTPUT_DIR} diff --git a/.prow/scripts/install_feast_and_run_e2e_test.sh b/.prow/scripts/install_feast_and_run_e2e_test.sh deleted file mode 100755 index 029f2e3e31a..00000000000 --- a/.prow/scripts/install_feast_and_run_e2e_test.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "============================================================" -echo "Installing Feast Release" -echo "============================================================" - -helm install --name ${FEAST_RELEASE_NAME} --wait --timeout 210 ${FEAST_HOME}/charts/feast -f integration-tests/feast-helm-values.yaml - -echo "============================================================" -echo "Testing Batch Import" -echo "============================================================" - -cd ${FEAST_HOME}/integration-tests/testdata - -feast apply entity entity_specs/entity_1.yaml -feast apply feature feature_specs/entity_1*.yaml -feast jobs run import_specs/batch_from_gcs.yaml --wait - -cd $FEAST_HOME/integration-tests - -python -m testutils.validate_feature_values \ - --entity_spec_file=testdata/entity_specs/entity_1.yaml \ - --feature_spec_files=testdata/feature_specs/entity_1*.yaml \ - --expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \ - --expected-serving-values-file=testdata/feature_values/serving_1.csv \ - --bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \ - --feast-serving-url=${FEAST_SERVING_URL} - -echo "============================================================" -echo "Testing Streaming Import" -echo "============================================================" - -cd $FEAST_HOME/integration-tests/testdata - -feast apply entity entity_specs/entity_2.yaml -feast apply feature feature_specs/entity_2*.yaml -feast jobs run import_specs/stream_from_kafka.yaml & - -IMPORT_JOB_PID=$! -sleep 20 - -cd $FEAST_HOME/integration-tests - -python -m testutils.kafka_producer \ - --bootstrap_servers=$KAFKA_BROKERS \ - --topic=$KAFKA_TOPICS \ - --entity_spec_file=testdata/entity_specs/entity_2.yaml \ - --feature_spec_files=testdata/feature_specs/entity_2*.yaml \ - --feature_values_file=testdata/feature_values/ingestion_2.csv -sleep 20 - -python -m testutils.validate_feature_values \ - --entity_spec_file=testdata/entity_specs/entity_2.yaml \ - --feature_spec_files=testdata/feature_specs/entity_2*.yaml \ - --expected-serving-values-file=testdata/feature_values/serving_2.csv \ - --feast-serving-url=$FEAST_SERVING_URL - -kill -9 ${IMPORT_JOB_PID} diff --git a/.prow/scripts/install_feast_sdk.sh b/.prow/scripts/install_feast_sdk.sh deleted file mode 100755 index 723199b9e51..00000000000 --- a/.prow/scripts/install_feast_sdk.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This script ensures latest Feast Python SDK and Feast CLI are installed - -pip install -qe ${FEAST_HOME}/sdk/python -pip install -qr ${FEAST_HOME}/integration-tests/testutils/requirements.txt -go build -o /usr/local/bin/feast ./cli/feast &> /dev/null -feast config set coreURI ${FEAST_CORE_URL} diff --git a/.prow/scripts/install_google_cloud_sdk.sh b/.prow/scripts/install_google_cloud_sdk.sh index 0865f6085c3..c6356557cec 100755 --- a/.prow/scripts/install_google_cloud_sdk.sh +++ b/.prow/scripts/install_google_cloud_sdk.sh @@ -23,14 +23,14 @@ while [ "$1" != "" ]; do shift done -GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz +GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-266.0.0-linux-x86_64.tar.gz GOOGLE_PROJECT_ID=kf-feast KUBE_CLUSTER_NAME=primary-test-cluster KUBE_CLUSTER_ZONE=us-central1-a curl -s ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz -C / export PATH=/google-cloud-sdk/bin:${PATH} -gcloud -q components install kubectl +gcloud -q components install kubectl &> /var/log/kubectl.install.log if [[ ${KEY_FILE} ]]; then gcloud -q auth activate-service-account --key-file=${KEY_FILE} diff --git a/.prow/scripts/install_test_tools.sh b/.prow/scripts/install_test_tools.sh deleted file mode 100755 index a2e30bbc3a2..00000000000 --- a/.prow/scripts/install_test_tools.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This script installs the following Feast test utilities: -# ============================================================ -# - gettext package so we can use envsubst command to provide values to helm template file -# - Python 3.6 because Feast requires Python version 3.6 and above -# - Golang if we need to build Feast CLI from source -# - Helm if we want to install Feast release - -apt-get -qq update -apt-get -y install curl wget gettext &> /dev/null - -curl -s https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh -o /tmp/miniconda.sh -bash /tmp/miniconda.sh -b -p /miniconda &> /dev/null -export PATH=/miniconda/bin:$PATH - -wget -qO- https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz | tar xzf - -mv go /usr/local/ -export PATH=/usr/local/go/bin:$PATH -export GO111MODULE=on - -wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz -mv linux-amd64/helm /usr/local/bin/helm diff --git a/.prow/scripts/prepare_integration_test.sh b/.prow/scripts/prepare_integration_test.sh deleted file mode 100755 index 1a08f26475b..00000000000 --- a/.prow/scripts/prepare_integration_test.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -set -e - -usage() -{ - echo "usage: prepare_integration_test.sh [--skip-build true]" -} - -while [ "$1" != "" ]; do - case "$1" in - --skip-build ) SKIP_BUILD=true; shift;; - * ) usage; exit 1 - esac - shift -done - -# Authenticate to Google Cloud and GKE -# ============================================================ -GOOGLE_PROJECT_ID=kf-feast -KUBE_CLUSTER_NAME=primary-test-cluster -KUBE_CLUSTER_ZONE=us-central1-a -KEY_FILE=/etc/service-account/service-account.json - -gcloud -q auth activate-service-account --key-file=${KEY_FILE} -gcloud -q auth configure-docker -gcloud -q config set project ${GOOGLE_PROJECT_ID} -gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID} -export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE} - -# Install Python 3.6, Golang 1.12, Helm and Feast SDK -# ============================================================ -. .prow/scripts/install_test_tools.sh -. .prow/scripts/install_feast_sdk.sh -.prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir ${FEAST_HOME} - -# Prepare Feast test data and config -# ============================================================ - -bq -q mk --dataset ${FEAST_WAREHOUSE_DATASET} -gsutil -q cp ${FEAST_HOME}/integration-tests/testdata/feature_values/ingestion_1.csv ${BATCH_IMPORT_DATA_GCS_PATH} - -BUILD_ID=${BUILD_ID:0:5} -envsubst < integration-tests/feast-helm-values.yaml.template > integration-tests/feast-helm-values.yaml -cd ${FEAST_HOME}/integration-tests/testdata/import_specs -envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml -envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml - -if [[ ! ${SKIP_BUILD} ]]; then - -echo "============================================================" -echo "Building Feast for Testing" -echo "============================================================" -cd ${FEAST_HOME} -docker build -t us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} -f Dockerfiles/core/Dockerfile . & -docker build -t us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} -f Dockerfiles/serving/Dockerfile . & -wait -docker push us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} & -docker push us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} & -wait - -fi - -# Switch back context to original directory -set +ex -cd ${FEAST_HOME} \ No newline at end of file diff --git a/.prow/scripts/run_unit_test.sh b/.prow/scripts/run_unit_test.sh deleted file mode 100755 index d2f6ea3255c..00000000000 --- a/.prow/scripts/run_unit_test.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -# This script will run unit test for a specific Feast component: -# - core, ingestion, serving or cli -# -# This script includes the pre and post test scripts, such as -# - downloading maven cache repository -# - saving the test output report so it can be viewed with Spyglass in Prow - -# Bucket in GCS used for running unit tests, when the unit tests need an -# actual running GCS (e.g. because there is no existing mock implementation of the function to test) -TEST_BUCKET=feast-templocation-kf-feast - -usage() -{ - echo "usage: run_unit_test.sh - --component {core, ingestion, serving, cli}" -} - -while [ "$1" != "" ]; do - case "$1" in - --component ) COMPONENT="$2"; shift;; - * ) usage; exit 1 - esac - shift -done - -if [[ ! ${COMPONENT} ]]; then - usage; exit 1; -fi - -. .prow/scripts/install_google_cloud_sdk.sh - -if [[ ${COMPONENT} == "core" ]] || [[ ${COMPONENT} == "ingestion" ]] || [[ ${COMPONENT} == "serving" ]]; then - - .prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ - mvn --projects ${COMPONENT} -Dtestbucket=feast-templocation-kf-feast test - TEST_EXIT_CODE=$? - cp -r ${COMPONENT}/target/surefire-reports /logs/artifacts/surefire-reports - -elif [[ ${COMPONENT} == "cli" ]]; then - - # https://stackoverflow.com/questions/6871859/piping-command-output-to-tee-but-also-save-exit-code-of-command - set -o pipefail - - go get -u github.com/jstemmer/go-junit-report - go test -v ./cli/feast/... 2>&1 | tee test_output - TEST_EXIT_CODE=$? - cat test_output | ${GOPATH}/bin/go-junit-report > ${ARTIFACTS}/unittest-cli-report.xml - -elif [[ ${COMPONENT} == "python-sdk" ]]; then - - cd sdk/python - pip install -r requirements-test.txt - pip install . - pytest ./tests --junitxml=${ARTIFACTS}/unittest-pythonsdk-report.xml - TEST_EXIT_CODE=$? - -else - usage; exit 1 -fi - -exit ${TEST_EXIT_CODE} diff --git a/.prow/scripts/test-core-ingestion.sh b/.prow/scripts/test-core-ingestion.sh new file mode 100755 index 00000000000..98a47ca68c9 --- /dev/null +++ b/.prow/scripts/test-core-ingestion.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +.prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir /root/ + +# Core depends on Ingestion so they are tested together +# Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level +mvn --projects core,ingestion --batch-mode --define skipTests=true \ + --define enforcer.skip=true clean install +mvn --projects core,ingestion --define enforcer.skip=true test +TEST_EXIT_CODE=$? + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts +mkdir -p ${LOGS_ARTIFACT_PATH}/surefire-reports +cp core/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/ +cp ingestion/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/ + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh new file mode 100755 index 00000000000..89ddd826e33 --- /dev/null +++ b/.prow/scripts/test-end-to-end.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +if ! cat /etc/*release | grep -q stretch; then + echo ${BASH_SOURCE} only supports Debian stretch. + echo Please change your operating system to use this script. + exit 1 +fi + +echo " +This script will run end-to-end tests for Feast Core and Online Serving. + +1. Install Redis as the store for Feast Online Serving. +2. Install Postgres for persisting Feast metadata. +3. Install Kafka and Zookeeper as the Source in Feast. +4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from + tests/e2e via pytest. +" + +echo " +============================================================ +Installing Redis at localhost:6379 +============================================================ +" +apt-get -qq update +# Allow starting serving in this Maven Docker image. Default set to not allowed. +echo "exit 0" > /usr/sbin/policy-rc.d +apt-get -y install redis-server wget > /var/log/redis.install.log +redis-server --daemonize yes +redis-cli ping + +echo " +============================================================ +Installing Postgres at localhost:5432 +============================================================ +" +apt-get -y install postgresql > /var/log/postgresql.install.log +service postgresql start +# Initialize with database: 'postgres', user: 'postgres', password: 'password' +cat < /tmp/update-postgres-role.sh +psql -c "ALTER USER postgres PASSWORD 'password';" +EOF +chmod +x /tmp/update-postgres-role.sh +su -s /bin/bash -c /tmp/update-postgres-role.sh postgres +export PGPASSWORD=password +pg_isready + +echo " +============================================================ +Installing Zookeeper at localhost:2181 +Installing Kafka at localhost:9092 +============================================================ +" +wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz +mv kafka_2.12-2.3.0/ /tmp/kafka +nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 & +sleep 5 +tail -n10 /var/log/zookeeper.log +nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 & +sleep 5 +tail -n10 /var/log/kafka.log + +echo " +============================================================ +Building jars for Feast +============================================================ +" + +.prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir /root/ + +# Build jars for Feast +mvn --quiet --batch-mode --define skipTests=true clean package + +echo " +============================================================ +Starting Feast Core +============================================================ +" +# Start Feast Core in background +cat < /tmp/core.application.yml +grpc: + port: 6565 + enable-reflection: true + +feast: + version: 0.3 + jobs: + runner: DirectRunner + options: {} + metrics: + enabled: false + + stream: + type: kafka + options: + bootstrapServers: localhost:9092 + replicationFactor: 1 + partitions: 1 + +spring: + jpa: + properties.hibernate.format_sql: true + hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl + hibernate.ddl-auto: update + datasource: + url: jdbc:postgresql://localhost:5432/postgres + username: postgres + password: password + +management: + metrics: + export: + simple: + enabled: false + statsd: + enabled: false +EOF + +nohup java -jar core/target/feast-core-0.3.0-SNAPSHOT.jar \ + --spring.config.location=file:///tmp/core.application.yml \ + &> /var/log/feast-core.log & +sleep 20 +tail -n10 /var/log/feast-core.log + +echo " +============================================================ +Starting Feast Online Serving +============================================================ +" +# Start Feast Online Serving in background +cat < /tmp/serving.store.redis.yml +name: serving +type: REDIS +redis_config: + host: localhost + port: 6379 +subscriptions: + - name: .* + version: ">0" +EOF + +cat < /tmp/serving.online.application.yml +feast: + version: 0.3 + core-host: localhost + core-grpc-port: 6565 + + tracing: + enabled: false + + store: + config-path: /tmp/serving.store.redis.yml + redis-pool-max-size: 128 + redis-pool-max-idle: 16 + + jobs: + staging-location: gs://feast-templocation-kf-feast/staging-location + store-type: + store-options: {} + +grpc: + port: 6566 + enable-reflection: true + +spring: + main: + web-environment: false +EOF + +nohup java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ + --spring.config.location=file:///tmp/serving.online.application.yml \ + &> /var/log/feast-serving-online.log & +sleep 15 +tail -n10 /var/log/feast-serving-online.log + +echo " +============================================================ +Installing Python 3.7 with Miniconda and Feast SDK +============================================================ +" +# Install Python 3.7 with Miniconda +wget -q https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh \ + -O /tmp/miniconda.sh +bash /tmp/miniconda.sh -b -p /root/miniconda -f +/root/miniconda/bin/conda init +source ~/.bashrc + +# Install Feast Python SDK and test requirements +pip install -q sdk/python +pip install -qr tests/e2e/requirements.txt + +echo " +============================================================ +Running end-to-end tests with pytest at 'tests/e2e' +============================================================ +" +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +ORIGINAL_DIR=$(pwd) +cd tests/e2e + +set +e +pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +TEST_EXIT_CODE=$? + +cd ${ORIGINAL_DIR} +exit ${TEST_EXIT_CODE} diff --git a/.prow/scripts/test-golang-sdk.sh b/.prow/scripts/test-golang-sdk.sh new file mode 100755 index 00000000000..b586927a512 --- /dev/null +++ b/.prow/scripts/test-golang-sdk.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -o pipefail + +cd sdk/go +go test -v 2>&1 | tee /tmp/test_output +TEST_EXIT_CODE=$? + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +go get -u github.com/jstemmer/go-junit-report +cat /tmp/test_output | ${GOPATH}/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/.prow/scripts/test-java-sdk.sh b/.prow/scripts/test-java-sdk.sh new file mode 100755 index 00000000000..0731b77976f --- /dev/null +++ b/.prow/scripts/test-java-sdk.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level +mvn --projects sdk/java --batch-mode --define skipTests=true \ + --define enforcer.skip=true clean install +mvn --projects sdk/java --define enforcer.skip=true test +TEST_EXIT_CODE=$? + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts +cp -r sdk/java/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/.prow/scripts/test-python-sdk.sh b/.prow/scripts/test-python-sdk.sh new file mode 100755 index 00000000000..eb40f921c7b --- /dev/null +++ b/.prow/scripts/test-python-sdk.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +cd sdk/python +pip install -r requirements-test.txt +pip install . +pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml diff --git a/.prow/scripts/test-serving.sh b/.prow/scripts/test-serving.sh new file mode 100755 index 00000000000..d105f733827 --- /dev/null +++ b/.prow/scripts/test-serving.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +.prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir /root/ + +# Skip Maven enforcer: https://stackoverflow.com/questions/50647223/maven-enforcer-issue-when-running-from-reactor-level +mvn --projects serving --batch-mode --define skipTests=true \ + --define enforcer.skip=true clean install +mvn --projects serving --define enforcer.skip=true test +TEST_EXIT_CODE=$? + +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts +cp -r serving/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports + +exit ${TEST_EXIT_CODE} \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml index db92236c830..0dbbeb82b62 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -102,9 +102,9 @@ org.springframework.boot spring-boot-starter-log4j2 - + - org.lognet + io.github.lognet grpc-spring-boot-starter diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java index 93f2519c383..ad58bf2f300 100644 --- a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java +++ b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java @@ -73,6 +73,7 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { expectedPipelineOptions.setAppName("DirectRunnerJobManager"); expectedPipelineOptions.setRunner(DirectRunner.class); expectedPipelineOptions.setBlockOnRun(false); + expectedPipelineOptions.setProject(""); expectedPipelineOptions.setStoreJson(Lists.newArrayList(printer.print(store))); expectedPipelineOptions .setFeatureSetSpecJson(Lists.newArrayList(printer.print(featureSetSpec))); diff --git a/infra/charts/feast/values.yaml b/infra/charts/feast/values.yaml index c879363611c..be3b4138290 100644 --- a/infra/charts/feast/values.yaml +++ b/infra/charts/feast/values.yaml @@ -205,8 +205,9 @@ warehouse-serving: name: warehouse type: BIGQUERY bigquery_config: - projectId: the-big-data-staging-007 - datasetId: feast + # Replace with your Google Cloud project configuration + projectId: google-project-id + datasetId: bigquery-dataset subscriptions: - name: .* version: ">0" diff --git a/ingestion/pom.xml b/ingestion/pom.xml index b7e8033c23a..8811613ede3 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -86,18 +86,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java index 1e9f0bf0129..d08e569518c 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java @@ -116,6 +116,7 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() options.setStoreJson( Collections.singletonList( JsonFormat.printer().omittingInsignificantWhitespace().print(redis))); + options.setProject(""); options.setBlockOnRun(false); int inputSize = 4096; diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java index 8e30edb9aef..8c03d6da172 100644 --- a/ingestion/src/test/java/feast/test/TestUtil.java +++ b/ingestion/src/test/java/feast/test/TestUtil.java @@ -92,7 +92,11 @@ public static void start(String kafkaHost, int kafkaPort, short kafkaReplication public static void stop() { if (server != null) { - server.shutdown(); + try { + server.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } } } } diff --git a/pom.xml b/pom.xml index 4d24212cb91..c88606df6f5 100644 --- a/pom.xml +++ b/pom.xml @@ -250,9 +250,9 @@ - org.lognet + io.github.lognet grpc-spring-boot-starter - 2.4.1 + 3.0.2 @@ -392,10 +392,10 @@ - 3.0.5 + [3.5,4.0) - 1.8.0 + [1.8,1.9) @@ -432,7 +432,7 @@ maven-surefire-plugin 2.22.1 - -Djdk.net.URLClassPath.disableClassPathURLCheck=true + -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true IntegrationTest diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto index 24939055fc2..a80ae36f088 100644 --- a/protos/feast/core/FeatureSet.proto +++ b/protos/feast/core/FeatureSet.proto @@ -47,7 +47,8 @@ message FeatureSetSpec { // as nulls and indicated to end user google.protobuf.Duration max_age = 5; - // Source on which feature rows can be found + // Optional. Source on which feature rows can be found. + // If not set, source will be set to the default value configured in Feast Core. Source source = 6; } diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index 87b95684d03..3760300056a 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -69,7 +69,7 @@ def __init__( if entities is not None: self.entities = entities if source is None: - self._source = KafkaSource() + self._source = None else: self._source = source self._max_age = max_age @@ -504,7 +504,7 @@ def to_proto(self) -> FeatureSetSpecProto: name=self.name, version=self.version, max_age=self.max_age, - source=self.source.to_proto(), + source=self.source.to_proto() if self.source is not None else None, features=[ field.to_proto() for field in self._fields.values() diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 30a97169e4d..d48acdf3842 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -106,7 +106,7 @@ def dtype_to_value_type(dtype): # TODO: to pass test_importer def pandas_dtype_to_feast_value_type(dtype: pd.DataFrame.dtypes) -> ValueType: type_map = { - "float64": ValueType.FLOAT, + "float64": ValueType.DOUBLE, "float32": ValueType.FLOAT, "int64": ValueType.INT64, "uint64": ValueType.INT64, @@ -247,7 +247,7 @@ def pd_value_to_proto_value(feast_value_type, value) -> ProtoValue: return ProtoValue(float_val=float(value)) elif feast_value_type == ValueType.DOUBLE: assert type(value) is float - return ProtoValue(float_val=value) + return ProtoValue(double_val=value) elif feast_value_type == ValueType.STRING: return ProtoValue(string_val=str(value)) elif feast_value_type == ValueType.BYTES: diff --git a/sdk/python/setup.py b/sdk/python/setup.py index b78cb8a6c86..24f9c3f2402 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -36,7 +36,7 @@ "grpcio==1.*", "pandas==0.*", "pandavro==1.5.1", - "protobuf==3.*", + "protobuf==3.10.*", "PyYAML==5.1.2", "fastavro==0.*", "kafka-python==1.4.*", diff --git a/serving/pom.xml b/serving/pom.xml index 2bec449afb0..248f453d36c 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -100,9 +100,9 @@ true - + - org.lognet + io.github.lognet grpc-spring-boot-starter diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 6d862ec32a5..60c0076bce4 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,4 +1,4 @@ def pytest_addoption(parser): parser.addoption("--core_url", action="store", default="localhost:6565") - parser.addoption("--serving_url", action="store", default="localhost:6565") - parser.addoption("--allow_dirty", action="store", default="false") + parser.addoption("--serving_url", action="store", default="localhost:6566") + parser.addoption("--allow_dirty", action="store", default="true") diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index e602ef89369..a529dd1a24b 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -71,8 +71,19 @@ def test_basic(client): # Register feature set client.apply(cust_trans_fs) + # Feast Core needs some time to fully commit the FeatureSet applied + # when there is no existing job yet for the Featureset + time.sleep(15) cust_trans_fs = client.get_feature_set(name="customer_transactions", version=1) + if cust_trans_fs is None: + raise Exception( + "Client cannot retrieve 'customer_transactions' FeatureSet " + "after registration. Either Feast Core does not save the " + "FeatureSet correctly or the client needs to wait longer for FeatureSet " + "to be committed." + ) + offset = random.randint(1000, 100000) # ensure a unique key space is used customer_data = pd.DataFrame( { @@ -88,6 +99,8 @@ def test_basic(client): # Poll serving for feature values until the correct values are returned while True: + time.sleep(1) + response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -103,8 +116,8 @@ def test_basic(client): "customer_transactions:1:total_transactions", ], ) # type: GetOnlineFeaturesResponse + if response is None: - time.sleep(1) continue returned_daily_transactions = float( @@ -124,7 +137,7 @@ def test_basic(client): @pytest.mark.timeout(300) def test_all_types(client): - all_types_fs = client.get_feature_set(name="all_types", version="1") + all_types_fs = client.get_feature_set(name="all_types", version=1) if all_types_fs is None: # Register new feature set if it doesnt exist @@ -152,7 +165,19 @@ def test_all_types(client): # Register feature set client.apply(all_types_fs) - all_types_fs = client.get_feature_set(name="all_types", version="1") + + # Feast Core needs some time to fully commit the FeatureSet applied + # when there is no existing job yet for the Featureset + time.sleep(10) + all_types_fs = client.get_feature_set(name="all_types", version=1) + + if all_types_fs is None: + raise Exception( + "Client cannot retrieve 'all_types_fs' FeatureSet " + "after registration. Either Feast Core does not save the " + "FeatureSet correctly or the client needs to wait longer for FeatureSet " + "to be committed." + ) all_types_df = pd.DataFrame( { @@ -205,9 +230,12 @@ def test_all_types(client): # Ingest user embedding data all_types_fs.ingest(dataframe=all_types_df) + time.sleep(3) # Poll serving for feature values until the correct values are returned while True: + time.sleep(1) + response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -233,7 +261,6 @@ def test_all_types(client): ) # type: GetOnlineFeaturesResponse if response is None: - time.sleep(1) continue returned_float_list = ( @@ -268,10 +295,21 @@ def test_large_volume(client): # Register feature set client.apply(cust_trans_fs) + # Feast Core needs some time to fully commit the FeatureSet applied + # when there is no existing job yet for the Featureset + time.sleep(10) cust_trans_fs = client.get_feature_set( name="customer_transactions_large", version=1 ) + if cust_trans_fs is None: + raise Exception( + "Client cannot retrieve 'customer_transactions' FeatureSet " + "after registration. Either Feast Core does not save the " + "FeatureSet correctly or the client needs to wait longer for FeatureSet " + "to be committed." + ) + offset = random.randint(1000000, 10000000) # ensure a unique key space customer_data = pd.DataFrame( { @@ -289,6 +327,8 @@ def test_large_volume(client): # Poll serving for feature values until the correct values are returned while True: + time.sleep(1) + response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -306,7 +346,6 @@ def test_large_volume(client): ) # type: GetOnlineFeaturesResponse if response is None: - time.sleep(1) continue returned_daily_transactions = float(