diff --git a/.prow/config.yaml b/.prow/config.yaml new file mode 100644 index 00000000000..29878749cde --- /dev/null +++ b/.prow/config.yaml @@ -0,0 +1,140 @@ +prowjob_namespace: default +pod_namespace: test-pods + +plank: + allow_cancellations: true + job_url_prefix: http://prow.feast.ai/view/gcs + report_template: '[Full PR test history](https://prow.feast.ai/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}})' + default_decoration_config: + timeout: 7200000000000 # 2h + grace_period: 15000000000 # 15s + utility_images: + clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a + initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a + entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a + sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a + gcs_configuration: + bucket: feast-templocation-kf-feast + path_strategy: explicit + gcs_credentials_secret: prow-service-account + +deck: + tide_update_period: 1s + spyglass: + size_limit: 100e+6 # 100MB + viewers: + "started.json|finished.json": ["metadata"] + "build-log.txt": ["buildlog"] + "report.xml": ["junit"] + "artifacts/.*\\.xml": ["junit"] + "surefire-reports/.*\\.xml": ["junit"] + +tide: + queries: + - repos: + - gojek/feast + labels: + - lgtm + - approved + missingLabels: + - do-not-merge + - do-not-merge/hold + - do-not-merge/invalid-owners-file + - do-not-merge/work-in-progress + - needs-rebase + merge_method: + gojek/feast: squash + blocker_label: merge-blocker + squash_label: tide/squash + +# presubmits list Prow jobs that run on pull requests +presubmits: + gojek/feast: + - name: unit-test-core + decorate: true + always_run: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/run_unit_test.sh", "--component", "core"] + + - name: unit-test-ingestion + decorate: true + always_run: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/run_unit_test.sh", "--component", "ingestion"] + + - name: unit-test-serving + decorate: true + always_run: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/run_unit_test.sh", "--component", "serving"] + + - name: unit-test-cli + decorate: true + always_run: true + spec: + containers: + - image: golang:1.12 + env: + - name: GO111MODULE + value: "on" + command: [".prow/scripts/run_unit_test.sh", "--component", "cli"] + + - name: integration-test + decorate: true + always_run: true + spec: + volumes: + - name: docker-socket-volume + hostPath: + path: /var/run/docker.sock + type: File + - name: service-account + secret: + secretName: prow-service-account + nodeSelector: + os: ubuntu + containers: + - image: google/cloud-sdk + # securityContext and docker socket vol mounts are needed because we are building + # Docker images in this job + securityContext: + privileged: true + volumeMounts: + - name: docker-socket-volume + mountPath: /var/run/docker.sock + - name: service-account + mountPath: /etc/service-account + readOnly: true + command: + - bash + - -c + - | + export FEAST_HOME=${PWD} + export FEAST_IMAGE_REGISTRY=us.gcr.io + export FEAST_IMAGE_TAG=${PULL_PULL_SHA} + export FEAST_WAREHOUSE_DATASET=feast_build_${BUILD_ID:0:5} + export FEAST_CORE_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 + export FEAST_SERVING_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80 + export FEAST_RELEASE_NAME=feast-${BUILD_ID:0:5} + export BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${BUILD_ID:0:5}/integration-tests/testdata/feature_values/ingestion_1.csv + export KAFKA_BROKERS=10.128.0.201:9092 + export KAFKA_TOPICS=feast_build_${BUILD_ID:0:5} + + . .prow/scripts/prepare_integration_test.sh + .prow/scripts/install_feast_and_run_e2e_test.sh + TEST_EXIT_CODE=$? + .prow/scripts/cleanup_feast_installation.sh + + exit ${TEST_EXIT_CODE} + +# TODO: do a release when a git tag is pushed +# postsubmits list Prow jobs that run on every push +# +# postsubmits: +# gojek/feast: diff --git a/.prow/plugins.yaml b/.prow/plugins.yaml new file mode 100644 index 00000000000..80195222903 --- /dev/null +++ b/.prow/plugins.yaml @@ -0,0 +1,25 @@ +plugins: + gojek/feast: + - approve + - assign + - help + - hold + - label + - lgtm + - lifecycle + - size + - verify-owners + - wip + - trigger + - config-updater + +config_updater: + maps: + .prow/config.yaml: + name: config + +external_plugins: + gojek/feast: + - name: needs-rebase + events: + - pull_request diff --git a/.prow/scripts/cleanup_feast_installation.sh b/.prow/scripts/cleanup_feast_installation.sh new file mode 100755 index 00000000000..78b7d83a327 --- /dev/null +++ b/.prow/scripts/cleanup_feast_installation.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e + +bq -q rm -rf --dataset ${FEAST_WAREHOUSE_DATASET} +gsutil -q rm ${BATCH_IMPORT_DATA_GCS_PATH} +helm delete --purge $FEAST_RELEASE_NAME diff --git a/.prow/scripts/install_feast_and_run_e2e_test.sh b/.prow/scripts/install_feast_and_run_e2e_test.sh new file mode 100755 index 00000000000..029f2e3e31a --- /dev/null +++ b/.prow/scripts/install_feast_and_run_e2e_test.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -e + +echo "============================================================" +echo "Installing Feast Release" +echo "============================================================" + +helm install --name ${FEAST_RELEASE_NAME} --wait --timeout 210 ${FEAST_HOME}/charts/feast -f integration-tests/feast-helm-values.yaml + +echo "============================================================" +echo "Testing Batch Import" +echo "============================================================" + +cd ${FEAST_HOME}/integration-tests/testdata + +feast apply entity entity_specs/entity_1.yaml +feast apply feature feature_specs/entity_1*.yaml +feast jobs run import_specs/batch_from_gcs.yaml --wait + +cd $FEAST_HOME/integration-tests + +python -m testutils.validate_feature_values \ + --entity_spec_file=testdata/entity_specs/entity_1.yaml \ + --feature_spec_files=testdata/feature_specs/entity_1*.yaml \ + --expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \ + --expected-serving-values-file=testdata/feature_values/serving_1.csv \ + --bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \ + --feast-serving-url=${FEAST_SERVING_URL} + +echo "============================================================" +echo "Testing Streaming Import" +echo "============================================================" + +cd $FEAST_HOME/integration-tests/testdata + +feast apply entity entity_specs/entity_2.yaml +feast apply feature feature_specs/entity_2*.yaml +feast jobs run import_specs/stream_from_kafka.yaml & + +IMPORT_JOB_PID=$! +sleep 20 + +cd $FEAST_HOME/integration-tests + +python -m testutils.kafka_producer \ + --bootstrap_servers=$KAFKA_BROKERS \ + --topic=$KAFKA_TOPICS \ + --entity_spec_file=testdata/entity_specs/entity_2.yaml \ + --feature_spec_files=testdata/feature_specs/entity_2*.yaml \ + --feature_values_file=testdata/feature_values/ingestion_2.csv +sleep 20 + +python -m testutils.validate_feature_values \ + --entity_spec_file=testdata/entity_specs/entity_2.yaml \ + --feature_spec_files=testdata/feature_specs/entity_2*.yaml \ + --expected-serving-values-file=testdata/feature_values/serving_2.csv \ + --feast-serving-url=$FEAST_SERVING_URL + +kill -9 ${IMPORT_JOB_PID} diff --git a/.prow/scripts/install_feast_sdk.sh b/.prow/scripts/install_feast_sdk.sh new file mode 100755 index 00000000000..723199b9e51 --- /dev/null +++ b/.prow/scripts/install_feast_sdk.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +# This script ensures latest Feast Python SDK and Feast CLI are installed + +pip install -qe ${FEAST_HOME}/sdk/python +pip install -qr ${FEAST_HOME}/integration-tests/testutils/requirements.txt +go build -o /usr/local/bin/feast ./cli/feast &> /dev/null +feast config set coreURI ${FEAST_CORE_URL} diff --git a/.prow/scripts/install_google_cloud_sdk.sh b/.prow/scripts/install_google_cloud_sdk.sh new file mode 100755 index 00000000000..18fb96834f2 --- /dev/null +++ b/.prow/scripts/install_google_cloud_sdk.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -e + +usage() +{ + echo "usage: . install_google_cloud_sdk.sh + [--with-key-file local file path to service account json] + +NOTE: requires 'dot' before install_google_cloud_sdk.sh + so that the PATH variable is exported succesfully to + the calling process, i.e. you don't need to provide + full path to gcloud command after installation + + --with-key-file is optional, + if no authentication is required" +} + +while [ "$1" != "" ]; do + case "$1" in + --with-key-file ) KEY_FILE="$2"; shift;; + * ) usage; exit 1 + esac + shift +done + +GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz +GOOGLE_PROJECT_ID=kf-feast +KUBE_CLUSTER_NAME=primary-test-cluster +KUBE_CLUSTER_ZONE=us-central1-a + +curl -s ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz -C / +export PATH=/google-cloud-sdk/bin:${PATH} +gcloud -q components install kubectl + +if [[ ${KEY_FILE} ]]; then + gcloud -q auth activate-service-account --key-file=${KEY_FILE} + gcloud -q auth configure-docker + gcloud -q config set project ${GOOGLE_PROJECT_ID} + gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID} + export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE} +fi diff --git a/.prow/scripts/install_test_tools.sh b/.prow/scripts/install_test_tools.sh new file mode 100755 index 00000000000..a2e30bbc3a2 --- /dev/null +++ b/.prow/scripts/install_test_tools.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +# This script installs the following Feast test utilities: +# ============================================================ +# - gettext package so we can use envsubst command to provide values to helm template file +# - Python 3.6 because Feast requires Python version 3.6 and above +# - Golang if we need to build Feast CLI from source +# - Helm if we want to install Feast release + +apt-get -qq update +apt-get -y install curl wget gettext &> /dev/null + +curl -s https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh -o /tmp/miniconda.sh +bash /tmp/miniconda.sh -b -p /miniconda &> /dev/null +export PATH=/miniconda/bin:$PATH + +wget -qO- https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz | tar xzf - +mv go /usr/local/ +export PATH=/usr/local/go/bin:$PATH +export GO111MODULE=on + +wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz +mv linux-amd64/helm /usr/local/bin/helm diff --git a/.prow/scripts/prepare_integration_test.sh b/.prow/scripts/prepare_integration_test.sh new file mode 100755 index 00000000000..1a08f26475b --- /dev/null +++ b/.prow/scripts/prepare_integration_test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -e + +usage() +{ + echo "usage: prepare_integration_test.sh [--skip-build true]" +} + +while [ "$1" != "" ]; do + case "$1" in + --skip-build ) SKIP_BUILD=true; shift;; + * ) usage; exit 1 + esac + shift +done + +# Authenticate to Google Cloud and GKE +# ============================================================ +GOOGLE_PROJECT_ID=kf-feast +KUBE_CLUSTER_NAME=primary-test-cluster +KUBE_CLUSTER_ZONE=us-central1-a +KEY_FILE=/etc/service-account/service-account.json + +gcloud -q auth activate-service-account --key-file=${KEY_FILE} +gcloud -q auth configure-docker +gcloud -q config set project ${GOOGLE_PROJECT_ID} +gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID} +export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE} + +# Install Python 3.6, Golang 1.12, Helm and Feast SDK +# ============================================================ +. .prow/scripts/install_test_tools.sh +. .prow/scripts/install_feast_sdk.sh +.prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir ${FEAST_HOME} + +# Prepare Feast test data and config +# ============================================================ + +bq -q mk --dataset ${FEAST_WAREHOUSE_DATASET} +gsutil -q cp ${FEAST_HOME}/integration-tests/testdata/feature_values/ingestion_1.csv ${BATCH_IMPORT_DATA_GCS_PATH} + +BUILD_ID=${BUILD_ID:0:5} +envsubst < integration-tests/feast-helm-values.yaml.template > integration-tests/feast-helm-values.yaml +cd ${FEAST_HOME}/integration-tests/testdata/import_specs +envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml +envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml + +if [[ ! ${SKIP_BUILD} ]]; then + +echo "============================================================" +echo "Building Feast for Testing" +echo "============================================================" +cd ${FEAST_HOME} +docker build -t us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} -f Dockerfiles/core/Dockerfile . & +docker build -t us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} -f Dockerfiles/serving/Dockerfile . & +wait +docker push us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} & +docker push us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} & +wait + +fi + +# Switch back context to original directory +set +ex +cd ${FEAST_HOME} \ No newline at end of file diff --git a/.prow/scripts/prepare_maven_cache.sh b/.prow/scripts/prepare_maven_cache.sh new file mode 100755 index 00000000000..e5113389426 --- /dev/null +++ b/.prow/scripts/prepare_maven_cache.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -e + +# This script downloads previous maven packages that have been downloaded +# from Google Cloud Storage to local path for faster build + +usage() +{ + echo "usage: prepare_maven_cache.sh + --archive-uri gcs uri to retrieve maven .m2 archive + --output-dir output directory for .m2 directory" +} + +while [ "$1" != "" ]; do + case "$1" in + --archive-uri ) ARCHIVE_URI="$2"; shift;; + --output-dir ) OUTPUT_DIR="$2"; shift;; + * ) usage; exit 1 + esac + shift +done + +if [[ ! ${ARCHIVE_URI} ]]; then usage; exit 1; fi +if [[ ! ${OUTPUT_DIR} ]]; then usage; exit 1; fi + +gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar +tar xf /tmp/.m2.tar -C ${OUTPUT_DIR} diff --git a/.prow/scripts/run_unit_test.sh b/.prow/scripts/run_unit_test.sh new file mode 100755 index 00000000000..f36fa58f90d --- /dev/null +++ b/.prow/scripts/run_unit_test.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -e + +# This script will run unit test for a specific Feast component: +# - core, ingestion, serving or cli +# +# This script includes the pre and post test scripts, such as +# - downloading maven cache repository +# - saving the test output report so it can be viewed with Spyglass in Prow + +usage() +{ + echo "usage: run_unit_test.sh + --component {core, ingestion, serving, cli}" +} + +while [ "$1" != "" ]; do + case "$1" in + --component ) COMPONENT="$2"; shift;; + * ) usage; exit 1 + esac + shift +done + +if [[ ! ${COMPONENT} ]]; then + usage; exit 1; +fi + +. .prow/scripts/install_google_cloud_sdk.sh + +if [[ ${COMPONENT} == "core" ]] || [[ ${COMPONENT} == "ingestion" ]] || [[ ${COMPONENT} == "serving" ]]; then + + .prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/ + mvn --projects ${COMPONENT} test + TEST_EXIT_CODE=$? + cp -r ${COMPONENT}/target/surefire-reports /logs/artifacts/surefire-reports + +elif [[ ${COMPONENT} == "cli" ]]; then + + go get -u github.com/jstemmer/go-junit-report + go test -v ./cli/feast/... 2>&1 | tee test_output + TEST_EXIT_CODE=$? + cat test_output | ${GOPATH}/bin/go-junit-report > ${ARTIFACTS}/report.xml + +else + usage; exit 1 +fi + +exit ${TEST_EXIT_CODE} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e3c30728552..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,114 +0,0 @@ -cache: - directories: - - $HOME/.m2 - -matrix: - include: - - stage: unit test - name: test ingestion - language: java - jdk: openjdk8 - script: mvn --projects ingestion --batch-mode test - - - stage: unit test - name: test core - language: java - jdk: openjdk8 - script: mvn --projects core --batch-mode test - - - stage: unit test - name: test serving - language: java - jdk: openjdk8 - script: mvn --projects serving --batch-mode test - - - stage: unit test - name: test cli - language: go - go: 1.12.x - env: GO111MODULE=on - script: go test ./cli/feast/... - - - stage: build - name: build core - language: java - jdk: openjdk8 - services: docker - before_install: .travis/decrypt_secrets.sh - install: . .travis/install_google_cloud_sdk.sh - before_script: .travis/prepare_maven_cache_for_docker.sh - script: - - docker build --tag=us.gcr.io/kf-feast/feast-core:${TRAVIS_COMMIT} --build-arg=REVISION=${TRAVIS_COMMIT} --file Dockerfiles/core/Dockerfile . - - docker push us.gcr.io/kf-feast/feast-core:${TRAVIS_COMMIT} - if: type != pull_request - - - stage: build - name: build serving - language: java - jdk: openjdk8 - services: docker - before_install: .travis/decrypt_secrets.sh - install: . .travis/install_google_cloud_sdk.sh - before_script: .travis/prepare_maven_cache_for_docker.sh - script: - - docker build --tag=us.gcr.io/kf-feast/feast-serving:${TRAVIS_COMMIT} --build-arg=REVISION=${TRAVIS_COMMIT} --file Dockerfiles/serving/Dockerfile . - - docker push us.gcr.io/kf-feast/feast-serving:${TRAVIS_COMMIT} - if: type != pull_request - - - stage: build - name: build cli - language: go - go: 1.12.x - env: - - GO111MODULE=on - - FEAST_CLI_GCS_URI=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/cli/feast - before_install: .travis/decrypt_secrets.sh - install: . .travis/install_google_cloud_sdk.sh - script: - - go build -o ./cli/build/feast ./cli/feast - - gsutil cp ./cli/build/feast ${FEAST_CLI_GCS_URI} - if: type != pull_request - - - stage: integration test - name: test batch and streaming import job - language: python - python: 3.6 - env: - - BATCH_IMPORT_DATA_LOCAL_PATH=${TRAVIS_BUILD_DIR}/integration-tests/testdata/feature_values/ingestion_1.csv - - BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/integration-tests/testdata/feature_values/ingestion_1.csv - - FEAST_IMAGE_TAG=${TRAVIS_COMMIT} - - FEAST_CLI_GCS_URI=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/cli/feast - - FEAST_WAREHOUSE_DATASET=feast_build_${TRAVIS_BUILD_NUMBER} - - FEAST_CORE_URI=localhost:6565 - - FEAST_SERVING_URI=localhost:6566 - - KAFKA_BROKERS=localhost:9092 - - KAFKA_TOPICS=feast-topic - before_install: .travis/decrypt_secrets.sh - install: - - . .travis/install_google_cloud_sdk.sh - - . .travis/install_feast_sdk.sh - before_script: - - .travis/start_local_feast.sh - - .travis/prepare_testdata.sh - script: - - .travis/run_batch_import_and_validate.sh - - .travis/run_streaming_import_and_validate.sh - after_script: .travis/cleanup_testdata.sh - if: type != pull_request - - - stage: deployment test - name: test helm deployment - language: minimal - env: - - BUILD_NUMBER=${TRAVIS_BUILD_NUMBER} - - FEAST_IMAGE_TAG=${TRAVIS_COMMIT} - - FEAST_WAREHOUSE_DATASET=feast_build_${TRAVIS_BUILD_NUMBER} - - RELEASE_NAME=feast-build-${TRAVIS_BUILD_NUMBER} - before_install: .travis/decrypt_secrets.sh - install: - - . .travis/install_google_cloud_sdk.sh - - . .travis/install_helm.sh - before_script: envsubst < ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml.template > ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml - script: helm install --name ${RELEASE_NAME} --wait --timeout 300 ./charts/feast -f ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml - after_script: helm delete --purge ${RELEASE_NAME} - if: type != pull_request diff --git a/.travis/cleanup_testdata.sh b/.travis/cleanup_testdata.sh deleted file mode 100755 index ecfd202f05f..00000000000 --- a/.travis/cleanup_testdata.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -bq --project_id=kf-feast rm -rf --dataset ${FEAST_WAREHOUSE_DATASET} -gsutil rm -r ${BATCH_IMPORT_DATA_GCS_PATH} diff --git a/.travis/decrypt_secrets.sh b/.travis/decrypt_secrets.sh deleted file mode 100755 index 8d3a50074c5..00000000000 --- a/.travis/decrypt_secrets.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -# Decrypt Google Cloud service account JSON key for BigQuery and GKE access -openssl aes-256-cbc \ --K $encrypted_deb1d355a9cd_key \ --iv $encrypted_deb1d355a9cd_iv \ --in ${SCRIPT_DIR}/service_account.json.enc \ --out ${SCRIPT_DIR}/service_account.json \ --d diff --git a/.travis/install_feast_sdk.sh b/.travis/install_feast_sdk.sh deleted file mode 100755 index 502fec2f2f8..00000000000 --- a/.travis/install_feast_sdk.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -# This assumes Feast CLI has been built and pushed to GCS in the previous build step -gsutil cp ${FEAST_CLI_GCS_URI} ${TRAVIS_BUILD_DIR}/cli/build/feast -chmod +x ${TRAVIS_BUILD_DIR}/cli/build/feast -export PATH=${TRAVIS_BUILD_DIR}/cli/build:$PATH -feast config set coreURI ${FEAST_CORE_URI} - -pip install -qe ${TRAVIS_BUILD_DIR}/sdk/python -pip install -qr ${TRAVIS_BUILD_DIR}/integration-tests/testutils/requirements.txt -export GOOGLE_APPLICATION_CREDENTIALS=${SCRIPT_DIR}/service_account.json diff --git a/.travis/install_google_cloud_sdk.sh b/.travis/install_google_cloud_sdk.sh deleted file mode 100755 index 1101cde1c2e..00000000000 --- a/.travis/install_google_cloud_sdk.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz - -wget -qO- ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz -export PATH=$PWD/google-cloud-sdk/bin:$PATH -gcloud -q components install kubectl -gcloud config set project kf-feast -gcloud -q auth activate-service-account --key-file=${SCRIPT_DIR}/service_account.json -gcloud -q auth configure-docker -gcloud -q container clusters get-credentials feast-test-cluster --zone us-central1-a --project kf-feast \ No newline at end of file diff --git a/.travis/install_helm.sh b/.travis/install_helm.sh deleted file mode 100755 index d90a920d263..00000000000 --- a/.travis/install_helm.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz -sudo mv linux-amd64/helm /usr/local/bin/helm \ No newline at end of file diff --git a/.travis/prepare_maven_cache_for_docker.sh b/.travis/prepare_maven_cache_for_docker.sh deleted file mode 100755 index 128960374f7..00000000000 --- a/.travis/prepare_maven_cache_for_docker.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -# This works because Dockerfile for core and serving assumes Maven repository -# is located at .m2 directory inside Feast repository root directory -cp -r $HOME/.m2 ${TRAVIS_BUILD_DIR}/.m2 \ No newline at end of file diff --git a/.travis/prepare_testdata.sh b/.travis/prepare_testdata.sh deleted file mode 100755 index 61a40d65ab9..00000000000 --- a/.travis/prepare_testdata.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -cd ${TRAVIS_BUILD_DIR}/integration-tests/testdata/import_specs -envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml -envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml - -bq --project_id=kf-feast mk --dataset $FEAST_WAREHOUSE_DATASET -gsutil cp ${BATCH_IMPORT_DATA_LOCAL_PATH} ${BATCH_IMPORT_DATA_GCS_PATH} \ No newline at end of file diff --git a/.travis/run_batch_import_and_validate.sh b/.travis/run_batch_import_and_validate.sh deleted file mode 100755 index 9ce6d4d7916..00000000000 --- a/.travis/run_batch_import_and_validate.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -cd ${TRAVIS_BUILD_DIR}/integration-tests - -feast apply entity testdata/entity_specs/entity_1.yaml -feast apply feature testdata/feature_specs/entity_1.feature_*.yaml -feast jobs run testdata/import_specs/batch_from_gcs.yaml --wait - -python -m testutils.validate_feature_values \ ---entity_spec_file=testdata/entity_specs/entity_1.yaml \ ---feature_spec_files=testdata/feature_specs/entity_1*.yaml \ ---expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \ ---expected-serving-values-file=testdata/feature_values/serving_1.csv \ ---bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \ ---feast-serving-url=${FEAST_SERVING_URI} \ ---project=kf-feast \ No newline at end of file diff --git a/.travis/run_streaming_import_and_validate.sh b/.travis/run_streaming_import_and_validate.sh deleted file mode 100755 index a567c90ded7..00000000000 --- a/.travis/run_streaming_import_and_validate.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -cd ${TRAVIS_BUILD_DIR}/integration-tests -feast apply entity testdata/entity_specs/entity_2.yaml -feast apply feature testdata/feature_specs/entity_2.feature_*.yaml -feast jobs run testdata/import_specs/stream_from_kafka.yaml & - -sleep 20 - -python -m testutils.kafka_producer \ ---bootstrap_servers=${KAFKA_BROKERS} \ ---topic=${KAFKA_TOPICS} \ ---entity_spec_file=testdata/entity_specs/entity_2.yaml \ ---feature_spec_files=testdata/feature_specs/entity_2*.yaml \ ---feature_values_file=testdata/feature_values/ingestion_2.csv - -sleep 20 - -python -m testutils.validate_feature_values \ ---entity_spec_file=testdata/entity_specs/entity_2.yaml \ ---feature_spec_files=testdata/feature_specs/entity_2*.yaml \ ---expected-serving-values-file=testdata/feature_values/serving_2.csv \ ---feast-serving-url=${FEAST_SERVING_URI} \ No newline at end of file diff --git a/.travis/service_account.json.enc b/.travis/service_account.json.enc deleted file mode 100644 index 628a87686e5..00000000000 Binary files a/.travis/service_account.json.enc and /dev/null differ diff --git a/.travis/start_local_feast.sh b/.travis/start_local_feast.sh deleted file mode 100755 index ea60291249a..00000000000 --- a/.travis/start_local_feast.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -COLOR="\033[0;32m$(tput bold)" -NO_COLOR="\033[0m$(tput sgr0)" -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -echo -e "${COLOR}Starting Postgres...${NO_COLOR}" -docker run --net=host --env=POSTGRES_PASSWORD=password --detach postgres:11.2-alpine - -echo -e "${COLOR}Starting Redis...${NO_COLOR}" -docker run --net=host --detach redis:5.0-alpine - -echo -e "${COLOR}Starting Zookeeper...${NO_COLOR}" -docker run --net=host --env=ZOOKEEPER_CLIENT_PORT=2181 \ ---detach confluentinc/cp-zookeeper:5.2.1 - -echo -e "${COLOR}Starting Kafka...${NO_COLOR}" -docker run --net=host \ ---env=KAFKA_ZOOKEEPER_CONNECT=localhost:2181 \ ---env=KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \ ---env=KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ ---detach confluentinc/cp-kafka:5.2.1 - -echo -e "${COLOR}Starting Feast Core...${NO_COLOR}" -docker run --name core --net host \ ---env=JOB_WORKSPACE=/tmp \ ---env=STORE_WAREHOUSE_TYPE=bigquery \ ---env=STORE_WAREHOUSE_OPTIONS='{"project":"kf-feast","dataset":"'${FEAST_WAREHOUSE_DATASET}'","tempLocation":"gs://feast-templocation-kf-feast"}' \ ---env=STORE_SERVING_TYPE=redis \ ---env=STORE_SERVING_OPTIONS='{"host": "localhost", "port": "6379"}' \ ---env=GOOGLE_APPLICATION_CREDENTIALS=/etc/google_cloud/service_account.json \ ---volume=${SCRIPT_DIR}/service_account.json:/etc/google_cloud/service_account.json \ ---detach us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} -sleep 20 - -echo -e "${COLOR}Starting Feast Serving...${NO_COLOR}" -docker run --name serving --net host \ ---env=FEAST_SERVING_HTTP_PORT=8081 \ ---detach us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1f30ec334e2..a80d21b6f92 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,8 +2,8 @@ ## Code reviews -All submissions, including submissions by project members, require review. We use GitHub pull -requests for this purpose. Consult GitHub Help for more information on using pull requests. +Code submission to Feast (including submission from project maintainers) requires review and approval. +Please submit a **pull request** to initiate the code review process. We use [prow](https://github.com/kubernetes/test-infra/tree/master/prow) to manage the testing and reviewing of pull requests. Please refer to [config.yaml](../.prow/config.yaml) for details on the test jobs. ## Code conventions @@ -19,3 +19,5 @@ https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xm Make sure you apply `go fmt`. ### JavaScript + +TODO \ No newline at end of file diff --git a/OWNERS b/OWNERS index 3153f0c86bb..38d0d706063 100644 --- a/OWNERS +++ b/OWNERS @@ -5,6 +5,7 @@ approvers: - tims - thirteen37 - davidheryanto + - budi reviewers: - zhilingc - pradithya @@ -12,3 +13,4 @@ reviewers: - tims - thirteen37 - davidheryanto + - budi diff --git a/README.md b/README.md index 283af204e6c..cdd9d4ce0b1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Feast - Feature Store for Machine Learning [![Build Status](https://travis-ci.com/gojek/feast.svg?branch=master)](https://travis-ci.com/gojek/feast) +# Feast - Feature Store for Machine Learning [![Build Status](http://prow.feast.ai/badge.svg?jobs=integration-test)](http://prow.feast.ai) ## Overview @@ -46,7 +46,7 @@ For Feast administrators: ## Notice -Feast is still under active development. Your feedback and contributions are important to us. +Feast is still under active development. Your feedback and contributions are important to us. Please check our [contributing guide](CONTRIBUTING.md) for details. ## Source Code Headers diff --git a/integration-tests/feast-helm-values.yaml.template b/integration-tests/feast-helm-values.yaml.template index e645a5f9e89..03caef820aa 100644 --- a/integration-tests/feast-helm-values.yaml.template +++ b/integration-tests/feast-helm-values.yaml.template @@ -1,6 +1,6 @@ # Required environment variables to supply for this template: # ============================================================ -# - BUILD_NUMBER +# - BUILD_ID # - FEAST_IMAGE_TAG # - FEAST_WAREHOUSE_DATASET # @@ -27,45 +27,45 @@ core: --- apiVersion: ambassador/v1 kind: Mapping - name: feast_core_grpc_core_service_mapping_build_${BUILD_NUMBER} + name: feast_core_grpc_core_service_mapping_build_${BUILD_ID} grpc: True headers: :authority: ${FEAST_CORE_URL} prefix: /feast.core.CoreService rewrite: /feast.core.CoreService - service: feast-${BUILD_NUMBER}-core:6565 + service: feast-${BUILD_ID}-core:6565 timeout_ms: 60000 --- apiVersion: ambassador/v1 kind: Mapping - name: feast_core_grpc_job_service_mapping_build_${BUILD_NUMBER} + name: feast_core_grpc_job_service_mapping_build_${BUILD_ID} grpc: True headers: :authority: ${FEAST_CORE_URL} prefix: /feast.core.JobService rewrite: /feast.core.JobService - service: feast-${BUILD_NUMBER}-core:6565 + service: feast-${BUILD_ID}-core:6565 timeout_ms: 120000 --- apiVersion: ambassador/v1 kind: Mapping - name: feast_core_grpc_ui_service_mapping_build_${BUILD_NUMBER} + name: feast_core_grpc_ui_service_mapping_build_${BUILD_ID} grpc: True headers: :authority: ${FEAST_CORE_URL} prefix: /feast.core.UIService rewrite: /feast.core.UIService - service: feast-${BUILD_NUMBER}-core:6565 + service: feast-${BUILD_ID}-core:6565 --- apiVersion: ambassador/v1 kind: Mapping - name: feast_core_grpc_dataset_service_mapping_build_${BUILD_NUMBER} + name: feast_core_grpc_dataset_service_mapping_build_${BUILD_ID} grpc: True headers: :authority: ${FEAST_CORE_URL} prefix: /feast.core.DatasetService rewrite: /feast.core.DatasetService - service: feast-${BUILD_NUMBER}-core:6565 + service: feast-${BUILD_ID}-core:6565 grpc: port: 6565 targetPort: 6565 @@ -86,10 +86,10 @@ store: type: "stdout" warehouse: type: "bigquery" - options: '{"project": "kf-feast", "dataset": "${FEAST_WAREHOUSE_DATASET}", "tempLocation": "gs://feast-templocation-kf-feast/warehouse_build_${BUILD_NUMBER}"}' + options: '{"project": "kf-feast", "dataset": "${FEAST_WAREHOUSE_DATASET}", "tempLocation": "gs://feast-templocation-kf-feast/warehouse_build_${BUILD_ID}"}' serving: type: "redis" - options: '{"host": "feast-${BUILD_NUMBER}-redis-master", "port": "6379"}' + options: '{"host": "feast-${BUILD_ID}-redis-master", "port": "6379"}' postgresql: provision: true @@ -121,13 +121,13 @@ serving: --- apiVersion: ambassador/v1 kind: Mapping - name: feast_serving_grpc_serving_api_mapping_build_${BUILD_NUMBER} + name: feast_serving_grpc_serving_api_mapping_build_${BUILD_ID} grpc: True headers: :authority: ${FEAST_SERVING_URL} prefix: /feast.serving.ServingAPI rewrite: /feast.serving.ServingAPI - service: feast-${BUILD_NUMBER}-serving:6565 + service: feast-${BUILD_ID}-serving:6565 timeout_ms: 30000 grpc: port: 6565 @@ -140,4 +140,4 @@ serving: # want to mount the secret of. serviceAccount: name: feast-service-account - key: feast-service-account.json + key: service-account.json diff --git a/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java b/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java index db95dad859a..3b3ae6f35a9 100644 --- a/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java +++ b/serving/src/main/java/feast/serving/config/ServingApiConfiguration.java @@ -110,14 +110,13 @@ public FeatureStorageRegistry getFeatureStorageRegistry( AppConfig appConfig, Tracer tracer) { storageOptions = Strings.isNullOrEmpty(storageOptions) ? "{}" : storageOptions; Map optionsMap = convertJsonStringToMap(storageOptions); - StorageSpec storageSpec = StorageSpec.getDefaultInstance(); - if (Strings.isNullOrEmpty(storageType)) { - storageSpec = StorageSpec.newBuilder() - .setId("SERVING") - .setType(storageType) - .putAllOptions(optionsMap) - .build(); - } + + StorageSpec storageSpec = StorageSpec.newBuilder() + .setId("SERVING") + .setType(storageType) + .putAllOptions(optionsMap) + .build(); + FeatureStorageRegistry registry = new FeatureStorageRegistry(appConfig, tracer); try { registry.connect(storageSpec);