diff --git a/core/src/main/java/feast/core/dao/FeatureSetRepository.java b/core/src/main/java/feast/core/dao/FeatureSetRepository.java index 3eba2108889..b136650dfdf 100644 --- a/core/src/main/java/feast/core/dao/FeatureSetRepository.java +++ b/core/src/main/java/feast/core/dao/FeatureSetRepository.java @@ -25,25 +25,16 @@ public interface FeatureSetRepository extends JpaRepository long count(); - // Find single feature set by project, name, and version - FeatureSet findFeatureSetByNameAndProject_NameAndVersion( - String name, String project, Integer version); + // Find single feature set by project and name + FeatureSet findFeatureSetByNameAndProject_Name(String name, String project); - // Find single latest version of a feature set by project and name (LIKE) - FeatureSet findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc( - String name, String project); + // find all feature sets and order by name + List findAllByOrderByNameAsc(); - // find all feature sets and order by name and version - List findAllByOrderByNameAscVersionAsc(); + // find all feature sets matching the given name pattern with a specific project. + List findAllByNameLikeAndProject_NameOrderByNameAsc(String name, String project_name); - // find all feature sets within a project and order by name and version - List findAllByProject_NameOrderByNameAscVersionAsc(String project_name); - - // find all versions of feature sets matching the given name pattern with a specific project. - List findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - String name, String project_name); - - // find all versions of feature sets matching the given name pattern and project pattern - List findAllByNameLikeAndProject_NameLikeOrderByNameAscVersionAsc( + // find all feature sets matching the given name pattern and project pattern + List findAllByNameLikeAndProject_NameLikeOrderByNameAsc( String name, String project_name); } diff --git a/core/src/main/java/feast/core/job/JobUpdateTask.java b/core/src/main/java/feast/core/job/JobUpdateTask.java index bb876f47f22..b508aa46d2f 100644 --- a/core/src/main/java/feast/core/job/JobUpdateTask.java +++ b/core/src/main/java/feast/core/job/JobUpdateTask.java @@ -17,6 +17,7 @@ package feast.core.job; import com.google.common.collect.Sets; +import feast.core.FeatureSetProto.FeatureSetStatus; import feast.core.log.Action; import feast.core.log.AuditLogger; import feast.core.log.Resource; @@ -28,7 +29,6 @@ import java.time.Instant; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -84,7 +84,7 @@ public Job call() { } else { Job job = currentJob.get(); - if (featureSetsChangedFor(job)) { + if (requiresUpdate(job)) { submittedJob = executorService.submit(() -> updateJob(job)); } else { return updateStatus(job); @@ -101,11 +101,19 @@ public Job call() { } } - boolean featureSetsChangedFor(Job job) { - Set existingFeatureSetsPopulatedByJob = Sets.newHashSet(job.getFeatureSets()); - Set newFeatureSetsPopulatedByJob = Sets.newHashSet(featureSets); + boolean requiresUpdate(Job job) { + // If set of feature sets has changed + if (!Sets.newHashSet(featureSets).equals(Sets.newHashSet(job.getFeatureSets()))) { + return true; + } - return !newFeatureSetsPopulatedByJob.equals(existingFeatureSetsPopulatedByJob); + // If any existing feature set populated by the job has its status as pending + for (FeatureSet featureSet : job.getFeatureSets()) { + if (featureSet.getStatus().equals(FeatureSetStatus.STATUS_PENDING)) { + return true; + } + } + return false; } private Job createJob() { diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java index 2adedbefd9f..7b160b2c3db 100644 --- a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java +++ b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java @@ -79,7 +79,7 @@ public Job startJob(Job job) { featureSetProtos.add(featureSet.toProto()); } ImportOptions pipelineOptions = - getPipelineOptions(featureSetProtos, job.getStore().toProto()); + getPipelineOptions(job.getId(), featureSetProtos, job.getStore().toProto()); PipelineResult pipelineResult = runPipeline(pipelineOptions); DirectJob directJob = new DirectJob(job.getId(), pipelineResult); jobs.add(directJob); @@ -93,7 +93,8 @@ public Job startJob(Job job) { } private ImportOptions getPipelineOptions( - List featureSets, StoreProto.Store sink) throws IOException { + String jobName, List featureSets, StoreProto.Store sink) + throws IOException { String[] args = TypeConversion.convertMapToArgs(defaultOptions); ImportOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).as(ImportOptions.class); @@ -101,6 +102,7 @@ private ImportOptions getPipelineOptions( new BZip2Compressor<>(new FeatureSetJsonByteConverter()); pipelineOptions.setFeatureSetJson(featureSetJsonCompressor.compress(featureSets)); + pipelineOptions.setJobName(jobName); pipelineOptions.setStoreJson(Collections.singletonList(JsonFormat.printer().print(sink))); pipelineOptions.setRunner(DirectRunner.class); pipelineOptions.setProject(""); // set to default value to satisfy validation diff --git a/core/src/main/java/feast/core/model/Entity.java b/core/src/main/java/feast/core/model/Entity.java index 791e280d481..190f2781d58 100644 --- a/core/src/main/java/feast/core/model/Entity.java +++ b/core/src/main/java/feast/core/model/Entity.java @@ -54,6 +54,10 @@ public static Entity fromProto(EntitySpec entitySpec) { return entity; } + public EntitySpec toProto() { + return EntitySpec.newBuilder().setName(name).setValueType(ValueType.Enum.valueOf(type)).build(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/core/src/main/java/feast/core/model/Feature.java b/core/src/main/java/feast/core/model/Feature.java index 487de3ce378..2ae91b1997a 100644 --- a/core/src/main/java/feast/core/model/Feature.java +++ b/core/src/main/java/feast/core/model/Feature.java @@ -16,7 +16,9 @@ */ package feast.core.model; +import com.google.protobuf.InvalidProtocolBufferException; import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.FeatureSetProto.FeatureSpec.Builder; import feast.core.util.TypeConversion; import feast.types.ValueProto.ValueType; import java.util.Arrays; @@ -26,6 +28,7 @@ import javax.persistence.Entity; import lombok.Getter; import lombok.Setter; +import org.tensorflow.metadata.v0.*; /** * Feature belonging to a featureset. Contains name, type as well as domain metadata about the @@ -79,6 +82,9 @@ public class Feature { private byte[] timeOfDayDomain; public Feature() {} + // Whether this feature has been archived. A archived feature cannot be + // retrieved from or written to. + private boolean archived = false; private Feature(String name, ValueType.Enum type) { this.setName(name); @@ -88,13 +94,66 @@ private Feature(String name, ValueType.Enum type) { public static Feature fromProto(FeatureSpec featureSpec) { Feature feature = new Feature(featureSpec.getName(), featureSpec.getValueType()); feature.labels = TypeConversion.convertMapToJsonString(featureSpec.getLabelsMap()); + feature.updateSchema(featureSpec); + return feature; + } + + public FeatureSpec toProto() throws InvalidProtocolBufferException { + Builder featureSpecBuilder = + FeatureSpec.newBuilder().setName(getName()).setValueType(ValueType.Enum.valueOf(getType())); + + if (getPresence() != null) { + featureSpecBuilder.setPresence(FeaturePresence.parseFrom(getPresence())); + } else if (getGroupPresence() != null) { + featureSpecBuilder.setGroupPresence(FeaturePresenceWithinGroup.parseFrom(getGroupPresence())); + } + if (getShape() != null) { + featureSpecBuilder.setShape(FixedShape.parseFrom(getShape())); + } else if (getValueCount() != null) { + featureSpecBuilder.setValueCount(ValueCount.parseFrom(getValueCount())); + } + + if (getDomain() != null) { + featureSpecBuilder.setDomain(getDomain()); + } else if (getIntDomain() != null) { + featureSpecBuilder.setIntDomain(IntDomain.parseFrom(getIntDomain())); + } else if (getFloatDomain() != null) { + featureSpecBuilder.setFloatDomain(FloatDomain.parseFrom(getFloatDomain())); + } else if (getStringDomain() != null) { + featureSpecBuilder.setStringDomain(StringDomain.parseFrom(getStringDomain())); + } else if (getBoolDomain() != null) { + featureSpecBuilder.setBoolDomain(BoolDomain.parseFrom(getBoolDomain())); + } else if (getStructDomain() != null) { + featureSpecBuilder.setStructDomain(StructDomain.parseFrom(getStructDomain())); + } else if (getNaturalLanguageDomain() != null) { + featureSpecBuilder.setNaturalLanguageDomain( + NaturalLanguageDomain.parseFrom(getNaturalLanguageDomain())); + } else if (getImageDomain() != null) { + featureSpecBuilder.setImageDomain(ImageDomain.parseFrom(getImageDomain())); + } else if (getMidDomain() != null) { + featureSpecBuilder.setMidDomain(MIDDomain.parseFrom(getMidDomain())); + } else if (getUrlDomain() != null) { + featureSpecBuilder.setUrlDomain(URLDomain.parseFrom(getUrlDomain())); + } else if (getTimeDomain() != null) { + featureSpecBuilder.setTimeDomain(TimeDomain.parseFrom(getTimeDomain())); + } else if (getTimeOfDayDomain() != null) { + featureSpecBuilder.setTimeOfDayDomain(TimeOfDayDomain.parseFrom(getTimeOfDayDomain())); + } + + if (getLabels() != null) { + featureSpecBuilder.putAllLabels(getLabels()); + } + return featureSpecBuilder.build(); + } + + private void updateSchema(FeatureSpec featureSpec) { switch (featureSpec.getPresenceConstraintsCase()) { case PRESENCE: - feature.setPresence(featureSpec.getPresence().toByteArray()); + setPresence(featureSpec.getPresence().toByteArray()); break; case GROUP_PRESENCE: - feature.setGroupPresence(featureSpec.getGroupPresence().toByteArray()); + setGroupPresence(featureSpec.getGroupPresence().toByteArray()); break; case PRESENCECONSTRAINTS_NOT_SET: break; @@ -102,10 +161,10 @@ public static Feature fromProto(FeatureSpec featureSpec) { switch (featureSpec.getShapeTypeCase()) { case SHAPE: - feature.setShape(featureSpec.getShape().toByteArray()); + setShape(featureSpec.getShape().toByteArray()); break; case VALUE_COUNT: - feature.setValueCount(featureSpec.getValueCount().toByteArray()); + setValueCount(featureSpec.getValueCount().toByteArray()); break; case SHAPETYPE_NOT_SET: break; @@ -113,45 +172,70 @@ public static Feature fromProto(FeatureSpec featureSpec) { switch (featureSpec.getDomainInfoCase()) { case DOMAIN: - feature.setDomain(featureSpec.getDomain()); + setDomain(featureSpec.getDomain()); break; case INT_DOMAIN: - feature.setIntDomain(featureSpec.getIntDomain().toByteArray()); + setIntDomain(featureSpec.getIntDomain().toByteArray()); break; case FLOAT_DOMAIN: - feature.setFloatDomain(featureSpec.getFloatDomain().toByteArray()); + setFloatDomain(featureSpec.getFloatDomain().toByteArray()); break; case STRING_DOMAIN: - feature.setStringDomain(featureSpec.getStringDomain().toByteArray()); + setStringDomain(featureSpec.getStringDomain().toByteArray()); break; case BOOL_DOMAIN: - feature.setBoolDomain(featureSpec.getBoolDomain().toByteArray()); + setBoolDomain(featureSpec.getBoolDomain().toByteArray()); break; case STRUCT_DOMAIN: - feature.setStructDomain(featureSpec.getStructDomain().toByteArray()); + setStructDomain(featureSpec.getStructDomain().toByteArray()); break; case NATURAL_LANGUAGE_DOMAIN: - feature.setNaturalLanguageDomain(featureSpec.getNaturalLanguageDomain().toByteArray()); + setNaturalLanguageDomain(featureSpec.getNaturalLanguageDomain().toByteArray()); break; case IMAGE_DOMAIN: - feature.setImageDomain(featureSpec.getImageDomain().toByteArray()); + setImageDomain(featureSpec.getImageDomain().toByteArray()); break; case MID_DOMAIN: - feature.setMidDomain(featureSpec.getMidDomain().toByteArray()); + setMidDomain(featureSpec.getMidDomain().toByteArray()); break; case URL_DOMAIN: - feature.setUrlDomain(featureSpec.getUrlDomain().toByteArray()); + setUrlDomain(featureSpec.getUrlDomain().toByteArray()); break; case TIME_DOMAIN: - feature.setTimeDomain(featureSpec.getTimeDomain().toByteArray()); + setTimeDomain(featureSpec.getTimeDomain().toByteArray()); break; case TIME_OF_DAY_DOMAIN: - feature.setTimeOfDayDomain(featureSpec.getTimeOfDayDomain().toByteArray()); + setTimeOfDayDomain(featureSpec.getTimeOfDayDomain().toByteArray()); break; case DOMAININFO_NOT_SET: break; } - return feature; + } + + /** Archive this feature. */ + public void archive() { + this.archived = true; + } + + /** + * Update the feature object with a valid feature spec. Only schema changes are allowed. + * + * @param featureSpec {@link FeatureSpec} containing schema changes. + */ + public void updateFromProto(FeatureSpec featureSpec) { + if (isArchived()) { + throw new IllegalArgumentException( + String.format( + "You are attempting to create a feature %s that was previously archived. This isn't allowed. Please create a new feature with a different name.", + featureSpec.getName())); + } + if (ValueType.Enum.valueOf(type) != featureSpec.getValueType()) { + throw new IllegalArgumentException( + String.format( + "You are attempting to change the type of feature %s from %s to %s. This isn't allowed. Please create a new feature.", + featureSpec.getName(), type, featureSpec.getValueType())); + } + updateSchema(featureSpec); } public Map getLabels() { @@ -167,7 +251,9 @@ public boolean equals(Object o) { return false; } Feature feature = (Feature) o; - return Objects.equals(getName(), feature.getName()) + return getName().equals(feature.getName()) + && getType().equals(feature.getType()) + && isArchived() == (feature.isArchived()) && Objects.equals(getLabels(), feature.getLabels()) && Arrays.equals(getPresence(), feature.getPresence()) && Arrays.equals(getGroupPresence(), feature.getGroupPresence()) diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java index 91bb2bea89f..1c8351f4790 100644 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ b/core/src/main/java/feast/core/model/FeatureSet.java @@ -16,14 +16,15 @@ */ package feast.core.model; +import com.google.common.collect.Sets; import com.google.protobuf.Duration; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; import feast.core.FeatureSetProto; import feast.core.FeatureSetProto.*; import feast.core.util.TypeConversion; -import feast.types.ValueProto.ValueType.Enum; import java.util.*; +import java.util.stream.Collectors; import javax.persistence.*; import lombok.Getter; import lombok.Setter; @@ -35,8 +36,8 @@ @javax.persistence.Entity @Table( name = "feature_sets", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "version", "project_name"})) -public class FeatureSet extends AbstractTimestampEntity implements Comparable { + uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project_name"})) +public class FeatureSet extends AbstractTimestampEntity { // Id of the featureSet, defined as project/feature_set_name:feature_set_version @Id @GeneratedValue private long id; @@ -45,10 +46,6 @@ public class FeatureSet extends AbstractTimestampEntity implements Comparable entities, List features, @@ -103,21 +100,16 @@ public FeatureSet( FeatureSetStatus status) { this.maxAgeSeconds = maxAgeSeconds; this.source = source; - this.status = status.toString(); + this.status = status; this.entities = new HashSet<>(); this.features = new HashSet<>(); this.name = name; this.project = new Project(project); - this.version = version; this.labels = TypeConversion.convertMapToJsonString(labels); addEntities(entities); addFeatures(features); } - public void setVersion(int version) { - this.version = version; - } - public void setName(String name) { this.name = name; } @@ -151,7 +143,6 @@ public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) { return new FeatureSet( featureSetProto.getSpec().getName(), featureSetProto.getSpec().getProject(), - featureSetProto.getSpec().getVersion(), featureSetSpec.getMaxAge().getSeconds(), entitySpecs, featureSpecs, @@ -160,6 +151,64 @@ public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) { featureSetProto.getMeta().getStatus()); } + // Updates the existing feature set from a proto. + public void updateFromProto(FeatureSetProto.FeatureSet featureSetProto) + throws InvalidProtocolBufferException { + FeatureSetSpec spec = featureSetProto.getSpec(); + if (this.toProto().getSpec().equals(spec)) { + return; + } + + // 1. validate + // 1a. check no change to identifiers + if (!name.equals(spec.getName())) { + throw new IllegalArgumentException( + String.format("Given feature set name %s does not match name %s.", spec.getName(), name)); + } + if (!project.getName().equals(spec.getProject())) { + throw new IllegalArgumentException( + String.format( + "You are attempting to change the project of feature set %s from %s to %s. This isn't allowed. Please create a new feature set under the desired project.", + spec.getName(), project, spec.getProject())); + } + + Set existingEntities = + entities.stream().map(Entity::toProto).collect(Collectors.toSet()); + + // 1b. check no change to entities + if (!Sets.newHashSet(spec.getEntitiesList()).equals(existingEntities)) { + throw new IllegalArgumentException( + String.format( + "You are attempting to change the entities of this feature set: Given set of entities \n{%s}\n does not match existing set of entities\n {%s}. This isn't allowed. Please create a new feature set. ", + spec.getEntitiesList(), existingEntities)); + } + + // 4. Update max age and source. + maxAgeSeconds = spec.getMaxAge().getSeconds(); + source = Source.fromProto(spec.getSource()); + + Map updatedFeatures = + spec.getFeaturesList().stream().collect(Collectors.toMap(FeatureSpec::getName, fs -> fs)); + + // 3. Tombstone features that are gone, update features that have changed + for (Feature existingFeature : features) { + String existingFeatureName = existingFeature.getName(); + FeatureSpec updatedFeatureSpec = updatedFeatures.get(existingFeatureName); + if (updatedFeatureSpec == null) { + existingFeature.archive(); + } else { + existingFeature.updateFromProto(updatedFeatureSpec); + updatedFeatures.remove(existingFeatureName); + } + } + + // 4. Add new features + for (FeatureSpec featureSpec : updatedFeatures.values()) { + Feature newFeature = Feature.fromProto(featureSpec); + addFeature(newFeature); + } + } + public void addEntities(List entities) { for (Entity entity : entities) { addEntity(entity); @@ -185,28 +234,25 @@ public void addFeature(Feature feature) { public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferException { List entitySpecs = new ArrayList<>(); for (Entity entityField : entities) { - EntitySpec.Builder entitySpecBuilder = EntitySpec.newBuilder(); - setEntitySpecFields(entitySpecBuilder, entityField); - entitySpecs.add(entitySpecBuilder.build()); + entitySpecs.add(entityField.toProto()); } List featureSpecs = new ArrayList<>(); for (Feature featureField : features) { - FeatureSpec.Builder featureSpecBuilder = FeatureSpec.newBuilder(); - setFeatureSpecFields(featureSpecBuilder, featureField); - featureSpecs.add(featureSpecBuilder.build()); + if (!featureField.isArchived()) { + featureSpecs.add(featureField.toProto()); + } } FeatureSetMeta.Builder meta = FeatureSetMeta.newBuilder() .setCreatedTimestamp( Timestamp.newBuilder().setSeconds(super.getCreated().getTime() / 1000L)) - .setStatus(FeatureSetStatus.valueOf(status)); + .setStatus(status); FeatureSetSpec.Builder spec = FeatureSetSpec.newBuilder() .setName(getName()) - .setVersion(getVersion()) .setProject(project.getName()) .setMaxAge(Duration.newBuilder().setSeconds(maxAgeSeconds)) .addAllEntities(entitySpecs) @@ -217,71 +263,24 @@ public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferExceptio return FeatureSetProto.FeatureSet.newBuilder().setMeta(meta).setSpec(spec).build(); } - private void setEntitySpecFields(EntitySpec.Builder entitySpecBuilder, Entity entityField) { - entitySpecBuilder - .setName(entityField.getName()) - .setValueType(Enum.valueOf(entityField.getType())); + @Override + public int hashCode() { + HashCodeBuilder hcb = new HashCodeBuilder(); + hcb.append(project.getName()); + hcb.append(getName()); + return hcb.toHashCode(); } - private void setFeatureSpecFields(FeatureSpec.Builder featureSpecBuilder, Feature featureField) - throws InvalidProtocolBufferException { - featureSpecBuilder - .setName(featureField.getName()) - .setValueType(Enum.valueOf(featureField.getType())); - - if (featureField.getPresence() != null) { - featureSpecBuilder.setPresence(FeaturePresence.parseFrom(featureField.getPresence())); - } else if (featureField.getGroupPresence() != null) { - featureSpecBuilder.setGroupPresence( - FeaturePresenceWithinGroup.parseFrom(featureField.getGroupPresence())); - } - - if (featureField.getShape() != null) { - featureSpecBuilder.setShape(FixedShape.parseFrom(featureField.getShape())); - } else if (featureField.getValueCount() != null) { - featureSpecBuilder.setValueCount(ValueCount.parseFrom(featureField.getValueCount())); - } - - if (featureField.getDomain() != null) { - featureSpecBuilder.setDomain(featureField.getDomain()); - } else if (featureField.getIntDomain() != null) { - featureSpecBuilder.setIntDomain(IntDomain.parseFrom(featureField.getIntDomain())); - } else if (featureField.getFloatDomain() != null) { - featureSpecBuilder.setFloatDomain(FloatDomain.parseFrom(featureField.getFloatDomain())); - } else if (featureField.getStringDomain() != null) { - featureSpecBuilder.setStringDomain(StringDomain.parseFrom(featureField.getStringDomain())); - } else if (featureField.getBoolDomain() != null) { - featureSpecBuilder.setBoolDomain(BoolDomain.parseFrom(featureField.getBoolDomain())); - } else if (featureField.getStructDomain() != null) { - featureSpecBuilder.setStructDomain(StructDomain.parseFrom(featureField.getStructDomain())); - } else if (featureField.getNaturalLanguageDomain() != null) { - featureSpecBuilder.setNaturalLanguageDomain( - NaturalLanguageDomain.parseFrom(featureField.getNaturalLanguageDomain())); - } else if (featureField.getImageDomain() != null) { - featureSpecBuilder.setImageDomain(ImageDomain.parseFrom(featureField.getImageDomain())); - } else if (featureField.getMidDomain() != null) { - featureSpecBuilder.setMidDomain(MIDDomain.parseFrom(featureField.getMidDomain())); - } else if (featureField.getUrlDomain() != null) { - featureSpecBuilder.setUrlDomain(URLDomain.parseFrom(featureField.getUrlDomain())); - } else if (featureField.getTimeDomain() != null) { - featureSpecBuilder.setTimeDomain(TimeDomain.parseFrom(featureField.getTimeDomain())); - } else if (featureField.getTimeOfDayDomain() != null) { - featureSpecBuilder.setTimeOfDayDomain( - TimeOfDayDomain.parseFrom(featureField.getTimeOfDayDomain())); + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - if (featureField.getLabels() != null) { - featureSpecBuilder.putAllLabels(featureField.getLabels()); + if (!(obj instanceof FeatureSet)) { + return false; } - } - /** - * Checks if the given featureSet's schema and source has is different from this one. - * - * @param other FeatureSet to compare to - * @return boolean denoting if the source or schema have changed. - */ - public boolean equalTo(FeatureSet other) { + FeatureSet other = (FeatureSet) obj; if (!getName().equals(other.getName())) { return false; } @@ -343,29 +342,4 @@ public boolean equalTo(FeatureSet other) { return true; } - - @Override - public int hashCode() { - HashCodeBuilder hcb = new HashCodeBuilder(); - hcb.append(project.getName()); - hcb.append(getName()); - hcb.append(getVersion()); - return hcb.toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof FeatureSet)) { - return false; - } - return this.equalTo(((FeatureSet) obj)); - } - - @Override - public int compareTo(FeatureSet o) { - return Integer.compare(getVersion(), o.getVersion()); - } } diff --git a/core/src/main/java/feast/core/model/Store.java b/core/src/main/java/feast/core/model/Store.java index debf211ec8d..a08c50b28a6 100644 --- a/core/src/main/java/feast/core/model/Store.java +++ b/core/src/main/java/feast/core/model/Store.java @@ -125,22 +125,18 @@ public List getSubscriptions() { } private static String convertSubscriptionToString(Subscription sub) { - if (sub.getVersion().isEmpty() || sub.getName().isEmpty() || sub.getProject().isEmpty()) { + if (sub.getName().isEmpty() || sub.getProject().isEmpty()) { throw new IllegalArgumentException( String.format("Missing arguments in subscription string: %s", sub.toString())); } - return String.format("%s:%s:%s", sub.getProject(), sub.getName(), sub.getVersion()); + return String.format("%s:%s", sub.getProject(), sub.getName()); } private Subscription convertStringToSubscription(String sub) { if (sub.equals("")) { return Subscription.newBuilder().build(); } - String[] split = sub.split(":", 3); - return Subscription.newBuilder() - .setProject(split[0]) - .setName(split[1]) - .setVersion(split[2]) - .build(); + String[] split = sub.split(":", 2); + return Subscription.newBuilder().setProject(split[0]).setName(split[1]).build(); } } diff --git a/core/src/main/java/feast/core/service/JobCoordinatorService.java b/core/src/main/java/feast/core/service/JobCoordinatorService.java index c0215767905..f61c1b829d4 100644 --- a/core/src/main/java/feast/core/service/JobCoordinatorService.java +++ b/core/src/main/java/feast/core/service/JobCoordinatorService.java @@ -17,10 +17,8 @@ package feast.core.service; import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.CoreServiceProto.ListFeatureSetsRequest; import feast.core.CoreServiceProto.ListStoresRequest.Filter; import feast.core.CoreServiceProto.ListStoresResponse; -import feast.core.FeatureSetProto; import feast.core.FeatureSetProto.FeatureSetStatus; import feast.core.StoreProto; import feast.core.StoreProto.Store.Subscription; @@ -99,16 +97,11 @@ public void Poll() throws InvalidProtocolBufferException { Store store = Store.fromProto(storeSpec); for (Subscription subscription : store.getSubscriptions()) { - var featureSetSpecs = - specService - .listFeatureSets( - ListFeatureSetsRequest.Filter.newBuilder() - .setFeatureSetName(subscription.getName()) - .setFeatureSetVersion(subscription.getVersion()) - .setProject(subscription.getProject()) - .build()) - .getFeatureSetsList(); - featureSets.addAll(featureSetsFromProto(featureSetSpecs)); + List featureSetsForSub = + featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc( + subscription.getName().replace('*', '%'), + subscription.getProject().replace('*', '%')); + featureSets.addAll(featureSetsForSub); } featureSets.stream() @@ -171,12 +164,12 @@ private void updateFeatureSetStatuses(List jobUpdateTasks) { ready.removeAll(pending); ready.forEach( fs -> { - fs.setStatus(FeatureSetStatus.STATUS_READY.toString()); + fs.setStatus(FeatureSetStatus.STATUS_READY); featureSetRepository.save(fs); }); pending.forEach( fs -> { - fs.setStatus(FeatureSetStatus.STATUS_PENDING.toString()); + fs.setStatus(FeatureSetStatus.STATUS_PENDING); featureSetRepository.save(fs); }); featureSetRepository.flush(); @@ -194,15 +187,4 @@ public Optional getJob(Source source, Store store) { // return the latest return Optional.of(jobs.get(0)); } - - // TODO: optimize this to make less calls to the database. - private List featureSetsFromProto(List protos) { - return protos.stream() - .map(FeatureSetProto.FeatureSet::getSpec) - .map( - fs -> - featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion( - fs.getName(), fs.getProject(), fs.getVersion())) - .collect(Collectors.toList()); - } } diff --git a/core/src/main/java/feast/core/service/JobService.java b/core/src/main/java/feast/core/service/JobService.java index 33c118999cc..246ca91a4bc 100644 --- a/core/src/main/java/feast/core/service/JobService.java +++ b/core/src/main/java/feast/core/service/JobService.java @@ -248,7 +248,6 @@ private ListFeatureSetsRequest.Filter toListFeatureSetFilter(FeatureSetReference // match featuresets using contents of featureset reference String fsName = fsReference.getName(); String fsProject = fsReference.getProject(); - Integer fsVersion = fsReference.getVersion(); // construct list featureset request filter using feature set reference // for proto3, default value for missing values: @@ -258,7 +257,6 @@ private ListFeatureSetsRequest.Filter toListFeatureSetFilter(FeatureSetReference ListFeatureSetsRequest.Filter.newBuilder() .setFeatureSetName((fsName != "") ? fsName : "*") .setProject((fsProject != "") ? fsProject : "*") - .setFeatureSetVersion((fsVersion != 0) ? fsVersion.toString() : "*") .build(); return filter; diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 4a068cba353..d8efde1eace 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -19,7 +19,6 @@ import static feast.core.validators.Matchers.checkValidCharacters; import static feast.core.validators.Matchers.checkValidCharactersAllowAsterisk; -import com.google.common.collect.Ordering; import com.google.protobuf.InvalidProtocolBufferException; import feast.core.CoreServiceProto.ApplyFeatureSetResponse; import feast.core.CoreServiceProto.ApplyFeatureSetResponse.Status; @@ -49,7 +48,6 @@ import java.util.ArrayList; import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -99,43 +97,24 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) if (request.getProject().isEmpty()) { throw new IllegalArgumentException("No project provided"); } - if (request.getVersion() < 0) { - throw new IllegalArgumentException("Version number cannot be less than 0"); - } FeatureSet featureSet; - // Filter the list based on version - if (request.getVersion() == 0) { - featureSet = - featureSetRepository.findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc( - request.getName(), request.getProject()); + featureSet = + featureSetRepository.findFeatureSetByNameAndProject_Name( + request.getName(), request.getProject()); - if (featureSet == null) { - throw new RetrievalException( - String.format("Feature set with name \"%s\" could not be found.", request.getName())); - } - } else { - featureSet = - featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion( - request.getName(), request.getProject(), request.getVersion()); - - if (featureSet == null) { - throw new RetrievalException( - String.format( - "Feature set with name \"%s\" and version \"%s\" could " + "not be found.", - request.getName(), request.getVersion())); - } + if (featureSet == null) { + throw new RetrievalException( + String.format("Feature set with name \"%s\" could not be found.", request.getName())); } - - // Only a single item in list, return successfully return GetFeatureSetResponse.newBuilder().setFeatureSet(featureSet.toProto()).build(); } /** - * Return a list of feature sets matching the feature set name, version, and project provided in - * the filter. All fields are requried. Use '*' for all three arguments in order to return all - * feature sets and versions in all projects. + * Return a list of feature sets matching the feature set name and project provided in the filter. + * All fields are requried. Use '*' for all arguments in order to return all feature sets in all + * projects. * *

Project name can be explicitly provided, or an asterisk can be provided to match all * projects. It is not possible to provide a combination of asterisks/wildcards and text. @@ -144,25 +123,17 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) * sets will be returned. Regex is not supported. Explicitly defining a feature set name is not * possible if a project name is not set explicitly * - *

The version field can be one of - '*' - This will match all versions - 'latest' - This will - * match the latest feature set version - '<number>' - This will match a specific feature - * set version. This property can only be set if both the feature set name and project name are - * explicitly set. - * - * @param filter filter containing the desired featureSet name and version filter + * @param filter filter containing the desired featureSet name * @return ListFeatureSetsResponse with list of featureSets found matching the filter */ public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter filter) throws InvalidProtocolBufferException { String name = filter.getFeatureSetName(); String project = filter.getProject(); - String version = filter.getFeatureSetVersion(); - if (project.isEmpty() || name.isEmpty() || version.isEmpty()) { + if (project.isEmpty() || name.isEmpty()) { throw new IllegalArgumentException( - String.format( - "Invalid listFeatureSetRequest, missing arguments. Must provide project, feature set name, and version.", - filter.toString())); + "Invalid listFeatureSetRequest, missing arguments. Must provide project and feature set name."); } checkValidCharactersAllowAsterisk(name, "featureSetName"); @@ -170,56 +141,34 @@ public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter fil List featureSets = new ArrayList() {}; - if (project.equals("*")) { - // Matching all projects - - if (name.equals("*") && version.equals("*")) { + if (project.contains("*")) { + // Matching a wildcard project + if (name.contains("*")) { featureSets = - featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAscVersionAsc( + featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc( name.replace('*', '%'), project.replace('*', '%')); } else { throw new IllegalArgumentException( String.format( - "Invalid listFeatureSetRequest. Version and feature set name must be set to " + "Invalid listFeatureSetRequest. Feature set name must be set to " + "\"*\" if the project name and feature set name aren't set explicitly: \n%s", filter.toString())); } } else if (!project.contains("*")) { // Matching a specific project - - if (name.contains("*") && version.equals("*")) { - // Find all feature sets matching a pattern and versions in a specific project - featureSets = - featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - name.replace('*', '%'), project); - - } else if (!name.contains("*") && version.equals("*")) { - // Find all versions of a specific feature set in a specific project + if (name.contains("*")) { + // Find all feature sets matching a pattern in a specific project featureSets = - featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - name, project); - - } else if (version.equals("latest")) { - // Find the latest version of a feature set matching a specific pattern in a specific - // project - FeatureSet latestFeatureSet = - featureSetRepository.findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc( + featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc( name.replace('*', '%'), project); - featureSets.add(latestFeatureSet); - } else if (!name.contains("*") && StringUtils.isNumeric(version)) { - // Find a specific version of a feature set matching a specific name in a specific project - FeatureSet specificFeatureSet = - featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion( - name, project, Integer.parseInt(version)); - featureSets.add(specificFeatureSet); - - } else { - throw new IllegalArgumentException( - String.format( - "Invalid listFeatureSetRequest. Version must be set to \"*\" if the project " - + "name and feature set name aren't set explicitly: \n%s", - filter.toString())); + } else if (!name.contains("*")) { + // Find a specific feature set in a specific project + FeatureSet featureSet = + featureSetRepository.findFeatureSetByNameAndProject_Name(name, project); + if (featureSet != null) { + featureSets.add(featureSet); + } } } else { throw new IllegalArgumentException( @@ -274,8 +223,7 @@ public ListStoresResponse listStores(ListStoresRequest.Filter filter) { } /** - * Creates or updates a feature set in the repository. If there is a change in the feature set - * schema, then the feature set version will be incremented. + * Creates or updates a feature set in the repository. * *

This function is idempotent. If no changes are detected in the incoming featureSet's schema, * this method will update the incoming featureSet spec with the latest version stored in the @@ -301,54 +249,48 @@ public ApplyFeatureSetResponse applyFeatureSet(FeatureSetProto.FeatureSet newFea throw new IllegalArgumentException(String.format("Project is archived: %s", project_name)); } - // Retrieve all existing FeatureSet objects - List existingFeatureSets = - featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - newFeatureSet.getSpec().getName(), project_name); - - if (existingFeatureSets.size() == 0) { - // Create new feature set since it doesn't exist + // Set source to default if not set in proto + if (newFeatureSet.getSpec().getSource() == SourceProto.Source.getDefaultInstance()) { newFeatureSet = newFeatureSet .toBuilder() - .setSpec(newFeatureSet.getSpec().toBuilder().setVersion(1)) + .setSpec( + newFeatureSet.getSpec().toBuilder().setSource(defaultSource.toProto()).build()) .build(); - } else { - // Retrieve the latest feature set if the name does exist - existingFeatureSets = Ordering.natural().reverse().sortedCopy(existingFeatureSets); - FeatureSet latest = existingFeatureSets.get(0); - FeatureSet featureSet = FeatureSet.fromProto(newFeatureSet); + } + + // Retrieve existing FeatureSet + FeatureSet featureSet = + featureSetRepository.findFeatureSetByNameAndProject_Name( + newFeatureSet.getSpec().getName(), project_name); + Status status; + if (featureSet == null) { + // Create new feature set since it doesn't exist + newFeatureSet = newFeatureSet.toBuilder().setSpec(newFeatureSet.getSpec()).build(); + featureSet = FeatureSet.fromProto(newFeatureSet); + status = Status.CREATED; + } else { // If the featureSet remains unchanged, we do nothing. - if (featureSet.equalTo(latest)) { + if (featureSet.toProto().getSpec().equals(newFeatureSet.getSpec())) { return ApplyFeatureSetResponse.newBuilder() - .setFeatureSet(latest.toProto()) + .setFeatureSet(featureSet.toProto()) .setStatus(Status.NO_CHANGE) .build(); } - // TODO: There is a race condition here with incrementing the version - newFeatureSet = - newFeatureSet - .toBuilder() - .setSpec(newFeatureSet.getSpec().toBuilder().setVersion(latest.getVersion() + 1)) - .build(); - } - - // Build a new FeatureSet object which includes the new properties - FeatureSet featureSet = FeatureSet.fromProto(newFeatureSet); - featureSet.setStatus(FeatureSetStatus.STATUS_PENDING.toString()); - if (newFeatureSet.getSpec().getSource() == SourceProto.Source.getDefaultInstance()) { - featureSet.setSource(defaultSource); + featureSet.updateFromProto(newFeatureSet); + status = Status.UPDATED; } // Persist the FeatureSet object + featureSet.setStatus(FeatureSetStatus.STATUS_PENDING); project.addFeatureSet(featureSet); projectRepository.saveAndFlush(project); // Build ApplyFeatureSetResponse return ApplyFeatureSetResponse.newBuilder() .setFeatureSet(featureSet.toProto()) - .setStatus(Status.CREATED) + .setStatus(status) .build(); } @@ -366,7 +308,7 @@ public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) List subs = newStoreProto.getSubscriptionsList(); for (Subscription sub : subs) { // Ensure that all fields in a subscription contain values - if ((sub.getVersion().isEmpty() || sub.getName().isEmpty()) || sub.getProject().isEmpty()) { + if ((sub.getName().isEmpty()) || sub.getProject().isEmpty()) { throw new IllegalArgumentException( String.format("Missing parameter in subscription: %s", sub)); } diff --git a/core/src/test/java/feast/core/job/JobUpdateTaskTest.java b/core/src/test/java/feast/core/job/JobUpdateTaskTest.java index 8d179baebb1..5570c71a99a 100644 --- a/core/src/test/java/feast/core/job/JobUpdateTaskTest.java +++ b/core/src/test/java/feast/core/job/JobUpdateTaskTest.java @@ -54,7 +54,7 @@ public class JobUpdateTaskTest { private static final FeatureSetProto.FeatureSet.Builder fsBuilder = FeatureSetProto.FeatureSet.newBuilder().setMeta(FeatureSetMeta.newBuilder()); private static final FeatureSetSpec.Builder specBuilder = - FeatureSetSpec.newBuilder().setProject("project1").setVersion(1); + FeatureSetSpec.newBuilder().setProject("project1"); @Mock private JobManager jobManager; @@ -73,8 +73,7 @@ public void setUp() { .setName("test") .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder().build()) - .addSubscriptions( - Subscription.newBuilder().setProject("*").setName("*").setVersion("*").build()) + .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build()) .build()); source = diff --git a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java index 72b921ef694..55e8a573771 100644 --- a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java +++ b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java @@ -103,8 +103,7 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException { .setName("SERVING") .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder().setHost("localhost").setPort(6379).build()) - .addSubscriptions( - Subscription.newBuilder().setProject("*").setName("*").setVersion("*").build()) + .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build()) .build(); SourceProto.Source source = @@ -124,7 +123,6 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException { FeatureSetSpec.newBuilder() .setSource(source) .setName("featureSet") - .setVersion(1) .setMaxAge(Duration.newBuilder().build())) .build(); @@ -220,12 +218,7 @@ public void shouldThrowExceptionWhenJobStateTerminal() throws IOException { FeatureSetProto.FeatureSet featureSet = FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setVersion(1) - .setSource(source) - .build()) + .setSpec(FeatureSetSpec.newBuilder().setName("featureSet").setSource(source).build()) .build(); dfJobManager = Mockito.spy(dfJobManager); diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java index 6980450ca4d..914177385e4 100644 --- a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java +++ b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java @@ -91,8 +91,7 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { .setName("SERVING") .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder().setHost("localhost").setPort(6379).build()) - .addSubscriptions( - Subscription.newBuilder().setProject("*").setName("*").setVersion("*").build()) + .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build()) .build(); SourceProto.Source source = @@ -110,7 +109,6 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { .setSpec( FeatureSetSpec.newBuilder() .setName("featureSet") - .setVersion(1) .setMaxAge(Duration.newBuilder()) .setSource(source) .build()) @@ -118,8 +116,10 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { Printer printer = JsonFormat.printer(); + String expectedJobId = "feast-job-0"; ImportOptions expectedPipelineOptions = PipelineOptionsFactory.fromArgs("").as(ImportOptions.class); + expectedPipelineOptions.setJobName(expectedJobId); expectedPipelineOptions.setAppName("DirectRunnerJobManager"); expectedPipelineOptions.setRunner(DirectRunner.class); expectedPipelineOptions.setBlockOnRun(false); @@ -132,7 +132,6 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { expectedPipelineOptions.setFeatureSetJson( featureSetJsonCompressor.compress(Collections.singletonList(featureSet))); - String expectedJobId = "feast-job-0"; ArgumentCaptor pipelineOptionsCaptor = ArgumentCaptor.forClass(ImportOptions.class); ArgumentCaptor directJobCaptor = ArgumentCaptor.forClass(DirectJob.class); diff --git a/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java b/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java index 2dfeef1d969..df9044e6331 100644 --- a/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java +++ b/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java @@ -29,7 +29,7 @@ public class FeatureSetJsonByteConverterTest { - private FeatureSetProto.FeatureSet newFeatureSet(Integer version, Integer numberOfFeatures) { + private FeatureSetProto.FeatureSet newFeatureSet(Integer numberOfFeatures) { List features = IntStream.range(1, numberOfFeatures + 1) .mapToObj( @@ -51,7 +51,6 @@ private FeatureSetProto.FeatureSet newFeatureSet(Integer version, Integer number .setBootstrapServers("somebrokers:9092") .setTopic("sometopic"))) .addAllFeatures(features) - .setVersion(version) .addEntities( FeatureSetProto.EntitySpec.newBuilder() .setName("entity") @@ -65,12 +64,11 @@ public void shouldConvertFeatureSetsAsJsonStringBytes() throws InvalidProtocolBu int nrOfFeatures = 1; List featureSets = IntStream.range(1, nrOfFeatureSet + 1) - .mapToObj(i -> newFeatureSet(i, nrOfFeatures)) + .mapToObj(i -> newFeatureSet(nrOfFeatures)) .collect(Collectors.toList()); String expectedOutputString = - "{\"version\":1," - + "\"entities\":[{\"name\":\"entity\",\"valueType\":2}]," + "{\"entities\":[{\"name\":\"entity\",\"valueType\":2}]," + "\"features\":[{\"name\":\"feature1\",\"valueType\":6}]," + "\"source\":{" + "\"type\":1," diff --git a/core/src/test/java/feast/core/model/FeatureSetTest.java b/core/src/test/java/feast/core/model/FeatureSetTest.java new file mode 100644 index 00000000000..70d160e875c --- /dev/null +++ b/core/src/test/java/feast/core/model/FeatureSetTest.java @@ -0,0 +1,205 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.model; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +import com.google.protobuf.Duration; +import com.google.protobuf.InvalidProtocolBufferException; +import feast.core.FeatureSetProto; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSetStatus; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.SourceProto; +import feast.core.SourceProto.KafkaSourceConfig; +import feast.core.SourceProto.SourceType; +import feast.types.ValueProto.ValueType.Enum; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.tensorflow.metadata.v0.IntDomain; + +public class FeatureSetTest { + @Rule public final ExpectedException expectedException = ExpectedException.none(); + + private FeatureSetProto.FeatureSet oldFeatureSetProto; + + @Before + public void setUp() { + SourceProto.Source oldSource = + SourceProto.Source.newBuilder() + .setType(SourceType.KAFKA) + .setKafkaSourceConfig( + KafkaSourceConfig.newBuilder() + .setBootstrapServers("kafka:9092") + .setTopic("mytopic")) + .build(); + + oldFeatureSetProto = + FeatureSetProto.FeatureSet.newBuilder() + .setSpec( + FeatureSetSpec.newBuilder() + .setName("featureSet") + .setProject("project") + .setMaxAge(Duration.newBuilder().setSeconds(100)) + .setSource(oldSource) + .addFeatures( + FeatureSpec.newBuilder().setName("feature1").setValueType(Enum.INT64)) + .addFeatures( + FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING)) + .addEntities( + EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) + .build()) + .build(); + } + + @Test + public void shouldUpdateFromProto() throws InvalidProtocolBufferException { + SourceProto.Source newSource = + SourceProto.Source.newBuilder() + .setType(SourceType.KAFKA) + .setKafkaSourceConfig( + KafkaSourceConfig.newBuilder() + .setBootstrapServers("kafka:9092") + .setTopic("mytopic-changed")) + .build(); + + FeatureSetProto.FeatureSet newFeatureSetProto = + FeatureSetProto.FeatureSet.newBuilder() + .setSpec( + FeatureSetSpec.newBuilder() + .setName("featureSet") + .setProject("project") + .setMaxAge(Duration.newBuilder().setSeconds(101)) + .setSource(newSource) + .addFeatures( + FeatureSpec.newBuilder() + .setName("feature1") + .setValueType(Enum.INT64) + .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) + .addFeatures( + FeatureSpec.newBuilder().setName("feature3").setValueType(Enum.STRING)) + .addEntities( + EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) + .build()) + .build(); + + FeatureSet actual = FeatureSet.fromProto(oldFeatureSetProto); + actual.updateFromProto(newFeatureSetProto); + + FeatureSet expected = FeatureSet.fromProto(newFeatureSetProto); + Feature archivedFeature = + Feature.fromProto( + FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING).build()); + archivedFeature.setArchived(true); + expected.addFeature(archivedFeature); + assertThat(actual, equalTo(expected)); + } + + @Test + public void shouldNotUpdateIfNoChange() throws InvalidProtocolBufferException { + FeatureSet actual = FeatureSet.fromProto(oldFeatureSetProto); + actual.setStatus(FeatureSetStatus.STATUS_READY); + actual.updateFromProto(oldFeatureSetProto); + + FeatureSet expected = FeatureSet.fromProto(oldFeatureSetProto); + expected.setStatus(FeatureSetStatus.STATUS_READY); + + assertThat(actual, equalTo(expected)); + } + + @Test + public void shouldThrowExceptionIfUpdateWithEntitiesChanged() + throws InvalidProtocolBufferException { + SourceProto.Source newSource = + SourceProto.Source.newBuilder() + .setType(SourceType.KAFKA) + .setKafkaSourceConfig( + KafkaSourceConfig.newBuilder() + .setBootstrapServers("kafka:9092") + .setTopic("mytopic-changed")) + .build(); + + FeatureSetProto.FeatureSet newFeatureSetProto = + FeatureSetProto.FeatureSet.newBuilder() + .setSpec( + FeatureSetSpec.newBuilder() + .setName("featureSet") + .setProject("project") + .setMaxAge(Duration.newBuilder().setSeconds(101)) + .setSource(newSource) + .addFeatures( + FeatureSpec.newBuilder() + .setName("feature1") + .setValueType(Enum.INT64) + .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) + .addFeatures( + FeatureSpec.newBuilder().setName("feature3").setValueType(Enum.STRING)) + .addEntities(EntitySpec.newBuilder().setName("entity").setValueType(Enum.FLOAT)) + .build()) + .build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage(containsString("does not match existing set of entities")); + FeatureSet existingFeatureSet = FeatureSet.fromProto(oldFeatureSetProto); + existingFeatureSet.updateFromProto(newFeatureSetProto); + } + + @Test + public void shouldThrowExceptionIfUpdateWithFeatureTypesChanged() + throws InvalidProtocolBufferException { + SourceProto.Source newSource = + SourceProto.Source.newBuilder() + .setType(SourceType.KAFKA) + .setKafkaSourceConfig( + KafkaSourceConfig.newBuilder() + .setBootstrapServers("kafka:9092") + .setTopic("mytopic-changed")) + .build(); + + FeatureSetProto.FeatureSet newFeatureSetProto = + FeatureSetProto.FeatureSet.newBuilder() + .setSpec( + FeatureSetSpec.newBuilder() + .setName("featureSet") + .setProject("project") + .setMaxAge(Duration.newBuilder().setSeconds(101)) + .setSource(newSource) + .addFeatures( + FeatureSpec.newBuilder() + .setName("feature1") + .setValueType(Enum.INT64) + .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) + .addFeatures( + FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.FLOAT)) + .addEntities( + EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) + .build()) + .build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage( + containsString( + "You are attempting to change the type of feature feature2 from STRING to FLOAT.")); + FeatureSet existingFeatureSet = FeatureSet.fromProto(oldFeatureSetProto); + existingFeatureSet.updateFromProto(newFeatureSetProto); + } +} diff --git a/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java b/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java index 26cf331c13b..38683c7bd59 100644 --- a/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java +++ b/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java @@ -95,17 +95,12 @@ public void shouldDoNothingIfNoMatchingFeatureSetsFound() throws InvalidProtocol .setName("test") .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder().build()) - .addSubscriptions( - Subscription.newBuilder().setProject("*").setName("*").setVersion("*").build()) + .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build()) .build(); when(specService.listStores(any())) .thenReturn(ListStoresResponse.newBuilder().addStore(store).build()); when(specService.listFeatureSets( - Filter.newBuilder() - .setProject("*") - .setFeatureSetName("*") - .setFeatureSetVersion("*") - .build())) + Filter.newBuilder().setProject("*").setFeatureSetName("*").build())) .thenReturn(ListFeatureSetsResponse.newBuilder().build()); JobCoordinatorService jcs = new JobCoordinatorService( @@ -121,12 +116,7 @@ public void shouldGenerateAndSubmitJobsIfAny() throws InvalidProtocolBufferExcep .setName("test") .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder().build()) - .addSubscriptions( - Subscription.newBuilder() - .setProject("project1") - .setName("features") - .setVersion("*") - .build()) + .addSubscriptions(Subscription.newBuilder().setProject("project1").setName("*").build()) .build(); Source source = Source.newBuilder() @@ -138,26 +128,26 @@ public void shouldGenerateAndSubmitJobsIfAny() throws InvalidProtocolBufferExcep .build()) .build(); - FeatureSetProto.FeatureSet featureSet1 = + FeatureSetProto.FeatureSet featureSetProto1 = FeatureSetProto.FeatureSet.newBuilder() .setSpec( FeatureSetSpec.newBuilder() .setSource(source) .setProject("project1") - .setName("features") - .setVersion(1)) + .setName("features1")) .setMeta(FeatureSetMeta.newBuilder()) .build(); - FeatureSetProto.FeatureSet featureSet2 = + FeatureSet featureSet1 = FeatureSet.fromProto(featureSetProto1); + FeatureSetProto.FeatureSet featureSetProto2 = FeatureSetProto.FeatureSet.newBuilder() .setSpec( FeatureSetSpec.newBuilder() .setSource(source) .setProject("project1") - .setName("features") - .setVersion(2)) + .setName("features2")) .setMeta(FeatureSetMeta.newBuilder()) .build(); + FeatureSet featureSet2 = FeatureSet.fromProto(featureSetProto2); String extId = "ext"; ArgumentCaptor jobArgCaptor = ArgumentCaptor.forClass(Job.class); @@ -168,7 +158,7 @@ public void shouldGenerateAndSubmitJobsIfAny() throws InvalidProtocolBufferExcep Runner.DATAFLOW, feast.core.model.Source.fromProto(source), feast.core.model.Store.fromProto(store), - Arrays.asList(FeatureSet.fromProto(featureSet1), FeatureSet.fromProto(featureSet2)), + Arrays.asList(featureSet1, featureSet2), JobStatus.PENDING); Job expected = @@ -178,30 +168,14 @@ public void shouldGenerateAndSubmitJobsIfAny() throws InvalidProtocolBufferExcep Runner.DATAFLOW, feast.core.model.Source.fromProto(source), feast.core.model.Store.fromProto(store), - Arrays.asList(FeatureSet.fromProto(featureSet1), FeatureSet.fromProto(featureSet2)), + Arrays.asList(featureSet1, featureSet2), JobStatus.RUNNING); - when(specService.listFeatureSets( - Filter.newBuilder() - .setProject("project1") - .setFeatureSetName("features") - .setFeatureSetVersion("*") - .build())) - .thenReturn( - ListFeatureSetsResponse.newBuilder() - .addFeatureSets(featureSet1) - .addFeatureSets(featureSet2) - .build()); + when(featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc("%", "project1")) + .thenReturn(Lists.newArrayList(featureSet1, featureSet2)); when(specService.listStores(any())) .thenReturn(ListStoresResponse.newBuilder().addStore(store).build()); - for (FeatureSetProto.FeatureSet fs : Lists.newArrayList(featureSet1, featureSet2)) { - FeatureSetSpec spec = fs.getSpec(); - when(featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion( - spec.getName(), spec.getProject(), spec.getVersion())) - .thenReturn(FeatureSet.fromProto(fs)); - } - when(jobManager.startJob(argThat(new JobMatcher(expectedInput)))).thenReturn(expected); when(jobManager.getRunnerType()).thenReturn(Runner.DATAFLOW); @@ -221,12 +195,7 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException { .setName("test") .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder().build()) - .addSubscriptions( - Subscription.newBuilder() - .setProject("project1") - .setName("features") - .setVersion("*") - .build()) + .addSubscriptions(Subscription.newBuilder().setProject("project1").setName("*").build()) .build(); Source source1 = Source.newBuilder() @@ -247,26 +216,27 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException { .build()) .build(); - FeatureSetProto.FeatureSet featureSet1 = + FeatureSetProto.FeatureSet featureSetProto1 = FeatureSetProto.FeatureSet.newBuilder() .setSpec( FeatureSetSpec.newBuilder() .setSource(source1) .setProject("project1") - .setName("features") - .setVersion(1)) + .setName("features1")) .setMeta(FeatureSetMeta.newBuilder()) .build(); - FeatureSetProto.FeatureSet featureSet2 = + FeatureSet featureSet1 = FeatureSet.fromProto(featureSetProto1); + + FeatureSetProto.FeatureSet featureSetProto2 = FeatureSetProto.FeatureSet.newBuilder() .setSpec( FeatureSetSpec.newBuilder() .setSource(source2) .setProject("project1") - .setName("features") - .setVersion(2)) + .setName("features2")) .setMeta(FeatureSetMeta.newBuilder()) .build(); + FeatureSet featureSet2 = FeatureSet.fromProto(featureSetProto2); Job expectedInput1 = new Job( @@ -275,7 +245,7 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException { Runner.DATAFLOW, feast.core.model.Source.fromProto(source1), feast.core.model.Store.fromProto(store), - Arrays.asList(FeatureSet.fromProto(featureSet1)), + Arrays.asList(featureSet1), JobStatus.PENDING); Job expected1 = @@ -285,7 +255,7 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException { Runner.DATAFLOW, feast.core.model.Source.fromProto(source1), feast.core.model.Store.fromProto(store), - Arrays.asList(FeatureSet.fromProto(featureSet1)), + Arrays.asList(featureSet1), JobStatus.RUNNING); Job expectedInput2 = @@ -295,7 +265,7 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException { Runner.DATAFLOW, feast.core.model.Source.fromProto(source2), feast.core.model.Store.fromProto(store), - Arrays.asList(FeatureSet.fromProto(featureSet2)), + Arrays.asList(featureSet2), JobStatus.PENDING); Job expected2 = @@ -305,33 +275,19 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException { Runner.DATAFLOW, feast.core.model.Source.fromProto(source2), feast.core.model.Store.fromProto(store), - Arrays.asList(FeatureSet.fromProto(featureSet2)), + Arrays.asList(featureSet2), JobStatus.RUNNING); ArgumentCaptor jobArgCaptor = ArgumentCaptor.forClass(Job.class); - when(specService.listFeatureSets( - Filter.newBuilder() - .setProject("project1") - .setFeatureSetName("features") - .setFeatureSetVersion("*") - .build())) - .thenReturn( - ListFeatureSetsResponse.newBuilder() - .addFeatureSets(featureSet1) - .addFeatureSets(featureSet2) - .build()); + when(featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc("%", "project1")) + .thenReturn(Lists.newArrayList(featureSet1, featureSet2)); + when(specService.listStores(any())) .thenReturn(ListStoresResponse.newBuilder().addStore(store).build()); when(jobManager.startJob(argThat(new JobMatcher(expectedInput1)))).thenReturn(expected1); when(jobManager.startJob(argThat(new JobMatcher(expectedInput2)))).thenReturn(expected2); when(jobManager.getRunnerType()).thenReturn(Runner.DATAFLOW); - for (FeatureSetProto.FeatureSet fs : Lists.newArrayList(featureSet1, featureSet2)) { - FeatureSetSpec spec = fs.getSpec(); - when(featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion( - spec.getName(), spec.getProject(), spec.getVersion())) - .thenReturn(FeatureSet.fromProto(fs)); - } JobCoordinatorService jcs = new JobCoordinatorService( diff --git a/core/src/test/java/feast/core/service/JobServiceTest.java b/core/src/test/java/feast/core/service/JobServiceTest.java index ba663020191..0fe8e22f0a9 100644 --- a/core/src/test/java/feast/core/service/JobServiceTest.java +++ b/core/src/test/java/feast/core/service/JobServiceTest.java @@ -148,7 +148,7 @@ private FeatureSet newDummyFeatureSet(String name, int version, String project) FeatureSet fs = TestObjectFactory.CreateFeatureSet( - name, project, version, Arrays.asList(entity), Arrays.asList(feature)); + name, project, Arrays.asList(entity), Arrays.asList(feature)); fs.setCreated(Date.from(Instant.ofEpochSecond(10L))); return fs; } @@ -168,7 +168,6 @@ private List newDummyFeatureSetReferences() { return Arrays.asList( // all provided: name, version and project FeatureSetReference.newBuilder() - .setVersion(this.featureSet.getVersion()) .setName(this.featureSet.getName()) .setProject(this.featureSet.getProject().toString()) .build(), @@ -180,10 +179,7 @@ private List newDummyFeatureSetReferences() { .build(), // name and version - FeatureSetReference.newBuilder() - .setName(this.featureSet.getName()) - .setVersion(this.featureSet.getVersion()) - .build()); + FeatureSetReference.newBuilder().setName(this.featureSet.getName()).build()); } private List newDummyListRequestFilters() { @@ -192,21 +188,18 @@ private List newDummyListRequestFilters() { ListFeatureSetsRequest.Filter.newBuilder() .setFeatureSetName(this.featureSet.getName()) .setProject(this.featureSet.getProject().toString()) - .setFeatureSetVersion(String.valueOf(this.featureSet.getVersion())) .build(), // name and project ListFeatureSetsRequest.Filter.newBuilder() .setFeatureSetName(this.featureSet.getName()) .setProject(this.featureSet.getProject().toString()) - .setFeatureSetVersion("*") .build(), // name and project ListFeatureSetsRequest.Filter.newBuilder() .setFeatureSetName(this.featureSet.getName()) .setProject("*") - .setFeatureSetVersion(String.valueOf(this.featureSet.getVersion())) .build()); } diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index 413a97e64b0..576bdeb8926 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -29,7 +29,6 @@ import feast.core.CoreServiceProto.ApplyFeatureSetResponse; import feast.core.CoreServiceProto.ApplyFeatureSetResponse.Status; import feast.core.CoreServiceProto.GetFeatureSetRequest; -import feast.core.CoreServiceProto.GetFeatureSetResponse; import feast.core.CoreServiceProto.ListFeatureSetsRequest.Filter; import feast.core.CoreServiceProto.ListFeatureSetsResponse; import feast.core.CoreServiceProto.ListStoresRequest; @@ -52,15 +51,8 @@ import feast.types.ValueProto.ValueType.Enum; import java.sql.Date; import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Optional; import java.util.stream.Collectors; import org.junit.Before; import org.junit.Rule; @@ -100,49 +92,37 @@ public class SpecServiceTest { private List stores; private Source defaultSource; + // TODO: Updates update features in place, so if tests follow the wrong order they might break. + // Refactor this maybe? @Before public void setUp() { initMocks(this); defaultSource = TestObjectFactory.defaultSource; - FeatureSet featureSet1v1 = newDummyFeatureSet("f1", 1, "project1"); - FeatureSet featureSet1v2 = newDummyFeatureSet("f1", 2, "project1"); - FeatureSet featureSet1v3 = newDummyFeatureSet("f1", 3, "project1"); - FeatureSet featureSet2v1 = newDummyFeatureSet("f2", 1, "project1"); + FeatureSet featureSet1 = newDummyFeatureSet("f1", "project1"); + FeatureSet featureSet2 = newDummyFeatureSet("f2", "project1"); Feature f3f1 = TestObjectFactory.CreateFeature("f3f1", Enum.INT64); Feature f3f2 = TestObjectFactory.CreateFeature("f3f2", Enum.INT64); Entity f3e1 = TestObjectFactory.CreateEntity("f3e1", Enum.STRING); FeatureSet featureSet3v1 = TestObjectFactory.CreateFeatureSet( - "f3", "project1", 1, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1)); + "f3", "project1", Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1)); - featureSets = - Arrays.asList(featureSet1v1, featureSet1v2, featureSet1v3, featureSet2v1, featureSet3v1); + featureSets = Arrays.asList(featureSet1, featureSet2); when(featureSetRepository.findAll()).thenReturn(featureSets); - when(featureSetRepository.findAllByOrderByNameAscVersionAsc()).thenReturn(featureSets); - when(featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion("f1", "project1", 1)) + when(featureSetRepository.findAllByOrderByNameAsc()).thenReturn(featureSets); + when(featureSetRepository.findFeatureSetByNameAndProject_Name("f1", "project1")) .thenReturn(featureSets.get(0)); - when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - "f1", "project1")) - .thenReturn(featureSets.subList(0, 3)); - when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - "f3", "project1")) - .thenReturn(featureSets.subList(4, 5)); - when(featureSetRepository.findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc( - "f1", "project1")) - .thenReturn(featureSet1v3); - when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - "f1", "project1")) - .thenReturn(featureSets.subList(0, 3)); - when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - "asd", "project1")) + when(featureSetRepository.findFeatureSetByNameAndProject_Name("f2", "project1")) + .thenReturn(featureSets.get(1)); + when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc("f1", "project1")) + .thenReturn(featureSets.subList(0, 1)); + when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc("asd", "project1")) .thenReturn(Lists.newArrayList()); - when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - "f%", "project1")) + when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc("f%", "project1")) .thenReturn(featureSets); - when(featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAscVersionAsc( - "%", "%")) + when(featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc("%", "%")) .thenReturn(featureSets); when(projectRepository.findAllByArchivedIsFalse()) @@ -169,11 +149,7 @@ public void shouldGetAllFeatureSetsIfOnlyWildcardsProvided() throws InvalidProtocolBufferException { ListFeatureSetsResponse actual = specService.listFeatureSets( - Filter.newBuilder() - .setFeatureSetName("*") - .setProject("*") - .setFeatureSetVersion("*") - .build()); + Filter.newBuilder().setFeatureSetName("*").setProject("*").build()); List list = new ArrayList<>(); for (FeatureSet featureSet : featureSets) { FeatureSetProto.FeatureSet toProto = featureSet.toProto(); @@ -189,31 +165,8 @@ public void listFeatureSetShouldFailIfFeatureSetProvidedWithoutProject() throws InvalidProtocolBufferException { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( - "Invalid listFeatureSetRequest, missing arguments. Must provide project, feature set name, and version."); - specService.listFeatureSets( - Filter.newBuilder().setFeatureSetName("f1").setFeatureSetVersion("1").build()); - } - - @Test - public void shouldGetAllFeatureSetsMatchingNameIfWildcardVersionProvided() - throws InvalidProtocolBufferException { - ListFeatureSetsResponse actual = - specService.listFeatureSets( - Filter.newBuilder() - .setProject("project1") - .setFeatureSetName("f1") - .setFeatureSetVersion("*") - .build()); - List expectedFeatureSets = - featureSets.stream().filter(fs -> fs.getName().equals("f1")).collect(Collectors.toList()); - List list = new ArrayList<>(); - for (FeatureSet expectedFeatureSet : expectedFeatureSets) { - FeatureSetProto.FeatureSet toProto = expectedFeatureSet.toProto(); - list.add(toProto); - } - ListFeatureSetsResponse expected = - ListFeatureSetsResponse.newBuilder().addAllFeatureSets(list).build(); - assertThat(actual, equalTo(expected)); + "Invalid listFeatureSetRequest, missing arguments. Must provide project and feature set name."); + specService.listFeatureSets(Filter.newBuilder().setFeatureSetName("f1").build()); } @Test @@ -221,11 +174,7 @@ public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch() throws InvalidProtocolBufferException { ListFeatureSetsResponse actual = specService.listFeatureSets( - Filter.newBuilder() - .setProject("project1") - .setFeatureSetName("f*") - .setFeatureSetVersion("*") - .build()); + Filter.newBuilder().setProject("project1").setFeatureSetName("f*").build()); List expectedFeatureSets = featureSets.stream() .filter(fs -> fs.getName().startsWith("f")) @@ -241,20 +190,12 @@ public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch() } @Test - public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator() - throws InvalidProtocolBufferException { + public void shouldGetFeatureSetsByNameAndProject() throws InvalidProtocolBufferException { ListFeatureSetsResponse actual = specService.listFeatureSets( - Filter.newBuilder() - .setProject("project1") - .setFeatureSetName("f1") - .setFeatureSetVersion("1") - .build()); + Filter.newBuilder().setProject("project1").setFeatureSetName("f1").build()); List expectedFeatureSets = - featureSets.stream() - .filter(fs -> fs.getName().equals("f1")) - .filter(fs -> fs.getVersion() == 1) - .collect(Collectors.toList()); + featureSets.stream().filter(fs -> fs.getName().equals("f1")).collect(Collectors.toList()); List list = new ArrayList<>(); for (FeatureSet expectedFeatureSet : expectedFeatureSets) { FeatureSetProto.FeatureSet toProto = expectedFeatureSet.toProto(); @@ -265,80 +206,20 @@ public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator() assertThat(actual, equalTo(expected)); } - @Test - public void shouldThrowExceptionIfGetAllFeatureSetsGivenVersionWithComparator() - throws InvalidProtocolBufferException { - expectedException.expect(IllegalArgumentException.class); - specService.listFeatureSets( - Filter.newBuilder() - .setProject("project1") - .setFeatureSetName("f1") - .setFeatureSetVersion(">1") - .build()); - } - - @Test - public void shouldGetLatestFeatureSetGivenMissingVersionFilter() - throws InvalidProtocolBufferException { - GetFeatureSetResponse actual = - specService.getFeatureSet( - GetFeatureSetRequest.newBuilder().setName("f1").setProject("project1").build()); - FeatureSet expected = featureSets.get(2); - assertThat(actual.getFeatureSet(), equalTo(expected.toProto())); - } - - @Test - public void shouldGetSpecificFeatureSetGivenSpecificVersionFilter() - throws InvalidProtocolBufferException { - when(featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion("f1", "project1", 2)) - .thenReturn(featureSets.get(1)); - GetFeatureSetResponse actual = - specService.getFeatureSet( - GetFeatureSetRequest.newBuilder() - .setProject("project1") - .setName("f1") - .setVersion(2) - .build()); - FeatureSet expected = featureSets.get(1); - assertThat(actual.getFeatureSet(), equalTo(expected.toProto())); - } - @Test public void shouldThrowExceptionGivenMissingFeatureSetName() throws InvalidProtocolBufferException { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("No feature set name provided"); - specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setVersion(2).build()); + specService.getFeatureSet(GetFeatureSetRequest.newBuilder().build()); } @Test public void shouldThrowExceptionGivenMissingFeatureSet() throws InvalidProtocolBufferException { expectedException.expect(RetrievalException.class); - expectedException.expectMessage( - "Feature set with name \"f1000\" and version \"2\" could not be found."); + expectedException.expectMessage("Feature set with name \"f1000\" could not be found."); specService.getFeatureSet( - GetFeatureSetRequest.newBuilder() - .setName("f1000") - .setProject("project1") - .setVersion(2) - .build()); - } - - @Test - public void shouldThrowRetrievalExceptionGivenInvalidFeatureSetVersionComparator() - throws InvalidProtocolBufferException { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage( - "Invalid listFeatureSetRequest. Version must be set to \"*\" if the project name and feature set name aren't set explicitly: \n" - + "feature_set_name: \"f1\"\n" - + "feature_set_version: \">1\"\n" - + "project: \"project1\""); - specService.listFeatureSets( - Filter.newBuilder() - .setProject("project1") - .setFeatureSetName("f1") - .setFeatureSetVersion(">1") - .build()); + GetFeatureSetRequest.newBuilder().setName("f1000").setProject("project1").build()); } @Test @@ -373,10 +254,10 @@ public void shouldThrowRetrievalExceptionIfNoStoresFoundWithName() { } @Test - public void applyFeatureSetShouldReturnFeatureSetWithLatestVersionIfFeatureSetHasNotChanged() + public void applyFeatureSetShouldReturnFeatureSetIfFeatureSetHasNotChanged() throws InvalidProtocolBufferException { FeatureSetSpec incomingFeatureSetSpec = - featureSets.get(2).toProto().getSpec().toBuilder().clearVersion().build(); + featureSets.get(0).toProto().getSpec().toBuilder().build(); ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet( @@ -384,21 +265,19 @@ public void applyFeatureSetShouldReturnFeatureSetWithLatestVersionIfFeatureSetHa verify(featureSetRepository, times(0)).save(ArgumentMatchers.any(FeatureSet.class)); assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.NO_CHANGE)); - assertThat(applyFeatureSetResponse.getFeatureSet(), equalTo(featureSets.get(2).toProto())); + assertThat(applyFeatureSetResponse.getFeatureSet(), equalTo(featureSets.get(0).toProto())); } @Test - public void applyFeatureSetShouldApplyFeatureSetWithInitVersionIfNotExists() + public void applyFeatureSetShouldApplyFeatureSetIfNotExists() throws InvalidProtocolBufferException { - when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( - "f2", "project1")) - .thenReturn(Lists.newArrayList()); + when(featureSetRepository.findFeatureSetByNameAndProject_Name("f2", "project1")) + .thenReturn(null); - FeatureSetProto.FeatureSet incomingFeatureSet = - newDummyFeatureSet("f2", 1, "project1").toProto(); + FeatureSetProto.FeatureSet incomingFeatureSet = newDummyFeatureSet("f2", "project1").toProto(); FeatureSetProto.FeatureSetSpec incomingFeatureSetSpec = - incomingFeatureSet.getSpec().toBuilder().clearVersion().build(); + incomingFeatureSet.getSpec().toBuilder().build(); ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet( @@ -407,24 +286,16 @@ public void applyFeatureSetShouldApplyFeatureSetWithInitVersionIfNotExists() FeatureSetProto.FeatureSet expected = FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - incomingFeatureSetSpec - .toBuilder() - .setVersion(1) - .setSource(defaultSource.toProto()) - .build()) + .setSpec(incomingFeatureSetSpec.toBuilder().setSource(defaultSource.toProto()).build()) .build(); assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.CREATED)); assertThat(applyFeatureSetResponse.getFeatureSet().getSpec(), equalTo(expected.getSpec())); - assertThat( - applyFeatureSetResponse.getFeatureSet().getSpec().getVersion(), - equalTo(expected.getSpec().getVersion())); } @Test - public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists() + public void applyFeatureSetShouldUpdateAndSaveFeatureSetIfAlreadyExists() throws InvalidProtocolBufferException { - FeatureSetProto.FeatureSet incomingFeatureSet = featureSets.get(2).toProto(); + FeatureSetProto.FeatureSet incomingFeatureSet = featureSets.get(0).toProto(); incomingFeatureSet = incomingFeatureSet .toBuilder() @@ -433,7 +304,6 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists() incomingFeatureSet .getSpec() .toBuilder() - .clearVersion() .addFeatures( FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING)) .build()) @@ -444,37 +314,27 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists() .toBuilder() .setMeta(incomingFeatureSet.getMeta().toBuilder().build()) .setSpec( - incomingFeatureSet - .getSpec() - .toBuilder() - .setVersion(4) - .setSource(defaultSource.toProto()) - .build()) + incomingFeatureSet.getSpec().toBuilder().setSource(defaultSource.toProto()).build()) .build(); ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet(incomingFeatureSet); verify(projectRepository).saveAndFlush(ArgumentMatchers.any(Project.class)); - assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.CREATED)); + assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.UPDATED)); assertEquals( FeatureSet.fromProto(applyFeatureSetResponse.getFeatureSet()), FeatureSet.fromProto(expected)); - assertThat( - applyFeatureSetResponse.getFeatureSet().getSpec().getVersion(), - equalTo(expected.getSpec().getVersion())); } @Test public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered() throws InvalidProtocolBufferException { - Feature f3f1 = TestObjectFactory.CreateFeature("f3f1", Enum.INT64); - Feature f3f2 = TestObjectFactory.CreateFeature("f3f2", Enum.INT64); - Entity f3e1 = TestObjectFactory.CreateEntity("f3e1", Enum.STRING); - FeatureSetProto.FeatureSet incomingFeatureSet = - (TestObjectFactory.CreateFeatureSet( - "f3", "project1", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) - .toProto(); + FeatureSet featureSet = featureSets.get(1); + List features = Lists.newArrayList(featureSet.getFeatures()); + Collections.shuffle(features); + featureSet.setFeatures(Set.copyOf(features)); + FeatureSetProto.FeatureSet incomingFeatureSet = featureSet.toProto(); ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet(incomingFeatureSet); @@ -573,12 +433,6 @@ public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints() @Test public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated() throws InvalidProtocolBufferException { - FeatureSetProto.FeatureSet existingFeatureSet = featureSets.get(2).toProto(); - assertThat( - "Existing feature set has version 3", existingFeatureSet.getSpec().getVersion() == 3); - assertThat( - "Existing feature set has at least 1 feature", - existingFeatureSet.getSpec().getFeaturesList().size() > 0); // Map of constraint field name -> value, e.g. "shape" -> FixedShape object. // If any of these fields are updated, SpecService should update the FeatureSet. @@ -603,6 +457,10 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated() contraintUpdates.put("time_of_day_domain", TimeOfDayDomain.getDefaultInstance()); for (Entry constraint : contraintUpdates.entrySet()) { + FeatureSet featureSet = newDummyFeatureSet("constraints", "project1"); + FeatureSetProto.FeatureSet existingFeatureSet = featureSet.toProto(); + when(featureSetRepository.findFeatureSetByNameAndProject_Name("constraints", "project1")) + .thenReturn(featureSet); String name = constraint.getKey(); Object value = constraint.getValue(); FeatureSpec newFeatureSpec = @@ -621,12 +479,8 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated() assertEquals( "Response should have CREATED status when field '" + name + "' is updated", - Status.CREATED, + Status.UPDATED, response.getStatus()); - assertEquals( - "FeatureSet should have new version when field '" + name + "' is updated", - existingFeatureSet.getSpec().getVersion() + 1, - response.getFeatureSet().getSpec().getVersion()); assertEquals( "Feature should have field '" + name + "' set correctly", constraint.getValue(), @@ -645,8 +499,8 @@ public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists() Feature f3f2 = TestObjectFactory.CreateFeature("f3f2", Enum.INT64); Entity f3e1 = TestObjectFactory.CreateEntity("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = - (TestObjectFactory.CreateFeatureSet( - "f3", "newproject", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) + TestObjectFactory.CreateFeatureSet( + "f3", "project", Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1)) .toProto(); ApplyFeatureSetResponse applyFeatureSetResponse = @@ -664,8 +518,8 @@ public void applyFeatureSetShouldFailWhenProjectIsArchived() Feature f3f2 = TestObjectFactory.CreateFeature("f3f2", Enum.INT64); Entity f3e1 = TestObjectFactory.CreateEntity("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = - (TestObjectFactory.CreateFeatureSet( - "f3", "archivedproject", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) + TestObjectFactory.CreateFeatureSet( + "f3", "archivedproject", Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1)) .toProto(); expectedException.expect(IllegalArgumentException.class); @@ -776,8 +630,7 @@ public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferExcep .setName("SERVING") .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder()) - .addSubscriptions( - Subscription.newBuilder().setProject("project1").setName("a").setVersion(">1")) + .addSubscriptions(Subscription.newBuilder().setProject("project1").setName("a")) .build(); UpdateStoreResponse actual = specService.updateStore(UpdateStoreRequest.newBuilder().setStore(newStore).build()); @@ -814,7 +667,7 @@ public void shouldFailIfGetFeatureSetWithoutProject() throws InvalidProtocolBuff specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setName("f1").build()); } - private FeatureSet newDummyFeatureSet(String name, int version, String project) { + private FeatureSet newDummyFeatureSet(String name, String project) { FeatureSpec f1 = FeatureSpec.newBuilder() .setName("feature") @@ -826,7 +679,7 @@ private FeatureSet newDummyFeatureSet(String name, int version, String project) FeatureSet fs = TestObjectFactory.CreateFeatureSet( - name, project, version, Arrays.asList(entity), Arrays.asList(feature)); + name, project, Arrays.asList(entity), Arrays.asList(feature)); fs.setCreated(Date.from(Instant.ofEpochSecond(10L))); return fs; } @@ -836,7 +689,7 @@ private Store newDummyStore(String name) { Store store = new Store(); store.setName(name); store.setType(StoreType.REDIS.toString()); - store.setSubscriptions("*:*:*"); + store.setSubscriptions("*:*"); store.setConfig(RedisConfig.newBuilder().setPort(6379).build().toByteArray()); return store; } diff --git a/core/src/test/java/feast/core/service/TestObjectFactory.java b/core/src/test/java/feast/core/service/TestObjectFactory.java index 0476dbe5c2e..2723db1d2ed 100644 --- a/core/src/test/java/feast/core/service/TestObjectFactory.java +++ b/core/src/test/java/feast/core/service/TestObjectFactory.java @@ -38,11 +38,10 @@ public class TestObjectFactory { true); public static FeatureSet CreateFeatureSet( - String name, String project, int version, List entities, List features) { + String name, String project, List entities, List features) { return new FeatureSet( name, project, - version, 100L, entities, features, diff --git a/examples/basic/basic.ipynb b/examples/basic/basic.ipynb index 921577fb085..b9e0ba9e1a4 100644 --- a/examples/basic/basic.ipynb +++ b/examples/basic/basic.ipynb @@ -250,14 +250,7 @@ "metadata": {}, "outputs": [], "source": [ - "client.apply(customer_fs)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We test the retrieval of this feature set object (not its data), to ensure that we have the latest version" + "client.apply(customer_fs)\n" ] }, { @@ -465,4 +458,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index 15160e57786..ec7cd20f804 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200320220750-118fecf932d8 golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae // indirect - golang.org/x/tools v0.0.0-20200414032229-332987a829c3 // indirect + golang.org/x/tools v0.0.0-20200504022951-6b6965ac5dd1 // indirect google.golang.org/genproto v0.0.0-20200319113533-08878b785e9c // indirect google.golang.org/grpc v1.28.0 gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 1b53b39cf40..49996e36f4f 100644 --- a/go.sum +++ b/go.sum @@ -446,6 +446,8 @@ golang.org/x/tools v0.0.0-20200321224714-0d839f3cf2ed h1:OCZDlBlLYiUK6T33/8+3Bno golang.org/x/tools v0.0.0-20200321224714-0d839f3cf2ed/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200414032229-332987a829c3 h1:Z68UA+HA9shnGhQbAFXKqL1Rk/tfiTHJ57bNm/MUL/A= golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200504022951-6b6965ac5dd1 h1:C8rdnd6KieI73Z2Av0sS0t4kW+geIH/M8kNX8Hmvn9E= +golang.org/x/tools v0.0.0-20200504022951-6b6965ac5dd1/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java index ef6039e5536..5afa66b5e14 100644 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ b/ingestion/src/main/java/feast/ingestion/ImportJob.java @@ -27,8 +27,8 @@ import feast.ingestion.options.BZip2Decompressor; import feast.ingestion.options.ImportOptions; import feast.ingestion.options.StringListStreamConverter; +import feast.ingestion.transform.ProcessAndValidateFeatureRows; import feast.ingestion.transform.ReadFromSource; -import feast.ingestion.transform.ValidateFeatureRows; import feast.ingestion.transform.metrics.WriteFailureMetricsTransform; import feast.ingestion.transform.metrics.WriteSuccessMetricsTransform; import feast.ingestion.utils.SpecUtil; @@ -124,12 +124,12 @@ public static PipelineResult runPipeline(ImportOptions options) throws IOExcepti .setFailureTag(DEADLETTER_OUT) .build()); - // Step 2. Validate incoming FeatureRows + // Step 2. Process and validate incoming FeatureRows PCollectionTuple validatedRows = convertedFeatureRows .get(FEATURE_ROW_OUT) .apply( - ValidateFeatureRows.newBuilder() + ProcessAndValidateFeatureRows.newBuilder() .setFeatureSetSpecs(featureSetSpecsByKey) .setSuccessTag(FEATURE_ROW_OUT) .setFailureTag(DEADLETTER_OUT) diff --git a/ingestion/src/main/java/feast/ingestion/transform/ValidateFeatureRows.java b/ingestion/src/main/java/feast/ingestion/transform/ProcessAndValidateFeatureRows.java similarity index 75% rename from ingestion/src/main/java/feast/ingestion/transform/ValidateFeatureRows.java rename to ingestion/src/main/java/feast/ingestion/transform/ProcessAndValidateFeatureRows.java index 06df06c074c..53d56c667d6 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ValidateFeatureRows.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ProcessAndValidateFeatureRows.java @@ -18,6 +18,7 @@ import com.google.auto.value.AutoValue; import feast.core.FeatureSetProto; +import feast.ingestion.transform.fn.ProcessFeatureRowDoFn; import feast.ingestion.transform.fn.ValidateFeatureRowDoFn; import feast.ingestion.values.FeatureSet; import feast.storage.api.writer.FailedElement; @@ -33,7 +34,7 @@ import org.apache.commons.lang3.tuple.Pair; @AutoValue -public abstract class ValidateFeatureRows +public abstract class ProcessAndValidateFeatureRows extends PTransform, PCollectionTuple> { public abstract Map getFeatureSetSpecs(); @@ -43,7 +44,7 @@ public abstract class ValidateFeatureRows public abstract TupleTag getFailureTag(); public static Builder newBuilder() { - return new AutoValue_ValidateFeatureRows.Builder(); + return new AutoValue_ProcessAndValidateFeatureRows.Builder(); } @AutoValue.Builder @@ -56,7 +57,7 @@ public abstract Builder setFeatureSetSpecs( public abstract Builder setFailureTag(TupleTag failureTag); - public abstract ValidateFeatureRows build(); + public abstract ProcessAndValidateFeatureRows build(); } @Override @@ -67,14 +68,16 @@ public PCollectionTuple expand(PCollection input) { .map(e -> Pair.of(e.getKey(), new FeatureSet(e.getValue()))) .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); - return input.apply( - "ValidateFeatureRows", - ParDo.of( - ValidateFeatureRowDoFn.newBuilder() - .setFeatureSets(featureSets) - .setSuccessTag(getSuccessTag()) - .setFailureTag(getFailureTag()) - .build()) - .withOutputTags(getSuccessTag(), TupleTagList.of(getFailureTag()))); + return input + .apply("ProcessFeatureRows", ParDo.of(new ProcessFeatureRowDoFn())) + .apply( + "ValidateFeatureRows", + ParDo.of( + ValidateFeatureRowDoFn.newBuilder() + .setFeatureSets(featureSets) + .setSuccessTag(getSuccessTag()) + .setFailureTag(getFailureTag()) + .build()) + .withOutputTags(getSuccessTag(), TupleTagList.of(getFailureTag()))); } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ProcessFeatureRowDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ProcessFeatureRowDoFn.java new file mode 100644 index 00000000000..70e173d5db1 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ProcessFeatureRowDoFn.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform.fn; + +import feast.types.FeatureRowProto.FeatureRow; +import org.apache.beam.sdk.transforms.DoFn; + +public class ProcessFeatureRowDoFn extends DoFn { + + @ProcessElement + public void processElement(ProcessContext context) { + FeatureRow featureRow = context.element(); + featureRow = + featureRow.toBuilder().setFeatureSet(stripVersion(featureRow.getFeatureSet())).build(); + context.output(featureRow); + } + + // For backward compatibility. Will be deprecated eventually. + private String stripVersion(String featureSetId) { + String[] split = featureSetId.split(":"); + return split[0]; + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java index 85ac3c86faa..d3e0475abf3 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java @@ -58,14 +58,14 @@ public abstract static class Builder { public void processElement(ProcessContext context) { String error = null; FeatureRow featureRow = context.element(); - FeatureSet featureSet = getFeatureSets().getOrDefault(featureRow.getFeatureSet(), null); + FeatureSet featureSet = getFeatureSets().get(featureRow.getFeatureSet()); List fields = new ArrayList<>(); if (featureSet != null) { for (FieldProto.Field field : featureRow.getFieldsList()) { Field fieldSpec = featureSet.getField(field.getName()); if (fieldSpec == null) { // skip - break; + continue; } // If value is set in the FeatureRow, make sure the value type matches // that defined in FeatureSetSpec @@ -99,13 +99,8 @@ public void processElement(ProcessContext context) { .setPayload(featureRow.toString()) .setErrorMessage(error); if (featureSet != null) { - String[] split = featureSet.getReference().split(":"); - String[] nameSplit = split[0].split("/"); - failedElement = - failedElement - .setProjectName(nameSplit[0]) - .setFeatureSetName(nameSplit[1]) - .setFeatureSetVersion(split[1]); + String[] split = featureSet.getReference().split("/"); + failedElement = failedElement.setProjectName(split[0]).setFeatureSetName(split[1]); } context.output(getFailureTag(), failedElement.build()); } else { diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java index b4338cda09b..828fed6ceda 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java @@ -76,7 +76,6 @@ public void processElement(ProcessContext c) { STORE_TAG_KEY + ":" + getStoreName(), PROJECT_TAG_KEY + ":" + ignored.getProjectName(), FEATURE_SET_NAME_TAG_KEY + ":" + ignored.getFeatureSetName(), - FEATURE_SET_VERSION_TAG_KEY + ":" + ignored.getFeatureSetVersion(), INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName()); } catch (StatsDClientException e) { log.warn("Unable to push metrics to server", e); diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java index cfecb858dcf..aa8c7a7e89a 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java @@ -18,7 +18,6 @@ import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.FEATURE_SET_NAME_TAG_KEY; import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.FEATURE_SET_PROJECT_TAG_KEY; -import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.FEATURE_SET_VERSION_TAG_KEY; import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.FEATURE_TAG_KEY; import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.INGESTION_JOB_NAME_KEY; import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.METRIC_PREFIX; @@ -131,25 +130,17 @@ public void processElement( "Feature set reference in the feature row is null. Please check the input feature rows from previous steps"); return; } - String[] colonSplits = featureSetRef.split(":"); - if (colonSplits.length != 2) { - log.error( - "Skip writing feature value metrics because the feature set reference '{}' does not" - + "follow the required format /:", - featureSetRef); - return; - } - String[] slashSplits = colonSplits[0].split("/"); + + String[] slashSplits = featureSetRef.split("/"); if (slashSplits.length != 2) { log.error( "Skip writing feature value metrics because the feature set reference '{}' does not" - + "follow the required format /:", + + "follow the required format /", featureSetRef); return; } String projectName = slashSplits[0]; String featureSetName = slashSplits[1]; - String version = colonSplits[1]; Map featureNameToStats = new HashMap<>(); Map> featureNameToValues = new HashMap<>(); @@ -166,7 +157,6 @@ public void processElement( STORE_TAG_KEY + ":" + getStoreName(), FEATURE_SET_PROJECT_TAG_KEY + ":" + projectName, FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, - FEATURE_SET_VERSION_TAG_KEY + ":" + version, FEATURE_TAG_KEY + ":" + featureName, INGESTION_JOB_NAME_KEY + ":" + context.getPipelineOptions().getJobName() }; diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java index 2fe1f2e7f01..d8cc7635730 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java @@ -142,15 +142,7 @@ public void processElement( "Feature set reference in the feature row is null. Please check the input feature rows from previous steps"); return; } - String[] colonSplits = featureSetRef.split(":"); - if (colonSplits.length != 2) { - log.error( - "Skip writing feature row metrics because the feature set reference '{}' does not" - + "follow the required format /:", - featureSetRef); - return; - } - String[] slashSplits = colonSplits[0].split("/"); + String[] slashSplits = featureSetRef.split("/"); if (slashSplits.length != 2) { log.error( "Skip writing feature row metrics because the feature set reference '{}' does not" @@ -161,7 +153,6 @@ public void processElement( String featureSetProject = slashSplits[0]; String featureSetName = slashSplits[1]; - String featureSetVersion = colonSplits[1]; // featureRowLagStats is stats for feature row lag for feature set "featureSetName" DescriptiveStatistics featureRowLagStats = new DescriptiveStatistics(); @@ -201,7 +192,6 @@ public void processElement( STORE_TAG_KEY + ":" + getStoreName(), FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, - FEATURE_SET_VERSION_TAG_KEY + ":" + featureSetVersion, INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName(), }; diff --git a/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java b/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java index f28dfc9ee39..9aac352fe45 100644 --- a/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java +++ b/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java @@ -34,9 +34,7 @@ public class SpecUtil { public static String getFeatureSetReference(FeatureSetSpec featureSetSpec) { - return String.format( - "%s/%s:%d", - featureSetSpec.getProject(), featureSetSpec.getName(), featureSetSpec.getVersion()); + return String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); } /** Get only feature set specs that matches the subscription */ @@ -46,28 +44,17 @@ public static List getSubscribedFeatureSets( for (FeatureSet featureSet : featureSets) { for (Subscription sub : subscriptions) { // If configuration missing, fail - if (sub.getProject().isEmpty() || sub.getName().isEmpty() || sub.getVersion().isEmpty()) { + if (sub.getProject().isEmpty() || sub.getName().isEmpty()) { throw new IllegalArgumentException( String.format("Subscription is missing arguments: %s", sub.toString())); } // If all wildcards, subscribe to everything - if (sub.getProject().equals("*") - || sub.getName().equals("*") - || sub.getVersion().equals("*")) { + if (sub.getProject().equals("*") || sub.getName().equals("*")) { subscribed.add(featureSet); break; } - // If all wildcards, subscribe to everything - if (sub.getProject().equals("*") - && (!sub.getName().equals("*") || !sub.getVersion().equals("*"))) { - throw new IllegalArgumentException( - String.format( - "Subscription cannot have feature set name and/or version set if project is not defined: %s", - sub.toString())); - } - // Match project name if (!featureSet.getSpec().getProject().equals(sub.getProject())) { continue; @@ -84,26 +71,7 @@ public static List getSubscribedFeatureSets( if (!pattern.matcher(featureSet.getSpec().getName()).matches()) { continue; } - - // If version is '*', match all - if (sub.getVersion().equals("*")) { - subscribed.add(featureSet); - break; - } else if (sub.getVersion().equals("latest")) { - // if version is "latest" - throw new RuntimeException( - String.format( - "Support for latest feature set subscription has not been implemented yet: %s", - sub.toString())); - - } else { - // If a specific version, match that version alone - int version = Integer.parseInt(sub.getVersion()); - if (featureSet.getSpec().getVersion() == version) { - subscribed.add(featureSet); - break; - } - } + subscribed.add(featureSet); } } return subscribed; diff --git a/ingestion/src/main/java/feast/ingestion/values/FailedElement.java b/ingestion/src/main/java/feast/ingestion/values/FailedElement.java deleted file mode 100644 index 9606c27d190..00000000000 --- a/ingestion/src/main/java/feast/ingestion/values/FailedElement.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.values; - -import com.google.auto.value.AutoValue; -import javax.annotation.Nullable; -import org.apache.beam.sdk.schemas.AutoValueSchema; -import org.apache.beam.sdk.schemas.annotations.DefaultSchema; -import org.joda.time.Instant; - -@AutoValue -// Use DefaultSchema annotation so this AutoValue class can be serialized by Beam -// https://issues.apache.org/jira/browse/BEAM-1891 -// https://github.com/apache/beam/pull/7334 -@DefaultSchema(AutoValueSchema.class) -public abstract class FailedElement { - public abstract Instant getTimestamp(); - - @Nullable - public abstract String getJobName(); - - @Nullable - public abstract String getProjectName(); - - @Nullable - public abstract String getFeatureSetName(); - - @Nullable - public abstract String getFeatureSetVersion(); - - @Nullable - public abstract String getTransformName(); - - @Nullable - public abstract String getPayload(); - - @Nullable - public abstract String getErrorMessage(); - - @Nullable - public abstract String getStackTrace(); - - public static Builder newBuilder() { - return new AutoValue_FailedElement.Builder().setTimestamp(Instant.now()); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setTimestamp(Instant timestamp); - - public abstract Builder setProjectName(String projectName); - - public abstract Builder setFeatureSetName(String featureSetName); - - public abstract Builder setFeatureSetVersion(String featureSetVersion); - - public abstract Builder setJobName(String jobName); - - public abstract Builder setTransformName(String transformName); - - public abstract Builder setPayload(String payload); - - public abstract Builder setErrorMessage(String errorMessage); - - public abstract Builder setStackTrace(String stackTrace); - - public abstract FailedElement build(); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java index 13df73e96a4..cd25cdf380a 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java @@ -120,7 +120,6 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() FeatureSetSpec spec = FeatureSetSpec.newBuilder() .setName("feature_set") - .setVersion(3) .setProject("myproject") .addEntities( EntitySpec.newBuilder() @@ -164,7 +163,6 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() Subscription.newBuilder() .setProject(spec.getProject()) .setName(spec.getName()) - .setVersion(String.valueOf(spec.getVersion())) .build()) .build(); diff --git a/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java b/ingestion/src/test/java/feast/ingestion/transform/ProcessAndValidateFeatureRowsTest.java similarity index 74% rename from ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java rename to ingestion/src/test/java/feast/ingestion/transform/ProcessAndValidateFeatureRowsTest.java index 3737a736168..be082988716 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/ProcessAndValidateFeatureRowsTest.java @@ -39,7 +39,7 @@ import org.junit.Rule; import org.junit.Test; -public class ValidateFeatureRowsTest { +public class ProcessAndValidateFeatureRowsTest { @Rule public transient TestPipeline p = TestPipeline.create(); @@ -52,7 +52,6 @@ public void shouldWriteSuccessAndFailureTagsCorrectly() { FeatureSetSpec fs1 = FeatureSetSpec.newBuilder() .setName("feature_set") - .setVersion(1) .setProject("myproject") .addEntities( EntitySpec.newBuilder() @@ -72,8 +71,7 @@ public void shouldWriteSuccessAndFailureTagsCorrectly() { FeatureSetSpec fs2 = FeatureSetSpec.newBuilder() - .setName("feature_set") - .setVersion(2) + .setName("feature_set_2") .setProject("myproject") .addEntities( EntitySpec.newBuilder() @@ -92,8 +90,8 @@ public void shouldWriteSuccessAndFailureTagsCorrectly() { .build(); Map featureSetSpecs = new HashMap<>(); - featureSetSpecs.put("myproject/feature_set:1", fs1); - featureSetSpecs.put("myproject/feature_set:2", fs2); + featureSetSpecs.put("myproject/feature_set", fs1); + featureSetSpecs.put("myproject/feature_set_2", fs2); List input = new ArrayList<>(); List expected = new ArrayList<>(); @@ -110,7 +108,7 @@ public void shouldWriteSuccessAndFailureTagsCorrectly() { p.apply(Create.of(input)) .setCoder(ProtoCoder.of(FeatureRow.class)) .apply( - ValidateFeatureRows.newBuilder() + ProcessAndValidateFeatureRows.newBuilder() .setFailureTag(FAILURE_TAG) .setSuccessTag(SUCCESS_TAG) .setFeatureSetSpecs(featureSetSpecs) @@ -122,12 +120,59 @@ public void shouldWriteSuccessAndFailureTagsCorrectly() { p.run(); } + @Test + public void shouldStripVersions() { + FeatureSetSpec fs1 = + FeatureSetSpec.newBuilder() + .setName("feature_set") + .setProject("myproject") + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_primary") + .setValueType(Enum.INT32) + .build()) + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_secondary") + .setValueType(Enum.STRING) + .build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) + .build(); + + Map featureSetSpecs = new HashMap<>(); + featureSetSpecs.put("myproject/feature_set", fs1); + + List input = new ArrayList<>(); + List expected = new ArrayList<>(); + + FeatureRow randomRow = TestUtil.createRandomFeatureRow(fs1); + expected.add(randomRow); + randomRow = randomRow.toBuilder().setFeatureSet("myproject/feature_set:1").build(); + input.add(randomRow); + + PCollectionTuple output = + p.apply(Create.of(input)) + .setCoder(ProtoCoder.of(FeatureRow.class)) + .apply( + ProcessAndValidateFeatureRows.newBuilder() + .setFailureTag(FAILURE_TAG) + .setSuccessTag(SUCCESS_TAG) + .setFeatureSetSpecs(featureSetSpecs) + .build()); + + PAssert.that(output.get(SUCCESS_TAG)).containsInAnyOrder(expected); + + p.run(); + } + @Test public void shouldExcludeUnregisteredFields() { FeatureSetSpec fs1 = FeatureSetSpec.newBuilder() .setName("feature_set") - .setVersion(1) .setProject("myproject") .addEntities( EntitySpec.newBuilder() @@ -146,7 +191,7 @@ public void shouldExcludeUnregisteredFields() { .build(); Map featureSets = new HashMap<>(); - featureSets.put("myproject/feature_set:1", fs1); + featureSets.put("myproject/feature_set", fs1); List input = new ArrayList<>(); List expected = new ArrayList<>(); @@ -166,7 +211,7 @@ public void shouldExcludeUnregisteredFields() { p.apply(Create.of(input)) .setCoder(ProtoCoder.of(FeatureRow.class)) .apply( - ValidateFeatureRows.newBuilder() + ProcessAndValidateFeatureRows.newBuilder() .setFailureTag(FAILURE_TAG) .setSuccessTag(SUCCESS_TAG) .setFeatureSetSpecs(featureSets) diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input index d2985711cee..42731b9fe1c 100644 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input +++ b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input @@ -1,4 +1,4 @@ featuresetref,int32,int64,double,float,bool,int32list,int64list,doublelist,floatlist,boollist,bytes,byteslist,string,stringlist -project/featureset:1,1,5,8,5,true,1|4|3,5|1|12,5|7|3,-2.0,true|false,,,, -project/featureset:1,5,-10,8,10.0,true,1|12|5,,,-1.0|-3.0,false|true,,,, -project/featureset:1,6,-4,8,0.0,true,2,2|5,,,true|false,,,, \ No newline at end of file +project/featureset,1,5,8,5,true,1|4|3,5|1|12,5|7|3,-2.0,true|false,,,, +project/featureset,5,-10,8,10.0,true,1|12|5,,,-1.0|-3.0,false|true,,,, +project/featureset,6,-4,8,0.0,true,2,2|5,,,true|false,,,, \ No newline at end of file diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output index 63bc7bbfa4e..12ed4b7e1f2 100644 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output +++ b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output @@ -1,66 +1,66 @@ -feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:6|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:4|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:6|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:6|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:4|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:6|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:-10|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:5|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:0|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:-3|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:-4|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:5|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:-10|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:5|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:0|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:-3|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:-4|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:5|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:10|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:10|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:10|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:10|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:12|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:4|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:3|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:12|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:12|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:4|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:3|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:12|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:12|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:12|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:12|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:12|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:3|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:7|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:7|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:3|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:7|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:7|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:-3|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:-1|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:-2|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:-2|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:-1|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:-3|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:-1|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:-2|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:-2|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:-1|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:1|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:0.5|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:0.5|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:1|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store \ No newline at end of file +feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:1|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:0.5|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:0.5|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:1|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store \ No newline at end of file diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.input b/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.input index 4d42f5bc4c4..c5543d2889d 100644 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.input +++ b/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.input @@ -1,4 +1,4 @@ featuresetref,int32,int64,timestamp -project/featureset:1,1,5,2020-03-30T06:10:38Z -project/featureset:1,5,8,2020-03-30T06:10:43Z -project/featureset:1,6,,2020-03-30T06:10:42Z \ No newline at end of file +project/featureset,1,5,2020-03-30T06:10:38Z +project/featureset,5,8,2020-03-30T06:10:43Z +project/featureset,6,,2020-03-30T06:10:42Z \ No newline at end of file diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.output b/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.output index 318ce8eb08b..954215764f3 100644 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.output +++ b/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.output @@ -1,23 +1,23 @@ -feast_ingestion.feature_row_ingested_count:3|c|#ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_min:2000|g|#ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_max:7000|g|#ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_mean:4000|g|#ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_percentile_90:7000|g|#ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_percentile_95:7000|g|#ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_percentile_99:7000|g|#ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_row_ingested_count:3|c|#ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_row_lag_ms_min:2000|g|#ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_row_lag_ms_max:7000|g|#ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_row_lag_ms_mean:4000|g|#ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_row_lag_ms_percentile_90:7000|g|#ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_row_lag_ms_percentile_95:7000|g|#ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_row_lag_ms_percentile_99:7000|g|#ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_min:2000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_max:7000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_mean:4000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_90:7000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_95:7000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_99:7000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_missing_count:0|c|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_min:2000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_max:7000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_mean:4000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_percentile_90:7000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_percentile_95:7000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_percentile_99:7000|g|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_missing_count:0|c|#feast_feature_name:int32,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_min:2000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_max:7000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_mean:4500|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_90:7000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_95:7000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_99:7000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_missing_count:1|c|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store \ No newline at end of file +feast_ingestion.feature_value_lag_ms_min:2000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_max:7000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_mean:4500|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_percentile_90:7000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_percentile_95:7000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_lag_ms_percentile_99:7000|g|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_missing_count:1|c|#feast_feature_name:int64,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store \ No newline at end of file diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index b7760d0b9aa..9cdcbf5c657 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -52,8 +52,11 @@ service CoreService { // Create or update and existing feature set. // // This function is idempotent - it will not create a new feature set if schema does not change. - // If an existing feature set is updated, core will advance the version number, which will be - // returned in response. + // Schema changes will update the feature set if the changes are valid. + // All changes except the following are valid: + // - Changes to feature set id (name, project) + // - Changes to entities + // - Changes to feature name and type rpc ApplyFeatureSet (ApplyFeatureSetRequest) returns (ApplyFeatureSetResponse); // Updates core with the configuration of the store. @@ -102,9 +105,6 @@ message GetFeatureSetRequest { // Name of feature set (required). string name = 1; - - // Version of feature set (optional). If omitted then latest feature set will be returned. - int32 version = 2; } // Response containing a single feature set @@ -133,15 +133,6 @@ message ListFeatureSetsRequest { // - my-feature-set* can be used to match all features prefixed by "my-feature-set" // - my-feature-set-6 can be used to select a single feature set string feature_set_name = 1; - - - // Versions of the given feature sets that will be returned. - // Valid options for version: - // "latest": only the latest version is returned. - // "*": Subscribe to all versions - // [version number]: pin to a specific version. Project and feature set name must be - // explicitly defined if a specific version is pinned. - string feature_set_version = 2; } } @@ -163,23 +154,25 @@ message ListStoresResponse { } message ApplyFeatureSetRequest { - // Feature set version and source will be ignored feast.core.FeatureSet feature_set = 1; } message ApplyFeatureSetResponse { + // TODO: 0 should correspond to invalid rather than NO_CHANGE enum Status { - // Latest feature set version is consistent with provided feature set + // Latest feature set is consistent with provided feature set NO_CHANGE = 0; - // New feature set or feature set version created + // New feature set created CREATED = 1; // Error occurred while trying to apply changes ERROR = 2; + + // Changes detected and updated successfully + UPDATED = 3; } - // Feature set response has been enriched with version and source information feast.core.FeatureSet feature_set = 1; Status status = 2; } diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto index e7e69ede562..f4315256a67 100644 --- a/protos/feast/core/FeatureSet.proto +++ b/protos/feast/core/FeatureSet.proto @@ -40,8 +40,8 @@ message FeatureSetSpec { // Name of the feature set. Must be unique. string name = 1; - // Feature set version. - int32 version = 2; + // Feature set version was removed in v0.5.0. + reserved 2; // List of entities contained within this featureSet. // This allows the feature to be used during joins between feature sets. diff --git a/protos/feast/core/FeatureSetReference.proto b/protos/feast/core/FeatureSetReference.proto index 2501ec0931c..fb2f56425c3 100644 --- a/protos/feast/core/FeatureSetReference.proto +++ b/protos/feast/core/FeatureSetReference.proto @@ -28,6 +28,6 @@ message FeatureSetReference { string project = 1; // Name of the FeatureSet string name = 2; - // Version no. of the FeatureSet - int32 version = 3; + // Feature set version was removed in v0.5.0. + reserved 3; } diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto index f35561467e1..3e5855753c7 100644 --- a/protos/feast/core/Store.proto +++ b/protos/feast/core/Store.proto @@ -48,12 +48,7 @@ message Store { // BigQuery stores a FeatureRow element as a row in a BigQuery table. // - // Table name is derived from the feature set name and version as: - // [feature_set_name]_v[feature_set_version] - // - // For example: - // A feature row for feature set "driver" and version "1" will be written - // to table "driver_v1". + // Table name is derived is the same as the feature set name. // // The entities and features in a FeatureSetSpec corresponds to the // fields in the BigQuery table (these make up the BigQuery schema). @@ -74,11 +69,6 @@ message Store { // // BigQuery table created will be partitioned by the field "event_timestamp" // of the FeatureRow (https://cloud.google.com/bigquery/docs/partitioned-tables). - // - // Since newer version of feature set can introduce breaking, non backward- - // compatible BigQuery schema updates, incrementing the version of a - // feature set will result in the creation of a new empty BigQuery table - // with the new schema. // // The following table shows how ValueType in Feast is mapped to // BigQuery Standard SQL data types @@ -149,7 +139,6 @@ message Store { // pattern matching. string project = 3; - // Name of the desired feature set. Asterisks can be used as wildcards in the name. // Matching on names is only permitted if a specific project is defined. It is disallowed // If the project name is set to "*" @@ -159,13 +148,8 @@ message Store { // - my-feature-set-6 can be used to select a single feature set string name = 1; - // Versions of the given feature sets that will be returned. - // Valid options for version: - // "latest": only the latest version is returned. - // "*": Subscribe to all versions - // [version number]: pin to a specific version. Project and feature set name must be - // explicitly defined if a specific version is pinned. - string version = 2; + // Feature set version was removed in v0.5.0. + reserved 2; } // Name of the store. diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index 5145670ec9a..bd302a83b46 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -69,9 +69,6 @@ message FeatureReference { // Feature name string name = 2; - // Feature version - int32 version = 3; - // The features will be retrieved if: // entity_timestamp - max_age <= event_timestamp <= entity_timestamp // diff --git a/protos/feast/storage/Redis.proto b/protos/feast/storage/Redis.proto index f58b137e9c1..c373f1b0524 100644 --- a/protos/feast/storage/Redis.proto +++ b/protos/feast/storage/Redis.proto @@ -28,7 +28,7 @@ message RedisKey { // Field number 1 is reserved for a future distributing hash if needed // (for when redis is clustered). - // FeatureSet this row belongs to, this is defined as featureSetName:version. + // FeatureSet this row belongs to, this is defined as featureSetName. string feature_set = 2; // List of fields containing entity names and their respective values diff --git a/protos/feast/types/FeatureRow.proto b/protos/feast/types/FeatureRow.proto index c19a393fde5..3e9056d12c2 100644 --- a/protos/feast/types/FeatureRow.proto +++ b/protos/feast/types/FeatureRow.proto @@ -36,7 +36,7 @@ message FeatureRow { google.protobuf.Timestamp event_timestamp = 3; // Complete reference to the featureSet this featureRow belongs to, in the form of - // /:. This value will be used by the feast ingestion job to filter + // /. This value will be used by the feast ingestion job to filter // rows, and write the values to the correct tables. string feature_set = 6; diff --git a/sdk/go/README.md b/sdk/go/README.md index 6084f909931..464eccf93d6 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -17,7 +17,7 @@ func main() { ctx := context.Background() req := feast.OnlineFeaturesRequest{ - Features: []string{"my_project_1/feature1:1", "my_project_2/feature1:1", "my_project_4/feature3", "feature2:2", "feature2"}, + Features: []string{"my_project_1/feature1", "my_project_2/feature1", "my_project_4/feature3", "feature2", "feature2"}, Entities: []feast.Row{ {"entity1": feast.Int64Val(1), "entity2": feast.StrVal("bob")}, {"entity1": feast.Int64Val(1), "entity2": feast.StrVal("annie")}, @@ -40,10 +40,10 @@ func main() { If all features retrieved are of a single type, Feast provides convenience functions to retrieve your features as a vector of feature values: ```{go} arr, err := resp.Int64Arrays( - []string{"my_project_1/feature1:1", - "my_project_2/feature1:1", + []string{"my_project_1/feature1", + "my_project_2/feature1", "my_project_4/feature3", - "feature2:2", + "feature2", "feature2"}, // order of features []int64{1,2,3,4,5}) // fillNa values ``` diff --git a/sdk/go/protos/feast/core/CoreService.pb.go b/sdk/go/protos/feast/core/CoreService.pb.go index 90e5f7d2408..1af820f50bb 100644 --- a/sdk/go/protos/feast/core/CoreService.pb.go +++ b/sdk/go/protos/feast/core/CoreService.pb.go @@ -48,12 +48,14 @@ const _ = proto.ProtoPackageIsVersion4 type ApplyFeatureSetResponse_Status int32 const ( - // Latest feature set version is consistent with provided feature set + // Latest feature set is consistent with provided feature set ApplyFeatureSetResponse_NO_CHANGE ApplyFeatureSetResponse_Status = 0 - // New feature set or feature set version created + // New feature set created ApplyFeatureSetResponse_CREATED ApplyFeatureSetResponse_Status = 1 // Error occurred while trying to apply changes ApplyFeatureSetResponse_ERROR ApplyFeatureSetResponse_Status = 2 + // Changes detected and updated successfully + ApplyFeatureSetResponse_UPDATED ApplyFeatureSetResponse_Status = 3 ) // Enum value maps for ApplyFeatureSetResponse_Status. @@ -62,11 +64,13 @@ var ( 0: "NO_CHANGE", 1: "CREATED", 2: "ERROR", + 3: "UPDATED", } ApplyFeatureSetResponse_Status_value = map[string]int32{ "NO_CHANGE": 0, "CREATED": 1, "ERROR": 2, + "UPDATED": 3, } ) @@ -155,8 +159,6 @@ type GetFeatureSetRequest struct { Project string `protobuf:"bytes,3,opt,name=project,proto3" json:"project,omitempty"` // Name of feature set (required). Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // Version of feature set (optional). If omitted then latest feature set will be returned. - Version int32 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"` } func (x *GetFeatureSetRequest) Reset() { @@ -205,13 +207,6 @@ func (x *GetFeatureSetRequest) GetName() string { return "" } -func (x *GetFeatureSetRequest) GetVersion() int32 { - if x != nil { - return x.Version - } - return 0 -} - // Response containing a single feature set type GetFeatureSetResponse struct { state protoimpl.MessageState @@ -454,7 +449,6 @@ type ApplyFeatureSetRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Feature set version and source will be ignored FeatureSet *FeatureSet `protobuf:"bytes,1,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` } @@ -502,7 +496,6 @@ type ApplyFeatureSetResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Feature set response has been enriched with version and source information FeatureSet *FeatureSet `protobuf:"bytes,1,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` Status ApplyFeatureSetResponse_Status `protobuf:"varint,2,opt,name=status,proto3,enum=feast.core.ApplyFeatureSetResponse_Status" json:"status,omitempty"` } @@ -1296,13 +1289,6 @@ type ListFeatureSetsRequest_Filter struct { // - my-feature-set* can be used to match all features prefixed by "my-feature-set" // - my-feature-set-6 can be used to select a single feature set FeatureSetName string `protobuf:"bytes,1,opt,name=feature_set_name,json=featureSetName,proto3" json:"feature_set_name,omitempty"` - // Versions of the given feature sets that will be returned. - // Valid options for version: - // "latest": only the latest version is returned. - // "*": Subscribe to all versions - // [version number]: pin to a specific version. Project and feature set name must be - // explicitly defined if a specific version is pinned. - FeatureSetVersion string `protobuf:"bytes,2,opt,name=feature_set_version,json=featureSetVersion,proto3" json:"feature_set_version,omitempty"` } func (x *ListFeatureSetsRequest_Filter) Reset() { @@ -1351,13 +1337,6 @@ func (x *ListFeatureSetsRequest_Filter) GetFeatureSetName() string { return "" } -func (x *ListFeatureSetsRequest_Filter) GetFeatureSetVersion() string { - if x != nil { - return x.FeatureSetVersion - } - return "" -} - type ListStoresRequest_Filter struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1485,202 +1464,199 @@ var file_feast_core_CoreService_proto_rawDesc = []byte{ 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1d, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x5e, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x44, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x50, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x50, 0x0a, 0x15, 0x47, 0x65, + 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, + 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, + 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x22, 0xa9, 0x01, 0x0a, + 0x16, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, + 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, + 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, 0x4c, 0x0a, 0x06, 0x46, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x28, + 0x0a, 0x10, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x53, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x54, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, + 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, + 0x74, 0x52, 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x22, 0x6f, + 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, + 0x72, 0x1a, 0x1c, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, + 0x3d, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x51, + 0x0a, 0x16, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, + 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x17, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x22, 0xd9, 0x01, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x46, + 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x42, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x3c, 0x0a, 0x06, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x47, + 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, + 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x55, + 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x03, 0x22, 0x1c, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x46, + 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x37, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, + 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, + 0x3d, 0x0a, 0x12, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0xa4, + 0x01, 0x0a, 0x13, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, + 0x3e, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, + 0x24, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, + 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x50, 0x44, 0x41, + 0x54, 0x45, 0x44, 0x10, 0x01, 0x22, 0x2a, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x22, 0x17, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, 0x0a, 0x15, 0x41, 0x72, + 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x41, 0x72, 0x63, 0x68, 0x69, + 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x32, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, + 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x22, 0xee, 0x01, 0x0a, + 0x18, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, + 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x06, 0x66, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, + 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, 0x8c, + 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x53, 0x0a, 0x15, 0x66, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, + 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, + 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x13, 0x66, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x1d, + 0x0a, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, + 0x19, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, + 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x6a, 0x6f, + 0x62, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, + 0x6f, 0x62, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x2c, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1d, 0x0a, 0x1b, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x29, 0x0a, 0x17, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, + 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x22, 0x1a, 0x0a, 0x18, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, + 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xcb, 0x08, 0x0a, + 0x0b, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x13, + 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, + 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, + 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x69, + 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, 0x22, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x41, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, + 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x1a, 0x7c, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x18, - 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0e, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, - 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x11, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x22, 0x54, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, - 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x0b, 0x66, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x22, 0x6f, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, - 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, - 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, 0x1c, 0x0a, 0x06, 0x46, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x3d, 0x0a, 0x12, 0x4c, 0x69, 0x73, - 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, - 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x51, 0x0a, 0x16, 0x41, 0x70, 0x70, 0x6c, - 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, - 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x22, 0xc7, 0x01, 0x0a, 0x17, - 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x53, 0x65, 0x74, 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, - 0x12, 0x42, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, - 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x22, 0x2f, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, - 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, - 0x07, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, - 0x52, 0x4f, 0x52, 0x10, 0x02, 0x22, 0x1c, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, - 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x22, 0x37, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, - 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x0a, 0x12, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, - 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x13, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x3e, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x24, 0x0a, 0x06, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x48, 0x41, - 0x4e, 0x47, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, - 0x10, 0x01, 0x22, 0x2a, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x17, - 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, 0x0a, 0x15, 0x41, 0x72, 0x63, 0x68, 0x69, - 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x15, - 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x32, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, - 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x22, 0xee, 0x01, 0x0a, 0x18, 0x4c, 0x69, - 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, - 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, - 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, 0x8c, 0x01, 0x0a, 0x06, - 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x53, 0x0a, 0x15, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x13, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, - 0x65, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, 0x19, 0x4c, 0x69, - 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, - 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x2c, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x22, 0x1d, 0x0a, 0x1b, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x29, 0x0a, 0x17, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, - 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1a, 0x0a, - 0x18, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, - 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xcb, 0x08, 0x0a, 0x0b, 0x43, 0x6f, - 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x13, 0x47, 0x65, 0x74, - 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, - 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, - 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, - 0x65, 0x74, 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, - 0x73, 0x12, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x5a, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x53, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0b, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1e, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, - 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, - 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x20, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, + 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x4e, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1e, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x54, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, + 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x4c, - 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, - 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, - 0x6f, 0x62, 0x73, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, - 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, - 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x66, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, - 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, - 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x70, - 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x23, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, - 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x4f, 0x0a, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x10, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, - 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, + 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x60, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, + 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, + 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, + 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, + 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, + 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, + 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, + 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, + 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, + 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, + 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x4f, 0x0a, 0x0a, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x10, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -2166,8 +2142,11 @@ type CoreServiceClient interface { // Create or update and existing feature set. // // This function is idempotent - it will not create a new feature set if schema does not change. - // If an existing feature set is updated, core will advance the version number, which will be - // returned in response. + // Schema changes will update the feature set if the changes are valid. + // All changes except the following are valid: + // - Changes to feature set id (name, project) + // - Changes to entities + // - Changes to feature type ApplyFeatureSet(ctx context.Context, in *ApplyFeatureSetRequest, opts ...grpc.CallOption) (*ApplyFeatureSetResponse, error) // Updates core with the configuration of the store. // @@ -2339,8 +2318,11 @@ type CoreServiceServer interface { // Create or update and existing feature set. // // This function is idempotent - it will not create a new feature set if schema does not change. - // If an existing feature set is updated, core will advance the version number, which will be - // returned in response. + // Schema changes will update the feature set if the changes are valid. + // All changes except the following are valid: + // - Changes to feature set id (name, project) + // - Changes to entities + // - Changes to feature type ApplyFeatureSet(context.Context, *ApplyFeatureSetRequest) (*ApplyFeatureSetResponse, error) // Updates core with the configuration of the store. // diff --git a/sdk/go/protos/feast/core/FeatureSet.pb.go b/sdk/go/protos/feast/core/FeatureSet.pb.go index bbf79e7d2a4..091dd1ca9fb 100644 --- a/sdk/go/protos/feast/core/FeatureSet.pb.go +++ b/sdk/go/protos/feast/core/FeatureSet.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.21.0 -// protoc v3.10.1 +// protoc v3.10.0 // source: feast/core/FeatureSet.proto package core @@ -160,8 +160,6 @@ type FeatureSetSpec struct { Project string `protobuf:"bytes,7,opt,name=project,proto3" json:"project,omitempty"` // Name of the feature set. Must be unique. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // Feature set version. - Version int32 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"` // List of entities contained within this featureSet. // This allows the feature to be used during joins between feature sets. // If the featureSet is ingested into a store that supports keys, this value @@ -176,6 +174,8 @@ type FeatureSetSpec struct { // Optional. Source on which feature rows can be found. // If not set, source will be set to the default value configured in Feast Core. Source *Source `protobuf:"bytes,6,opt,name=source,proto3" json:"source,omitempty"` + // User defined metadata + Labels map[string]string `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *FeatureSetSpec) Reset() { @@ -224,13 +224,6 @@ func (x *FeatureSetSpec) GetName() string { return "" } -func (x *FeatureSetSpec) GetVersion() int32 { - if x != nil { - return x.Version - } - return 0 -} - func (x *FeatureSetSpec) GetEntities() []*EntitySpec { if x != nil { return x.Entities @@ -259,6 +252,13 @@ func (x *FeatureSetSpec) GetSource() *Source { return nil } +func (x *FeatureSetSpec) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + type EntitySpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -266,35 +266,8 @@ type EntitySpec struct { // Name of the entity. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // Value type of the feature. + // Value type of the entity. ValueType types.ValueType_Enum `protobuf:"varint,2,opt,name=value_type,json=valueType,proto3,enum=feast.types.ValueType_Enum" json:"value_type,omitempty"` - // Types that are assignable to PresenceConstraints: - // *EntitySpec_Presence - // *EntitySpec_GroupPresence - PresenceConstraints isEntitySpec_PresenceConstraints `protobuf_oneof:"presence_constraints"` - // The shape of the feature which governs the number of values that appear in - // each example. - // - // Types that are assignable to ShapeType: - // *EntitySpec_Shape - // *EntitySpec_ValueCount - ShapeType isEntitySpec_ShapeType `protobuf_oneof:"shape_type"` - // Domain for the values of the feature. - // - // Types that are assignable to DomainInfo: - // *EntitySpec_Domain - // *EntitySpec_IntDomain - // *EntitySpec_FloatDomain - // *EntitySpec_StringDomain - // *EntitySpec_BoolDomain - // *EntitySpec_StructDomain - // *EntitySpec_NaturalLanguageDomain - // *EntitySpec_ImageDomain - // *EntitySpec_MidDomain - // *EntitySpec_UrlDomain - // *EntitySpec_TimeDomain - // *EntitySpec_TimeOfDayDomain - DomainInfo isEntitySpec_DomainInfo `protobuf_oneof:"domain_info"` } func (x *EntitySpec) Reset() { @@ -343,256 +316,6 @@ func (x *EntitySpec) GetValueType() types.ValueType_Enum { return types.ValueType_INVALID } -func (m *EntitySpec) GetPresenceConstraints() isEntitySpec_PresenceConstraints { - if m != nil { - return m.PresenceConstraints - } - return nil -} - -func (x *EntitySpec) GetPresence() *v0.FeaturePresence { - if x, ok := x.GetPresenceConstraints().(*EntitySpec_Presence); ok { - return x.Presence - } - return nil -} - -func (x *EntitySpec) GetGroupPresence() *v0.FeaturePresenceWithinGroup { - if x, ok := x.GetPresenceConstraints().(*EntitySpec_GroupPresence); ok { - return x.GroupPresence - } - return nil -} - -func (m *EntitySpec) GetShapeType() isEntitySpec_ShapeType { - if m != nil { - return m.ShapeType - } - return nil -} - -func (x *EntitySpec) GetShape() *v0.FixedShape { - if x, ok := x.GetShapeType().(*EntitySpec_Shape); ok { - return x.Shape - } - return nil -} - -func (x *EntitySpec) GetValueCount() *v0.ValueCount { - if x, ok := x.GetShapeType().(*EntitySpec_ValueCount); ok { - return x.ValueCount - } - return nil -} - -func (m *EntitySpec) GetDomainInfo() isEntitySpec_DomainInfo { - if m != nil { - return m.DomainInfo - } - return nil -} - -func (x *EntitySpec) GetDomain() string { - if x, ok := x.GetDomainInfo().(*EntitySpec_Domain); ok { - return x.Domain - } - return "" -} - -func (x *EntitySpec) GetIntDomain() *v0.IntDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_IntDomain); ok { - return x.IntDomain - } - return nil -} - -func (x *EntitySpec) GetFloatDomain() *v0.FloatDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_FloatDomain); ok { - return x.FloatDomain - } - return nil -} - -func (x *EntitySpec) GetStringDomain() *v0.StringDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_StringDomain); ok { - return x.StringDomain - } - return nil -} - -func (x *EntitySpec) GetBoolDomain() *v0.BoolDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_BoolDomain); ok { - return x.BoolDomain - } - return nil -} - -func (x *EntitySpec) GetStructDomain() *v0.StructDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_StructDomain); ok { - return x.StructDomain - } - return nil -} - -func (x *EntitySpec) GetNaturalLanguageDomain() *v0.NaturalLanguageDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_NaturalLanguageDomain); ok { - return x.NaturalLanguageDomain - } - return nil -} - -func (x *EntitySpec) GetImageDomain() *v0.ImageDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_ImageDomain); ok { - return x.ImageDomain - } - return nil -} - -func (x *EntitySpec) GetMidDomain() *v0.MIDDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_MidDomain); ok { - return x.MidDomain - } - return nil -} - -func (x *EntitySpec) GetUrlDomain() *v0.URLDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_UrlDomain); ok { - return x.UrlDomain - } - return nil -} - -func (x *EntitySpec) GetTimeDomain() *v0.TimeDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_TimeDomain); ok { - return x.TimeDomain - } - return nil -} - -func (x *EntitySpec) GetTimeOfDayDomain() *v0.TimeOfDayDomain { - if x, ok := x.GetDomainInfo().(*EntitySpec_TimeOfDayDomain); ok { - return x.TimeOfDayDomain - } - return nil -} - -type isEntitySpec_PresenceConstraints interface { - isEntitySpec_PresenceConstraints() -} - -type EntitySpec_Presence struct { - // Constraints on the presence of this feature in the examples. - Presence *v0.FeaturePresence `protobuf:"bytes,3,opt,name=presence,proto3,oneof"` -} - -type EntitySpec_GroupPresence struct { - // Only used in the context of a "group" context, e.g., inside a sequence. - GroupPresence *v0.FeaturePresenceWithinGroup `protobuf:"bytes,4,opt,name=group_presence,json=groupPresence,proto3,oneof"` -} - -func (*EntitySpec_Presence) isEntitySpec_PresenceConstraints() {} - -func (*EntitySpec_GroupPresence) isEntitySpec_PresenceConstraints() {} - -type isEntitySpec_ShapeType interface { - isEntitySpec_ShapeType() -} - -type EntitySpec_Shape struct { - // The feature has a fixed shape corresponding to a multi-dimensional - // tensor. - Shape *v0.FixedShape `protobuf:"bytes,5,opt,name=shape,proto3,oneof"` -} - -type EntitySpec_ValueCount struct { - // The feature doesn't have a well defined shape. All we know are limits on - // the minimum and maximum number of values. - ValueCount *v0.ValueCount `protobuf:"bytes,6,opt,name=value_count,json=valueCount,proto3,oneof"` -} - -func (*EntitySpec_Shape) isEntitySpec_ShapeType() {} - -func (*EntitySpec_ValueCount) isEntitySpec_ShapeType() {} - -type isEntitySpec_DomainInfo interface { - isEntitySpec_DomainInfo() -} - -type EntitySpec_Domain struct { - // Reference to a domain defined at the schema level. - Domain string `protobuf:"bytes,7,opt,name=domain,proto3,oneof"` -} - -type EntitySpec_IntDomain struct { - // Inline definitions of domains. - IntDomain *v0.IntDomain `protobuf:"bytes,8,opt,name=int_domain,json=intDomain,proto3,oneof"` -} - -type EntitySpec_FloatDomain struct { - FloatDomain *v0.FloatDomain `protobuf:"bytes,9,opt,name=float_domain,json=floatDomain,proto3,oneof"` -} - -type EntitySpec_StringDomain struct { - StringDomain *v0.StringDomain `protobuf:"bytes,10,opt,name=string_domain,json=stringDomain,proto3,oneof"` -} - -type EntitySpec_BoolDomain struct { - BoolDomain *v0.BoolDomain `protobuf:"bytes,11,opt,name=bool_domain,json=boolDomain,proto3,oneof"` -} - -type EntitySpec_StructDomain struct { - StructDomain *v0.StructDomain `protobuf:"bytes,12,opt,name=struct_domain,json=structDomain,proto3,oneof"` -} - -type EntitySpec_NaturalLanguageDomain struct { - // Supported semantic domains. - NaturalLanguageDomain *v0.NaturalLanguageDomain `protobuf:"bytes,13,opt,name=natural_language_domain,json=naturalLanguageDomain,proto3,oneof"` -} - -type EntitySpec_ImageDomain struct { - ImageDomain *v0.ImageDomain `protobuf:"bytes,14,opt,name=image_domain,json=imageDomain,proto3,oneof"` -} - -type EntitySpec_MidDomain struct { - MidDomain *v0.MIDDomain `protobuf:"bytes,15,opt,name=mid_domain,json=midDomain,proto3,oneof"` -} - -type EntitySpec_UrlDomain struct { - UrlDomain *v0.URLDomain `protobuf:"bytes,16,opt,name=url_domain,json=urlDomain,proto3,oneof"` -} - -type EntitySpec_TimeDomain struct { - TimeDomain *v0.TimeDomain `protobuf:"bytes,17,opt,name=time_domain,json=timeDomain,proto3,oneof"` -} - -type EntitySpec_TimeOfDayDomain struct { - TimeOfDayDomain *v0.TimeOfDayDomain `protobuf:"bytes,18,opt,name=time_of_day_domain,json=timeOfDayDomain,proto3,oneof"` -} - -func (*EntitySpec_Domain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_IntDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_FloatDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_StringDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_BoolDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_StructDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_NaturalLanguageDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_ImageDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_MidDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_UrlDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_TimeDomain) isEntitySpec_DomainInfo() {} - -func (*EntitySpec_TimeOfDayDomain) isEntitySpec_DomainInfo() {} - type FeatureSpec struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -629,6 +352,8 @@ type FeatureSpec struct { // *FeatureSpec_TimeDomain // *FeatureSpec_TimeOfDayDomain DomainInfo isFeatureSpec_DomainInfo `protobuf_oneof:"domain_info"` + // Labels for user defined metadata on a feature + Labels map[string]string `protobuf:"bytes,19,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *FeatureSpec) Reset() { @@ -810,6 +535,13 @@ func (x *FeatureSpec) GetTimeOfDayDomain() *v0.TimeOfDayDomain { return nil } +func (x *FeatureSpec) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + type isFeatureSpec_PresenceConstraints interface { isFeatureSpec_PresenceConstraints() } @@ -1011,30 +743,42 @@ var file_feast_core_FeatureSet_proto_rawDesc = []byte{ 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x2e, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x52, - 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0xa1, 0x02, 0x0a, 0x0e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0x82, 0x03, 0x0a, 0x0e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x32, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x70, 0x65, 0x63, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, - 0x74, 0x69, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x70, 0x65, 0x63, 0x52, - 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61, 0x78, - 0x5f, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, 0x41, 0x67, 0x65, 0x12, 0x2a, 0x0a, - 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x9b, 0x0a, 0x0a, 0x0a, 0x45, 0x6e, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, + 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x70, 0x65, 0x63, + 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x08, 0x66, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x53, 0x70, 0x65, 0x63, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, + 0x32, 0x0a, 0x07, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, + 0x41, 0x67, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, + 0x3e, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x4c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, + 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5c, 0x0a, 0x0a, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x09, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x22, 0x94, 0x0b, 0x0a, 0x0b, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x0a, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x45, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, @@ -1108,111 +852,37 @@ var file_feast_core_FeatureSet_proto_rawDesc = []byte{ 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x4f, 0x66, 0x44, 0x61, 0x79, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x66, 0x44, 0x61, 0x79, 0x44, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x42, 0x16, 0x0a, 0x14, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x73, 0x68, - 0x61, 0x70, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x0d, 0x0a, 0x0b, 0x64, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x9c, 0x0a, 0x0a, 0x0b, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x0a, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x09, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x45, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6e, 0x73, - 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x76, 0x30, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x63, 0x65, 0x48, 0x00, 0x52, 0x08, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x5b, - 0x0a, 0x0e, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, - 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x57, - 0x69, 0x74, 0x68, 0x69, 0x6e, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0d, 0x67, 0x72, - 0x6f, 0x75, 0x70, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x3a, 0x0a, 0x05, 0x73, - 0x68, 0x61, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6e, - 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x76, 0x30, 0x2e, 0x46, 0x69, 0x78, 0x65, 0x64, 0x53, 0x68, 0x61, 0x70, 0x65, 0x48, 0x01, - 0x52, 0x05, 0x73, 0x68, 0x61, 0x70, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, - 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, - 0x48, 0x01, 0x52, 0x0a, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x18, - 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, - 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x42, 0x0a, 0x0a, 0x69, 0x6e, 0x74, 0x5f, - 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, - 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, - 0x02, 0x52, 0x09, 0x69, 0x6e, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x48, 0x0a, 0x0c, - 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x46, 0x6c, 0x6f, 0x61, - 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x0b, 0x66, 0x6c, 0x6f, 0x61, 0x74, - 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x4b, 0x0a, 0x0d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, - 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x6f, 0x6d, - 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x44, 0x6f, 0x6d, - 0x61, 0x69, 0x6e, 0x12, 0x45, 0x0a, 0x0b, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x64, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, - 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, - 0x30, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x0a, - 0x62, 0x6f, 0x6f, 0x6c, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x4b, 0x0a, 0x0d, 0x73, 0x74, - 0x72, 0x75, 0x63, 0x74, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x24, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x0c, 0x73, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x67, 0x0a, 0x17, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x61, 0x6c, 0x5f, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, - 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, - 0x30, 0x2e, 0x4e, 0x61, 0x74, 0x75, 0x72, 0x61, 0x6c, 0x4c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, - 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x15, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x61, 0x6c, 0x4c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x12, 0x48, 0x0a, 0x0c, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, - 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, - 0x49, 0x6d, 0x61, 0x67, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x0b, 0x69, - 0x6d, 0x61, 0x67, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x42, 0x0a, 0x0a, 0x6d, 0x69, - 0x64, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, - 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x4d, 0x49, 0x44, 0x44, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x48, 0x02, 0x52, 0x09, 0x6d, 0x69, 0x64, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x42, - 0x0a, 0x0a, 0x75, 0x72, 0x6c, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x10, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x55, 0x52, 0x4c, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x09, 0x75, 0x72, 0x6c, 0x44, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x12, 0x45, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, - 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, 0x52, 0x0a, 0x74, - 0x69, 0x6d, 0x65, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x56, 0x0a, 0x12, 0x74, 0x69, 0x6d, - 0x65, 0x5f, 0x6f, 0x66, 0x5f, 0x64, 0x61, 0x79, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, - 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, - 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x4f, 0x66, 0x44, 0x61, 0x79, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x48, 0x02, - 0x52, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x66, 0x44, 0x61, 0x79, 0x44, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x42, 0x16, 0x0a, 0x14, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x63, 0x6f, - 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x73, 0x68, 0x61, - 0x70, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x0d, 0x0a, 0x0b, 0x64, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x8f, 0x01, 0x0a, 0x0e, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x53, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x47, 0x0a, 0x11, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x12, 0x34, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2a, 0x4c, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, - 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, - 0x4e, 0x47, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, - 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x42, 0x4e, 0x0a, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, - 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x6e, 0x12, 0x3b, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x13, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x4c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, + 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x16, 0x0a, 0x14, 0x70, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, + 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x70, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x42, 0x0d, 0x0a, 0x0b, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x22, + 0x8f, 0x01, 0x0a, 0x0e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x4d, 0x65, + 0x74, 0x61, 0x12, 0x47, 0x0a, 0x11, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x34, 0x0a, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x2a, 0x4c, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x10, 0x0a, + 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x42, + 0x4e, 0x0a, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1228,7 +898,7 @@ func file_feast_core_FeatureSet_proto_rawDescGZIP() []byte { } var file_feast_core_FeatureSet_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_feast_core_FeatureSet_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_feast_core_FeatureSet_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_feast_core_FeatureSet_proto_goTypes = []interface{}{ (FeatureSetStatus)(0), // 0: feast.core.FeatureSetStatus (*FeatureSet)(nil), // 1: feast.core.FeatureSet @@ -1236,72 +906,61 @@ var file_feast_core_FeatureSet_proto_goTypes = []interface{}{ (*EntitySpec)(nil), // 3: feast.core.EntitySpec (*FeatureSpec)(nil), // 4: feast.core.FeatureSpec (*FeatureSetMeta)(nil), // 5: feast.core.FeatureSetMeta - (*duration.Duration)(nil), // 6: google.protobuf.Duration - (*Source)(nil), // 7: feast.core.Source - (types.ValueType_Enum)(0), // 8: feast.types.ValueType.Enum - (*v0.FeaturePresence)(nil), // 9: tensorflow.metadata.v0.FeaturePresence - (*v0.FeaturePresenceWithinGroup)(nil), // 10: tensorflow.metadata.v0.FeaturePresenceWithinGroup - (*v0.FixedShape)(nil), // 11: tensorflow.metadata.v0.FixedShape - (*v0.ValueCount)(nil), // 12: tensorflow.metadata.v0.ValueCount - (*v0.IntDomain)(nil), // 13: tensorflow.metadata.v0.IntDomain - (*v0.FloatDomain)(nil), // 14: tensorflow.metadata.v0.FloatDomain - (*v0.StringDomain)(nil), // 15: tensorflow.metadata.v0.StringDomain - (*v0.BoolDomain)(nil), // 16: tensorflow.metadata.v0.BoolDomain - (*v0.StructDomain)(nil), // 17: tensorflow.metadata.v0.StructDomain - (*v0.NaturalLanguageDomain)(nil), // 18: tensorflow.metadata.v0.NaturalLanguageDomain - (*v0.ImageDomain)(nil), // 19: tensorflow.metadata.v0.ImageDomain - (*v0.MIDDomain)(nil), // 20: tensorflow.metadata.v0.MIDDomain - (*v0.URLDomain)(nil), // 21: tensorflow.metadata.v0.URLDomain - (*v0.TimeDomain)(nil), // 22: tensorflow.metadata.v0.TimeDomain - (*v0.TimeOfDayDomain)(nil), // 23: tensorflow.metadata.v0.TimeOfDayDomain - (*timestamp.Timestamp)(nil), // 24: google.protobuf.Timestamp + nil, // 6: feast.core.FeatureSetSpec.LabelsEntry + nil, // 7: feast.core.FeatureSpec.LabelsEntry + (*duration.Duration)(nil), // 8: google.protobuf.Duration + (*Source)(nil), // 9: feast.core.Source + (types.ValueType_Enum)(0), // 10: feast.types.ValueType.Enum + (*v0.FeaturePresence)(nil), // 11: tensorflow.metadata.v0.FeaturePresence + (*v0.FeaturePresenceWithinGroup)(nil), // 12: tensorflow.metadata.v0.FeaturePresenceWithinGroup + (*v0.FixedShape)(nil), // 13: tensorflow.metadata.v0.FixedShape + (*v0.ValueCount)(nil), // 14: tensorflow.metadata.v0.ValueCount + (*v0.IntDomain)(nil), // 15: tensorflow.metadata.v0.IntDomain + (*v0.FloatDomain)(nil), // 16: tensorflow.metadata.v0.FloatDomain + (*v0.StringDomain)(nil), // 17: tensorflow.metadata.v0.StringDomain + (*v0.BoolDomain)(nil), // 18: tensorflow.metadata.v0.BoolDomain + (*v0.StructDomain)(nil), // 19: tensorflow.metadata.v0.StructDomain + (*v0.NaturalLanguageDomain)(nil), // 20: tensorflow.metadata.v0.NaturalLanguageDomain + (*v0.ImageDomain)(nil), // 21: tensorflow.metadata.v0.ImageDomain + (*v0.MIDDomain)(nil), // 22: tensorflow.metadata.v0.MIDDomain + (*v0.URLDomain)(nil), // 23: tensorflow.metadata.v0.URLDomain + (*v0.TimeDomain)(nil), // 24: tensorflow.metadata.v0.TimeDomain + (*v0.TimeOfDayDomain)(nil), // 25: tensorflow.metadata.v0.TimeOfDayDomain + (*timestamp.Timestamp)(nil), // 26: google.protobuf.Timestamp } var file_feast_core_FeatureSet_proto_depIdxs = []int32{ 2, // 0: feast.core.FeatureSet.spec:type_name -> feast.core.FeatureSetSpec 5, // 1: feast.core.FeatureSet.meta:type_name -> feast.core.FeatureSetMeta 3, // 2: feast.core.FeatureSetSpec.entities:type_name -> feast.core.EntitySpec 4, // 3: feast.core.FeatureSetSpec.features:type_name -> feast.core.FeatureSpec - 6, // 4: feast.core.FeatureSetSpec.max_age:type_name -> google.protobuf.Duration - 7, // 5: feast.core.FeatureSetSpec.source:type_name -> feast.core.Source - 8, // 6: feast.core.EntitySpec.value_type:type_name -> feast.types.ValueType.Enum - 9, // 7: feast.core.EntitySpec.presence:type_name -> tensorflow.metadata.v0.FeaturePresence - 10, // 8: feast.core.EntitySpec.group_presence:type_name -> tensorflow.metadata.v0.FeaturePresenceWithinGroup - 11, // 9: feast.core.EntitySpec.shape:type_name -> tensorflow.metadata.v0.FixedShape - 12, // 10: feast.core.EntitySpec.value_count:type_name -> tensorflow.metadata.v0.ValueCount - 13, // 11: feast.core.EntitySpec.int_domain:type_name -> tensorflow.metadata.v0.IntDomain - 14, // 12: feast.core.EntitySpec.float_domain:type_name -> tensorflow.metadata.v0.FloatDomain - 15, // 13: feast.core.EntitySpec.string_domain:type_name -> tensorflow.metadata.v0.StringDomain - 16, // 14: feast.core.EntitySpec.bool_domain:type_name -> tensorflow.metadata.v0.BoolDomain - 17, // 15: feast.core.EntitySpec.struct_domain:type_name -> tensorflow.metadata.v0.StructDomain - 18, // 16: feast.core.EntitySpec.natural_language_domain:type_name -> tensorflow.metadata.v0.NaturalLanguageDomain - 19, // 17: feast.core.EntitySpec.image_domain:type_name -> tensorflow.metadata.v0.ImageDomain - 20, // 18: feast.core.EntitySpec.mid_domain:type_name -> tensorflow.metadata.v0.MIDDomain - 21, // 19: feast.core.EntitySpec.url_domain:type_name -> tensorflow.metadata.v0.URLDomain - 22, // 20: feast.core.EntitySpec.time_domain:type_name -> tensorflow.metadata.v0.TimeDomain - 23, // 21: feast.core.EntitySpec.time_of_day_domain:type_name -> tensorflow.metadata.v0.TimeOfDayDomain - 8, // 22: feast.core.FeatureSpec.value_type:type_name -> feast.types.ValueType.Enum - 9, // 23: feast.core.FeatureSpec.presence:type_name -> tensorflow.metadata.v0.FeaturePresence - 10, // 24: feast.core.FeatureSpec.group_presence:type_name -> tensorflow.metadata.v0.FeaturePresenceWithinGroup - 11, // 25: feast.core.FeatureSpec.shape:type_name -> tensorflow.metadata.v0.FixedShape - 12, // 26: feast.core.FeatureSpec.value_count:type_name -> tensorflow.metadata.v0.ValueCount - 13, // 27: feast.core.FeatureSpec.int_domain:type_name -> tensorflow.metadata.v0.IntDomain - 14, // 28: feast.core.FeatureSpec.float_domain:type_name -> tensorflow.metadata.v0.FloatDomain - 15, // 29: feast.core.FeatureSpec.string_domain:type_name -> tensorflow.metadata.v0.StringDomain - 16, // 30: feast.core.FeatureSpec.bool_domain:type_name -> tensorflow.metadata.v0.BoolDomain - 17, // 31: feast.core.FeatureSpec.struct_domain:type_name -> tensorflow.metadata.v0.StructDomain - 18, // 32: feast.core.FeatureSpec.natural_language_domain:type_name -> tensorflow.metadata.v0.NaturalLanguageDomain - 19, // 33: feast.core.FeatureSpec.image_domain:type_name -> tensorflow.metadata.v0.ImageDomain - 20, // 34: feast.core.FeatureSpec.mid_domain:type_name -> tensorflow.metadata.v0.MIDDomain - 21, // 35: feast.core.FeatureSpec.url_domain:type_name -> tensorflow.metadata.v0.URLDomain - 22, // 36: feast.core.FeatureSpec.time_domain:type_name -> tensorflow.metadata.v0.TimeDomain - 23, // 37: feast.core.FeatureSpec.time_of_day_domain:type_name -> tensorflow.metadata.v0.TimeOfDayDomain - 24, // 38: feast.core.FeatureSetMeta.created_timestamp:type_name -> google.protobuf.Timestamp - 0, // 39: feast.core.FeatureSetMeta.status:type_name -> feast.core.FeatureSetStatus - 40, // [40:40] is the sub-list for method output_type - 40, // [40:40] is the sub-list for method input_type - 40, // [40:40] is the sub-list for extension type_name - 40, // [40:40] is the sub-list for extension extendee - 0, // [0:40] is the sub-list for field type_name + 8, // 4: feast.core.FeatureSetSpec.max_age:type_name -> google.protobuf.Duration + 9, // 5: feast.core.FeatureSetSpec.source:type_name -> feast.core.Source + 6, // 6: feast.core.FeatureSetSpec.labels:type_name -> feast.core.FeatureSetSpec.LabelsEntry + 10, // 7: feast.core.EntitySpec.value_type:type_name -> feast.types.ValueType.Enum + 10, // 8: feast.core.FeatureSpec.value_type:type_name -> feast.types.ValueType.Enum + 11, // 9: feast.core.FeatureSpec.presence:type_name -> tensorflow.metadata.v0.FeaturePresence + 12, // 10: feast.core.FeatureSpec.group_presence:type_name -> tensorflow.metadata.v0.FeaturePresenceWithinGroup + 13, // 11: feast.core.FeatureSpec.shape:type_name -> tensorflow.metadata.v0.FixedShape + 14, // 12: feast.core.FeatureSpec.value_count:type_name -> tensorflow.metadata.v0.ValueCount + 15, // 13: feast.core.FeatureSpec.int_domain:type_name -> tensorflow.metadata.v0.IntDomain + 16, // 14: feast.core.FeatureSpec.float_domain:type_name -> tensorflow.metadata.v0.FloatDomain + 17, // 15: feast.core.FeatureSpec.string_domain:type_name -> tensorflow.metadata.v0.StringDomain + 18, // 16: feast.core.FeatureSpec.bool_domain:type_name -> tensorflow.metadata.v0.BoolDomain + 19, // 17: feast.core.FeatureSpec.struct_domain:type_name -> tensorflow.metadata.v0.StructDomain + 20, // 18: feast.core.FeatureSpec.natural_language_domain:type_name -> tensorflow.metadata.v0.NaturalLanguageDomain + 21, // 19: feast.core.FeatureSpec.image_domain:type_name -> tensorflow.metadata.v0.ImageDomain + 22, // 20: feast.core.FeatureSpec.mid_domain:type_name -> tensorflow.metadata.v0.MIDDomain + 23, // 21: feast.core.FeatureSpec.url_domain:type_name -> tensorflow.metadata.v0.URLDomain + 24, // 22: feast.core.FeatureSpec.time_domain:type_name -> tensorflow.metadata.v0.TimeDomain + 25, // 23: feast.core.FeatureSpec.time_of_day_domain:type_name -> tensorflow.metadata.v0.TimeOfDayDomain + 7, // 24: feast.core.FeatureSpec.labels:type_name -> feast.core.FeatureSpec.LabelsEntry + 26, // 25: feast.core.FeatureSetMeta.created_timestamp:type_name -> google.protobuf.Timestamp + 0, // 26: feast.core.FeatureSetMeta.status:type_name -> feast.core.FeatureSetStatus + 27, // [27:27] is the sub-list for method output_type + 27, // [27:27] is the sub-list for method input_type + 27, // [27:27] is the sub-list for extension type_name + 27, // [27:27] is the sub-list for extension extendee + 0, // [0:27] is the sub-list for field type_name } func init() { file_feast_core_FeatureSet_proto_init() } @@ -1372,24 +1031,6 @@ func file_feast_core_FeatureSet_proto_init() { } } } - file_feast_core_FeatureSet_proto_msgTypes[2].OneofWrappers = []interface{}{ - (*EntitySpec_Presence)(nil), - (*EntitySpec_GroupPresence)(nil), - (*EntitySpec_Shape)(nil), - (*EntitySpec_ValueCount)(nil), - (*EntitySpec_Domain)(nil), - (*EntitySpec_IntDomain)(nil), - (*EntitySpec_FloatDomain)(nil), - (*EntitySpec_StringDomain)(nil), - (*EntitySpec_BoolDomain)(nil), - (*EntitySpec_StructDomain)(nil), - (*EntitySpec_NaturalLanguageDomain)(nil), - (*EntitySpec_ImageDomain)(nil), - (*EntitySpec_MidDomain)(nil), - (*EntitySpec_UrlDomain)(nil), - (*EntitySpec_TimeDomain)(nil), - (*EntitySpec_TimeOfDayDomain)(nil), - } file_feast_core_FeatureSet_proto_msgTypes[3].OneofWrappers = []interface{}{ (*FeatureSpec_Presence)(nil), (*FeatureSpec_GroupPresence)(nil), @@ -1414,7 +1055,7 @@ func file_feast_core_FeatureSet_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_FeatureSet_proto_rawDesc, NumEnums: 1, - NumMessages: 5, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/feast/core/FeatureSetReference.pb.go b/sdk/go/protos/feast/core/FeatureSetReference.pb.go index 52a63b6a8c1..1667565bd23 100644 --- a/sdk/go/protos/feast/core/FeatureSetReference.pb.go +++ b/sdk/go/protos/feast/core/FeatureSetReference.pb.go @@ -51,8 +51,6 @@ type FeatureSetReference struct { Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` // Name of the FeatureSet Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - // Version no. of the FeatureSet - Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` } func (x *FeatureSetReference) Reset() { @@ -101,32 +99,23 @@ func (x *FeatureSetReference) GetName() string { return "" } -func (x *FeatureSetReference) GetVersion() int32 { - if x != nil { - return x.Version - } - return 0 -} - var File_feast_core_FeatureSetReference_proto protoreflect.FileDescriptor var file_feast_core_FeatureSetReference_proto_rawDesc = []byte{ 0x0a, 0x24, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x22, 0x5d, 0x0a, 0x13, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, + 0x72, 0x65, 0x22, 0x43, 0x0a, 0x13, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x42, 0x57, 0x0a, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, - 0x18, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x57, 0x0a, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x18, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, + 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, + 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, + 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/go/protos/feast/core/Runner.pb.go b/sdk/go/protos/feast/core/Runner.pb.go new file mode 100644 index 00000000000..ae9f7c4e7d3 --- /dev/null +++ b/sdk/go/protos/feast/core/Runner.pb.go @@ -0,0 +1,375 @@ +// +// * Copyright 2020 The Feast Authors +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * https://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.21.0 +// protoc v3.10.0 +// source: feast/core/Runner.proto + +package core + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type DirectRunnerConfigOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + //* + // Controls the amount of target parallelism the DirectRunner will use. + // Defaults to the greater of the number of available processors and 3. Must be a value + // greater than zero. + TargetParallelism int32 `protobuf:"varint,1,opt,name=targetParallelism,proto3" json:"targetParallelism,omitempty"` + // BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID + DeadLetterTableSpec string `protobuf:"bytes,2,opt,name=deadLetterTableSpec,proto3" json:"deadLetterTableSpec,omitempty"` +} + +func (x *DirectRunnerConfigOptions) Reset() { + *x = DirectRunnerConfigOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_Runner_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DirectRunnerConfigOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DirectRunnerConfigOptions) ProtoMessage() {} + +func (x *DirectRunnerConfigOptions) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_Runner_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DirectRunnerConfigOptions.ProtoReflect.Descriptor instead. +func (*DirectRunnerConfigOptions) Descriptor() ([]byte, []int) { + return file_feast_core_Runner_proto_rawDescGZIP(), []int{0} +} + +func (x *DirectRunnerConfigOptions) GetTargetParallelism() int32 { + if x != nil { + return x.TargetParallelism + } + return 0 +} + +func (x *DirectRunnerConfigOptions) GetDeadLetterTableSpec() string { + if x != nil { + return x.DeadLetterTableSpec + } + return "" +} + +type DataflowRunnerConfigOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Project id to use when launching jobs. + Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` + // The Google Compute Engine region for creating Dataflow jobs. + Region string `protobuf:"bytes,2,opt,name=region,proto3" json:"region,omitempty"` + // GCP availability zone for operations. + Zone string `protobuf:"bytes,3,opt,name=zone,proto3" json:"zone,omitempty"` + // Run the job as a specific service account, instead of the default GCE robot. + ServiceAccount string `protobuf:"bytes,4,opt,name=serviceAccount,proto3" json:"serviceAccount,omitempty"` + // GCE network for launching workers. + Network string `protobuf:"bytes,5,opt,name=network,proto3" json:"network,omitempty"` + // GCE subnetwork for launching workers. e.g. regions/asia-east1/subnetworks/mysubnetwork + Subnetwork string `protobuf:"bytes,6,opt,name=subnetwork,proto3" json:"subnetwork,omitempty"` + // Machine type to create Dataflow worker VMs as. + WorkerMachineType string `protobuf:"bytes,7,opt,name=workerMachineType,proto3" json:"workerMachineType,omitempty"` + // The autoscaling algorithm to use for the workerpool. + AutoscalingAlgorithm string `protobuf:"bytes,8,opt,name=autoscalingAlgorithm,proto3" json:"autoscalingAlgorithm,omitempty"` + // Specifies whether worker pools should be started with public IP addresses. + UsePublicIps bool `protobuf:"varint,9,opt,name=usePublicIps,proto3" json:"usePublicIps,omitempty"` + // A pipeline level default location for storing temporary files. Support Google Cloud Storage locations, + // e.g. gs://bucket/object + TempLocation string `protobuf:"bytes,10,opt,name=tempLocation,proto3" json:"tempLocation,omitempty"` + // The maximum number of workers to use for the workerpool. + MaxNumWorkers int32 `protobuf:"varint,11,opt,name=maxNumWorkers,proto3" json:"maxNumWorkers,omitempty"` + // BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID + DeadLetterTableSpec string `protobuf:"bytes,12,opt,name=deadLetterTableSpec,proto3" json:"deadLetterTableSpec,omitempty"` +} + +func (x *DataflowRunnerConfigOptions) Reset() { + *x = DataflowRunnerConfigOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_Runner_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DataflowRunnerConfigOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DataflowRunnerConfigOptions) ProtoMessage() {} + +func (x *DataflowRunnerConfigOptions) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_Runner_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DataflowRunnerConfigOptions.ProtoReflect.Descriptor instead. +func (*DataflowRunnerConfigOptions) Descriptor() ([]byte, []int) { + return file_feast_core_Runner_proto_rawDescGZIP(), []int{1} +} + +func (x *DataflowRunnerConfigOptions) GetProject() string { + if x != nil { + return x.Project + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetZone() string { + if x != nil { + return x.Zone + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetServiceAccount() string { + if x != nil { + return x.ServiceAccount + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetNetwork() string { + if x != nil { + return x.Network + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetSubnetwork() string { + if x != nil { + return x.Subnetwork + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetWorkerMachineType() string { + if x != nil { + return x.WorkerMachineType + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetAutoscalingAlgorithm() string { + if x != nil { + return x.AutoscalingAlgorithm + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetUsePublicIps() bool { + if x != nil { + return x.UsePublicIps + } + return false +} + +func (x *DataflowRunnerConfigOptions) GetTempLocation() string { + if x != nil { + return x.TempLocation + } + return "" +} + +func (x *DataflowRunnerConfigOptions) GetMaxNumWorkers() int32 { + if x != nil { + return x.MaxNumWorkers + } + return 0 +} + +func (x *DataflowRunnerConfigOptions) GetDeadLetterTableSpec() string { + if x != nil { + return x.DeadLetterTableSpec + } + return "" +} + +var File_feast_core_Runner_proto protoreflect.FileDescriptor + +var file_feast_core_Runner_proto_rawDesc = []byte{ + 0x0a, 0x17, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x52, 0x75, 0x6e, + 0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x7b, 0x0a, 0x19, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x52, + 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, 0x61, + 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, + 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, + 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, + 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, + 0x65, 0x63, 0x22, 0xc7, 0x03, 0x0a, 0x1b, 0x44, 0x61, 0x74, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x52, + 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, + 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, + 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x75, + 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x2c, 0x0a, 0x11, 0x77, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x61, 0x63, + 0x68, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x32, 0x0a, 0x14, 0x61, 0x75, 0x74, 0x6f, + 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, + 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x22, 0x0a, 0x0c, + 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, 0x73, 0x18, 0x09, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0c, 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, 0x73, + 0x12, 0x22, 0x0a, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, 0x57, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6d, 0x61, 0x78, + 0x4e, 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, + 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, + 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, + 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x42, 0x4a, 0x0a, 0x0a, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0b, 0x52, 0x75, 0x6e, 0x6e, + 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, + 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_feast_core_Runner_proto_rawDescOnce sync.Once + file_feast_core_Runner_proto_rawDescData = file_feast_core_Runner_proto_rawDesc +) + +func file_feast_core_Runner_proto_rawDescGZIP() []byte { + file_feast_core_Runner_proto_rawDescOnce.Do(func() { + file_feast_core_Runner_proto_rawDescData = protoimpl.X.CompressGZIP(file_feast_core_Runner_proto_rawDescData) + }) + return file_feast_core_Runner_proto_rawDescData +} + +var file_feast_core_Runner_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_feast_core_Runner_proto_goTypes = []interface{}{ + (*DirectRunnerConfigOptions)(nil), // 0: feast.core.DirectRunnerConfigOptions + (*DataflowRunnerConfigOptions)(nil), // 1: feast.core.DataflowRunnerConfigOptions +} +var file_feast_core_Runner_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_feast_core_Runner_proto_init() } +func file_feast_core_Runner_proto_init() { + if File_feast_core_Runner_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_feast_core_Runner_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DirectRunnerConfigOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_Runner_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataflowRunnerConfigOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_feast_core_Runner_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_feast_core_Runner_proto_goTypes, + DependencyIndexes: file_feast_core_Runner_proto_depIdxs, + MessageInfos: file_feast_core_Runner_proto_msgTypes, + }.Build() + File_feast_core_Runner_proto = out.File + file_feast_core_Runner_proto_rawDesc = nil + file_feast_core_Runner_proto_goTypes = nil + file_feast_core_Runner_proto_depIdxs = nil +} diff --git a/sdk/go/protos/feast/core/Source.pb.go b/sdk/go/protos/feast/core/Source.pb.go index 30bd2362723..368f50c5acb 100644 --- a/sdk/go/protos/feast/core/Source.pb.go +++ b/sdk/go/protos/feast/core/Source.pb.go @@ -169,10 +169,14 @@ type KafkaSourceConfig struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // - bootstrapServers: [comma delimited value of host[:port]] + // Comma separated list of Kafka bootstrap servers. Used for feature sets without a defined source host[:port]] BootstrapServers string `protobuf:"bytes,1,opt,name=bootstrap_servers,json=bootstrapServers,proto3" json:"bootstrap_servers,omitempty"` - // - topics: [Kafka topic name. This value is provisioned by core and should not be set by the user.] + // Kafka topic to use for feature sets without user defined topics Topic string `protobuf:"bytes,2,opt,name=topic,proto3" json:"topic,omitempty"` + // Number of Kafka partitions to to use for managed feature stream. + Partitions int32 `protobuf:"varint,3,opt,name=partitions,proto3" json:"partitions,omitempty"` + // Defines the number of copies of managed feature stream Kafka. + ReplicationFactor int32 `protobuf:"varint,4,opt,name=replicationFactor,proto3" json:"replicationFactor,omitempty"` } func (x *KafkaSourceConfig) Reset() { @@ -221,6 +225,20 @@ func (x *KafkaSourceConfig) GetTopic() string { return "" } +func (x *KafkaSourceConfig) GetPartitions() int32 { + if x != nil { + return x.Partitions + } + return 0 +} + +func (x *KafkaSourceConfig) GetReplicationFactor() int32 { + if x != nil { + return x.ReplicationFactor + } + return 0 +} + var File_feast_core_Source_proto protoreflect.FileDescriptor var file_feast_core_Source_proto_rawDesc = []byte{ @@ -235,20 +253,25 @@ var file_feast_core_Source_proto_rawDesc = []byte{ 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x11, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x0f, 0x0a, - 0x0d, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x56, - 0x0a, 0x11, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, - 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, - 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x2a, 0x24, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, - 0x00, 0x12, 0x09, 0x0a, 0x05, 0x4b, 0x41, 0x46, 0x4b, 0x41, 0x10, 0x01, 0x42, 0x4a, 0x0a, 0x0a, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0b, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, - 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x0d, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xa4, + 0x01, 0x0a, 0x11, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, + 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x10, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x11, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, + 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2a, 0x24, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, + 0x12, 0x09, 0x0a, 0x05, 0x4b, 0x41, 0x46, 0x4b, 0x41, 0x10, 0x01, 0x42, 0x4a, 0x0a, 0x0a, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0b, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, + 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/go/protos/feast/core/Store.pb.go b/sdk/go/protos/feast/core/Store.pb.go index 55c699d788b..f3b7728ab97 100644 --- a/sdk/go/protos/feast/core/Store.pb.go +++ b/sdk/go/protos/feast/core/Store.pb.go @@ -58,12 +58,7 @@ const ( Store_REDIS Store_StoreType = 1 // BigQuery stores a FeatureRow element as a row in a BigQuery table. // - // Table name is derived from the feature set name and version as: - // [feature_set_name]_v[feature_set_version] - // - // For example: - // A feature row for feature set "driver" and version "1" will be written - // to table "driver_v1". + // Table name is derived is the same as the feature set name. // // The entities and features in a FeatureSetSpec corresponds to the // fields in the BigQuery table (these make up the BigQuery schema). @@ -84,11 +79,6 @@ const ( // BigQuery table created will be partitioned by the field "event_timestamp" // of the FeatureRow (https://cloud.google.com/bigquery/docs/partitioned-tables). // - // Since newer version of feature set can introduce breaking, non backward- - // compatible BigQuery schema updates, incrementing the version of a - // feature set will result in the creation of a new empty BigQuery table - // with the new schema. - // // The following table shows how ValueType in Feast is mapped to // BigQuery Standard SQL data types // (https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types): @@ -113,7 +103,8 @@ const ( // Store_BIGQUERY Store_StoreType = 2 // Unsupported in Feast 0.3 - Store_CASSANDRA Store_StoreType = 3 + Store_CASSANDRA Store_StoreType = 3 + Store_REDIS_CLUSTER Store_StoreType = 4 ) // Enum value maps for Store_StoreType. @@ -123,12 +114,14 @@ var ( 1: "REDIS", 2: "BIGQUERY", 3: "CASSANDRA", + 4: "REDIS_CLUSTER", } Store_StoreType_value = map[string]int32{ - "INVALID": 0, - "REDIS": 1, - "BIGQUERY": 2, - "CASSANDRA": 3, + "INVALID": 0, + "REDIS": 1, + "BIGQUERY": 2, + "CASSANDRA": 3, + "REDIS_CLUSTER": 4, } ) @@ -184,6 +177,7 @@ type Store struct { // *Store_RedisConfig_ // *Store_BigqueryConfig // *Store_CassandraConfig_ + // *Store_RedisClusterConfig_ Config isStore_Config `protobuf_oneof:"config"` } @@ -268,6 +262,13 @@ func (x *Store) GetCassandraConfig() *Store_CassandraConfig { return nil } +func (x *Store) GetRedisClusterConfig() *Store_RedisClusterConfig { + if x, ok := x.GetConfig().(*Store_RedisClusterConfig_); ok { + return x.RedisClusterConfig + } + return nil +} + type isStore_Config interface { isStore_Config() } @@ -284,12 +285,18 @@ type Store_CassandraConfig_ struct { CassandraConfig *Store_CassandraConfig `protobuf:"bytes,13,opt,name=cassandra_config,json=cassandraConfig,proto3,oneof"` } +type Store_RedisClusterConfig_ struct { + RedisClusterConfig *Store_RedisClusterConfig `protobuf:"bytes,14,opt,name=redis_cluster_config,json=redisClusterConfig,proto3,oneof"` +} + func (*Store_RedisConfig_) isStore_Config() {} func (*Store_BigqueryConfig) isStore_Config() {} func (*Store_CassandraConfig_) isStore_Config() {} +func (*Store_RedisClusterConfig_) isStore_Config() {} + type Store_RedisConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -369,8 +376,11 @@ type Store_BigQueryConfig struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` - DatasetId string `protobuf:"bytes,2,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"` + ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` + DatasetId string `protobuf:"bytes,2,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"` + StagingLocation string `protobuf:"bytes,3,opt,name=staging_location,json=stagingLocation,proto3" json:"staging_location,omitempty"` + InitialRetryDelaySeconds int32 `protobuf:"varint,4,opt,name=initial_retry_delay_seconds,json=initialRetryDelaySeconds,proto3" json:"initial_retry_delay_seconds,omitempty"` + TotalTimeoutSeconds int32 `protobuf:"varint,5,opt,name=total_timeout_seconds,json=totalTimeoutSeconds,proto3" json:"total_timeout_seconds,omitempty"` } func (x *Store_BigQueryConfig) Reset() { @@ -419,6 +429,27 @@ func (x *Store_BigQueryConfig) GetDatasetId() string { return "" } +func (x *Store_BigQueryConfig) GetStagingLocation() string { + if x != nil { + return x.StagingLocation + } + return "" +} + +func (x *Store_BigQueryConfig) GetInitialRetryDelaySeconds() int32 { + if x != nil { + return x.InitialRetryDelaySeconds + } + return 0 +} + +func (x *Store_BigQueryConfig) GetTotalTimeoutSeconds() int32 { + if x != nil { + return x.TotalTimeoutSeconds + } + return 0 +} + type Store_CassandraConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -474,6 +505,70 @@ func (x *Store_CassandraConfig) GetPort() int32 { return 0 } +type Store_RedisClusterConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // List of Redis Uri for all the nodes in Redis Cluster, comma separated. Eg. host1:6379, host2:6379 + ConnectionString string `protobuf:"bytes,1,opt,name=connection_string,json=connectionString,proto3" json:"connection_string,omitempty"` + InitialBackoffMs int32 `protobuf:"varint,2,opt,name=initial_backoff_ms,json=initialBackoffMs,proto3" json:"initial_backoff_ms,omitempty"` + MaxRetries int32 `protobuf:"varint,3,opt,name=max_retries,json=maxRetries,proto3" json:"max_retries,omitempty"` +} + +func (x *Store_RedisClusterConfig) Reset() { + *x = Store_RedisClusterConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_Store_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Store_RedisClusterConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Store_RedisClusterConfig) ProtoMessage() {} + +func (x *Store_RedisClusterConfig) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_Store_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Store_RedisClusterConfig.ProtoReflect.Descriptor instead. +func (*Store_RedisClusterConfig) Descriptor() ([]byte, []int) { + return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 3} +} + +func (x *Store_RedisClusterConfig) GetConnectionString() string { + if x != nil { + return x.ConnectionString + } + return "" +} + +func (x *Store_RedisClusterConfig) GetInitialBackoffMs() int32 { + if x != nil { + return x.InitialBackoffMs + } + return 0 +} + +func (x *Store_RedisClusterConfig) GetMaxRetries() int32 { + if x != nil { + return x.MaxRetries + } + return 0 +} + type Store_Subscription struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -494,19 +589,12 @@ type Store_Subscription struct { // - my-feature-set* can be used to match all features prefixed by "my-feature-set" // - my-feature-set-6 can be used to select a single feature set Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // Versions of the given feature sets that will be returned. - // Valid options for version: - // "latest": only the latest version is returned. - // "*": Subscribe to all versions - // [version number]: pin to a specific version. Project and feature set name must be - // explicitly defined if a specific version is pinned. - Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` } func (x *Store_Subscription) Reset() { *x = Store_Subscription{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_Store_proto_msgTypes[4] + mi := &file_feast_core_Store_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -519,7 +607,7 @@ func (x *Store_Subscription) String() string { func (*Store_Subscription) ProtoMessage() {} func (x *Store_Subscription) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_Store_proto_msgTypes[4] + mi := &file_feast_core_Store_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -532,7 +620,7 @@ func (x *Store_Subscription) ProtoReflect() protoreflect.Message { // Deprecated: Use Store_Subscription.ProtoReflect.Descriptor instead. func (*Store_Subscription) Descriptor() ([]byte, []int) { - return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 3} + return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 4} } func (x *Store_Subscription) GetProject() string { @@ -549,19 +637,12 @@ func (x *Store_Subscription) GetName() string { return "" } -func (x *Store_Subscription) GetVersion() string { - if x != nil { - return x.Version - } - return "" -} - var File_feast_core_Store_proto protoreflect.FileDescriptor var file_feast_core_Store_proto_rawDesc = []byte{ 0x0a, 0x16, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x22, 0xa9, 0x06, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, + 0x63, 0x6f, 0x72, 0x65, 0x22, 0xae, 0x09, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, @@ -584,40 +665,64 @@ var file_feast_core_Store_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x61, 0x73, 0x73, 0x61, 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x0f, 0x63, 0x61, 0x73, 0x73, 0x61, - 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x84, 0x01, 0x0a, 0x0b, 0x52, - 0x65, 0x64, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, - 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, - 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, - 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, - 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, - 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, - 0x73, 0x1a, 0x4e, 0x0a, 0x0e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x49, - 0x64, 0x1a, 0x39, 0x0a, 0x0f, 0x43, 0x61, 0x73, 0x73, 0x61, 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, + 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x58, 0x0a, 0x14, 0x72, 0x65, + 0x64, 0x69, 0x73, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x64, 0x69, + 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, + 0x52, 0x12, 0x72, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x84, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x56, 0x0a, 0x0c, - 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, - 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x40, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x09, - 0x0a, 0x05, 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x49, 0x47, - 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x53, 0x53, 0x41, - 0x4e, 0x44, 0x52, 0x41, 0x10, 0x03, 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x42, 0x49, 0x0a, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0a, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, - 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x12, + 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, + 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, + 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, + 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0xec, 0x01, 0x0a, 0x0e, + 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, + 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, + 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, + 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3d, 0x0a, 0x1b, 0x69, 0x6e, 0x69, 0x74, 0x69, + 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x5f, 0x73, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x18, 0x69, 0x6e, + 0x69, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x74, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x53, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0f, 0x43, 0x61, + 0x73, 0x73, 0x61, 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, + 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, + 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x04, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x90, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x69, + 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, + 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, + 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, + 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0x3c, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, + 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x53, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, + 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x42, + 0x49, 0x47, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x53, + 0x53, 0x41, 0x4e, 0x44, 0x52, 0x41, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x44, 0x49, + 0x53, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x42, 0x08, 0x0a, 0x06, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x49, 0x0a, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x42, 0x0a, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, + 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, + 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -633,26 +738,28 @@ func file_feast_core_Store_proto_rawDescGZIP() []byte { } var file_feast_core_Store_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_feast_core_Store_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_feast_core_Store_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_feast_core_Store_proto_goTypes = []interface{}{ - (Store_StoreType)(0), // 0: feast.core.Store.StoreType - (*Store)(nil), // 1: feast.core.Store - (*Store_RedisConfig)(nil), // 2: feast.core.Store.RedisConfig - (*Store_BigQueryConfig)(nil), // 3: feast.core.Store.BigQueryConfig - (*Store_CassandraConfig)(nil), // 4: feast.core.Store.CassandraConfig - (*Store_Subscription)(nil), // 5: feast.core.Store.Subscription + (Store_StoreType)(0), // 0: feast.core.Store.StoreType + (*Store)(nil), // 1: feast.core.Store + (*Store_RedisConfig)(nil), // 2: feast.core.Store.RedisConfig + (*Store_BigQueryConfig)(nil), // 3: feast.core.Store.BigQueryConfig + (*Store_CassandraConfig)(nil), // 4: feast.core.Store.CassandraConfig + (*Store_RedisClusterConfig)(nil), // 5: feast.core.Store.RedisClusterConfig + (*Store_Subscription)(nil), // 6: feast.core.Store.Subscription } var file_feast_core_Store_proto_depIdxs = []int32{ 0, // 0: feast.core.Store.type:type_name -> feast.core.Store.StoreType - 5, // 1: feast.core.Store.subscriptions:type_name -> feast.core.Store.Subscription + 6, // 1: feast.core.Store.subscriptions:type_name -> feast.core.Store.Subscription 2, // 2: feast.core.Store.redis_config:type_name -> feast.core.Store.RedisConfig 3, // 3: feast.core.Store.bigquery_config:type_name -> feast.core.Store.BigQueryConfig 4, // 4: feast.core.Store.cassandra_config:type_name -> feast.core.Store.CassandraConfig - 5, // [5:5] is the sub-list for method output_type - 5, // [5:5] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 5, // 5: feast.core.Store.redis_cluster_config:type_name -> feast.core.Store.RedisClusterConfig + 6, // [6:6] is the sub-list for method output_type + 6, // [6:6] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name } func init() { file_feast_core_Store_proto_init() } @@ -710,6 +817,18 @@ func file_feast_core_Store_proto_init() { } } file_feast_core_Store_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Store_RedisClusterConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_Store_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Store_Subscription); i { case 0: return &v.state @@ -726,6 +845,7 @@ func file_feast_core_Store_proto_init() { (*Store_RedisConfig_)(nil), (*Store_BigqueryConfig)(nil), (*Store_CassandraConfig_)(nil), + (*Store_RedisClusterConfig_)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -733,7 +853,7 @@ func file_feast_core_Store_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_Store_proto_rawDesc, NumEnums: 1, - NumMessages: 5, + NumMessages: 6, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index 2485687d81e..a8659c55814 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -359,8 +359,6 @@ type FeatureReference struct { Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` // Feature name Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - // Feature version - Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` // The features will be retrieved if: // entity_timestamp - max_age <= event_timestamp <= entity_timestamp // @@ -415,13 +413,6 @@ func (x *FeatureReference) GetName() string { return "" } -func (x *FeatureReference) GetVersion() int32 { - if x != nil { - return x.Version - } - return 0 -} - func (x *FeatureReference) GetMaxAge() *duration.Duration { if x != nil { return x.MaxAge @@ -1095,166 +1086,165 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x6a, 0x6f, 0x62, 0x5f, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6a, 0x6f, 0x62, 0x53, 0x74, 0x61, - 0x67, 0x69, 0x6e, 0x67, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x8e, 0x01, 0x0a, - 0x10, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61, 0x78, - 0x5f, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, 0x41, 0x67, 0x65, 0x22, 0xe1, 0x03, - 0x0a, 0x18, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x66, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x52, 0x0a, 0x0b, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, - 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x52, - 0x0a, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x39, 0x0a, 0x19, 0x6f, - 0x6d, 0x69, 0x74, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x5f, - 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, - 0x6f, 0x6d, 0x69, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x49, 0x6e, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0xf8, 0x01, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x52, 0x6f, 0x77, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x65, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x55, 0x0a, 0x06, 0x66, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, - 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, - 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, + 0x67, 0x69, 0x6e, 0x67, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x74, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, 0x61, - 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, - 0xad, 0x02, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, - 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0xb6, 0x01, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x32, + 0x0a, 0x07, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, 0x41, + 0x67, 0x65, 0x22, 0xe1, 0x03, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x3b, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, + 0x63, 0x65, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x52, 0x0a, 0x0b, + 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x31, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x52, 0x6f, 0x77, 0x52, 0x0a, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x73, + 0x12, 0x39, 0x0a, 0x19, 0x6f, 0x6d, 0x69, 0x74, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, + 0x73, 0x5f, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x16, 0x6f, 0x6d, 0x69, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, + 0x73, 0x49, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0xf8, 0x01, 0x0a, 0x09, + 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x0f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x12, 0x55, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x3d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x52, 0x6f, 0x77, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, + 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, + 0x43, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x22, 0xad, 0x02, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, + 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, + 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0xb6, 0x01, 0x0a, 0x0b, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, + 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x73, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, + 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x40, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x40, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, - 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, - 0x62, 0x22, 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, - 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4a, - 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, - 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, - 0x22, 0xe2, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1b, 0x0a, 0x09, - 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, 0x64, 0x61, 0x74, - 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, - 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x46, - 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, - 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x01, + 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, + 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, + 0x0e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, + 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0xe2, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, - 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, + 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, - 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x64, 0x61, - 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x6f, 0x0a, 0x10, - 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, - 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, - 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, - 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, - 0x1c, 0x0a, 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x36, 0x0a, - 0x07, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x15, - 0x0a, 0x11, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, - 0x4f, 0x41, 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, - 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, - 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, - 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, - 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, - 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x2a, - 0x3b, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x17, 0x0a, - 0x13, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x49, 0x4e, 0x56, - 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, - 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x32, 0x92, 0x03, 0x0a, - 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x6c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, + 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, + 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, + 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, + 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, + 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, + 0x72, 0x69, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, + 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, + 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, + 0x10, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2a, 0x6f, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, + 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, + 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, + 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, + 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, + 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, + 0x10, 0x02, 0x2a, 0x36, 0x0a, 0x07, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, + 0x10, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, + 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, + 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, + 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, + 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, + 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, + 0x4e, 0x45, 0x10, 0x03, 0x2a, 0x3b, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x12, 0x17, 0x0a, 0x13, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, + 0x54, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, + 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, + 0x01, 0x32, 0x92, 0x03, 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, + 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, - 0x11, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, - 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, - 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x06, 0x47, 0x65, - 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x42, 0x54, 0x0a, 0x0d, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, 0x50, 0x49, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, - 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x66, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, + 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, + 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, + 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x45, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x54, 0x0a, 0x0d, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/go/protos/feast/storage/Redis.pb.go b/sdk/go/protos/feast/storage/Redis.pb.go index 1dca28e26af..354a0f045b6 100644 --- a/sdk/go/protos/feast/storage/Redis.pb.go +++ b/sdk/go/protos/feast/storage/Redis.pb.go @@ -46,7 +46,7 @@ type RedisKey struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // FeatureSet this row belongs to, this is defined as featureSetName:version. + // FeatureSet this row belongs to, this is defined as featureSetName. FeatureSet string `protobuf:"bytes,2,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` // List of fields containing entity names and their respective values // contained within this feature row. The entities should be sorted diff --git a/sdk/go/protos/feast/types/FeatureRow.pb.go b/sdk/go/protos/feast/types/FeatureRow.pb.go index 769f219d32e..696f138459f 100644 --- a/sdk/go/protos/feast/types/FeatureRow.pb.go +++ b/sdk/go/protos/feast/types/FeatureRow.pb.go @@ -53,7 +53,7 @@ type FeatureRow struct { // will use to perform joins, determine latest values, and coalesce rows. EventTimestamp *timestamp.Timestamp `protobuf:"bytes,3,opt,name=event_timestamp,json=eventTimestamp,proto3" json:"event_timestamp,omitempty"` // Complete reference to the featureSet this featureRow belongs to, in the form of - // /:. This value will be used by the feast ingestion job to filter + // /. This value will be used by the feast ingestion job to filter // rows, and write the values to the correct tables. FeatureSet string `protobuf:"bytes,6,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` } diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go index 5b55e5bca5a..d609c5e89c0 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go @@ -112,14 +112,14 @@ var file_tensorflow_metadata_proto_v0_path_proto_rawDesc = []byte{ 0x61, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x22, 0x1a, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x74, 0x65, - 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x42, 0x70, 0x0a, + 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x42, 0x64, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, 0x4d, 0x67, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x68, 0x69, 0x72, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x79, 0x2f, - 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0xf8, 0x01, 0x01, + 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, + 0xf8, 0x01, 0x01, } var ( diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go index 5eec2f259d0..ab7ffc7201d 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go @@ -3476,14 +3476,13 @@ var file_tensorflow_metadata_proto_v0_schema_proto_rawDesc = []byte{ 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x55, 0x43, 0x54, 0x10, 0x04, 0x42, - 0x70, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, + 0x64, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, - 0x4d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, + 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x68, 0x69, 0x72, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x74, - 0x79, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0xf8, 0x01, - 0x01, + 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, + 0x76, 0x30, 0xf8, 0x01, 0x01, } var ( diff --git a/sdk/go/request.go b/sdk/go/request.go index a3a3fe6d71a..09a00f8a239 100644 --- a/sdk/go/request.go +++ b/sdk/go/request.go @@ -3,22 +3,19 @@ package feast import ( "fmt" "github.com/gojek/feast/sdk/go/protos/feast/serving" - "strconv" "strings" ) var ( // ErrInvalidFeatureName indicates that the user has provided a feature reference with the wrong structure or contents - ErrInvalidFeatureName = "invalid feature references %s provided, feature names must be in the format /:" + ErrInvalidFeatureName = "invalid feature references %s provided, feature names must be in the format /" ) // OnlineFeaturesRequest wrapper on feast.serving.GetOnlineFeaturesRequest. type OnlineFeaturesRequest struct { // Features is the list of features to obtain from Feast. Each feature can be given as // - // : // / - // /: // The only required components are the feature name and project. Features []string @@ -50,7 +47,7 @@ func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequest }, nil } -// buildFeatures create a slice of FeatureReferences from a slice of "/:" +// buildFeatures create a slice of FeatureReferences from a slice of "/" // It returns an error when the format is invalid func buildFeatures(featureReferences []string, defaultProject string) ([]*serving.FeatureReference, error) { var features []*serving.FeatureReference @@ -58,41 +55,25 @@ func buildFeatures(featureReferences []string, defaultProject string) ([]*servin for _, featureRef := range featureReferences { var project string var name string - var version int - var featureSplit []string projectSplit := strings.Split(featureRef, "/") if len(projectSplit) == 2 { project = projectSplit[0] - featureSplit = strings.Split(projectSplit[1], ":") + name = projectSplit[1] } else if len(projectSplit) == 1 { project = defaultProject - featureSplit = strings.Split(projectSplit[0], ":") + name = projectSplit[0] } else { return nil, fmt.Errorf(ErrInvalidFeatureName, featureRef) } - if len(featureSplit) == 2 { - name = featureSplit[0] - v, err := strconv.Atoi(featureSplit[1]) - if err != nil { - return nil, fmt.Errorf(ErrInvalidFeatureName, featureRef) - } - version = v - } else if len(featureSplit) == 1 { - name = featureSplit[0] - } else { - return nil, fmt.Errorf(ErrInvalidFeatureName, featureRef) - } - - if project == "" || name == "" || version < 0 { + if project == "" || name == "" { return nil, fmt.Errorf(ErrInvalidFeatureName, featureRef) } features = append(features, &serving.FeatureReference{ Name: name, - Version: int32(version), Project: project, }) } diff --git a/sdk/go/request_test.go b/sdk/go/request_test.go index b6866638670..5c149fe1583 100644 --- a/sdk/go/request_test.go +++ b/sdk/go/request_test.go @@ -20,7 +20,7 @@ func TestGetOnlineFeaturesRequest(t *testing.T) { { name: "valid", req: OnlineFeaturesRequest{ - Features: []string{"my_project_1/feature1:1", "my_project_2/feature1:1", "my_project_4/feature3", "feature2:2", "feature2"}, + Features: []string{"my_project_1/feature1", "my_project_2/feature1", "my_project_4/feature3", "feature2", "feature2"}, Entities: []Row{ {"entity1": Int64Val(1), "entity2": StrVal("bob")}, {"entity1": Int64Val(1), "entity2": StrVal("annie")}, @@ -33,27 +33,22 @@ func TestGetOnlineFeaturesRequest(t *testing.T) { { Project: "my_project_1", Name: "feature1", - Version: 1, }, { Project: "my_project_2", Name: "feature1", - Version: 1, }, { Project: "my_project_4", Name: "feature3", - Version: 0, }, { Project: "my_project_3", Name: "feature2", - Version: 2, }, { Project: "my_project_3", Name: "feature2", - Version: 0, }, }, EntityRows: []*serving.GetOnlineFeaturesRequest_EntityRow{ @@ -92,7 +87,6 @@ func TestGetOnlineFeaturesRequest(t *testing.T) { { Project: "project", Name: "feature1", - Version: 0, }, }, EntityRows: []*serving.GetOnlineFeaturesRequest_EntityRow{}, @@ -113,21 +107,12 @@ func TestGetOnlineFeaturesRequest(t *testing.T) { { name: "invalid_feature_name/wrong_format", req: OnlineFeaturesRequest{ - Features: []string{"fs1:3:feature1"}, + Features: []string{"/fs1:feature1"}, Entities: []Row{}, Project: "my_project", }, wantErr: true, - err: fmt.Errorf(ErrInvalidFeatureName, "fs1:3:feature1"), - }, - { - name: "invalid_feature_name/invalid_version", - req: OnlineFeaturesRequest{ - Features: []string{"project/a:feature1"}, - Entities: []Row{}, - }, - wantErr: true, - err: fmt.Errorf(ErrInvalidFeatureName, "project/a:feature1"), + err: fmt.Errorf(ErrInvalidFeatureName, "/fs1:feature1"), }, } for _, tc := range tt { diff --git a/sdk/go/response_test.go b/sdk/go/response_test.go index 5aa2c276d61..f660130475f 100644 --- a/sdk/go/response_test.go +++ b/sdk/go/response_test.go @@ -67,7 +67,7 @@ func TestOnlineFeaturesResponseToInt64Array(t *testing.T) { { name: "length mismatch", args: args{ - order: []string{"fs:1:feature2", "fs:1:feature1"}, + order: []string{"fs:feature2", "fs:feature1"}, fillNa: []int64{-1}, }, want: nil, diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index 8014231836e..4e18b651657 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -62,7 +62,7 @@ public GetFeastServingInfoResponse getFeastServingInfo() { *

See {@link #getOnlineFeatures(List, List, String)} * * @param features list of string feature references to retrieve, feature reference follows this - * format [project]/[name]:[version] + * format [project]/[name] * @param rows list of {@link Row} to select the entities to retrieve the features for * @param defaultProject {@link String} Default project to find features in if not provided in * feature reference. @@ -76,11 +76,11 @@ public List getOnlineFeatures(List features, List rows, String * Get online features from Feast. * *

Example of retrieving online features for the driver project, with features driver_id and - * driver_name, both version 1 + * driver_name * * 

{@code
    * FeastClient client = FeastClient.create("localhost", 6566);
-   * List requestedFeatureIds = Arrays.asList("driver/driver_id:1", "driver/driver_name:1");
+   * List requestedFeatureIds = Arrays.asList("driver/driver_id", "driver/driver_name");
    * List requestedRows =
    *         Arrays.asList(Row.create().set("driver_id", 123), Row.create().set("driver_id", 456));
    * List retrievedFeatures = client.getOnlineFeatures(requestedFeatureIds, requestedRows);
@@ -88,7 +88,7 @@ public List getOnlineFeatures(List features, List rows, String
    * }
* * @param featureRefStrings list of feature refs to retrieve, feature refs follow this format - * [project]/[name]:[version] + * [project]/[name] * @param rows list of {@link Row} to select the entities to retrieve the features for * @param defaultProject {@link String} Default project to find features in if not provided in * feature reference. diff --git a/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java index 874196e92bd..ebea625ff1e 100644 --- a/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java +++ b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java @@ -34,58 +34,30 @@ public static List createFeatureRefs( for (String featureRefString : featureRefStrings) { String project; String name; - int version = 0; - String[] featureSplit; String[] projectSplit = featureRefString.split("/"); - if (projectSplit.length == 2) { - project = projectSplit[0]; - featureSplit = projectSplit[1].split(":"); - } else if (projectSplit.length == 1) { + if (projectSplit.length == 1) { project = defaultProject; - featureSplit = projectSplit[0].split(":"); - } else { - throw new IllegalArgumentException( - String.format( - "Feature id '%s' has invalid format. Expected format: ::.", - featureRefString)); - } - - if (featureSplit.length == 2) { - name = featureSplit[0]; - try { - version = Integer.parseInt(featureSplit[1]); - } catch (NumberFormatException e) { - throw new IllegalArgumentException( - String.format( - "Feature id '%s' contains invalid version. Expected format: /:.", - featureRefString)); - } - } else if (featureSplit.length == 1) { - name = featureSplit[0]; + name = projectSplit[0]; + } else if (projectSplit.length == 2) { + project = projectSplit[0]; + name = projectSplit[1]; } else { throw new IllegalArgumentException( String.format( - "Feature id '%s' has invalid format. Expected format: /:.", + "Feature id '%s' has invalid format. Expected format: /.", featureRefString)); } - if (project.isEmpty() || name.isEmpty() || version < 0) { + if (project.isEmpty() || name.isEmpty() || name.contains(":")) { throw new IllegalArgumentException( String.format( - "Feature id '%s' has invalid format. Expected format: /:.", + "Feature id '%s' has invalid format. Expected format: /.", featureRefString)); } - featureRefs.add( - FeatureReference.newBuilder() - .setName(name) - .setProject(project) - .setVersion(version) - .build()); + featureRefs.add(FeatureReference.newBuilder().setName(name).setProject(project).build()); } - - ; return featureRefs; } } diff --git a/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java index 3b9429ad8f6..a471609149b 100644 --- a/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java +++ b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java @@ -36,40 +36,35 @@ class RequestUtilTest { private static Stream provideValidFeatureIds() { return Stream.of( Arguments.of( - Collections.singletonList("driver_project/driver_id:1"), + Collections.singletonList("driver_project/driver_id"), Collections.singletonList( FeatureReference.newBuilder() .setProject("driver_project") .setName("driver_id") - .setVersion(1) .build())), Arguments.of( - Arrays.asList("driver_project/driver_id:1", "driver_project/driver_name:1"), + Arrays.asList("driver_project/driver_id", "driver_project/driver_name"), Arrays.asList( FeatureReference.newBuilder() .setProject("driver_project") .setName("driver_id") - .setVersion(1) .build(), FeatureReference.newBuilder() .setProject("driver_project") .setName("driver_name") - .setVersion(1) .build())), Arguments.of( Arrays.asList( - "driver_project/driver_id:1", - "driver_project/driver_name:1", + "driver_project/driver_id", + "driver_project/driver_name", "booking_project/driver_name"), Arrays.asList( FeatureReference.newBuilder() .setProject("driver_project") - .setVersion(1) .setName("driver_id") .build(), FeatureReference.newBuilder() .setProject("driver_project") - .setVersion(1) .setName("driver_name") .build(), FeatureReference.newBuilder() @@ -96,7 +91,7 @@ void createFeatureSets_ShouldReturnFeatureSetsForValidFeatureIds( private static Stream provideInvalidFeatureRefs() { return Stream.of( - Arguments.of(Collections.singletonList("missing:bad_version")), + Arguments.of(Collections.singletonList("/noproject")), Arguments.of(Collections.singletonList(""))); } diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 489d28fe833..2fd5a4cdf56 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -129,11 +129,11 @@ def feature_set_list(): table = [] for fs in feast_client.list_feature_sets(): - table.append([fs.name, fs.version, repr(fs)]) + table.append([fs.name, repr(fs)]) from tabulate import tabulate - print(tabulate(table, headers=["NAME", "VERSION", "REFERENCE"], tablefmt="plain")) + print(tabulate(table, headers=["NAME", "REFERENCE"], tablefmt="plain")) @feature_set.command("apply") @@ -155,17 +155,14 @@ def feature_set_create(filename): @feature_set.command("describe") @click.argument("name", type=click.STRING) -@click.argument("version", type=click.INT) -def feature_set_describe(name: str, version: int): +def feature_set_describe(name: str): """ Describe a feature set """ feast_client = Client() # type: Client - fs = feast_client.get_feature_set(name=name, version=version) + fs = feast_client.get_feature_set(name=name) if not fs: - print( - f'Feature set with name "{name}" and version "{version}" could not be found' - ) + print(f'Feature set with name "{name}" could not be found') return print(yaml.dump(yaml.safe_load(str(fs)), default_flow_style=False, sort_keys=False)) @@ -329,9 +326,6 @@ def ingest_job_restart(job_id: str): @click.option( "--name", "-n", help="Feature set name to ingest data into", required=True ) -@click.option( - "--version", "-v", help="Feature set version to ingest data into", type=int -) @click.option( "--filename", "-f", @@ -345,13 +339,13 @@ def ingest_job_restart(job_id: str): type=click.Choice(["CSV"], case_sensitive=False), help="Type of file to ingest. Defaults to CSV.", ) -def ingest(name, version, filename, file_type): +def ingest(name, filename, file_type): """ Ingest feature data into a feature set """ feast_client = Client() # type: Client - feature_set = feast_client.get_feature_set(name=name, version=version) + feature_set = feast_client.get_feature_set(name=name) feature_set.ingest_file(file_path=filename) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 8d66f58c06c..89a89fd2a8d 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -400,9 +400,10 @@ def _apply_feature_set(self, feature_set: FeatureSet): # If the feature set has changed, update the local copy if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED: - print( - f'Feature set updated/created: "{applied_fs.name}:{applied_fs.version}"' - ) + print(f'Feature set created: "{applied_fs.name}"') + + if apply_fs_response.status == ApplyFeatureSetResponse.Status.UPDATED: + print(f'Feature set updated: "{applied_fs.name}"') # If no change has been applied, do nothing if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE: @@ -412,7 +413,7 @@ def _apply_feature_set(self, feature_set: FeatureSet): feature_set._update_from_feature_set(applied_fs) def list_feature_sets( - self, project: str = None, name: str = None, version: str = None + self, project: str = None, name: str = None, ) -> List[FeatureSet]: """ Retrieve a list of feature sets from Feast Core @@ -420,7 +421,6 @@ def list_feature_sets( Args: project: Filter feature sets based on project name name: Filter feature sets based on feature set name - version: Filter feature sets based on version numbf, Returns: List of feature sets @@ -436,12 +436,7 @@ def list_feature_sets( if name is None: name = "*" - if version is None: - version = "*" - - filter = ListFeatureSetsRequest.Filter( - project=project, feature_set_name=name, feature_set_version=version - ) + filter = ListFeatureSetsRequest.Filter(project=project, feature_set_name=name) # Get latest feature sets from Feast Core feature_set_protos = self._core_service_stub.ListFeatureSets( @@ -457,16 +452,14 @@ def list_feature_sets( return feature_sets def get_feature_set( - self, name: str, version: int = None, project: str = None + self, name: str, project: str = None ) -> Union[FeatureSet, None]: """ - Retrieves a feature set. If no version is specified then the latest - version will be returned. + Retrieves a feature set. Args: project: Feast project that this feature set belongs to name: Name of feature set - version: Version of feature set Returns: Returns either the specified feature set, or raises an exception if @@ -480,14 +473,9 @@ def get_feature_set( else: raise ValueError("No project has been configured.") - if version is None: - version = 0 - try: get_feature_set_response = self._core_service_stub.GetFeatureSet( - GetFeatureSetRequest( - project=project, name=name.strip(), version=int(version) - ) + GetFeatureSetRequest(project=project, name=name.strip()) ) # type: GetFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) @@ -519,7 +507,7 @@ def get_batch_features( feature_refs (List[str]): List of feature references that will be returned for each entity. Each feature reference should have the following format - "project/feature:version". + "project/feature". entity_rows (Union[pd.DataFrame, str]): Pandas dataframe containing entities and a 'datetime' column. @@ -539,7 +527,7 @@ def get_batch_features( >>> from datetime import datetime >>> >>> feast_client = Client(core_url="localhost:6565", serving_url="localhost:6566") - >>> feature_refs = ["my_project/bookings_7d:1", "booking_14d"] + >>> feature_refs = ["my_project/bookings_7d", "booking_14d"] >>> entity_rows = pd.DataFrame( >>> { >>> "datetime": [pd.datetime.now() for _ in range(3)], @@ -626,11 +614,11 @@ def get_online_features( Args: feature_refs: List of feature references in the following format - [project]/[feature_name]:[version]. Only the feature name + [project]/[feature_name]. Only the feature name is a required component in the reference. example: - ["my_project/my_feature_1:3", - "my_project3/my_feature_4:1",] + ["my_project/my_feature_1", + "my_feature_4",] entity_rows: List of GetFeaturesRequest.EntityRow where each row contains entities. Timestamp should not be set for online retrieval. All entity types within a feature @@ -731,18 +719,16 @@ def ingest( feature_set: Union[str, FeatureSet], source: Union[pd.DataFrame, str], chunk_size: int = 10000, - version: int = None, max_workers: int = max(CPU_COUNT - 1, 1), disable_progress_bar: bool = False, timeout: int = KAFKA_CHUNK_PRODUCTION_TIMEOUT, - ) -> None: + ) -> str: """ Loads feature data into Feast for a specific feature set. Args: feature_set (typing.Union[str, feast.feature_set.FeatureSet]): Feature set object or the string name of the feature set - (without a version). source (typing.Union[pd.DataFrame, str]): Either a file path or Pandas Dataframe to ingest into Feast @@ -754,9 +740,6 @@ def ingest( chunk_size (int): Amount of rows to load and ingest at a time. - version (int): - Feature set version. - max_workers (int): Number of worker processes to use to encode values. @@ -767,14 +750,12 @@ def ingest( Timeout in seconds to wait for completion. Returns: - None: - None + str: + ingestion id for this dataset """ if isinstance(feature_set, FeatureSet): name = feature_set.name - if version is None: - version = feature_set.version elif isinstance(feature_set, str): name = feature_set else: @@ -793,7 +774,7 @@ def ingest( while True: if timeout is not None and time.time() - current_time >= timeout: raise TimeoutError("Timed out waiting for feature set to be ready") - feature_set = self.get_feature_set(name, version) + feature_set = self.get_feature_set(name) if ( feature_set is not None and feature_set.status == FeatureSetStatus.STATUS_READY @@ -849,7 +830,7 @@ def ingest( print("Removing temporary file(s)...") shutil.rmtree(dir_path) - return None + return ingestion_id def _build_feature_references( @@ -861,7 +842,7 @@ def _build_feature_references( Args: feature_refs: List of feature reference strings - ("project/feature:version") + ("project/feature") default_project: This project will be used if the project name is not provided in the feature reference """ @@ -870,12 +851,11 @@ def _build_feature_references( for feature_ref in feature_refs: project_split = feature_ref.split("/") - version = 0 if len(project_split) == 2: - project, feature_version = project_split + project, name = project_split elif len(project_split) == 1: - feature_version = project_split[0] + name = project_split[0] if default_project is None: raise ValueError( f"No project specified in {feature_ref} and no default project provided" @@ -883,26 +863,20 @@ def _build_feature_references( project = default_project else: raise ValueError( - f'Could not parse feature ref {feature_ref}, expecting "project/feature:version"' + f'Could not parse feature ref {feature_ref}, expecting "project/feature"' ) - feature_split = feature_version.split(":") - if len(feature_split) == 2: - name, version = feature_split - version = int(version) - elif len(feature_split) == 1: - name = feature_split[0] - else: + if len(project) == 0 or len(name) == 0: raise ValueError( - f'Could not parse feature ref {feature_ref}, expecting "project/feature:version"' + f'Could not parse feature ref {feature_ref}, expecting "project/feature"' ) - if len(project) == 0 or len(name) == 0 or version < 0: + if ":" in name: raise ValueError( - f'Could not parse feature ref {feature_ref}, expecting "project/feature:version"' + f'Could not parse feature ref {feature_ref}, expecting "project/feature". Versions were deprecated in v0.5.0.' ) - features.append(FeatureReference(project=project, name=name, version=version)) + features.append(FeatureReference(project=project, name=name)) return features @@ -916,7 +890,7 @@ def _generate_ingestion_id(feature_set: FeatureSet) -> str: Returns: UUID unique to current time and the feature set provided. """ - uuid_str = f"{feature_set.name}_{feature_set.version}_{int(time.time())}" + uuid_str = f"{feature_set.name}_{int(time.time())}" return str(uuid.uuid3(uuid.NAMESPACE_DNS, uuid_str)) diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index 973c2a52a57..3c77aa1db5e 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -68,7 +68,6 @@ def __init__( else: self._source = source self._max_age = max_age - self._version = None self._status = None self._created_timestamp = None @@ -195,20 +194,6 @@ def source(self, source: Source): """ self._source = source - @property - def version(self): - """ - Returns the version of this feature set - """ - return self._version - - @version.setter - def version(self, version): - """ - Sets the version of this feature set - """ - self._version = version - @property def max_age(self): """ @@ -621,7 +606,6 @@ def _update_from_feature_set(self, feature_set): self.name = feature_set.name self.project = feature_set.project - self.version = feature_set.version self.source = feature_set.source self.max_age = feature_set.max_age self.features = feature_set.features @@ -809,7 +793,6 @@ def from_proto(cls, feature_set_proto: FeatureSetProto): if len(feature_set_proto.spec.project) == 0 else feature_set_proto.spec.project, ) - feature_set._version = feature_set_proto.spec.version feature_set._status = feature_set_proto.meta.status feature_set._created_timestamp = feature_set_proto.meta.created_timestamp return feature_set @@ -828,7 +811,6 @@ def to_proto(self) -> FeatureSetProto: spec = FeatureSetSpecProto( name=self.name, - version=self.version, project=self.project, max_age=self.max_age, source=self.source.to_proto() if self.source is not None else None, @@ -852,10 +834,8 @@ class FeatureSetRef: Represents a reference to a featureset """ - def __init__(self, project: str = None, name: str = None, version: int = None): - self.proto = FeatureSetReferenceProto( - project=project, name=name, version=version - ) + def __init__(self, project: str = None, name: str = None): + self.proto = FeatureSetReferenceProto(project=project, name=name) @property def project(self) -> str: @@ -871,13 +851,6 @@ def name(self) -> str: """ return self.proto.name - @property - def version(self) -> int: - """ - Get the version of feature set referenced by this reference - """ - return self.proto.version - @classmethod def from_feature_set(cls, feature_set: FeatureSet): """ @@ -889,7 +862,7 @@ def from_feature_set(cls, feature_set: FeatureSet): Returns: FeatureSetRef that refers to the given feature set """ - return cls(feature_set.project, feature_set.name, feature_set.version) + return cls(feature_set.project, feature_set.name) @classmethod def from_str(cls, ref_str: str): @@ -903,15 +876,13 @@ def from_str(cls, ref_str: str): Returns: FeatureSetRef constructed from the string """ + project = "" if "/" in ref_str: project, ref_str = ref_str.split("/") - if ":" in ref_str: - ref_str, version_str = ref_str.split(":") - name = ref_str - return cls(project, name, int(version_str)) + return cls(project, ref_str) - def to_proto(self, arg1) -> FeatureSetReferenceProto: + def to_proto(self) -> FeatureSetReferenceProto: """ Convert and return this feature set reference to protobuf. @@ -926,14 +897,12 @@ def __str__(self): def __repr__(self): # return string representation of the reference - # [project/]name[:version] + # [project/]name ref_str = "" if self.proto.project: ref_str += self.proto.project + "/" if self.proto.name: ref_str += self.proto.name - if self.proto.version: - ref_str += ":" + str(self.proto.version).strip() return ref_str def __eq__(self, other): diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index 34d0356ea78..b439dbd3027 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -44,7 +44,7 @@ def _encode_pa_tables( Parquet file must have more than one row group. feature_set (str): - Feature set reference in the format f"{project}/{name}:{version}". + Feature set reference in the format f"{project}/{name}". fields (dict[str, enum.Enum.ValueType]): A mapping of field names to their value types. @@ -135,7 +135,7 @@ def get_feature_row_chunks( Iterable list of byte encoded FeatureRow(s). """ - feature_set = f"{fs.project}/{fs.name}:{fs.version}" + feature_set = f"{fs.project}/{fs.name}" field_map = {field.name: field.dtype for field in fs.fields.values()} diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 8df0499239a..85def25fcb9 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -147,7 +147,7 @@ def convert_series_to_proto_values(row: pd.Series): event_timestamp=_pd_datetime_to_timestamp_proto( dataframe[DATETIME_COLUMN].dtype, row[DATETIME_COLUMN] ), - feature_set=feature_set.name + ":" + str(feature_set.version), + feature_set=feature_set.project + "/" + feature_set.name, ) for field_name, field in feature_set.fields.items(): @@ -185,11 +185,7 @@ def convert_dict_to_proto_values( event_timestamp=_pd_datetime_to_timestamp_proto( df_datetime_dtype, row[DATETIME_COLUMN] ), - feature_set=feature_set.project - + "/" - + feature_set.name - + ":" - + str(feature_set.version), + feature_set=f"{feature_set.project}/{feature_set.name}", ) for field_name, field in feature_set.fields.items(): diff --git a/sdk/python/tests/feast_core_server.py b/sdk/python/tests/feast_core_server.py index b6efe2cb6d1..3ac1b17d003 100644 --- a/sdk/python/tests/feast_core_server.py +++ b/sdk/python/tests/feast_core_server.py @@ -40,21 +40,12 @@ def ListFeatureSets(self, request: ListFeatureSetsRequest, context): or request.filter.feature_set_name == "*" or fs.spec.name == request.filter.feature_set_name ) - and ( - not request.filter.feature_set_version - or str(fs.spec.version) == request.filter.feature_set_version - or request.filter.feature_set_version == "*" - ) ] return ListFeatureSetsResponse(feature_sets=filtered_feature_set_response) def ApplyFeatureSet(self, request: ApplyFeatureSetRequest, context): feature_set = request.feature_set - if feature_set.spec.version is None: - feature_set.spec.version = 1 - else: - feature_set.spec.version = feature_set.spec.version + 1 if feature_set.spec.source.type == SourceTypeProto.INVALID: feature_set.spec.source.kafka_source_config.CopyFrom( diff --git a/sdk/python/tests/feast_serving_server.py b/sdk/python/tests/feast_serving_server.py index 364c1907141..983e74e8850 100644 --- a/sdk/python/tests/feast_serving_server.py +++ b/sdk/python/tests/feast_serving_server.py @@ -67,7 +67,6 @@ def GetOnlineFeatures(self, request: GetOnlineFeaturesRequest, context): feature_data_sets=[ GetOnlineFeaturesResponse.FeatureDataSet( name="feature_set_1", - version="1", feature_rows=[ FeatureRowProto.FeatureRow( feature_set="feature_set_1", diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 3082265eccf..e87c8573353 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -198,7 +198,7 @@ def test_get_online_features(self, mocked_client, mocker): fields = dict() for feature_num in range(1, 10): - fields[f"my_project/feature_{str(feature_num)}:1"] = ValueProto.Value( + fields[f"my_project/feature_{str(feature_num)}"] = ValueProto.Value( int64_val=feature_num ) field_values = GetOnlineFeaturesResponse.FieldValues(fields=fields) @@ -222,21 +222,21 @@ def test_get_online_features(self, mocked_client, mocker): response = mocked_client.get_online_features( entity_rows=entity_rows, feature_refs=[ - "my_project/feature_1:1", - "my_project/feature_2:1", - "my_project/feature_3:1", - "my_project/feature_4:1", - "my_project/feature_5:1", - "my_project/feature_6:1", - "my_project/feature_7:1", - "my_project/feature_8:1", - "my_project/feature_9:1", + "my_project/feature_1", + "my_project/feature_2", + "my_project/feature_3", + "my_project/feature_4", + "my_project/feature_5", + "my_project/feature_6", + "my_project/feature_7", + "my_project/feature_8", + "my_project/feature_9", ], ) # type: GetOnlineFeaturesResponse assert ( - response.field_values[0].fields["my_project/feature_1:1"].int64_val == 1 - and response.field_values[0].fields["my_project/feature_9:1"].int64_val == 9 + response.field_values[0].fields["my_project/feature_1"].int64_val == 1 + and response.field_values[0].fields["my_project/feature_9"].int64_val == 9 ) @pytest.mark.parametrize( @@ -257,7 +257,6 @@ def test_get_feature_set(self, mocked_client, mocker): feature_set=FeatureSetProto( spec=FeatureSetSpecProto( name="my_feature_set", - version=2, max_age=Duration(seconds=3600), features=[ FeatureSpecProto( @@ -287,11 +286,10 @@ def test_get_feature_set(self, mocked_client, mocker): ), ) mocked_client.set_project("my_project") - feature_set = mocked_client.get_feature_set("my_feature_set", version=2) + feature_set = mocked_client.get_feature_set("my_feature_set") assert ( feature_set.name == "my_feature_set" - and feature_set.version == 2 and feature_set.fields["my_feature_1"].name == "my_feature_1" and feature_set.fields["my_feature_1"].dtype == ValueType.FLOAT and feature_set.fields["my_entity_1"].name == "my_entity_1" @@ -422,7 +420,6 @@ def test_stop_ingest_job(self, mocked_client, mocker): # feature_set=FeatureSetProto( # spec=FeatureSetSpecProto( # name="customer_fs", - # version=1, # project="my_project", # entities=[ # EntitySpecProto( @@ -454,8 +451,8 @@ def test_stop_ingest_job(self, mocked_client, mocker): # "datetime": [datetime.utcnow() for _ in range(3)], # "customer": [1001, 1002, 1003], # "transaction": [1001, 1002, 1003], - # "my_project/customer_feature_1:1": [1001, 1002, 1003], - # "my_project/customer_feature_2:1": [1001, 1002, 1003], + # "my_project/customer_feature_1": [1001, 1002, 1003], + # "my_project/customer_feature_2": [1001, 1002, 1003], # } # ) # @@ -511,8 +508,8 @@ def test_stop_ingest_job(self, mocked_client, mocker): # } # ), # feature_refs=[ - # "my_project/customer_feature_1:1", - # "my_project/customer_feature_2:1", + # "my_project/customer_feature_1", + # "my_project/customer_feature_2", # ], # ) # type: Job # @@ -521,10 +518,10 @@ def test_stop_ingest_job(self, mocked_client, mocker): # actual_dataframe = response.to_dataframe() # # assert actual_dataframe[ - # ["my_project/customer_feature_1:1", "my_project/customer_feature_2:1"] + # ["my_project/customer_feature_1", "my_project/customer_feature_2"] # ].equals( # expected_dataframe[ - # ["my_project/customer_feature_1:1", "my_project/customer_feature_2:1"] + # ["my_project/customer_feature_1", "my_project/customer_feature_2"] # ] # ) diff --git a/sdk/python/tests/test_feature_set.py b/sdk/python/tests/test_feature_set.py index a2cc12fe113..04e75c9e76d 100644 --- a/sdk/python/tests/test_feature_set.py +++ b/sdk/python/tests/test_feature_set.py @@ -268,15 +268,13 @@ def make_tfx_schema_domain_info_inline(schema): class TestFeatureSetRef: def test_from_feature_set(self): feature_set = FeatureSet("test", "test") - feature_set.version = 2 ref = FeatureSetRef.from_feature_set(feature_set) assert ref.name == "test" assert ref.project == "test" - assert ref.version == 2 def test_str_ref(self): - original_ref = FeatureSetRef(project="test", name="test", version=2) + original_ref = FeatureSetRef(project="test", name="test") ref_str = repr(original_ref) parsed_ref = FeatureSetRef.from_str(ref_str) assert original_ref == parsed_ref diff --git a/serving/README.md b/serving/README.md index f88f30923b2..39eef311033 100644 --- a/serving/README.md +++ b/serving/README.md @@ -28,7 +28,6 @@ grpc_cli call localhost:6566 GetFeastServingType '' grpc_cli call localhost:6565 ApplyFeatureSet ' feature_set { name: "driver" - version: 1 entities { name: "driver_id" value_type: STRING @@ -53,14 +52,12 @@ feature_set { grpc_cli call localhost:6565 GetFeatureSets ' filter { feature_set_name: "driver" - feature_set_version: "1" } ' grpc_cli call localhost:6566 GetBatchFeatures ' feature_sets { name: "driver" - version: 1 feature_names: "booking_completed_count" max_age { seconds: 86400 diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java index b7fd0a9fed7..9eec333a148 100644 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -386,7 +386,6 @@ public StoreProto.Store.Subscription toProto() { return StoreProto.Store.Subscription.newBuilder() .setName(getName()) .setProject(getProject()) - .setVersion(getVersion()) .build(); } } diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index 30addd2b9f2..28885ca4452 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -133,9 +133,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ } private void populateStaleKeyCountMetrics(String project, FeatureReference ref) { - Metrics.staleKeyCount - .labels(project, RefUtil.generateFeatureStringRefWithoutProject(ref)) - .inc(); + Metrics.staleKeyCount.labels(project, ref.getName()).inc(); } private void populateRequestCountMetrics(FeatureSetRequest featureSetRequest) { @@ -143,11 +141,7 @@ private void populateRequestCountMetrics(FeatureSetRequest featureSetRequest) { featureSetRequest .getFeatureReferences() .parallelStream() - .forEach( - ref -> - Metrics.requestCount - .labels(project, RefUtil.generateFeatureStringRefWithoutProject(ref)) - .inc()); + .forEach(ref -> Metrics.requestCount.labels(project, ref.getName()).inc()); } @Override diff --git a/serving/src/main/java/feast/serving/specs/CachedSpecService.java b/serving/src/main/java/feast/serving/specs/CachedSpecService.java index a117754e844..5dc57de7c2a 100644 --- a/serving/src/main/java/feast/serving/specs/CachedSpecService.java +++ b/serving/src/main/java/feast/serving/specs/CachedSpecService.java @@ -18,7 +18,6 @@ import static feast.serving.util.RefUtil.generateFeatureSetStringRef; import static feast.serving.util.RefUtil.generateFeatureStringRef; -import static java.util.Comparator.comparingInt; import static java.util.stream.Collectors.groupingBy; import com.google.common.cache.CacheBuilder; @@ -115,11 +114,8 @@ public List getFeatureSets(List featureRefe if (featureSet == null) { throw new SpecRetrievalException( String.format( - "Unable to find feature set for feature ref: " - + "(project: %s, name: %s, version: %d)", - featureReference.getProject(), - featureReference.getName(), - featureReference.getVersion())); + "Unable to find feature set for feature ref: " + "(project: %s, name: %s)", + featureReference.getProject(), featureReference.getName())); } return Pair.of(featureSet, featureReference); }) @@ -150,7 +146,11 @@ public List getFeatureSets(List featureRefe */ public void populateCache() { Map featureSetMap = getFeatureSetMap(); + + featureSetCache.invalidateAll(); featureSetCache.putAll(featureSetMap); + + featureToFeatureSetMapping.clear(); featureToFeatureSetMapping.putAll(getFeatureToFeatureSetMapping(featureSetMap)); featureSetsCount.set(featureSetCache.size()); @@ -176,8 +176,7 @@ private Map getFeatureSetMap() { .setFilter( ListFeatureSetsRequest.Filter.newBuilder() .setProject(subscription.getProject()) - .setFeatureSetName(subscription.getName()) - .setFeatureSetVersion(subscription.getVersion())) + .setFeatureSetName(subscription.getName())) .build()); for (FeatureSet featureSet : featureSetsResponse.getFeatureSetsList()) { @@ -196,39 +195,17 @@ private Map getFeatureToFeatureSetMapping( Map featureSets) { HashMap mapping = new HashMap<>(); - featureSets.values().stream() - .collect(groupingBy(featureSet -> Pair.of(featureSet.getProject(), featureSet.getName()))) - .forEach( - (group, groupedFeatureSets) -> { - groupedFeatureSets = - groupedFeatureSets.stream() - .sorted(comparingInt(FeatureSetSpec::getVersion)) - .collect(Collectors.toList()); - for (int i = 0; i < groupedFeatureSets.size(); i++) { - FeatureSetSpec featureSetSpec = groupedFeatureSets.get(i); - for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { - FeatureReference featureRef = - FeatureReference.newBuilder() - .setProject(featureSetSpec.getProject()) - .setName(featureSpec.getName()) - .setVersion(featureSetSpec.getVersion()) - .build(); - mapping.put( - generateFeatureStringRef(featureRef), - generateFeatureSetStringRef(featureSetSpec)); - if (i == groupedFeatureSets.size() - 1) { - featureRef = - FeatureReference.newBuilder() - .setProject(featureSetSpec.getProject()) - .setName(featureSpec.getName()) - .build(); - mapping.put( - generateFeatureStringRef(featureRef), - generateFeatureSetStringRef(featureSetSpec)); - } - } - } - }); + for (FeatureSetSpec featureSetSpec : featureSets.values()) { + for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { + FeatureReference featureRef = + FeatureReference.newBuilder() + .setProject(featureSetSpec.getProject()) + .setName(featureSpec.getName()) + .build(); + mapping.put( + generateFeatureStringRef(featureRef), generateFeatureSetStringRef(featureSetSpec)); + } + } return mapping; } } diff --git a/serving/src/main/java/feast/serving/util/RefUtil.java b/serving/src/main/java/feast/serving/util/RefUtil.java index c3bcb0827a2..7557bd12bf2 100644 --- a/serving/src/main/java/feast/serving/util/RefUtil.java +++ b/serving/src/main/java/feast/serving/util/RefUtil.java @@ -22,25 +22,11 @@ public class RefUtil { public static String generateFeatureStringRef(FeatureReference featureReference) { String ref = String.format("%s/%s", featureReference.getProject(), featureReference.getName()); - if (featureReference.getVersion() > 0) { - return ref + String.format(":%d", featureReference.getVersion()); - } - return ref; - } - - public static String generateFeatureStringRefWithoutProject(FeatureReference featureReference) { - String ref = String.format("%s", featureReference.getName()); - if (featureReference.getVersion() > 0) { - return ref + String.format(":%d", featureReference.getVersion()); - } return ref; } public static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) { String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); - if (featureSetSpec.getVersion() > 0) { - return ref + String.format(":%d", featureSetSpec.getVersion()); - } return ref; } } diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index f6eaccf3cd4..9158ee7fa58 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -22,7 +22,6 @@ feast: # Wildcards match all options. No filtering is done. - name: "*" project: "*" - version: "*" - name: historical type: BIGQUERY @@ -44,7 +43,6 @@ feast: subscriptions: - name: "*" project: "*" - version: "*" tracing: # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls diff --git a/serving/src/main/resources/templates/join_featuresets.sql b/serving/src/main/resources/templates/join_featuresets.sql deleted file mode 100644 index 60b7c7d7a12..00000000000 --- a/serving/src/main/resources/templates/join_featuresets.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* - Joins the outputs of multiple point-in-time-correctness joins to a single table. - */ -WITH joined as ( -SELECT * FROM `{{ leftTableName }}` -{% for featureSet in featureSets %} -LEFT JOIN ( - SELECT - uuid, - {% for featureName in featureSet.features %} - {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} - {% endfor %} - FROM `{{ featureSet.table }}` -) USING (uuid) -{% endfor %} -) SELECT - event_timestamp, - {{ entities | join(', ') }} - {% for featureSet in featureSets %} - {% for featureName in featureSet.features %} - ,{{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }} as {{ featureName }} - {% endfor %} - {% endfor %} -FROM joined \ No newline at end of file diff --git a/serving/src/main/resources/templates/single_featureset_pit_join.sql b/serving/src/main/resources/templates/single_featureset_pit_join.sql deleted file mode 100644 index f3f20828ff1..00000000000 --- a/serving/src/main/resources/templates/single_featureset_pit_join.sql +++ /dev/null @@ -1,90 +0,0 @@ -/* - This query template performs the point-in-time correctness join for a single feature set table - to the provided entity table. - - 1. Concatenate the timestamp and entities from the feature set table with the entity dataset. - Feature values are joined to this table later for improved efficiency. - featureset_timestamp is equal to null in rows from the entity dataset. - */ -WITH union_features AS ( -SELECT - -- uuid is a unique identifier for each row in the entity dataset. Generated by `QueryTemplater.createEntityTableUUIDQuery` - uuid, - -- event_timestamp contains the timestamps to join onto - event_timestamp, - -- the feature_timestamp, i.e. the latest occurrence of the requested feature relative to the entity_dataset timestamp - NULL as {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, - -- created timestamp of the feature at the corresponding feature_timestamp - NULL as created_timestamp, - -- select only entities belonging to this feature set - {{ featureSet.entities | join(', ')}}, - -- boolean for filtering the dataset later - true AS is_entity_table -FROM `{{leftTableName}}` -UNION ALL -SELECT - NULL as uuid, - event_timestamp, - event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, - created_timestamp, - {{ featureSet.entities | join(', ')}}, - false AS is_entity_table -FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` WHERE event_timestamp <= '{{maxTimestamp}}' -{% if featureSet.maxAge == 0 %}{% else %}AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second){% endif %} -), -/* - 2. Window the data in the unioned dataset, partitioning by entity and ordering by event_timestamp, as - well as is_entity_table. - Within each window, back-fill the feature_timestamp - as a result of this, the null feature_timestamps - in the rows from the entity table should now contain the latest timestamps relative to the row's - event_timestamp. - - For rows where event_timestamp(provided datetime) - feature_timestamp > max age, set the - feature_timestamp to null. - */ -joined AS ( -SELECT - uuid, - event_timestamp, - {{ featureSet.entities | join(', ')}}, - {% for featureName in featureSet.features %} - IF(event_timestamp >= {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp {% if featureSet.maxAge == 0 %}{% else %}AND Timestamp_sub(event_timestamp, interval {{ featureSet.maxAge }} second) < {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp{% endif %}, {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}, NULL) as {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} - {% endfor %} -FROM ( -SELECT - uuid, - event_timestamp, - {{ featureSet.entities | join(', ')}}, - FIRST_VALUE(created_timestamp IGNORE NULLS) over w AS created_timestamp, - FIRST_VALUE({{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp IGNORE NULLS) over w AS {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, - is_entity_table -FROM union_features -WINDOW w AS (PARTITION BY {{ featureSet.entities | join(', ') }} ORDER BY event_timestamp DESC, is_entity_table DESC, created_timestamp DESC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) -) -/* - 3. Select only the rows from the entity table, and join the features from the original feature set table - to the dataset using the entity values, feature_timestamp, and created_timestamps. - */ -LEFT JOIN ( -SELECT - event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, - created_timestamp, - {{ featureSet.entities | join(', ')}}, - {% for featureName in featureSet.features %} - {{ featureName }} as {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} - {% endfor %} -FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` WHERE event_timestamp <= '{{maxTimestamp}}' -{% if featureSet.maxAge == 0 %}{% else %}AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second){% endif %} -) USING ({{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, created_timestamp, {{ featureSet.entities | join(', ')}}) -WHERE is_entity_table -) -/* - 4. Finally, deduplicate the rows by selecting the first occurrence of each entity table row UUID. - */ -SELECT - k.* -FROM ( - SELECT ARRAY_AGG(row LIMIT 1)[OFFSET(0)] k - FROM joined row - GROUP BY uuid -) \ No newline at end of file diff --git a/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java b/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java index d23f9da1d25..d292c33fba9 100644 --- a/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java +++ b/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java @@ -52,17 +52,9 @@ public void setUp() { validRequest = GetOnlineFeaturesRequest.newBuilder() .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatures( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) diff --git a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java index 144b967c9f5..0987b95e956 100644 --- a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java @@ -69,62 +69,50 @@ public void setUp() { .setType(StoreType.REDIS) .setRedisConfig(RedisConfig.newBuilder().setHost("localhost").setPort(6379)) .addSubscriptions( - Subscription.newBuilder() - .setProject("project") - .setName("fs1") - .setVersion("*") - .build()) + Subscription.newBuilder().setProject("project").setName("fs1").build()) .addSubscriptions( - Subscription.newBuilder() - .setProject("project") - .setName("fs2") - .setVersion("*") - .build()) + Subscription.newBuilder().setProject("project").setName("fs2").build()) .build(); when(coreService.registerStore(store)).thenReturn(store); featureSetSpecs = new LinkedHashMap<>(); featureSetSpecs.put( - "fs1:1", + "fs1", FeatureSetSpec.newBuilder() .setProject("project") .setName("fs1") - .setVersion(1) .addFeatures(FeatureSpec.newBuilder().setName("feature")) .build()); featureSetSpecs.put( - "fs1:2", + "fs1", FeatureSetSpec.newBuilder() .setProject("project") .setName("fs1") - .setVersion(2) .addFeatures(FeatureSpec.newBuilder().setName("feature")) .addFeatures(FeatureSpec.newBuilder().setName("feature2")) .build()); featureSetSpecs.put( - "fs2:1", + "fs2", FeatureSetSpec.newBuilder() .setProject("project") .setName("fs2") - .setVersion(1) .addFeatures(FeatureSpec.newBuilder().setName("feature3")) .build()); List fs1FeatureSets = Lists.newArrayList( - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpecs.get("fs1:1")).build(), - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpecs.get("fs1:2")).build()); + FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpecs.get("fs1")).build(), + FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpecs.get("fs1")).build()); List fs2FeatureSets = Lists.newArrayList( - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpecs.get("fs2:1")).build()); + FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpecs.get("fs2")).build()); when(coreService.listFeatureSets( ListFeatureSetsRequest.newBuilder() .setFilter( ListFeatureSetsRequest.Filter.newBuilder() .setProject("project") .setFeatureSetName("fs1") - .setFeatureSetVersion("*") .build()) .build())) .thenReturn(ListFeatureSetsResponse.newBuilder().addAllFeatureSets(fs1FeatureSets).build()); @@ -134,7 +122,6 @@ public void setUp() { ListFeatureSetsRequest.Filter.newBuilder() .setProject("project") .setFeatureSetName("fs2") - .setFeatureSetVersion("*") .build()) .build())) .thenReturn(ListFeatureSetsResponse.newBuilder().addAllFeatureSets(fs2FeatureSets).build()); @@ -158,17 +145,9 @@ public void shouldPopulateAndReturnStore() { public void shouldPopulateAndReturnFeatureSets() { cachedSpecService.populateCache(); FeatureReference frv1 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature") - .setVersion(1) - .build(); + FeatureReference.newBuilder().setProject("project").setName("feature").build(); FeatureReference frv2 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature") - .setVersion(2) - .build(); + FeatureReference.newBuilder().setProject("project").setName("feature").build(); assertThat( cachedSpecService.getFeatureSets(Collections.singletonList(frv1)), @@ -176,7 +155,7 @@ public void shouldPopulateAndReturnFeatureSets() { Lists.newArrayList( FeatureSetRequest.newBuilder() .addFeatureReference(frv1) - .setSpec(featureSetSpecs.get("fs1:1")) + .setSpec(featureSetSpecs.get("fs1")) .build()))); assertThat( cachedSpecService.getFeatureSets(Collections.singletonList(frv2)), @@ -184,7 +163,7 @@ public void shouldPopulateAndReturnFeatureSets() { Lists.newArrayList( FeatureSetRequest.newBuilder() .addFeatureReference(frv2) - .setSpec(featureSetSpecs.get("fs1:2")) + .setSpec(featureSetSpecs.get("fs1")) .build()))); } @@ -200,7 +179,7 @@ public void shouldPopulateAndReturnLatestFeatureSetIfVersionsNotSupplied() { Lists.newArrayList( FeatureSetRequest.newBuilder() .addFeatureReference(frv1) - .setSpec(featureSetSpecs.get("fs1:2")) + .setSpec(featureSetSpecs.get("fs1")) .build()))); } @@ -208,17 +187,9 @@ public void shouldPopulateAndReturnLatestFeatureSetIfVersionsNotSupplied() { public void shouldPopulateAndReturnFeatureSetsGivenFeaturesFromDifferentFeatureSets() { cachedSpecService.populateCache(); FeatureReference frv1 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature") - .setVersion(1) - .build(); + FeatureReference.newBuilder().setProject("project").setName("feature").build(); FeatureReference fr3 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature3") - .setVersion(1) - .build(); + FeatureReference.newBuilder().setProject("project").setName("feature3").build(); assertThat( cachedSpecService.getFeatureSets(Lists.newArrayList(frv1, fr3)), @@ -226,11 +197,11 @@ public void shouldPopulateAndReturnFeatureSetsGivenFeaturesFromDifferentFeatureS Lists.newArrayList( FeatureSetRequest.newBuilder() .addFeatureReference(frv1) - .setSpec(featureSetSpecs.get("fs1:1")) + .setSpec(featureSetSpecs.get("fs1")) .build(), FeatureSetRequest.newBuilder() .addFeatureReference(fr3) - .setSpec(featureSetSpecs.get("fs2:1")) + .setSpec(featureSetSpecs.get("fs2")) .build()) .toArray())); } @@ -239,17 +210,9 @@ public void shouldPopulateAndReturnFeatureSetsGivenFeaturesFromDifferentFeatureS public void shouldPopulateAndReturnFeatureSetGivenFeaturesFromSameFeatureSet() { cachedSpecService.populateCache(); FeatureReference fr1 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature") - .setVersion(2) - .build(); + FeatureReference.newBuilder().setProject("project").setName("feature").build(); FeatureReference fr2 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature2") - .setVersion(2) - .build(); + FeatureReference.newBuilder().setProject("project").setName("feature2").build(); assertThat( cachedSpecService.getFeatureSets(Lists.newArrayList(fr1, fr2)), @@ -258,7 +221,7 @@ public void shouldPopulateAndReturnFeatureSetGivenFeaturesFromSameFeatureSet() { FeatureSetRequest.newBuilder() .addFeatureReference(fr1) .addFeatureReference(fr2) - .setSpec(featureSetSpecs.get("fs1:2")) + .setSpec(featureSetSpecs.get("fs1")) .build()))); } } diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index b78fcb69170..5b9421a3c13 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -70,17 +70,9 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatures( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -103,7 +95,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -113,7 +105,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build()); FeatureSetRequest featureSetRequest = @@ -135,14 +127,14 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { FieldValues.newBuilder() .putFields("entity1", intValue(1)) .putFields("entity2", strValue("a")) - .putFields("project/feature1:1", intValue(1)) - .putFields("project/feature2:1", intValue(1))) + .putFields("project/feature1", intValue(1)) + .putFields("project/feature2", intValue(1))) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) .putFields("entity2", strValue("b")) - .putFields("project/feature1:1", intValue(2)) - .putFields("project/feature2:1", intValue(2))) + .putFields("project/feature1", intValue(2)) + .putFields("project/feature2", intValue(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -154,11 +146,7 @@ public void shouldReturnKeysWithoutVersionIfNotProvided() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatures( FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( @@ -183,7 +171,7 @@ public void shouldReturnKeysWithoutVersionIfNotProvided() { Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -193,7 +181,7 @@ public void shouldReturnKeysWithoutVersionIfNotProvided() { Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build()); FeatureSetRequest featureSetRequest = @@ -215,13 +203,13 @@ public void shouldReturnKeysWithoutVersionIfNotProvided() { FieldValues.newBuilder() .putFields("entity1", intValue(1)) .putFields("entity2", strValue("a")) - .putFields("project/feature1:1", intValue(1)) + .putFields("project/feature1", intValue(1)) .putFields("project/feature2", intValue(1))) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) .putFields("entity2", strValue("b")) - .putFields("project/feature1:1", intValue(2)) + .putFields("project/feature1", intValue(2)) .putFields("project/feature2", intValue(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); @@ -235,17 +223,9 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatures( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -268,14 +248,14 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { Lists.newArrayList( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) .build(), FeatureRow.newBuilder() - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").build(), @@ -295,14 +275,14 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { FieldValues.newBuilder() .putFields("entity1", intValue(1)) .putFields("entity2", strValue("a")) - .putFields("project/feature1:1", intValue(1)) - .putFields("project/feature2:1", intValue(1))) + .putFields("project/feature1", intValue(1)) + .putFields("project/feature2", intValue(1))) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) .putFields("entity2", strValue("b")) - .putFields("project/feature1:1", Value.newBuilder().build()) - .putFields("project/feature2:1", Value.newBuilder().build())) + .putFields("project/feature1", Value.newBuilder().build()) + .putFields("project/feature2", Value.newBuilder().build())) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -315,17 +295,9 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatures( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -348,7 +320,7 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build(), FeatureRow.newBuilder() .setEventTimestamp( @@ -359,7 +331,7 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build()); FeatureSetSpec spec = @@ -383,14 +355,14 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { FieldValues.newBuilder() .putFields("entity1", intValue(1)) .putFields("entity2", strValue("a")) - .putFields("project/feature1:1", intValue(1)) - .putFields("project/feature2:1", intValue(1))) + .putFields("project/feature1", intValue(1)) + .putFields("project/feature2", intValue(1))) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) .putFields("entity2", strValue("b")) - .putFields("project/feature1:1", Value.newBuilder().build()) - .putFields("project/feature2:1", Value.newBuilder().build())) + .putFields("project/feature1", Value.newBuilder().build()) + .putFields("project/feature2", Value.newBuilder().build())) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -403,11 +375,7 @@ public void shouldFilterOutUndesiredRows() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -430,7 +398,7 @@ public void shouldFilterOutUndesiredRows() { Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -440,7 +408,7 @@ public void shouldFilterOutUndesiredRows() { Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build()); FeatureSetRequest featureSetRequest = @@ -462,12 +430,12 @@ public void shouldFilterOutUndesiredRows() { FieldValues.newBuilder() .putFields("entity1", intValue(1)) .putFields("entity2", strValue("a")) - .putFields("project/feature1:1", intValue(1))) + .putFields("project/feature1", intValue(1))) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) .putFields("entity2", strValue("b")) - .putFields("project/feature1:1", intValue(2))) + .putFields("project/feature1", intValue(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -492,21 +460,9 @@ private FeatureSetSpec getFeatureSetSpec() { return FeatureSetSpec.newBuilder() .setProject("project") .setName("featureSet") - .setVersion(1) .addEntities(EntitySpec.newBuilder().setName("entity1")) .addEntities(EntitySpec.newBuilder().setName("entity2")) .setMaxAge(Duration.newBuilder().setSeconds(30)) // default .build(); } - - private FeatureSetSpec getFeatureSetSpecWithNoMaxAge() { - return FeatureSetSpec.newBuilder() - .setProject("project") - .setName("featureSet") - .setVersion(1) - .addEntities(EntitySpec.newBuilder().setName("entity1")) - .addEntities(EntitySpec.newBuilder().setName("entity2")) - .setMaxAge(Duration.newBuilder().setSeconds(0).setNanos(0).build()) - .build(); - } } diff --git a/storage/api/src/main/java/feast/storage/api/writer/FailedElement.java b/storage/api/src/main/java/feast/storage/api/writer/FailedElement.java index d5823414772..c6db877216f 100644 --- a/storage/api/src/main/java/feast/storage/api/writer/FailedElement.java +++ b/storage/api/src/main/java/feast/storage/api/writer/FailedElement.java @@ -39,9 +39,6 @@ public abstract class FailedElement { @Nullable public abstract String getFeatureSetName(); - @Nullable - public abstract String getFeatureSetVersion(); - @Nullable public abstract String getTransformName(); @@ -66,8 +63,6 @@ public abstract static class Builder { public abstract Builder setFeatureSetName(String featureSetName); - public abstract Builder setFeatureSetVersion(String featureSetVersion); - public abstract Builder setJobName(String jobName); public abstract Builder setTransformName(String transformName); diff --git a/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java b/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java index 6047a93dc17..e1f5c116752 100644 --- a/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java +++ b/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java @@ -87,7 +87,7 @@ public static FeatureRow createRandomFeatureRow(FeatureSet featureSet, int rando private static String getFeatureSetReference(FeatureSet featureSet) { FeatureSetSpec spec = featureSet.getSpec(); - return String.format("%s/%s:%d", spec.getProject(), spec.getName(), spec.getVersion()); + return String.format("%s/%s:%d", spec.getProject(), spec.getName()); } /** diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/FeatureSetQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/FeatureSetQueryInfo.java index 5a7d56e9844..a99411f87b9 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/FeatureSetQueryInfo.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/FeatureSetQueryInfo.java @@ -22,7 +22,6 @@ public class FeatureSetQueryInfo { private final String project; private final String name; - private final int version; private final long maxAge; private final List entities; private final List features; @@ -31,14 +30,12 @@ public class FeatureSetQueryInfo { public FeatureSetQueryInfo( String project, String name, - int version, long maxAge, List entities, List features, String table) { this.project = project; this.name = name; - this.version = version; this.maxAge = maxAge; this.entities = entities; this.features = features; @@ -49,7 +46,6 @@ public FeatureSetQueryInfo(FeatureSetQueryInfo featureSetInfo, String table) { this.project = featureSetInfo.getProject(); this.name = featureSetInfo.getName(); - this.version = featureSetInfo.getVersion(); this.maxAge = featureSetInfo.getMaxAge(); this.entities = featureSetInfo.getEntities(); this.features = featureSetInfo.getFeatures(); @@ -64,10 +60,6 @@ public String getName() { return name; } - public int getVersion() { - return version; - } - public long getMaxAge() { return maxAge; } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java index cba997b6ab0..1fb0a617afb 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java @@ -85,13 +85,7 @@ public static List getFeatureSetInfos( .collect(Collectors.toList()); featureSetInfos.add( new FeatureSetQueryInfo( - spec.getProject(), - spec.getName(), - spec.getVersion(), - maxAge.getSeconds(), - fsEntities, - features, - "")); + spec.getProject(), spec.getName(), maxAge.getSeconds(), fsEntities, features, "")); } return featureSetInfos; } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java index 5d8f3d25cb7..20281783cf2 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java @@ -100,22 +100,19 @@ public void prepareWrite(FeatureSetProto.FeatureSet featureSet) { bigquery.create(DatasetInfo.of(datasetId)); } String tableName = - String.format( - "%s_%s_v%d", - featureSetSpec.getProject(), featureSetSpec.getName(), featureSetSpec.getVersion()) + String.format("%s_%s", featureSetSpec.getProject(), featureSetSpec.getName()) .replaceAll("-", "_"); TableId tableId = TableId.of(datasetId.getProject(), datasetId.getDataset(), tableName); - // Return if there is an existing table Table table = bigquery.getTable(tableId); + TableDefinition tableDefinition = createBigQueryTableDefinition(table, featureSet.getSpec()); + TableInfo tableInfo = TableInfo.of(tableId, tableDefinition); if (table != null) { log.info( "Updating and writing to existing BigQuery table '{}:{}.{}'", datasetId.getProject(), datasetId.getDataset(), tableName); - TableDefinition tableDefinition = createBigQueryTableDefinition(featureSet.getSpec()); - TableInfo tableInfo = TableInfo.of(tableId, tableDefinition); bigquery.update(tableInfo); return; } @@ -125,8 +122,6 @@ public void prepareWrite(FeatureSetProto.FeatureSet featureSet) { tableId.getTable(), datasetId.getDataset(), datasetId.getProject()); - TableDefinition tableDefinition = createBigQueryTableDefinition(featureSet.getSpec()); - TableInfo tableInfo = TableInfo.of(tableId, tableDefinition); bigquery.create(tableInfo); } @@ -134,8 +129,18 @@ public void prepareWrite(FeatureSetProto.FeatureSet featureSet) { public PTransform, WriteResult> writer() { return new BigQueryWrite(DatasetId.of(getProjectId(), getDatasetId())); } - - private TableDefinition createBigQueryTableDefinition(FeatureSetProto.FeatureSetSpec spec) { + /** + * Creates a BigQuery {@link TableDefinition} based on the provided FeatureSetSpec and the + * existing table, if any. If a table already exists, existing fields will be retained, and new + * fields present in the feature set will be appended to the existing FieldsList. + * + * @param existingTable existing {@link Table} retrieved using bigquery.GetTable(). If the table + * does not exist, will be null. + * @param spec FeatureSet spec that this table is for + * @return {@link TableDefinition} containing all tombstoned and active fields. + */ + private TableDefinition createBigQueryTableDefinition( + Table existingTable, FeatureSetProto.FeatureSetSpec spec) { List fields = new ArrayList<>(); log.info("Table will have the following fields:"); @@ -189,9 +194,21 @@ private TableDefinition createBigQueryTableDefinition(FeatureSetProto.FeatureSet TimePartitioning.newBuilder(TimePartitioning.Type.DAY).setField("event_timestamp").build(); log.info("Table partitioning: " + timePartitioning.toString()); + List fieldsList = new ArrayList<>(); + if (existingTable != null) { + Schema existingSchema = existingTable.getDefinition().getSchema(); + fieldsList.addAll(existingSchema.getFields()); + } + + for (Field field : fields) { + if (!fieldsList.contains(field)) { + fieldsList.add(field); + } + } + return StandardTableDefinition.newBuilder() .setTimePartitioning(timePartitioning) - .setSchema(Schema.of(fields)) + .setSchema(Schema.of(FieldList.of(fieldsList))) .build(); } } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/GetTableDestination.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/GetTableDestination.java index 5903d36b858..2007f8573fb 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/GetTableDestination.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/GetTableDestination.java @@ -35,8 +35,7 @@ public GetTableDestination(String projectId, String datasetId) { @Override public TableDestination apply(ValueInSingleWindow input) { - String[] split = input.getValue().getFeatureSet().split(":"); - String[] splitName = split[0].split("/"); + String[] splitName = input.getValue().getFeatureSet().split("/"); TimePartitioning timePartitioning = new TimePartitioning() @@ -44,8 +43,7 @@ public TableDestination apply(ValueInSingleWindow input) { .setField(FeatureRowToTableRow.getEventTimestampColumn()); return new TableDestination( - String.format( - "%s:%s.%s_%s_v%s", projectId, datasetId, splitName[0], splitName[1], split[1]), + String.format("%s:%s.%s_%s", projectId, datasetId, splitName[0], splitName[1]), String.format("Feast table for %s", input.getValue().getFeatureSet()), timePartitioning); } diff --git a/storage/connectors/bigquery/src/main/resources/templates/join_featuresets.sql b/storage/connectors/bigquery/src/main/resources/templates/join_featuresets.sql index 60b7c7d7a12..10aafb05092 100644 --- a/storage/connectors/bigquery/src/main/resources/templates/join_featuresets.sql +++ b/storage/connectors/bigquery/src/main/resources/templates/join_featuresets.sql @@ -8,7 +8,7 @@ LEFT JOIN ( SELECT uuid, {% for featureName in featureSet.features %} - {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} + {{ featureSet.project }}_{{ featureName }}{% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM `{{ featureSet.table }}` ) USING (uuid) @@ -18,7 +18,7 @@ LEFT JOIN ( {{ entities | join(', ') }} {% for featureSet in featureSets %} {% for featureName in featureSet.features %} - ,{{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }} as {{ featureName }} + ,{{ featureSet.project }}_{{ featureName }} as {{ featureName }} {% endfor %} {% endfor %} FROM joined \ No newline at end of file diff --git a/storage/connectors/bigquery/src/main/resources/templates/single_featureset_pit_join.sql b/storage/connectors/bigquery/src/main/resources/templates/single_featureset_pit_join.sql index fb4c555b529..c02bf6b46c1 100644 --- a/storage/connectors/bigquery/src/main/resources/templates/single_featureset_pit_join.sql +++ b/storage/connectors/bigquery/src/main/resources/templates/single_featureset_pit_join.sql @@ -13,7 +13,7 @@ SELECT -- event_timestamp contains the timestamps to join onto event_timestamp, -- the feature_timestamp, i.e. the latest occurrence of the requested feature relative to the entity_dataset timestamp - NULL as {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, + NULL as {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, -- created timestamp of the feature at the corresponding feature_timestamp NULL as created_timestamp, -- select only entities belonging to this feature set @@ -25,11 +25,11 @@ UNION ALL SELECT NULL as uuid, event_timestamp, - event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, + event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, created_timestamp, {{ featureSet.entities | join(', ')}}, false AS is_entity_table -FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` WHERE event_timestamp <= '{{maxTimestamp}}' +FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}` WHERE event_timestamp <= '{{maxTimestamp}}' {% if featureSet.maxAge == 0 %}{% else %}AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second){% endif %} ), /* @@ -48,7 +48,7 @@ SELECT event_timestamp, {{ featureSet.entities | join(', ')}}, {% for featureName in featureSet.features %} - IF(event_timestamp >= {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp {% if featureSet.maxAge == 0 %}{% else %}AND Timestamp_sub(event_timestamp, interval {{ featureSet.maxAge }} second) < {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp{% endif %}, {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}, NULL) as {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} + IF(event_timestamp >= {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp {% if featureSet.maxAge == 0 %}{% else %}AND Timestamp_sub(event_timestamp, interval {{ featureSet.maxAge }} second) < {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp{% endif %}, {{ featureSet.project }}_{{ featureName }}, NULL) as {{ featureSet.project }}_{{ featureName }}{% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM ( SELECT @@ -56,7 +56,7 @@ SELECT event_timestamp, {{ featureSet.entities | join(', ')}}, FIRST_VALUE(created_timestamp IGNORE NULLS) over w AS created_timestamp, - FIRST_VALUE({{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp IGNORE NULLS) over w AS {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, + FIRST_VALUE({{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp IGNORE NULLS) over w AS {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, is_entity_table FROM union_features WINDOW w AS (PARTITION BY {{ featureSet.entities | join(', ') }} ORDER BY event_timestamp DESC, is_entity_table DESC, created_timestamp DESC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) @@ -67,15 +67,15 @@ WINDOW w AS (PARTITION BY {{ featureSet.entities | join(', ') }} ORDER BY event_ */ LEFT JOIN ( SELECT - event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, + event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, created_timestamp, {{ featureSet.entities | join(', ')}}, {% for featureName in featureSet.features %} - {{ featureName }} as {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} + {{ featureName }} as {{ featureSet.project }}_{{ featureName }}{% if loop.last %}{% else %}, {% endif %} {% endfor %} -FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` WHERE event_timestamp <= '{{maxTimestamp}}' +FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}` WHERE event_timestamp <= '{{maxTimestamp}}' {% if featureSet.maxAge == 0 %}{% else %}AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second){% endif %} -) USING ({{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, created_timestamp, {{ featureSet.entities | join(', ')}}) +) USING ({{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, created_timestamp, {{ featureSet.entities | join(', ')}}) WHERE is_entity_table ) /* diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 0963731988c..f25bf558f22 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -112,7 +112,7 @@ private List buildRedisKeys(List entityRows, FeatureSetSpec /** * Create {@link RedisKey} * - * @param featureSet featureSet reference of the feature. E.g. feature_set_1:1 + * @param featureSet featureSet reference of the feature. E.g. feature_set_1 * @param featureSetEntityNames entity names that belong to the featureSet * @param entityRow entityRow to build the key from * @return {@link RedisKey} @@ -213,9 +213,6 @@ private List sendMultiGet(List keys) { // TODO: Refactor this out to common package? private static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) { String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); - if (featureSetSpec.getVersion() > 0) { - return ref + String.format(":%d", featureSetSpec.getVersion()); - } return ref; } } diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java index 41bbfaa74c4..9e9ccbe53f6 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java @@ -64,14 +64,14 @@ public void setUp() { redisKeyList = Lists.newArrayList( RedisKey.newBuilder() - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllEntities( Lists.newArrayList( Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), Field.newBuilder().setName("entity2").setValue(strValue("a")).build())) .build(), RedisKey.newBuilder() - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllEntities( Lists.newArrayList( Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), @@ -89,17 +89,9 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { FeatureSetRequest.newBuilder() .setSpec(getFeatureSetSpec()) .addFeatureReference( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatureReference( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .build(); List entityRows = ImmutableList.of( @@ -145,7 +137,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Lists.newArrayList( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), @@ -153,7 +145,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), @@ -171,17 +163,9 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { FeatureSetRequest.newBuilder() .setSpec(getFeatureSetSpec()) .addFeatureReference( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatureReference( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .build(); List entityRows = ImmutableList.of( @@ -221,14 +205,14 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { Lists.newArrayList( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) .build(), FeatureRow.newBuilder() - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").build(), @@ -252,7 +236,6 @@ private FeatureSetSpec getFeatureSetSpec() { return FeatureSetSpec.newBuilder() .setProject("project") .setName("featureSet") - .setVersion(1) .addEntities(EntitySpec.newBuilder().setName("entity1")) .addEntities(EntitySpec.newBuilder().setName("entity2")) .addFeatures(FeatureSpec.newBuilder().setName("feature1")) diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/writer/RedisFeatureSinkTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/writer/RedisFeatureSinkTest.java index beeabc2c884..c9d8cf4cfc5 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/writer/RedisFeatureSinkTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/writer/RedisFeatureSinkTest.java @@ -78,7 +78,6 @@ public void setUp() throws IOException { FeatureSetSpec spec1 = FeatureSetSpec.newBuilder() .setName("fs") - .setVersion(1) .setProject("myproject") .addEntities(EntitySpec.newBuilder().setName("entity").setValueType(Enum.INT64).build()) .addFeatures( @@ -89,7 +88,6 @@ public void setUp() throws IOException { FeatureSetSpec.newBuilder() .setName("feature_set") .setProject("myproject") - .setVersion(1) .addEntities( EntitySpec.newBuilder() .setName("entity_id_primary") @@ -107,7 +105,7 @@ public void setUp() throws IOException { .build(); Map specMap = - ImmutableMap.of("myproject/fs:1", spec1, "myproject/feature_set:1", spec2); + ImmutableMap.of("myproject/fs", spec1, "myproject/feature_set", spec2); StoreProto.Store.RedisConfig redisConfig = StoreProto.Store.RedisConfig.newBuilder().setHost(REDIS_HOST).setPort(REDIS_PORT).build(); @@ -127,7 +125,7 @@ public void shouldWriteToRedis() { HashMap kvs = new LinkedHashMap<>(); kvs.put( RedisKey.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addEntities(field("entity", 1, Enum.INT64)) .build(), FeatureRow.newBuilder() @@ -136,7 +134,7 @@ public void shouldWriteToRedis() { .build()); kvs.put( RedisKey.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addEntities(field("entity", 2, Enum.INT64)) .build(), FeatureRow.newBuilder() @@ -147,12 +145,12 @@ public void shouldWriteToRedis() { List featureRows = ImmutableList.of( FeatureRow.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addFields(field("entity", 1, Enum.INT64)) .addFields(field("feature", "one", Enum.STRING)) .build(), FeatureRow.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addFields(field("entity", 2, Enum.INT64)) .addFields(field("feature", "two", Enum.STRING)) .build()); @@ -181,7 +179,7 @@ public void shouldRetryFailConnection() throws InterruptedException { HashMap kvs = new LinkedHashMap<>(); kvs.put( RedisKey.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addEntities(field("entity", 1, Enum.INT64)) .build(), FeatureRow.newBuilder() @@ -192,7 +190,7 @@ public void shouldRetryFailConnection() throws InterruptedException { List featureRows = ImmutableList.of( FeatureRow.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addFields(field("entity", 1, Enum.INT64)) .addFields(field("feature", "one", Enum.STRING)) .build()); @@ -234,7 +232,7 @@ public void shouldProduceFailedElementIfRetryExceeded() { HashMap kvs = new LinkedHashMap<>(); kvs.put( RedisKey.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addEntities(field("entity", 1, Enum.INT64)) .build(), FeatureRow.newBuilder() @@ -245,7 +243,7 @@ public void shouldProduceFailedElementIfRetryExceeded() { List featureRows = ImmutableList.of( FeatureRow.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addFields(field("entity", 1, Enum.INT64)) .addFields(field("feature", "one", Enum.STRING)) .build()); @@ -266,7 +264,7 @@ public void shouldConvertRowWithDuplicateEntitiesToValidKey() { FeatureRow offendingRow = FeatureRow.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) .addFields( Field.newBuilder() @@ -292,7 +290,7 @@ public void shouldConvertRowWithDuplicateEntitiesToValidKey() { RedisKey expectedKey = RedisKey.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .addEntities( Field.newBuilder() .setName("entity_id_primary") @@ -322,7 +320,7 @@ public void shouldConvertRowWithDuplicateEntitiesToValidKey() { public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { FeatureRow offendingRow = FeatureRow.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) .addFields( Field.newBuilder() @@ -344,7 +342,7 @@ public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { RedisKey expectedKey = RedisKey.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .addEntities( Field.newBuilder() .setName("entity_id_primary") @@ -377,7 +375,7 @@ public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { public void shouldMergeDuplicateFeatureFields() { FeatureRow featureRowWithDuplicatedFeatureFields = FeatureRow.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) .addFields( Field.newBuilder() @@ -403,7 +401,7 @@ public void shouldMergeDuplicateFeatureFields() { RedisKey expectedKey = RedisKey.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .addEntities( Field.newBuilder() .setName("entity_id_primary") @@ -433,7 +431,7 @@ public void shouldMergeDuplicateFeatureFields() { public void shouldPopulateMissingFeatureValuesWithDefaultInstance() { FeatureRow featureRowWithDuplicatedFeatureFields = FeatureRow.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) .addFields( Field.newBuilder() @@ -451,7 +449,7 @@ public void shouldPopulateMissingFeatureValuesWithDefaultInstance() { RedisKey expectedKey = RedisKey.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .addEntities( Field.newBuilder() .setName("entity_id_primary") diff --git a/storage/connectors/rediscluster/src/main/java/feast/storage/connectors/rediscluster/retriever/RedisClusterOnlineRetriever.java b/storage/connectors/rediscluster/src/main/java/feast/storage/connectors/rediscluster/retriever/RedisClusterOnlineRetriever.java index 713b6897b2d..1141e4eeed5 100644 --- a/storage/connectors/rediscluster/src/main/java/feast/storage/connectors/rediscluster/retriever/RedisClusterOnlineRetriever.java +++ b/storage/connectors/rediscluster/src/main/java/feast/storage/connectors/rediscluster/retriever/RedisClusterOnlineRetriever.java @@ -218,9 +218,6 @@ private List sendMultiGet(List keys) { // TODO: Refactor this out to common package? private static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) { String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); - if (featureSetSpec.getVersion() > 0) { - return ref + String.format(":%d", featureSetSpec.getVersion()); - } return ref; } } diff --git a/storage/connectors/rediscluster/src/test/java/feast/storage/connectors/rediscluster/retriever/RedisClusterOnlineRetrieverTest.java b/storage/connectors/rediscluster/src/test/java/feast/storage/connectors/rediscluster/retriever/RedisClusterOnlineRetrieverTest.java index 567e92a3d41..b89ed8b2286 100644 --- a/storage/connectors/rediscluster/src/test/java/feast/storage/connectors/rediscluster/retriever/RedisClusterOnlineRetrieverTest.java +++ b/storage/connectors/rediscluster/src/test/java/feast/storage/connectors/rediscluster/retriever/RedisClusterOnlineRetrieverTest.java @@ -64,14 +64,14 @@ public void setUp() { redisKeyList = Lists.newArrayList( RedisKey.newBuilder() - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllEntities( Lists.newArrayList( Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), Field.newBuilder().setName("entity2").setValue(strValue("a")).build())) .build(), RedisKey.newBuilder() - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllEntities( Lists.newArrayList( Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), @@ -89,17 +89,9 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { FeatureSetRequest.newBuilder() .setSpec(getFeatureSetSpec()) .addFeatureReference( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatureReference( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .build(); List entityRows = ImmutableList.of( @@ -145,7 +137,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Lists.newArrayList( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), @@ -153,7 +145,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), @@ -171,17 +163,9 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { FeatureSetRequest.newBuilder() .setSpec(getFeatureSetSpec()) .addFeatureReference( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature1").setProject("project").build()) .addFeatureReference( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .build(); List entityRows = ImmutableList.of( @@ -221,14 +205,14 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { Lists.newArrayList( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) .build(), FeatureRow.newBuilder() - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").build(), @@ -252,7 +236,6 @@ private FeatureSetSpec getFeatureSetSpec() { return FeatureSetSpec.newBuilder() .setProject("project") .setName("featureSet") - .setVersion(1) .addEntities(EntitySpec.newBuilder().setName("entity1")) .addEntities(EntitySpec.newBuilder().setName("entity2")) .addFeatures(FeatureSpec.newBuilder().setName("feature1")) diff --git a/storage/connectors/rediscluster/src/test/java/feast/storage/connectors/rediscluster/writer/RedisClusterFeatureSinkTest.java b/storage/connectors/rediscluster/src/test/java/feast/storage/connectors/rediscluster/writer/RedisClusterFeatureSinkTest.java index cc1993636ee..c766d9b26c6 100644 --- a/storage/connectors/rediscluster/src/test/java/feast/storage/connectors/rediscluster/writer/RedisClusterFeatureSinkTest.java +++ b/storage/connectors/rediscluster/src/test/java/feast/storage/connectors/rediscluster/writer/RedisClusterFeatureSinkTest.java @@ -87,7 +87,6 @@ public void setUp() throws IOException { FeatureSetSpec spec1 = FeatureSetSpec.newBuilder() .setName("fs") - .setVersion(1) .setProject("myproject") .addEntities(EntitySpec.newBuilder().setName("entity").setValueType(Enum.INT64).build()) .addFeatures( @@ -98,7 +97,6 @@ public void setUp() throws IOException { FeatureSetSpec.newBuilder() .setName("feature_set") .setProject("myproject") - .setVersion(1) .addEntities( EntitySpec.newBuilder() .setName("entity_id_primary") @@ -116,7 +114,7 @@ public void setUp() throws IOException { .build(); Map specMap = - ImmutableMap.of("myproject/fs:1", spec1, "myproject/feature_set:1", spec2); + ImmutableMap.of("myproject/fs", spec1, "myproject/feature_set", spec2); RedisClusterConfig redisClusterConfig = RedisClusterConfig.newBuilder() .setConnectionString(CONNECTION_STRING) @@ -154,7 +152,7 @@ public void shouldWriteToRedis() { HashMap kvs = new LinkedHashMap<>(); kvs.put( RedisKey.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addEntities(field("entity", 1, Enum.INT64)) .build(), FeatureRow.newBuilder() @@ -163,7 +161,7 @@ public void shouldWriteToRedis() { .build()); kvs.put( RedisKey.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addEntities(field("entity", 2, Enum.INT64)) .build(), FeatureRow.newBuilder() @@ -174,12 +172,12 @@ public void shouldWriteToRedis() { List featureRows = ImmutableList.of( FeatureRow.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addFields(field("entity", 1, Enum.INT64)) .addFields(field("feature", "one", Enum.STRING)) .build(), FeatureRow.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addFields(field("entity", 2, Enum.INT64)) .addFields(field("feature", "two", Enum.STRING)) .build()); @@ -199,7 +197,7 @@ public void shouldRetryFailConnection() throws InterruptedException { HashMap kvs = new LinkedHashMap<>(); kvs.put( RedisKey.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addEntities(field("entity", 1, Enum.INT64)) .build(), FeatureRow.newBuilder() @@ -210,7 +208,7 @@ public void shouldRetryFailConnection() throws InterruptedException { List featureRows = ImmutableList.of( FeatureRow.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addFields(field("entity", 1, Enum.INT64)) .addFields(field("feature", "one", Enum.STRING)) .build()); @@ -254,13 +252,12 @@ public void shouldProduceFailedElementIfRetryExceeded() { FeatureSetSpec spec1 = FeatureSetSpec.newBuilder() .setName("fs") - .setVersion(1) .setProject("myproject") .addEntities(EntitySpec.newBuilder().setName("entity").setValueType(Enum.INT64).build()) .addFeatures( FeatureSpec.newBuilder().setName("feature").setValueType(Enum.STRING).build()) .build(); - Map specMap = ImmutableMap.of("myproject/fs:1", spec1); + Map specMap = ImmutableMap.of("myproject/fs", spec1); redisClusterFeatureSink = RedisClusterFeatureSink.builder() .setFeatureSetSpecs(specMap) @@ -271,7 +268,7 @@ public void shouldProduceFailedElementIfRetryExceeded() { List featureRows = ImmutableList.of( FeatureRow.newBuilder() - .setFeatureSet("myproject/fs:1") + .setFeatureSet("myproject/fs") .addFields(field("entity", 1, Enum.INT64)) .addFields(field("feature", "one", Enum.STRING)) .build()); @@ -291,7 +288,7 @@ public void shouldConvertRowWithDuplicateEntitiesToValidKey() { FeatureRow offendingRow = FeatureRow.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) .addFields( Field.newBuilder() @@ -317,7 +314,7 @@ public void shouldConvertRowWithDuplicateEntitiesToValidKey() { RedisKey expectedKey = RedisKey.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .addEntities( Field.newBuilder() .setName("entity_id_primary") @@ -347,7 +344,7 @@ public void shouldConvertRowWithDuplicateEntitiesToValidKey() { public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { FeatureRow offendingRow = FeatureRow.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) .addFields( Field.newBuilder() @@ -369,7 +366,7 @@ public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { RedisKey expectedKey = RedisKey.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .addEntities( Field.newBuilder() .setName("entity_id_primary") @@ -402,7 +399,7 @@ public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { public void shouldMergeDuplicateFeatureFields() { FeatureRow featureRowWithDuplicatedFeatureFields = FeatureRow.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) .addFields( Field.newBuilder() @@ -428,7 +425,7 @@ public void shouldMergeDuplicateFeatureFields() { RedisKey expectedKey = RedisKey.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .addEntities( Field.newBuilder() .setName("entity_id_primary") @@ -459,7 +456,7 @@ public void shouldMergeDuplicateFeatureFields() { public void shouldPopulateMissingFeatureValuesWithDefaultInstance() { FeatureRow featureRowWithDuplicatedFeatureFields = FeatureRow.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) .addFields( Field.newBuilder() @@ -477,7 +474,7 @@ public void shouldPopulateMissingFeatureValuesWithDefaultInstance() { RedisKey expectedKey = RedisKey.newBuilder() - .setFeatureSet("myproject/feature_set:1") + .setFeatureSet("myproject/feature_set") .addEntities( Field.newBuilder() .setName("entity_id_primary") diff --git a/tests/e2e/basic-ingest-redis-serving.py b/tests/e2e/basic-ingest-redis-serving.py index 50ec3854553..19c934e1067 100644 --- a/tests/e2e/basic-ingest-redis-serving.py +++ b/tests/e2e/basic-ingest-redis-serving.py @@ -557,7 +557,6 @@ def test_all_types_infer_register_ingest_file_success(client, # TODO: rewrite these using python SDK once the labels are implemented there class TestsBasedOnGrpc: - LAST_VERSION = 0 GRPC_CONNECTION_TIMEOUT = 3 LABEL_KEY = "my" LABEL_VALUE = "label" @@ -595,7 +594,7 @@ def get_feature_set(self, core_service_stub, name, project): try: get_feature_set_response = core_service_stub.GetFeatureSet( CoreService_pb2.GetFeatureSetRequest( - project=project, name=name.strip(), version=self.LAST_VERSION + project=project, name=name.strip(), ) ) # type: GetFeatureSetResponse except grpc.RpcError as e: diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bq-batch-retrieval.py index c1c6ab67805..a9a51894c77 100644 --- a/tests/e2e/bq-batch-retrieval.py +++ b/tests/e2e/bq-batch-retrieval.py @@ -1,28 +1,30 @@ +import math +import os import random import time +import uuid from datetime import datetime from datetime import timedelta from urllib.parse import urlparse -import os -import uuid import numpy as np import pandas as pd import pytest import pytz -from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.client import Client +from feast.core.CoreService_pb2 import ListStoresRequest +from feast.core.IngestionJob_pb2 import IngestionJobStatus from feast.entity import Entity from feast.feature import Feature from feast.feature_set import FeatureSet from feast.type_map import ValueType -from google.cloud import storage +from google.cloud import storage, bigquery from google.protobuf.duration_pb2 import Duration from pandavro import to_avro -pd.set_option('display.max_columns', None) +pd.set_option("display.max_columns", None) -PROJECT_NAME = 'batch_' + uuid.uuid4().hex.upper()[0:6] +PROJECT_NAME = "batch_" + uuid.uuid4().hex.upper()[0:6] @pytest.fixture(scope="module") @@ -56,62 +58,66 @@ def client(core_url, serving_url, allow_dirty): if not allow_dirty: feature_sets = client.list_feature_sets() if len(feature_sets) > 0: - raise Exception("Feast cannot have existing feature sets registered. Exiting tests.") + raise Exception( + "Feast cannot have existing feature sets registered. Exiting tests." + ) return client + @pytest.mark.first @pytest.mark.direct_runner @pytest.mark.dataflow_runner -def test_apply_all_featuresets(client): +@pytest.mark.run(order=1) +def test_batch_apply_all_featuresets(client): client.set_project(PROJECT_NAME) file_fs1 = FeatureSet( - "file_feature_set", - features=[Feature("feature_value1", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "file_feature_set", + features=[Feature("feature_value1", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(file_fs1) gcs_fs1 = FeatureSet( - "gcs_feature_set", - features=[Feature("feature_value2", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "gcs_feature_set", + features=[Feature("feature_value2", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(gcs_fs1) proc_time_fs = FeatureSet( - "processing_time", - features=[Feature("feature_value3", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "processing_time", + features=[Feature("feature_value3", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(proc_time_fs) add_cols_fs = FeatureSet( - "additional_columns", - features=[Feature("feature_value4", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "additional_columns", + features=[Feature("feature_value4", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(add_cols_fs) historical_fs = FeatureSet( - "historical", - features=[Feature("feature_value5", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "historical", + features=[Feature("feature_value5", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(historical_fs) fs1 = FeatureSet( - "feature_set_1", - features=[Feature("feature_value6", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "feature_set_1", + features=[Feature("feature_value6", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) fs2 = FeatureSet( "feature_set_2", @@ -133,8 +139,9 @@ def test_apply_all_featuresets(client): @pytest.mark.direct_runner @pytest.mark.dataflow_runner -def test_get_batch_features_with_file(client): - file_fs1 = client.get_feature_set(name="file_feature_set", version=1) +@pytest.mark.run(order=10) +def test_batch_get_batch_features_with_file(client): + file_fs1 = client.get_feature_set(name="file_feature_set") N_ROWS = 10 time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) @@ -150,23 +157,30 @@ def test_get_batch_features_with_file(client): # Rename column (datetime -> event_timestamp) features_1_df = features_1_df.rename(columns={"datetime": "event_timestamp"}) - to_avro(df=features_1_df[["event_timestamp", "entity_id"]], file_path_or_buffer="file_feature_set.avro") + to_avro( + df=features_1_df[["event_timestamp", "entity_id"]], + file_path_or_buffer="file_feature_set.avro", + ) time.sleep(15) feature_retrieval_job = client.get_batch_features( - entity_rows="file://file_feature_set.avro", feature_refs=[f"{PROJECT_NAME}/feature_value1:1"] + entity_rows="file://file_feature_set.avro", + feature_refs=[f"{PROJECT_NAME}/feature_value1"], ) output = feature_retrieval_job.to_dataframe() print(output.head()) - assert output["entity_id"].to_list() == [int(i) for i in output["feature_value1"].to_list()] + assert output["entity_id"].to_list() == [ + int(i) for i in output["feature_value1"].to_list() + ] @pytest.mark.direct_runner @pytest.mark.dataflow_runner -def test_get_batch_features_with_gs_path(client, gcs_path): - gcs_fs1 = client.get_feature_set(name="gcs_feature_set", version=1) +@pytest.mark.run(order=11) +def test_batch_get_batch_features_with_gs_path(client, gcs_path): + gcs_fs1 = client.get_feature_set(name="gcs_feature_set") N_ROWS = 10 time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) @@ -184,7 +198,10 @@ def test_get_batch_features_with_gs_path(client, gcs_path): # Output file to local file_name = "gcs_feature_set.avro" - to_avro(df=features_1_df[["event_timestamp", "entity_id"]], file_path_or_buffer=file_name) + to_avro( + df=features_1_df[["event_timestamp", "entity_id"]], + file_path_or_buffer=file_name, + ) uri = urlparse(gcs_path) bucket = uri.hostname @@ -199,19 +216,21 @@ def test_get_batch_features_with_gs_path(client, gcs_path): time.sleep(15) feature_retrieval_job = client.get_batch_features( - entity_rows=f"{gcs_path}{ts}/*", - feature_refs=[f"{PROJECT_NAME}/feature_value2:1"] + entity_rows=f"{gcs_path}{ts}/*", feature_refs=[f"{PROJECT_NAME}/feature_value2"] ) output = feature_retrieval_job.to_dataframe() print(output.head()) - assert output["entity_id"].to_list() == [int(i) for i in output["feature_value2"].to_list()] + assert output["entity_id"].to_list() == [ + int(i) for i in output["feature_value2"].to_list() + ] @pytest.mark.direct_runner -def test_order_by_creation_time(client): - proc_time_fs = client.get_feature_set(name="processing_time", version=1) +@pytest.mark.run(order=12) +def test_batch_order_by_creation_time(client): + proc_time_fs = client.get_feature_set(name="processing_time") time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) N_ROWS = 10 @@ -233,7 +252,8 @@ def test_order_by_creation_time(client): time.sleep(15) client.ingest(proc_time_fs, correct_df) feature_retrieval_job = client.get_batch_features( - entity_rows=incorrect_df[["datetime", "entity_id"]], feature_refs=[f"{PROJECT_NAME}/feature_value3:1"] + entity_rows=incorrect_df[["datetime", "entity_id"]], + feature_refs=[f"{PROJECT_NAME}/feature_value3"], ) output = feature_retrieval_job.to_dataframe() print(output.head()) @@ -242,13 +262,18 @@ def test_order_by_creation_time(client): @pytest.mark.direct_runner -def test_additional_columns_in_entity_table(client): - add_cols_fs = client.get_feature_set(name="additional_columns", version=1) +@pytest.mark.run(order=13) +def test_batch_additional_columns_in_entity_table(client): + add_cols_fs = client.get_feature_set(name="additional_columns") N_ROWS = 10 time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) features_df = pd.DataFrame( - {"datetime": [time_offset] * N_ROWS, "entity_id": [i for i in range(N_ROWS)], "feature_value4": ["abc"] * N_ROWS} + { + "datetime": [time_offset] * N_ROWS, + "entity_id": [i for i in range(N_ROWS)], + "feature_value4": ["abc"] * N_ROWS, + } ) client.ingest(add_cols_fs, features_df) @@ -263,19 +288,25 @@ def test_additional_columns_in_entity_table(client): time.sleep(15) feature_retrieval_job = client.get_batch_features( - entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value4:1"] + entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value4"] ) output = feature_retrieval_job.to_dataframe().sort_values(by=["entity_id"]) print(output.head(10)) - assert np.allclose(output["additional_float_col"], entity_df["additional_float_col"]) - assert output["additional_string_col"].to_list() == entity_df["additional_string_col"].to_list() + assert np.allclose( + output["additional_float_col"], entity_df["additional_float_col"] + ) + assert ( + output["additional_string_col"].to_list() + == entity_df["additional_string_col"].to_list() + ) assert output["feature_value4"].to_list() == features_df["feature_value4"].to_list() @pytest.mark.direct_runner -def test_point_in_time_correctness_join(client): - historical_fs = client.get_feature_set(name="historical", version=1) +@pytest.mark.run(order=14) +def test_batch_point_in_time_correctness_join(client): + historical_fs = client.get_feature_set(name="historical") time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) N_EXAMPLES = 10 @@ -292,13 +323,18 @@ def test_point_in_time_correctness_join(client): } ) entity_df = pd.DataFrame( - {"datetime": [time_offset - timedelta(seconds=10)] * N_EXAMPLES, "entity_id": [i for i in range(N_EXAMPLES)]} + { + "datetime": [time_offset - timedelta(seconds=10)] * N_EXAMPLES, + "entity_id": [i for i in range(N_EXAMPLES)], + } ) client.ingest(historical_fs, historical_df) time.sleep(15) - feature_retrieval_job = client.get_batch_features(entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value5"]) + feature_retrieval_job = client.get_batch_features( + entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value5"] + ) output = feature_retrieval_job.to_dataframe() print(output.head()) @@ -306,9 +342,10 @@ def test_point_in_time_correctness_join(client): @pytest.mark.direct_runner -def test_multiple_featureset_joins(client): - fs1 = client.get_feature_set(name="feature_set_1", version=1) - fs2 = client.get_feature_set(name="feature_set_2", version=1) +@pytest.mark.run(order=15) +def test_batch_multiple_featureset_joins(client): + fs1 = client.get_feature_set(name="feature_set_1") + fs2 = client.get_feature_set(name="feature_set_2") N_ROWS = 10 time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) @@ -340,18 +377,27 @@ def test_multiple_featureset_joins(client): time.sleep(15) feature_retrieval_job = client.get_batch_features( - entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value6:1", f"{PROJECT_NAME}/other_feature_value7:1"] + entity_rows=entity_df, + feature_refs=[ + f"{PROJECT_NAME}/feature_value6", + f"{PROJECT_NAME}/other_feature_value7", + ], ) output = feature_retrieval_job.to_dataframe() print(output.head()) - assert output["entity_id"].to_list() == [int(i) for i in output["feature_value6"].to_list()] - assert output["other_entity_id"].to_list() == output["other_feature_value7"].to_list() + assert output["entity_id"].to_list() == [ + int(i) for i in output["feature_value6"].to_list() + ] + assert ( + output["other_entity_id"].to_list() == output["other_feature_value7"].to_list() + ) @pytest.mark.direct_runner -def test_no_max_age(client): - no_max_age_fs = client.get_feature_set(name="no_max_age", version=1) +@pytest.mark.run(order=16) +def test_batch_no_max_age(client): + no_max_age_fs = client.get_feature_set(name="no_max_age") time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) N_ROWS = 10 @@ -366,7 +412,8 @@ def test_no_max_age(client): time.sleep(15) feature_retrieval_job = client.get_batch_features( - entity_rows=features_8_df[["datetime", "entity_id"]], feature_refs=[f"{PROJECT_NAME}/feature_value8:1"] + entity_rows=features_8_df[["datetime", "entity_id"]], + feature_refs=[f"{PROJECT_NAME}/feature_value8"], ) output = feature_retrieval_job.to_dataframe() @@ -377,18 +424,191 @@ def test_no_max_age(client): @pytest.fixture(scope="module", autouse=True) def infra_teardown(pytestconfig, core_url, serving_url): - client = Client(core_url=core_url, serving_url=serving_url) - client.set_project(PROJECT_NAME) - - marker = pytestconfig.getoption("-m") - yield marker - if marker == 'dataflow_runner': - ingest_jobs = client.list_ingest_jobs() - ingest_jobs = [client.list_ingest_jobs(job.id)[0].external_id for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING] - - cwd = os.getcwd() - with open(f"{cwd}/ingesting_jobs.txt", "w+") as output: - for job in ingest_jobs: - output.write('%s\n' % job) - else: - print('Cleaning up not required') + client = Client(core_url=core_url, serving_url=serving_url) + client.set_project(PROJECT_NAME) + + marker = pytestconfig.getoption("-m") + yield marker + if marker == "dataflow_runner": + ingest_jobs = client.list_ingest_jobs() + ingest_jobs = [ + client.list_ingest_jobs(job.id)[0].external_id + for job in ingest_jobs + if job.status == IngestionJobStatus.RUNNING + ] + + cwd = os.getcwd() + with open(f"{cwd}/ingesting_jobs.txt", "w+") as output: + for job in ingest_jobs: + output.write("%s\n" % job) + else: + print("Cleaning up not required") + + +@pytest.fixture(scope="module") +def update_featureset_dataframe(): + n_rows = 10 + time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) + return pd.DataFrame( + { + "datetime": [time_offset] * n_rows, + "entity_id": [i for i in range(n_rows)], + "update_feature1": ["a" for i in range(n_rows)], + "update_feature2": [i + 2 for i in range(n_rows)], + "update_feature3": [i for i in range(n_rows)], + "update_feature4": ["b" for i in range(n_rows)], + } + ) + + +@pytest.mark.direct_runner +@pytest.mark.run(order=20) +def test_update_featureset_apply_featureset_and_ingest_first_subset( + client, update_featureset_dataframe +): + subset_columns = ["datetime", "entity_id", "update_feature1", "update_feature2"] + subset_df = update_featureset_dataframe.iloc[:5][subset_columns] + update_fs = FeatureSet( + "update_fs", + entities=[Entity(name="entity_id", dtype=ValueType.INT64)], + max_age=Duration(seconds=432000), + ) + update_fs.infer_fields_from_df(subset_df) + client.apply(update_fs) + + client.ingest(feature_set=update_fs, source=subset_df) + + time.sleep(15) + feature_retrieval_job = client.get_batch_features( + entity_rows=update_featureset_dataframe[["datetime", "entity_id"]].iloc[:5], + feature_refs=[ + f"{PROJECT_NAME}/update_feature1", + f"{PROJECT_NAME}/update_feature2", + ], + ) + + output = feature_retrieval_job.to_dataframe().sort_values(by=["entity_id"]) + print(output.head()) + + assert output["update_feature1"].to_list() == subset_df["update_feature1"].to_list() + assert output["update_feature2"].to_list() == subset_df["update_feature2"].to_list() + + +@pytest.mark.direct_runner +@pytest.mark.timeout(600) +@pytest.mark.run(order=21) +def test_update_featureset_update_featureset_and_ingest_second_subset( + client, update_featureset_dataframe +): + subset_columns = [ + "datetime", + "entity_id", + "update_feature1", + "update_feature3", + "update_feature4", + ] + subset_df = update_featureset_dataframe.iloc[5:][subset_columns] + update_fs = FeatureSet( + "update_fs", + entities=[Entity(name="entity_id", dtype=ValueType.INT64)], + max_age=Duration(seconds=432000), + ) + update_fs.infer_fields_from_df(subset_df) + client.apply(update_fs) + + # We keep retrying this ingestion until all values make it into the buffer. + # This is a necessary step because bigquery streaming caches table schemas + # and as a result, rows may be lost. + while True: + ingestion_id = client.ingest(feature_set=update_fs, source=subset_df) + time.sleep(15) # wait for rows to get written to bq + rows_ingested = get_rows_ingested(client, update_fs, ingestion_id) + if rows_ingested == len(subset_df): + print(f"Number of rows successfully ingested: {rows_ingested}. Continuing.") + break + print( + f"Number of rows successfully ingested: {rows_ingested}. Retrying ingestion." + ) + time.sleep(30) + + feature_retrieval_job = client.get_batch_features( + entity_rows=update_featureset_dataframe[["datetime", "entity_id"]].iloc[5:], + feature_refs=[ + f"{PROJECT_NAME}/update_feature1", + f"{PROJECT_NAME}/update_feature3", + f"{PROJECT_NAME}/update_feature4", + ], + ) + + output = feature_retrieval_job.to_dataframe().sort_values(by=["entity_id"]) + print(output.head()) + + assert output["update_feature1"].to_list() == subset_df["update_feature1"].to_list() + assert output["update_feature3"].to_list() == subset_df["update_feature3"].to_list() + assert output["update_feature4"].to_list() == subset_df["update_feature4"].to_list() + + +@pytest.mark.direct_runner +@pytest.mark.run(order=22) +def test_update_featureset_retrieve_all_fields(client, update_featureset_dataframe): + with pytest.raises(Exception): + feature_retrieval_job = client.get_batch_features( + entity_rows=update_featureset_dataframe[["datetime", "entity_id"]], + feature_refs=[ + f"{PROJECT_NAME}/update_feature1", + f"{PROJECT_NAME}/update_feature2", + f"{PROJECT_NAME}/update_feature3", + f"{PROJECT_NAME}/update_feature4", + ], + ) + feature_retrieval_job.result() + + +@pytest.mark.direct_runner +@pytest.mark.run(order=23) +def test_update_featureset_retrieve_valid_fields(client, update_featureset_dataframe): + feature_retrieval_job = client.get_batch_features( + entity_rows=update_featureset_dataframe[["datetime", "entity_id"]], + feature_refs=[ + f"{PROJECT_NAME}/update_feature1", + f"{PROJECT_NAME}/update_feature3", + f"{PROJECT_NAME}/update_feature4", + ], + ) + output = feature_retrieval_job.to_dataframe().sort_values(by=["entity_id"]) + print(output.head(10)) + assert ( + output["update_feature1"].to_list() + == update_featureset_dataframe["update_feature1"].to_list() + ) + # we have to convert to float because the column contains np.NaN + assert [math.isnan(i) for i in output["update_feature3"].to_list()[:5]] == [ + True + ] * 5 + assert output["update_feature3"].to_list()[5:] == [ + float(i) for i in update_featureset_dataframe["update_feature3"].to_list()[5:] + ] + assert ( + output["update_feature4"].to_list() + == [None] * 5 + update_featureset_dataframe["update_feature4"].to_list()[5:] + ) + + +def get_rows_ingested( + client: Client, feature_set: FeatureSet, ingestion_id: str +) -> int: + response = client._core_service_stub.ListStores( + ListStoresRequest(filter=ListStoresRequest.Filter(name="historical")) + ) + bq_config = response.store[0].bigquery_config + project = bq_config.project_id + dataset = bq_config.dataset_id + table = f"{PROJECT_NAME}_{feature_set.name}" + + bq_client = bigquery.Client(project=project) + rows = bq_client.query( + f'SELECT COUNT(*) as count FROM `{project}.{dataset}.{table}` WHERE ingestion_id = "{ingestion_id}"' + ).result() + + for row in rows: + return row["count"]