Skip to content

Commit 1a0e215

Browse files
steFaizsteFaiz
committed
fix comment
1 parent 5172294 commit 1a0e215

File tree

7 files changed

+401
-152
lines changed

7 files changed

+401
-152
lines changed

‎paimon-common/src/main/java/org/apache/paimon/utils/PredicateUtils.java‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.utils;
20+
21+
import org.apache.paimon.predicate.Between;
22+
import org.apache.paimon.predicate.CompareUtils;
23+
import org.apache.paimon.predicate.CompoundPredicate;
24+
import org.apache.paimon.predicate.GreaterOrEqual;
25+
import org.apache.paimon.predicate.LeafPredicate;
26+
import org.apache.paimon.predicate.LessOrEqual;
27+
import org.apache.paimon.predicate.Or;
28+
import org.apache.paimon.predicate.Predicate;
29+
import org.apache.paimon.predicate.PredicateBuilder;
30+
import org.apache.paimon.predicate.Transform;
31+
32+
import java.util.ArrayList;
33+
import java.util.Arrays;
34+
import java.util.HashMap;
35+
import java.util.List;
36+
import java.util.Map;
37+
import java.util.Objects;
38+
import java.util.Optional;
39+
40+
/** Utils for {@link Predicate}. */
41+
public class PredicateUtils {
42+
43+
/**
44+
* Try to rewrite possible {@code GREATER_OR_EQUAL} and {@code LESS_OR_EQUAL} predicates to
45+
* {@code BETWEEN} leaf predicate. This method will recursively try to rewrite the children
46+
* predicates of an {@code AND}, for example: {@code OR(a >= 1, AND(b >= 1, b <= 2))} will be
47+
* rewritten to {@code OR(a >= 1, BETWEEN(b, 1, 2))}.
48+
*/
49+
public static Predicate tryRewriteBetweenPredicate(Predicate filter) {
50+
if (filter instanceof LeafPredicate) {
51+
return filter;
52+
}
53+
CompoundPredicate compoundPredicate = (CompoundPredicate) filter;
54+
boolean isOr = compoundPredicate.function() instanceof Or;
55+
56+
Map<Transform, List<LeafPredicate>> leavesByTransform = new HashMap<>();
57+
List<Predicate> resultChildren = new ArrayList<>();
58+
// Flatten the children predicates of an AND
59+
// For example, for AND(b >= 1, AND(a >= 1, b <= 2)), we will get [a >= 1, b >= 1, b <= 2]
60+
// After flattening, all children will be either LeafPredicate or ORPredicate
61+
List<Predicate> effectiveChildren =
62+
isOr ? compoundPredicate.children() : flattenChildren(compoundPredicate.children());
63+
for (Predicate child : effectiveChildren) {
64+
if (child instanceof LeafPredicate) {
65+
leavesByTransform
66+
.computeIfAbsent(
67+
((LeafPredicate) child).transform(), k -> new ArrayList<>())
68+
.add((LeafPredicate) child);
69+
} else {
70+
resultChildren.add(tryRewriteBetweenPredicate(child));
71+
}
72+
}
73+
74+
for (Map.Entry<Transform, List<LeafPredicate>> leaves : leavesByTransform.entrySet()) {
75+
if (isOr) {
76+
resultChildren.addAll(leaves.getValue());
77+
continue;
78+
}
79+
80+
Transform transform = leaves.getKey();
81+
82+
// for children predicates of an AND, we only need to reserve
83+
// the largest GREATER_OR_EQUAL and the smallest LESS_OR_EQUAL
84+
// For example, for AND(a >= 1, a >= 2, a <= 3, a <= 4), we only need to reserve a >= 2
85+
// and a <= 3
86+
LeafPredicate gtePredicate = null;
87+
LeafPredicate ltePredicate = null;
88+
for (LeafPredicate leaf : leaves.getValue()) {
89+
if (leaf.function() instanceof GreaterOrEqual) {
90+
if (gtePredicate == null
91+
|| CompareUtils.compareLiteral(
92+
transform.outputType(),
93+
leaf.literals().get(0),
94+
gtePredicate.literals().get(0))
95+
> 0) {
96+
gtePredicate = leaf;
97+
}
98+
} else if (leaf.function() instanceof LessOrEqual) {
99+
if (ltePredicate == null
100+
|| CompareUtils.compareLiteral(
101+
transform.outputType(),
102+
leaf.literals().get(0),
103+
ltePredicate.literals().get(0))
104+
< 0) {
105+
ltePredicate = leaf;
106+
}
107+
} else {
108+
resultChildren.add(leaf);
109+
}
110+
}
111+
112+
boolean converted = false;
113+
if (gtePredicate != null && ltePredicate != null) {
114+
Optional<Predicate> betweenLeaf = convertToBetweenLeaf(gtePredicate, ltePredicate);
115+
if (betweenLeaf.isPresent()) {
116+
converted = true;
117+
resultChildren.add(betweenLeaf.get());
118+
}
119+
}
120+
if (!converted) {
121+
if (gtePredicate != null) {
122+
resultChildren.add(gtePredicate);
123+
}
124+
if (ltePredicate != null) {
125+
resultChildren.add(ltePredicate);
126+
}
127+
}
128+
}
129+
130+
return isOr ? PredicateBuilder.or(resultChildren) : PredicateBuilder.and(resultChildren);
131+
}
132+
133+
private static List<Predicate> flattenChildren(List<Predicate> children) {
134+
List<Predicate> result = new ArrayList<>();
135+
for (Predicate child : children) {
136+
if (child instanceof LeafPredicate) {
137+
result.add(child);
138+
} else {
139+
CompoundPredicate compoundPredicate = (CompoundPredicate) child;
140+
if (compoundPredicate.function() instanceof Or) {
141+
result.add(child);
142+
} else {
143+
result.addAll(flattenChildren(compoundPredicate.children()));
144+
}
145+
}
146+
}
147+
return result;
148+
}
149+
150+
/**
151+
* Convert child predicates of an AND to a BETWEEN leaf predicate. Return `Optional.empty()` if
152+
* not possible.
153+
*/
154+
public static Optional<Predicate> convertToBetweenLeaf(
155+
Predicate leftChild, Predicate rightChild) {
156+
if (leftChild instanceof LeafPredicate && rightChild instanceof LeafPredicate) {
157+
LeafPredicate left = (LeafPredicate) leftChild;
158+
LeafPredicate right = (LeafPredicate) rightChild;
159+
if (Objects.equals(left.transform(), right.transform())) {
160+
if (left.function() instanceof GreaterOrEqual
161+
&& right.function() instanceof LessOrEqual) {
162+
return createBetweenLeaf(left, right);
163+
} else if (left.function() instanceof LessOrEqual
164+
&& right.function() instanceof GreaterOrEqual) {
165+
return createBetweenLeaf(right, left);
166+
}
167+
}
168+
}
169+
170+
return Optional.empty();
171+
}
172+
173+
private static Optional<Predicate> createBetweenLeaf(
174+
LeafPredicate gtePredicate, LeafPredicate ltePredicate) {
175+
// gtePredicate and ltePredicate should have the same transform
176+
Transform transform = gtePredicate.transform();
177+
Object lbLiteral = gtePredicate.literals().get(0);
178+
Object ubLiteral = ltePredicate.literals().get(0);
179+
180+
if (CompareUtils.compareLiteral(transform.outputType(), lbLiteral, ubLiteral) > 0) {
181+
return Optional.empty();
182+
}
183+
184+
return Optional.of(
185+
new LeafPredicate(
186+
transform, Between.INSTANCE, Arrays.asList(lbLiteral, ubLiteral)));
187+
}
188+
}

‎paimon-common/src/test/java/org/apache/paimon/utils/PredicateUtilsTest.java‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.utils;
20+
21+
import org.apache.paimon.predicate.And;
22+
import org.apache.paimon.predicate.Between;
23+
import org.apache.paimon.predicate.CompoundPredicate;
24+
import org.apache.paimon.predicate.LeafPredicate;
25+
import org.apache.paimon.predicate.Or;
26+
import org.apache.paimon.predicate.Predicate;
27+
import org.apache.paimon.predicate.PredicateBuilder;
28+
import org.apache.paimon.types.IntType;
29+
import org.apache.paimon.types.RowType;
30+
31+
import org.junit.jupiter.api.Test;
32+
33+
import java.util.Arrays;
34+
35+
import static org.assertj.core.api.Assertions.assertThat;
36+
37+
/** Test for {@link PredicateUtils}. */
38+
public class PredicateUtilsTest {
39+
40+
@Test
41+
public void testTryRewriteBetweenPredicateBasic() {
42+
// Test basic case: AND(a>=1, a<=10, a is not null) should be rewritten to BETWEEN
43+
PredicateBuilder builder = new PredicateBuilder(RowType.of(new IntType()));
44+
Predicate gte = builder.greaterOrEqual(0, 1);
45+
Predicate lte = builder.lessOrEqual(0, 10);
46+
Predicate isNotNull = builder.isNotNull(0);
47+
48+
Predicate andPredicate = PredicateBuilder.and(gte, isNotNull, lte);
49+
Predicate result = PredicateUtils.tryRewriteBetweenPredicate(andPredicate);
50+
51+
assertThat(result).isInstanceOf(CompoundPredicate.class);
52+
CompoundPredicate compoundResult = (CompoundPredicate) result;
53+
assertThat(compoundResult.function()).isInstanceOf(And.class);
54+
assertThat(compoundResult.children()).hasSize(2);
55+
56+
Predicate betweenChild = compoundResult.children().get(1);
57+
assertThat(betweenChild).isInstanceOf(LeafPredicate.class);
58+
LeafPredicate betweenLeaf = (LeafPredicate) betweenChild;
59+
assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
60+
assertThat(betweenLeaf.literals()).containsExactly(1, 10);
61+
62+
Predicate notNullChild = compoundResult.children().get(0);
63+
assertThat(notNullChild).isInstanceOf(LeafPredicate.class);
64+
assertThat(notNullChild.toString()).contains("IsNotNull");
65+
}
66+
67+
@Test
68+
public void testTryRewriteBetweenPredicateRecursive() {
69+
// Test recursive case: OR(b>=1, AND(a>=1, a<=10, a is not null)) should rewrite nested AND
70+
PredicateBuilder builder = new PredicateBuilder(RowType.of(new IntType(), new IntType()));
71+
72+
Predicate gteB = builder.greaterOrEqual(1, 1);
73+
Predicate gteA = builder.greaterOrEqual(0, 1);
74+
Predicate lteA = builder.lessOrEqual(0, 10);
75+
Predicate isNotNullA = builder.isNotNull(0);
76+
Predicate andPredicate = PredicateBuilder.and(gteA, isNotNullA, lteA);
77+
Predicate orPredicate = PredicateBuilder.or(gteB, andPredicate);
78+
79+
Predicate result = PredicateUtils.tryRewriteBetweenPredicate(orPredicate);
80+
81+
assertThat(result).isInstanceOf(CompoundPredicate.class);
82+
CompoundPredicate compoundResult = (CompoundPredicate) result;
83+
assertThat(compoundResult.function()).isInstanceOf(Or.class);
84+
assertThat(compoundResult.children()).hasSize(2);
85+
86+
Predicate secondChild = compoundResult.children().get(1);
87+
assertThat(secondChild).isInstanceOf(LeafPredicate.class);
88+
assertThat(secondChild.toString()).contains("GreaterOrEqual");
89+
90+
Predicate firstChild = compoundResult.children().get(0);
91+
assertThat(firstChild).isInstanceOf(CompoundPredicate.class);
92+
CompoundPredicate innerAnd = (CompoundPredicate) firstChild;
93+
assertThat(innerAnd.function()).isInstanceOf(And.class);
94+
assertThat(innerAnd.children()).hasSize(2);
95+
96+
Predicate betweenCandidate = innerAnd.children().get(1);
97+
assertThat(betweenCandidate).isInstanceOf(LeafPredicate.class);
98+
LeafPredicate betweenLeaf = (LeafPredicate) betweenCandidate;
99+
assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
100+
assertThat(betweenLeaf.literals()).containsExactly(1, 10);
101+
}
102+
103+
/**
104+
* Test this complicated scenario:
105+
*
106+
* <pre>{@code
107+
* AND
108+
* / | \
109+
* OR AND a>=1
110+
* /| || \
111+
* / | / | \
112+
* a>=1 a<=2 OR AND a>=2
113+
* / | | \
114+
* / | | \
115+
* a>=1 b<2 b>=1 a<=10
116+
*
117+
* }</pre>
118+
*/
119+
@Test
120+
public void testAnExtremeComplicatedPredicate() {
121+
PredicateBuilder builder = new PredicateBuilder(RowType.of(new IntType(), new IntType()));
122+
Predicate l3p1 = builder.greaterOrEqual(0, 1);
123+
Predicate l3p2 = builder.lessThan(1, 2);
124+
Predicate l3p3 = builder.greaterOrEqual(1, 1);
125+
Predicate l3p4 = builder.lessOrEqual(0, 10);
126+
Predicate l2p1 = builder.greaterOrEqual(0, 1);
127+
Predicate l2p2 = builder.lessOrEqual(1, 2);
128+
Predicate l2p3 = PredicateBuilder.or(l3p1, l3p2);
129+
Predicate l2p4 = PredicateBuilder.and(l3p3, l3p4);
130+
Predicate l2p5 = builder.greaterOrEqual(0, 2);
131+
Predicate l1p1 = PredicateBuilder.or(l2p1, l2p2);
132+
Predicate l1p2 = PredicateBuilder.and(l2p3, l2p4, l2p5);
133+
Predicate l1p3 = builder.greaterOrEqual(0, 1);
134+
Predicate root = PredicateBuilder.and(l1p1, l1p2, l1p3);
135+
136+
Predicate result = PredicateUtils.tryRewriteBetweenPredicate(root);
137+
assertThat(result).isInstanceOf(CompoundPredicate.class);
138+
139+
CompoundPredicate compoundResult = (CompoundPredicate) result;
140+
assertThat(compoundResult.function()).isInstanceOf(And.class);
141+
142+
// directly check the toString
143+
String resultString = compoundResult.toString();
144+
assertThat(resultString).contains("Between(f0, [2, 10])");
145+
}
146+
147+
@Test
148+
public void testTryRewriteBetweenPredicateIntersection() {
149+
// Test intersection case: AND(a>=1, a<=10, a>=2, a<=7) should use intersection (2, 7)
150+
PredicateBuilder builder = new PredicateBuilder(RowType.of(new IntType()));
151+
152+
Predicate gte1 = builder.greaterOrEqual(0, 1);
153+
Predicate lte10 = builder.lessOrEqual(0, 10);
154+
Predicate gte2 = builder.greaterOrEqual(0, 2);
155+
Predicate lte7 = builder.lessOrEqual(0, 7);
156+
157+
Predicate predicate =
158+
PredicateBuilder.and(
159+
PredicateBuilder.and(gte1, lte10), PredicateBuilder.and(gte2, lte7));
160+
Predicate result = PredicateUtils.tryRewriteBetweenPredicate(predicate);
161+
162+
assertThat(result).isInstanceOf(LeafPredicate.class);
163+
LeafPredicate betweenLeaf = (LeafPredicate) result;
164+
assertThat(betweenLeaf.function()).isInstanceOf(Between.class);
165+
assertThat(betweenLeaf.literals()).containsExactly(2, 7);
166+
}
167+
168+
@Test
169+
public void testTryRewriteBetweenPredicateDifferentColumns() {
170+
// Test different columns case: AND(a>=1, b<=10) should not be rewritten
171+
PredicateBuilder builder = new PredicateBuilder(RowType.of(new IntType(), new IntType()));
172+
173+
Predicate gteA = builder.greaterOrEqual(0, 1);
174+
Predicate lteB = builder.lessOrEqual(1, 10);
175+
Predicate predicate = PredicateBuilder.and(gteA, lteB);
176+
177+
Predicate result = PredicateUtils.tryRewriteBetweenPredicate(predicate);
178+
179+
assertThat(result).isInstanceOf(CompoundPredicate.class);
180+
CompoundPredicate compoundResult = (CompoundPredicate) result;
181+
assertThat(compoundResult.function()).isInstanceOf(And.class);
182+
assertThat(compoundResult.children()).hasSize(2);
183+
assertThat(compoundResult.children().stream().map(Predicate::toString))
184+
.containsExactlyInAnyOrderElementsOf(
185+
Arrays.asList("GreaterOrEqual(f0, 1)", "LessOrEqual(f1, 10)"));
186+
}
187+
188+
@Test
189+
public void testTryRewriteBetweenPredicateInvalidRange() {
190+
// Test invalid range case: AND(a>=10, a<=1) should not be rewritten to BETWEEN
191+
PredicateBuilder builder = new PredicateBuilder(RowType.of(new IntType()));
192+
193+
Predicate gte = builder.greaterOrEqual(0, 10);
194+
Predicate lte = builder.lessOrEqual(0, 1);
195+
Predicate predicate = PredicateBuilder.and(gte, lte);
196+
197+
Predicate result = PredicateUtils.tryRewriteBetweenPredicate(predicate);
198+
199+
assertThat(result).isInstanceOf(CompoundPredicate.class);
200+
CompoundPredicate compoundResult = (CompoundPredicate) result;
201+
assertThat(compoundResult.function()).isInstanceOf(And.class);
202+
assertThat(compoundResult.children()).hasSize(2);
203+
assertThat(compoundResult.children().stream().map(Predicate::toString))
204+
.containsExactlyInAnyOrderElementsOf(
205+
Arrays.asList("GreaterOrEqual(f0, 10)", "LessOrEqual(f0, 1)"));
206+
}
207+
}

0 commit comments

Comments
 (0)