Skip to content

Commit d9b1353

Browse files
Jibing-LiJibing-Li
committed
Agg table set preagg on when doing sample analyzing.
1 parent 14c4945 commit d9b1353

File tree

20 files changed

+659
-97
lines changed

20 files changed

+659
-97
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ public enum TableFrom {
158158
private final List<Expression> joinFilters = new ArrayList<>();
159159

160160
private final List<Hint> hints = new ArrayList<>();
161+
private boolean hintForcePreAggOn = false;
161162

162163
// the columns in Plan.getExpressions(), such as columns in join condition or filter condition, group by expression
163164
private final Set<SlotReference> keySlots = Sets.newHashSet();
@@ -252,6 +253,14 @@ public void setNeedLockTables(boolean needLockTables) {
252253
this.needLockTables = needLockTables;
253254
}
254255

256+
public void setHintForcePreAggOn(boolean preAggOn) {
257+
this.hintForcePreAggOn = preAggOn;
258+
}
259+
260+
public boolean isHintForcePreAggOn() {
261+
return hintForcePreAggOn;
262+
}
263+
255264
/**
256265
* cache view info to avoid view's def and sql mode changed before lock it.
257266
*

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.doris.nereids.rules.analysis.CompressedMaterialize;
3333
import org.apache.doris.nereids.rules.analysis.EliminateDistinctConstant;
3434
import org.apache.doris.nereids.rules.analysis.EliminateGroupByConstant;
35+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalCommonHint;
3536
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
3637
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
3738
import org.apache.doris.nereids.rules.analysis.FillUpQualifyMissingSlot;
@@ -97,7 +98,8 @@ private static List<RewriteJob> buildAnalyzerJobs() {
9798
return jobs(
9899
// we should eliminate hint before "Subquery unnesting".
99100
topDown(new AnalyzeCTE()),
100-
topDown(new EliminateLogicalSelectHint()),
101+
topDown(new EliminateLogicalSelectHint(),
102+
new EliminateLogicalCommonHint()),
101103
bottomUp(
102104
new BindRelation(),
103105
new CheckPolicy()

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 86 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -817,6 +817,7 @@
817817
import org.apache.doris.nereids.trees.plans.commands.use.UseCommand;
818818
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
819819
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
820+
import org.apache.doris.nereids.trees.plans.logical.LogicalCommonHint;
820821
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
821822
import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink;
822823
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -2029,12 +2030,15 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
20292030
return selectPlan;
20302031
}
20312032
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
2033+
List<ParserRuleContext> commonHintContexts = Lists.newArrayList();
20322034
for (Integer key : selectHintMap.keySet()) {
20332035
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
20342036
selectHintContexts.add(selectHintMap.get(key));
2037+
} else {
2038+
commonHintContexts.add(selectHintMap.get(key));
20352039
}
20362040
}
2037-
return withSelectHint(selectPlan, selectHintContexts);
2041+
return withHints(selectPlan, selectHintContexts, commonHintContexts);
20382042
});
20392043
}
20402044

@@ -3690,82 +3694,97 @@ private List<List<String>> getTableList(List<MultipartIdentifierContext> ctx) {
36903694
return tableList;
36913695
}
36923696

3693-
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
3694-
if (hintContexts.isEmpty()) {
3697+
private LogicalPlan withHints(LogicalPlan logicalPlan, List<ParserRuleContext> selectHintContexts,
3698+
List<ParserRuleContext> commonHintContexts) {
3699+
if (selectHintContexts.isEmpty() && commonHintContexts.isEmpty()) {
36953700
return logicalPlan;
36963701
}
3697-
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
3698-
for (ParserRuleContext hintContext : hintContexts) {
3699-
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3700-
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3701-
if (hintStatement.USE_MV() != null) {
3702-
hints.add(new SelectHintUseMv("USE_MV", getTableList(hintStatement.tableList), true));
3703-
continue;
3704-
} else if (hintStatement.NO_USE_MV() != null) {
3705-
hints.add(new SelectHintUseMv("NO_USE_MV", getTableList(hintStatement.tableList), false));
3706-
continue;
3707-
}
3708-
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3709-
switch (hintName) {
3710-
case "set_var":
3711-
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3712-
for (HintAssignmentContext kv : hintStatement.parameters) {
3713-
if (kv.key != null) {
3714-
String parameterName = visitIdentifierOrText(kv.key);
3715-
Optional<String> value = Optional.empty();
3716-
if (kv.constantValue != null) {
3717-
Literal literal = (Literal) visit(kv.constantValue);
3718-
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3719-
} else if (kv.identifierValue != null) {
3720-
// maybe we should throw exception when the identifierValue is quoted identifier
3721-
value = Optional.ofNullable(kv.identifierValue.getText());
3702+
LogicalPlan newPlan = logicalPlan;
3703+
if (!selectHintContexts.isEmpty()) {
3704+
ImmutableList.Builder<SelectHint> hints = ImmutableList.builder();
3705+
for (ParserRuleContext hintContext : selectHintContexts) {
3706+
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3707+
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3708+
if (hintStatement.USE_MV() != null) {
3709+
hints.add(new SelectHintUseMv("USE_MV", getTableList(hintStatement.tableList), true));
3710+
continue;
3711+
} else if (hintStatement.NO_USE_MV() != null) {
3712+
hints.add(new SelectHintUseMv("NO_USE_MV", getTableList(hintStatement.tableList), false));
3713+
continue;
3714+
}
3715+
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3716+
switch (hintName) {
3717+
case "set_var":
3718+
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3719+
for (HintAssignmentContext kv : hintStatement.parameters) {
3720+
if (kv.key != null) {
3721+
String parameterName = visitIdentifierOrText(kv.key);
3722+
Optional<String> value = Optional.empty();
3723+
if (kv.constantValue != null) {
3724+
Literal literal = (Literal) visit(kv.constantValue);
3725+
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3726+
} else if (kv.identifierValue != null) {
3727+
// maybe we should throw exception when the identifierValue is quoted identifier
3728+
value = Optional.ofNullable(kv.identifierValue.getText());
3729+
}
3730+
parameters.put(parameterName, value);
37223731
}
3723-
parameters.put(parameterName, value);
37243732
}
3725-
}
3726-
SelectHintSetVar setVar = new SelectHintSetVar(hintName, parameters);
3727-
setVar.setVarOnceInSql(ConnectContext.get().getStatementContext());
3728-
hints.add(setVar);
3729-
break;
3730-
case "leading":
3731-
List<String> leadingParameters = new ArrayList<>();
3732-
for (HintAssignmentContext kv : hintStatement.parameters) {
3733-
if (kv.key != null) {
3734-
String parameterName = visitIdentifierOrText(kv.key);
3735-
leadingParameters.add(parameterName);
3733+
SelectHintSetVar setVar = new SelectHintSetVar(hintName, parameters);
3734+
setVar.setVarOnceInSql(ConnectContext.get().getStatementContext());
3735+
hints.add(setVar);
3736+
break;
3737+
case "leading":
3738+
List<String> leadingParameters = new ArrayList<>();
3739+
for (HintAssignmentContext kv : hintStatement.parameters) {
3740+
if (kv.key != null) {
3741+
String parameterName = visitIdentifierOrText(kv.key);
3742+
leadingParameters.add(parameterName);
3743+
}
37363744
}
3737-
}
3738-
hints.add(new SelectHintLeading(hintName, leadingParameters));
3739-
break;
3740-
case "ordered":
3741-
hints.add(new SelectHintOrdered(hintName));
3742-
break;
3743-
case "use_cbo_rule":
3744-
List<String> useRuleParameters = new ArrayList<>();
3745-
for (HintAssignmentContext kv : hintStatement.parameters) {
3746-
if (kv.key != null) {
3747-
String parameterName = visitIdentifierOrText(kv.key);
3748-
useRuleParameters.add(parameterName);
3745+
hints.add(new SelectHintLeading(hintName, leadingParameters));
3746+
break;
3747+
case "ordered":
3748+
hints.add(new SelectHintOrdered(hintName));
3749+
break;
3750+
case "use_cbo_rule":
3751+
List<String> useRuleParameters = new ArrayList<>();
3752+
for (HintAssignmentContext kv : hintStatement.parameters) {
3753+
if (kv.key != null) {
3754+
String parameterName = visitIdentifierOrText(kv.key);
3755+
useRuleParameters.add(parameterName);
3756+
}
37493757
}
3750-
}
3751-
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
3752-
break;
3753-
case "no_use_cbo_rule":
3754-
List<String> noUseRuleParameters = new ArrayList<>();
3755-
for (HintAssignmentContext kv : hintStatement.parameters) {
3756-
String parameterName = visitIdentifierOrText(kv.key);
3757-
if (kv.key != null) {
3758-
noUseRuleParameters.add(parameterName);
3758+
hints.add(new SelectHintUseCboRule(hintName, useRuleParameters, false));
3759+
break;
3760+
case "no_use_cbo_rule":
3761+
List<String> noUseRuleParameters = new ArrayList<>();
3762+
for (HintAssignmentContext kv : hintStatement.parameters) {
3763+
String parameterName = visitIdentifierOrText(kv.key);
3764+
if (kv.key != null) {
3765+
noUseRuleParameters.add(parameterName);
3766+
}
37593767
}
3760-
}
3761-
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3762-
break;
3763-
default:
3764-
break;
3768+
hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3769+
break;
3770+
default:
3771+
break;
3772+
}
3773+
}
3774+
}
3775+
newPlan = new LogicalSelectHint<>(hints.build(), newPlan);
3776+
}
3777+
if (!commonHintContexts.isEmpty()) {
3778+
for (ParserRuleContext hintContext : commonHintContexts) {
3779+
SelectHintContext commonHintContext = (SelectHintContext) hintContext;
3780+
String text = commonHintContext.hintStatement.hintName.getText();
3781+
if (text.equalsIgnoreCase("PREAGGOPEN")) {
3782+
newPlan = new LogicalCommonHint<>(newPlan);
3783+
break;
37653784
}
37663785
}
37673786
}
3768-
return new LogicalSelectHint<>(hints.build(), logicalPlan);
3787+
return newPlan;
37693788
}
37703789

37713790
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ public enum RuleType {
138138
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
139139

140140
ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
141+
ELIMINATE_LOGICAL_COMMON_HINT(RuleTypeClass.REWRITE),
141142
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
142143
ELIMINATE_ORDER_BY_UNDER_SUBQUERY(RuleTypeClass.REWRITE),
143144
ELIMINATE_ORDER_BY_UNDER_VIEW(RuleTypeClass.REWRITE),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@
9696
import com.google.common.collect.Lists;
9797
import com.google.common.collect.Sets;
9898
import org.apache.commons.collections.CollectionUtils;
99+
import org.apache.logging.log4j.LogManager;
100+
import org.apache.logging.log4j.Logger;
99101

100102
import java.util.ArrayList;
101103
import java.util.List;
@@ -105,6 +107,7 @@
105107
* Rule to bind relations in query plan.
106108
*/
107109
public class BindRelation extends OneAnalysisRuleFactory {
110+
private static final Logger LOG = LogManager.getLogger(StatementContext.class);
108111

109112
public BindRelation() {}
110113

@@ -179,7 +182,8 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
179182
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
180183
}
181184

182-
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier) {
185+
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier,
186+
CascadesContext cascadesContext) {
183187
LogicalOlapScan scan;
184188
List<Long> partIds = getPartitionIds(table, unboundRelation, qualifier);
185189
List<Long> tabletIds = unboundRelation.getTabletIds();
@@ -217,6 +221,9 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
217221
// This tabletIds is set manually, so need to set specifiedTabletIds
218222
scan = scan.withManuallySpecifiedTabletIds(tabletIds);
219223
}
224+
if (cascadesContext.getStatementContext().isHintForcePreAggOn()) {
225+
return scan.withPreAggStatus(PreAggStatus.on());
226+
}
220227
if (needGenerateLogicalAggForRandomDistAggTable(scan)) {
221228
// it's a random distribution agg table
222229
// add agg on olap scan
@@ -383,7 +390,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
383390
switch (table.getType()) {
384391
case OLAP:
385392
case MATERIALIZED_VIEW:
386-
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName);
393+
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName, cascadesContext);
387394
case VIEW:
388395
View view = (View) table;
389396
isView = true;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalCommonHint.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.analysis;
19+
20+
import org.apache.doris.nereids.rules.Rule;
21+
import org.apache.doris.nereids.rules.RuleType;
22+
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
23+
import org.apache.doris.nereids.trees.plans.Plan;
24+
import org.apache.doris.nereids.trees.plans.logical.LogicalCommonHint;
25+
26+
import com.google.common.base.Preconditions;
27+
28+
/**
29+
* eliminate logical common hint and set them to cascade context
30+
*/
31+
public class EliminateLogicalCommonHint extends OneRewriteRuleFactory {
32+
33+
@Override
34+
public Rule build() {
35+
return logicalCommonHint().thenApply(ctx -> {
36+
LogicalCommonHint<Plan> commonHintPlan = ctx.root;
37+
Preconditions.checkArgument(commonHintPlan.preAggEnabledByHint(), "common hint only support PREAGGOPEN");
38+
ctx.statementContext.setHintForcePreAggOn(true);
39+
return commonHintPlan.child();
40+
}).toRule(RuleType.ELIMINATE_LOGICAL_SELECT_HINT);
41+
}
42+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public enum PlanType {
6262
LOGICAL_APPLY,
6363
LOGICAL_ASSERT_NUM_ROWS,
6464
LOGICAL_CHECK_POLICY,
65+
LOGICAL_COMMON_HINT,
6566
LOGICAL_CTE,
6667
LOGICAL_CTE_ANCHOR,
6768
LOGICAL_CTE_PRODUCER,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.doris.nereids.rules.analysis.BindExpression;
4242
import org.apache.doris.nereids.rules.analysis.BindRelation;
4343
import org.apache.doris.nereids.rules.analysis.CheckPolicy;
44+
import org.apache.doris.nereids.rules.analysis.EliminateLogicalCommonHint;
4445
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
4546
import org.apache.doris.nereids.trees.expressions.Expression;
4647
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -263,7 +264,8 @@ public List<RewriteJob> getJobs() {
263264

264265
private static List<RewriteJob> buildAnalyzeViewJobsForStar() {
265266
return jobs(
266-
topDown(new EliminateLogicalSelectHint()),
267+
topDown(new EliminateLogicalSelectHint(),
268+
new EliminateLogicalCommonHint()),
267269
topDown(new AnalyzeCTE()),
268270
bottomUp(
269271
new BindRelation(),

0 commit comments

Comments
 (0)