Skip to content

[fix] (mtmv) Fix topN rewrite by materialized view fail and cause rewrite result wrong #58406

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Dec 3, 2025
Merged

[fix] (mtmv) Fix topN rewrite by materialized view fail and cause rewrite result wrong #58406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
22e3b58
[fix] (mtmv) Fix topN rewrite by materialized view fail and cause rew…
seawinde Nov 26, 2025
7b486ff
add test
seawinde Nov 26, 2025
c369920
fix test
seawinde Nov 26, 2025
bd412f1
fix code
seawinde Nov 27, 2025
24a868b
fix regression test
seawinde Dec 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1066,9 +1066,9 @@ protected Plan tryRewriteTopN(LogicalTopN<Plan> queryTopNode, LogicalTopN<Plan>
// check the order keys of TopN between query and view is consistent
List<OrderKey> queryOrderKeys = queryTopNode.getOrderKeys();
List<OrderKey> viewOrderKeys = viewTopNode.getOrderKeys();
if (queryOrderKeys.size() != viewOrderKeys.size()) {
if (queryOrderKeys.size() > viewOrderKeys.size()) {
materializationContext.recordFailReason(queryStructInfo,
"query topN order keys size is not consistent with view topN order keys size",
"query topN order keys size is bigger than view topN order keys size",
() -> String.format("query topN order keys = %s,\n view topN order keys = %s,\n",
queryOrderKeys, viewOrderKeys));
return null;
Expand Down Expand Up @@ -1096,7 +1096,7 @@ protected Plan tryRewriteTopN(LogicalTopN<Plan> queryTopNode, LogicalTopN<Plan>
viewShuttledOrderKeys.add(new OrderKey(viewOrderByExpressionsQueryBasedSet.get(j), viewOrderKey.isAsc(),
viewOrderKey.isNullFirst()));
}
if (!queryShuttledOrderKeys.equals(viewShuttledOrderKeys)) {
if (!MaterializedViewUtils.isPrefixSameFromStart(queryShuttledOrderKeys, viewShuttledOrderKeys)) {
materializationContext.recordFailReason(queryStructInfo,
"view topN order key doesn't match query order key",
() -> String.format("queryShuttledOrderKeys = %s,\n viewShuttledOrderKeys = %s,\n",
Expand Down
4 changes: 3 additions & 1 deletion .../java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewScanRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) {
PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
&& !checkContext.isContainsTopAggregate();
&& !checkContext.isContainsTopAggregate()
&& !checkContext.isContainsTopLimit() && !checkContext.isContainsTopTopN()
&& !checkContext.isContainsTopWindow();
}
}
8 changes: 6 additions & 2 deletions ...ache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,17 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon
// any check result of join or scan is true, then return true
PlanCheckContext joinCheckContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
boolean joinCheckResult = structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, joinCheckContext)
&& !joinCheckContext.isContainsTopAggregate();
&& !joinCheckContext.isContainsTopAggregate()
&& !joinCheckContext.isContainsTopLimit() && !joinCheckContext.isContainsTopTopN()
&& !joinCheckContext.isContainsTopWindow();
if (joinCheckResult) {
return true;
}
PlanCheckContext scanCheckContext = PlanCheckContext.of(ImmutableSet.of());
return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, scanCheckContext)
&& !scanCheckContext.isContainsTopAggregate();
&& !scanCheckContext.isContainsTopAggregate()
&& !joinCheckContext.isContainsTopLimit() && !joinCheckContext.isContainsTopTopN()
&& !joinCheckContext.isContainsTopWindow();
}

@Override
Expand Down
1 change: 1 addition & 0 deletions ...ava/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitAggregateRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
&& checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() == 1
&& !checkContext.isContainsTopTopN()
&& !checkContext.isContainsTopWindow()
&& checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1;
}

Expand Down
1 change: 1 addition & 0 deletions ...ain/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitJoinRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
&& !checkContext.isContainsTopAggregate()
&& !checkContext.isContainsTopTopN()
&& !checkContext.isContainsTopWindow()
&& checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1;
}

Expand Down
6 changes: 6 additions & 0 deletions ...ain/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewLimitScanRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,15 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext)
&& !checkContext.isContainsTopAggregate()
&& !checkContext.isContainsTopTopN()
&& !checkContext.isContainsTopWindow()
&& checkContext.isContainsTopLimit() && checkContext.getTopLimitNum() == 1;
}

@Override
protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) {
return checkQueryPattern(structInfo, cascadesContext);
}

@Override
public List<Rule> buildRules() {
return ImmutableList.of(
Expand Down
1 change: 1 addition & 0 deletions ...java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNAggregateRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
&& checkContext.isContainsTopAggregate() && checkContext.getTopAggregateNum() == 1
&& !checkContext.isContainsTopLimit()
&& !checkContext.isContainsTopWindow()
&& checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1;
}

Expand Down
1 change: 1 addition & 0 deletions ...main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNJoinRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
return accept
&& !checkContext.isContainsTopAggregate()
&& !checkContext.isContainsTopLimit()
&& !checkContext.isContainsTopWindow()
&& checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1;
}

Expand Down
6 changes: 6 additions & 0 deletions ...main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewTopNScanRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,15 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
return accept
&& !checkContext.isContainsTopAggregate()
&& !checkContext.isContainsTopLimit()
&& !checkContext.isContainsTopWindow()
&& checkContext.isContainsTopTopN() && checkContext.getTopTopNNum() == 1;
}

@Override
protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) {
return checkQueryPattern(structInfo, cascadesContext);
}

@Override
public List<Rule> buildRules() {
return ImmutableList.of(
Expand Down
20 changes: 20 additions & 0 deletions ...re/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.StructInfoMap;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.analysis.BindRelation;
import org.apache.doris.nereids.rules.exploration.mv.PartitionIncrementMaintainer.PartitionIncrementCheckContext;
Expand Down Expand Up @@ -651,4 +652,23 @@ public Boolean visit(Plan plan, Void context) {
return false;
}
}

/**
* Check the prefix of two order key list is same from start
*/
public static boolean isPrefixSameFromStart(List<OrderKey> queryShuttledOrderKeys,
List<OrderKey> viewShuttledOrderKeys) {
if (queryShuttledOrderKeys == null || viewShuttledOrderKeys == null) {
return false;
}
if (queryShuttledOrderKeys.size() > viewShuttledOrderKeys.size()) {
return false;
}
for (int i = 0; i < queryShuttledOrderKeys.size(); i++) {
if (!java.util.Objects.equals(queryShuttledOrderKeys.get(i), viewShuttledOrderKeys.get(i))) {
return false;
}
}
return true;
}
}
3 changes: 2 additions & 1 deletion ...va/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowAggregateRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
&& checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow()
&& checkContext.getTopAggregateNum() <= 1 && checkContext.getTopWindowNum() <= 1
&& !checkContext.isWindowUnderAggregate();
&& !checkContext.isWindowUnderAggregate()
&& !checkContext.isContainsTopTopN() && !checkContext.isContainsTopLimit();
}

@Override
Expand Down
3 changes: 2 additions & 1 deletion ...in/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowJoinRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
&& !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow()
&& checkContext.getTopWindowNum() <= 1;
&& checkContext.getTopWindowNum() <= 1
&& !checkContext.isContainsTopTopN() && !checkContext.isContainsTopLimit();
}

@Override
Expand Down
3 changes: 2 additions & 1 deletion ...in/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewWindowScanRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext casca
PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of());
return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext)
&& !checkContext.isContainsTopAggregate() && checkContext.isContainsTopWindow()
&& checkContext.getTopWindowNum() <= 1;
&& checkContext.getTopWindowNum() <= 1
&& !checkContext.isContainsTopTopN() && !checkContext.isContainsTopLimit();
}

@Override
Expand Down
75 changes: 75 additions & 0 deletions ...rc/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,19 @@
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.rules.exploration.mv.RelatedTableInfo.RelatedTableColumnInfo;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;

import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.List;

/**
* Test for materialized view util
*/
Expand Down Expand Up @@ -952,6 +957,76 @@ public void getRelatedTableInfoWhenMultiPartitionExprs() {
});
}

// java
@Test
public void isPrefixSameFromStartNullAndEmptyTests() {
// both null
Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(null, null));
// query null
List<OrderKey> view = ImmutableList.of(
new OrderKey(new VarcharLiteral("a"), true, false)
);
Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(null, view));
// view null
List<OrderKey> query = ImmutableList.of(
new OrderKey(new VarcharLiteral("a"), true, false)
);
Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(query, null));
List<OrderKey> emptyQuery = ImmutableList.of();
List<OrderKey> emptyView = ImmutableList.of();
Assertions.assertTrue(MaterializedViewUtils.isPrefixSameFromStart(emptyQuery, emptyView));
}

@Test
public void isPrefixSameFromStart_queryLongerThanView() {
List<OrderKey> query = ImmutableList.of(
new OrderKey(new VarcharLiteral("a"), true, false),
new OrderKey(new VarcharLiteral("b"), true, false)
);
List<OrderKey> view = ImmutableList.of(
new OrderKey(new VarcharLiteral("a"), true, false)
);
Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(query, view));
}

@Test
public void isPrefixSameFromStart_emptyQueryIsPrefix() {
List<OrderKey> emptyQuery = ImmutableList.of();
List<OrderKey> view = ImmutableList.of(
new OrderKey(new VarcharLiteral("x"), true, false),
new OrderKey(new VarcharLiteral("y"), true, false)
);
Assertions.assertTrue(MaterializedViewUtils.isPrefixSameFromStart(emptyQuery, view));
}

@Test
public void isPrefixSameFromStart_prefixEqualReturnsTrue() {
List<OrderKey> query = ImmutableList.of(
new OrderKey(new VarcharLiteral("a"), true, false),
new OrderKey(new VarcharLiteral("b"), true, false)
);
List<OrderKey> view = ImmutableList.of(
new OrderKey(new VarcharLiteral("a"), true, false),
new OrderKey(new VarcharLiteral("b"), true, false),
new OrderKey(new VarcharLiteral("c"), true, false)
);
Assertions.assertTrue(MaterializedViewUtils.isPrefixSameFromStart(query, view));
}

@Test
public void isPrefixSameFromStart_prefixNotEqualReturnsFalse() {
List<OrderKey> query = ImmutableList.of(
new OrderKey(new VarcharLiteral("a"), true, false),
new OrderKey(new VarcharLiteral("x"), true, false)
);
List<OrderKey> view = ImmutableList.of(
new OrderKey(new VarcharLiteral("a"), true, false),
new OrderKey(new VarcharLiteral("b"), true, false),
new OrderKey(new VarcharLiteral("c"), true, false)
);
Assertions.assertFalse(MaterializedViewUtils.isPrefixSameFromStart(query, view));
}

private void checkRelatedTableInfo(RelatedTableInfo relatedTableInfo,
String expectTableName,
String expectColumnName,
Expand Down
22 changes: 22 additions & 0 deletions regression-test/data/nereids_rules_p0/mv/topN/topN_rewrite.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,28 @@
2023-12-12 1 1 5 1
2023-12-13 1 1 6 1

-- !query1_3_before --
2023-12-10 2 3
2023-12-11 3 4
2023-12-12 2 5
2023-12-13 2 6

-- !query1_3_after --
2023-12-10 2 3
2023-12-11 3 4
2023-12-12 2 5
2023-12-13 2 6

-- !query1_4_before --
2023-12-11 3 4
2023-12-12 2 5
2023-12-13 2 6

-- !query1_4_after --
2023-12-11 3 4
2023-12-12 2 5
2023-12-13 2 6

-- !query1_1_before --
2023-12-12 1 1 5 1
2023-12-13 1 1 6 1
Expand Down
9 changes: 6 additions & 3 deletions regression-test/suites/nereids_rules_p0/mv/limit/limit_rewrite.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -580,7 +580,8 @@ suite("limit_rewrite") {
where o_orderdate > '2023-12-08'
limit 2 offset 5;
"""
async_mv_rewrite_success(db, mv5_1, query5_1, "mv5_1")
// mv data can not cover query limit offset, should fail
async_mv_rewrite_fail(db, mv5_1, query5_1, "mv5_1")
sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_1"""


Expand All @@ -606,7 +607,8 @@ suite("limit_rewrite") {
where o_orderdate > '2023-12-09'
limit 4 offset 2;
"""
async_mv_rewrite_success(db, mv5_2, query5_2, "mv5_2")
// mv data can not cover query limit offset, should fail
async_mv_rewrite_fail(db, mv5_2, query5_2, "mv5_2")
sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_2"""


Expand Down Expand Up @@ -657,7 +659,8 @@ suite("limit_rewrite") {
orders
limit 2 offset 5;
"""
async_mv_rewrite_success(db, mv6_1, query6_1, "mv6_1")
// mv data can not cover query limit offset, should fail
async_mv_rewrite_fail(db, mv6_1, query6_1, "mv6_1")
sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1"""

}
Loading
Loading