Skip to content

HIVE-29612: LATERAL VIEW OUTER doesn't work with CBO enabled #6482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
soumyakanti3578 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

HIVE-29612: LATERAL VIEW OUTER doesn't work with CBO enabled #6482

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f2ec801
HIVE-29612: LATERAL VIEW OUTER doesn't work with CBO enabled
soumyakanti3578 May 13, 2026
da242cd
Refactor to address review comments, and update isCBOSupportedLateral…
soumyakanti3578 May 15, 2026
544b33f
Refactor again to capture missing calls to isASTNodeLateralViewOrOuter
soumyakanti3578 May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 33 additions & 3 deletions .../java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableFunctionScan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelInput;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.TableFunctionScan;
import org.apache.calcite.rel.metadata.RelColumnMapping;
import org.apache.calcite.rel.type.RelDataType;
Expand All @@ -34,6 +35,9 @@

public class HiveTableFunctionScan extends TableFunctionScan implements HiveRelNode {

// Whether this is a LATERAL VIEW OUTER
private final boolean outer;

/**
* @param cluster
* cluster - Cluster that this relational expression belongs to
Expand All @@ -49,28 +53,54 @@ public class HiveTableFunctionScan extends TableFunctionScan implements HiveRelN
* rowType - Row type produced by function
* @param columnMappings
* columnMappings - Column mappings associated with this function
* @param outer
* outer - true if this is a LATERAL VIEW OUTER
*/
protected HiveTableFunctionScan(RelOptCluster cluster, RelTraitSet traitSet, List<RelNode> inputs,
RexNode rexCall, Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings) {
RexNode rexCall, Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings,
boolean outer) {
super(cluster, traitSet, inputs, rexCall, elementType, rowType, columnMappings);
this.outer = outer;
}

public HiveTableFunctionScan(RelInput input) {
super(input);
this.outer = input.getBoolean("outer", false);
}

public static HiveTableFunctionScan create(RelOptCluster cluster, RelTraitSet traitSet,
List<RelNode> inputs, RexNode rexCall, Type elementType, RelDataType rowType,
Set<RelColumnMapping> columnMappings) throws CalciteSemanticException {
return new HiveTableFunctionScan(cluster, traitSet, inputs, rexCall, elementType, rowType,
columnMappings);
columnMappings, false);
}

public static HiveTableFunctionScan create(RelOptCluster cluster, RelTraitSet traitSet,
List<RelNode> inputs, RexNode rexCall, Type elementType, RelDataType rowType,
Set<RelColumnMapping> columnMappings, boolean outer) throws CalciteSemanticException {
return new HiveTableFunctionScan(cluster, traitSet, inputs, rexCall, elementType, rowType,
columnMappings, outer);
}

/** Returns true if this represents a LATERAL VIEW OUTER. */
public boolean isOuter() {
return outer;
}

@Override
public RelWriter explainTerms(RelWriter pw) {
super.explainTerms(pw);
if (outer) {
pw.item("outer", true);
}
return pw;
}

@Override
public TableFunctionScan copy(RelTraitSet traitSet, List<RelNode> inputs, RexNode rexCall,
Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings) {
return new HiveTableFunctionScan(getCluster(), traitSet, inputs, rexCall,
elementType, rowType, columnMappings);
elementType, rowType, columnMappings, outer);
}

/**
Expand Down
15 changes: 11 additions & 4 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,14 +703,21 @@ private static QueryBlockInfo createASTLateralView(TableFunctionScan tfs, QueryB
sel.add(selexpr.node());

// place the SELECT clause under the LATERAL VIEW clause
ASTBuilder lateralview = ASTBuilder.construct(HiveParser.TOK_LATERAL_VIEW, "TOK_LATERAL_VIEW");
lateralview.add(sel.node());
final boolean isOuterLateralView = tfs instanceof HiveTableFunctionScan htfs && htfs.isOuter();
final int lateralViewToken = isOuterLateralView
? HiveParser.TOK_LATERAL_VIEW_OUTER
: HiveParser.TOK_LATERAL_VIEW;
final String lateralViewText = isOuterLateralView
? "TOK_LATERAL_VIEW_OUTER"
: "TOK_LATERAL_VIEW";
ASTBuilder lateralView = ASTBuilder.construct(lateralViewToken, lateralViewText);
lateralView.add(sel.node());

// finally, add the LATERAL VIEW clause under the left side source which is the base table.
lateralview.add(tableFunctionSource.ast);
lateralView.add(tableFunctionSource.ast);

Schema outputSchema = new Schema(tableFunctionSource.schema, new Schema(alias, lvFields));
return new QueryBlockInfo(outputSchema, lateralview.node());
return new QueryBlockInfo(outputSchema, lateralView.node());
}

private boolean isLateralView(RelNode relNode) {
Expand Down
22 changes: 16 additions & 6 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1041,9 +1041,9 @@ boolean isCBOExecuted() {
}

@Override
boolean isCBOSupportedLateralView(ASTNode lateralView) {
// LATERAL VIEW OUTER not supported in CBO
return lateralView.getToken().getType() != HiveParser.TOK_LATERAL_VIEW_OUTER;
boolean isCBOSupportedLateralView() {
// Both LATERAL VIEW and LATERAL VIEW OUTER are supported in CBO.
return !this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP);
Copy link
Copy Markdown
Contributor

@thomasrebele thomasrebele May 21, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previously the result didn't depend on the HIVE_CBO_RETPATH_HIVEOP conf option. Maybe it should still return true if lateralView.getToken().getType() == HiveParser.TOK_LATERAL_VIEW?

Could you please explain why using that conf option here? It's description is Flag to control calcite plan to hive operator conversion, but I don't understand its purpose.

Copy link
Copy Markdown
Contributor

@kasakrisz kasakrisz May 21, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@thomasrebele
CBO return path is a different implementation of transforming the logical plan to Hive operators. It skips the AST conversion and converts Calcite operators directly to Hive operators. Maybe I'm missing something but I haven't found any change regarding this conversion in this patch.

@soumyakanti3578
Does lateral views in general is supported in CBO return path?
If yes could you please implement the lateral view outer too. I assume it would affect only 1-2 files and it would fit into the scope of this patch.

}

@Override
Expand Down Expand Up @@ -2980,7 +2980,7 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map<String, Rel
leftRel = aliasToRel.get(leftTableAlias);
} else if (SemanticAnalyzer.isJoinToken(left)) {
leftRel = genJoinLogicalPlan(qb, left, aliasToRel, outerNameToPosMap, outerRR);
} else if (left.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) {
} else if (isASTNodeLateralViewOrOuter(left)) {
leftRel = genLateralViewPlans(qb, left, aliasToRel);
} else {
assert (false);
Expand All @@ -2994,7 +2994,7 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map<String, Rel
|| (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) {
rightTableAlias = getTableAlias(right);
rightRel = aliasToRel.get(rightTableAlias);
} else if (right.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) {
} else if (isASTNodeLateralViewOrOuter(right)) {
rightRel = genLateralViewPlans(qb, right, aliasToRel);
} else {
assert (false);
Expand Down Expand Up @@ -3374,7 +3374,7 @@ private RelNode genLateralViewPlans(QB qb, ASTNode lateralView, Map<String, RelN

// next token is either the table alias name or another lateral view (which we will call
// recursively)
RelNode inputRel = next.getToken().getType() == HiveParser.TOK_LATERAL_VIEW
RelNode inputRel = isASTNodeLateralViewOrOuter(next)
? genLateralViewPlans(qb, next, aliasToRel)
: aliasToRel.get(getTableAlias(next));

Expand Down Expand Up @@ -5646,4 +5646,14 @@ private RelNode endGenOBLogicalPlan(RelNode sortRel) throws CalciteSemanticExcep
return sortRel;
}
}

/**
* Utility method to determine if an AST node represents a lateral view or lateral view outer.
* @param node AST node
* @return true if node is of lateral view or lateral view outer; false otherwise.
*/
private boolean isASTNodeLateralViewOrOuter(ASTNode node) {
return node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW
|| node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER;
}
}
6 changes: 2 additions & 4 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1713,9 +1713,7 @@ private String processLateralView(QB qb, ASTNode lateralView)
int numChildren = lateralView.getChildCount();
assert (numChildren == 2);

if (!isCBOSupportedLateralView(lateralView)) {
queryProperties.setCBOSupportedLateralViews(false);
}
queryProperties.setCBOSupportedLateralViews(isCBOSupportedLateralView());

ASTNode next = (ASTNode) lateralView.getChild(1);
String alias = null;
Expand Down Expand Up @@ -11123,7 +11121,7 @@ boolean isCBOExecuted() {
return false;
}

boolean isCBOSupportedLateralView(ASTNode lateralView) {
boolean isCBOSupportedLateralView() {
return false;
}

Expand Down
13 changes: 7 additions & 6 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/relnodegen/LateralViewPlan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
Expand Down Expand Up @@ -91,14 +90,15 @@ public class LateralViewPlan {

public LateralViewPlan(ASTNode lateralView, RelOptCluster cluster, RelNode inputRel,
RowResolver inputRR, UnparseTranslator unparseTranslator,
HiveConf conf, FunctionHelper functionHelper
) throws SemanticException {
HiveConf conf, FunctionHelper functionHelper) throws SemanticException {
// initialize global variables containing helper information
this.cluster = cluster;
this.unparseTranslator = unparseTranslator;
this.conf = conf;
this.functionHelper = functionHelper;

boolean isOuter = lateralView.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER;

// AST should have form of LATERAL_VIEW -> SELECT -> SELEXPR -> FUNCTION -> function info tree
ASTNode selExprAST = (ASTNode) lateralView.getChild(0).getChild(0);
ASTNode functionAST = (ASTNode) selExprAST.getChild(0);
Expand All @@ -118,7 +118,7 @@ public LateralViewPlan(ASTNode lateralView, RelOptCluster cluster, RelNode input

this.lateralViewRel = HiveTableFunctionScan.create(cluster,
TraitsUtil.getDefaultTraitSet(cluster), ImmutableList.of(inputRel), udtfCall,
null, retType, createColumnMappings(inputRel));
null, retType, createColumnMappings(inputRel), isOuter);
}

public static void validateLateralView(ASTNode lateralView) throws SemanticException {
Expand All @@ -128,8 +128,9 @@ public static void validateLateralView(ASTNode lateralView) throws SemanticExcep
}
ASTNode next = (ASTNode) lateralView.getChild(1);
if (!TABLE_ALIAS_TOKEN_TYPES.contains(next.getToken().getType()) &&
HiveParser.TOK_LATERAL_VIEW != next.getToken().getType()) {
throw new SemanticException(ASTErrorUtils.getMsg(
HiveParser.TOK_LATERAL_VIEW != next.getToken().getType() &&
Copy link
Copy Markdown
Contributor

@rubenada rubenada May 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess now this could also be rewritten using the new helper
... && !isASTNodeLateralViewOrOuter(next)

Copy link
Copy Markdown
Contributor Author

@soumyakanti3578 soumyakanti3578 May 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, thanks for catching this! I have refactored this and also other places in SemanticAnalyzer.

HiveParser.TOK_LATERAL_VIEW_OUTER != next.getToken().getType()) {
throw new SemanticException(ASTErrorUtils.getMsg(
ErrorMsg.LATERAL_VIEW_INVALID_CHILD.getMsg(), lateralView));
}
}
Expand Down
16 changes: 16 additions & 0 deletions ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
CREATE TABLE test (id string, items array<string>);
INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array());

CREATE VIEW v AS
SELECT test.id AS id, item
FROM test
LATERAL VIEW OUTER explode(test.items) lv AS item;

-- CBO plan should contain `outer=[true]` in HiveTableFunctionScan node.
EXPLAIN CBO
SELECT id, item FROM v ORDER BY id, item;
-- Explain plan should contain `outer lateral view: true` in the UDTF Operator
EXPLAIN
SELECT id, item FROM v ORDER BY id, item;
-- One of the output row should be ('D', NULL) since it's an outer lateral view.
SELECT id, item FROM v ORDER BY id, item;
Loading
Loading