HIVE-29612: LATERAL VIEW OUTER doesn't work with CBO enabled

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableFunctionScan.java

@@ -25,6 +25,7 @@
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelInput;
 import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
 import org.apache.calcite.rel.core.TableFunctionScan;
 import org.apache.calcite.rel.metadata.RelColumnMapping;
 import org.apache.calcite.rel.type.RelDataType;
@@ -34,6 +35,9 @@
 
 public class HiveTableFunctionScan extends TableFunctionScan implements HiveRelNode {
 
+  // Whether this is a LATERAL VIEW OUTER
+  private final boolean outer;
+
   /**
    * @param cluster
    *          cluster - Cluster that this relational expression belongs to
@@ -49,28 +53,54 @@ public class HiveTableFunctionScan extends TableFunctionScan implements HiveRelN
    *          rowType - Row type produced by function
    * @param columnMappings
    *          columnMappings - Column mappings associated with this function
+   * @param outer
+   *          outer - true if this is a LATERAL VIEW OUTER 
    */
   protected HiveTableFunctionScan(RelOptCluster cluster, RelTraitSet traitSet, List<RelNode> inputs,
-      RexNode rexCall, Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings) {
+      RexNode rexCall, Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings,
+      boolean outer) {
     super(cluster, traitSet, inputs, rexCall, elementType, rowType, columnMappings);
+    this.outer = outer;
   }
 
   public HiveTableFunctionScan(RelInput input) {
     super(input);
+    this.outer = input.getBoolean("outer", false);
   }
 
   public static HiveTableFunctionScan create(RelOptCluster cluster, RelTraitSet traitSet,
       List<RelNode> inputs, RexNode rexCall, Type elementType, RelDataType rowType,
       Set<RelColumnMapping> columnMappings) throws CalciteSemanticException {
     return new HiveTableFunctionScan(cluster, traitSet, inputs, rexCall, elementType, rowType,
-        columnMappings);
+        columnMappings, false);
+  }
+
+  public static HiveTableFunctionScan create(RelOptCluster cluster, RelTraitSet traitSet,
+      List<RelNode> inputs, RexNode rexCall, Type elementType, RelDataType rowType,
+      Set<RelColumnMapping> columnMappings, boolean outer) throws CalciteSemanticException {
+    return new HiveTableFunctionScan(cluster, traitSet, inputs, rexCall, elementType, rowType,
+        columnMappings, outer);
+  }
+
+  /** Returns true if this represents a LATERAL VIEW OUTER. */
+  public boolean isOuter() {
+    return outer;
+  }
+
+  @Override
+  public RelWriter explainTerms(RelWriter pw) {
+    super.explainTerms(pw);
+    if (outer) {
+      pw.item("outer", true);
+    }
+    return pw;
   }
 
   @Override
   public TableFunctionScan copy(RelTraitSet traitSet, List<RelNode> inputs, RexNode rexCall,
       Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings) {
     return new HiveTableFunctionScan(getCluster(), traitSet, inputs, rexCall,
-        elementType, rowType, columnMappings);
+        elementType, rowType, columnMappings, outer);
   }
 
   /**

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java

@@ -703,14 +703,21 @@ private static QueryBlockInfo createASTLateralView(TableFunctionScan tfs, QueryB
     sel.add(selexpr.node());
 
     // place the SELECT clause under the LATERAL VIEW clause
-    ASTBuilder lateralview = ASTBuilder.construct(HiveParser.TOK_LATERAL_VIEW, "TOK_LATERAL_VIEW");
-    lateralview.add(sel.node());
+    final boolean isOuterLateralView = tfs instanceof HiveTableFunctionScan htfs && htfs.isOuter();
+    final int lateralViewToken = isOuterLateralView
+        ? HiveParser.TOK_LATERAL_VIEW_OUTER
+        : HiveParser.TOK_LATERAL_VIEW;
+    final String lateralViewText = isOuterLateralView
+        ? "TOK_LATERAL_VIEW_OUTER"
+        : "TOK_LATERAL_VIEW";
+    ASTBuilder lateralView = ASTBuilder.construct(lateralViewToken, lateralViewText);
+    lateralView.add(sel.node());
 
     // finally, add the LATERAL VIEW clause under the left side source which is the base table.
-    lateralview.add(tableFunctionSource.ast);
+    lateralView.add(tableFunctionSource.ast);
 
     Schema outputSchema = new Schema(tableFunctionSource.schema, new Schema(alias, lvFields));
-    return new QueryBlockInfo(outputSchema, lateralview.node());
+    return new QueryBlockInfo(outputSchema, lateralView.node());
   }
 
   private boolean isLateralView(RelNode relNode) {

ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java

@@ -1041,9 +1041,9 @@ boolean isCBOExecuted() {
   }
 
   @Override
-  boolean isCBOSupportedLateralView(ASTNode lateralView) {
-    // LATERAL VIEW OUTER not supported in CBO
-    return lateralView.getToken().getType() != HiveParser.TOK_LATERAL_VIEW_OUTER;
+  boolean isCBOSupportedLateralView() {
+    // Both LATERAL VIEW and LATERAL VIEW OUTER are supported in CBO.
+    return !this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP);
   }
 
   @Override
@@ -2980,7 +2980,7 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map<String, Rel
         leftRel = aliasToRel.get(leftTableAlias);
       } else if (SemanticAnalyzer.isJoinToken(left)) {
         leftRel = genJoinLogicalPlan(qb, left, aliasToRel, outerNameToPosMap, outerRR);
-      } else if (left.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) {
+      } else if (isASTNodeLateralViewOrOuter(left)) {
         leftRel = genLateralViewPlans(qb, left, aliasToRel);
       } else {
         assert (false);
@@ -2994,7 +2994,7 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map<String, Rel
           || (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) {
         rightTableAlias = getTableAlias(right);
         rightRel = aliasToRel.get(rightTableAlias);
-      } else if (right.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) {
+      } else if (isASTNodeLateralViewOrOuter(right)) {
         rightRel = genLateralViewPlans(qb, right, aliasToRel);
       }  else {
         assert (false);
@@ -3374,7 +3374,7 @@ private RelNode genLateralViewPlans(QB qb, ASTNode lateralView, Map<String, RelN
 
       // next token is either the table alias name or another lateral view (which we will call
       // recursively)
-      RelNode inputRel = next.getToken().getType() == HiveParser.TOK_LATERAL_VIEW
+      RelNode inputRel = isASTNodeLateralViewOrOuter(next)
           ? genLateralViewPlans(qb, next, aliasToRel)
           : aliasToRel.get(getTableAlias(next));
 

ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java

@@ -1683,8 +1683,7 @@ private void processJoin(QB qb, ASTNode join) throws SemanticException {
               "PTF invocation in a Join must have an alias"));
         }
 
-      } else if (child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
-          child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
+      } else if (isASTNodeLateralViewOrOuter(child)) {
         // SELECT * FROM src1 LATERAL VIEW udtf() AS myTable JOIN src2 ...
         // is not supported. Instead, the lateral view must be in a subquery
         // SELECT * FROM (SELECT * FROM src1 LATERAL VIEW udtf() AS myTable) a
@@ -1713,9 +1712,7 @@ private String processLateralView(QB qb, ASTNode lateralView)
     int numChildren = lateralView.getChildCount();
     assert (numChildren == 2);
 
-    if (!isCBOSupportedLateralView(lateralView)) {
-      queryProperties.setCBOSupportedLateralViews(false);
-    }
+    queryProperties.setCBOSupportedLateralViews(isCBOSupportedLateralView());
 
     ASTNode next = (ASTNode) lateralView.getChild(1);
     String alias = null;
@@ -1889,8 +1886,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx,
           processTable(qb, frm);
         } else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) {
           processSubQuery(qb, frm);
-        } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
-            frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
+        } else if (isASTNodeLateralViewOrOuter(frm)) {
           queryProperties.setHasLateralViews(true);
           processLateralView(qb, frm);
         } else if (isJoinToken(frm)) {
@@ -11123,7 +11119,7 @@ boolean isCBOExecuted() {
     return false;
   }
 
-  boolean isCBOSupportedLateralView(ASTNode lateralView) {
+  boolean isCBOSupportedLateralView() {
     return false;
   }
 
@@ -15526,4 +15522,14 @@ public void startAnalysis() {
       queryState.createHMSCache();
     }
   }
+
+  /**
+   * Utility method to determine if an AST node represents a lateral view or lateral view outer.
+   * @param node AST node
+   * @return true if node is of lateral view or lateral view outer; false otherwise.
+   */
+  public static boolean isASTNodeLateralViewOrOuter(ASTNode node) {
+    return node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW
+        || node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER;
+  }
 }

ql/src/java/org/apache/hadoop/hive/ql/parse/relnodegen/LateralViewPlan.java

@@ -28,7 +28,6 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
@@ -91,14 +90,15 @@ public class LateralViewPlan {
 
   public LateralViewPlan(ASTNode lateralView, RelOptCluster cluster, RelNode inputRel,
       RowResolver inputRR, UnparseTranslator unparseTranslator,
-      HiveConf conf, FunctionHelper functionHelper
-      ) throws SemanticException {
+      HiveConf conf, FunctionHelper functionHelper) throws SemanticException {
     // initialize global variables containing helper information
     this.cluster = cluster;
     this.unparseTranslator = unparseTranslator;
     this.conf = conf;
     this.functionHelper = functionHelper;
 
+    boolean isOuter = lateralView.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER;
+
     // AST should have form of LATERAL_VIEW -> SELECT -> SELEXPR -> FUNCTION -> function info tree
     ASTNode selExprAST = (ASTNode) lateralView.getChild(0).getChild(0);
     ASTNode functionAST = (ASTNode) selExprAST.getChild(0);
@@ -118,7 +118,7 @@ public LateralViewPlan(ASTNode lateralView, RelOptCluster cluster, RelNode input
 
     this.lateralViewRel = HiveTableFunctionScan.create(cluster,
         TraitsUtil.getDefaultTraitSet(cluster), ImmutableList.of(inputRel), udtfCall,
-        null, retType, createColumnMappings(inputRel));
+        null, retType, createColumnMappings(inputRel), isOuter);
   }
 
   public static void validateLateralView(ASTNode lateralView) throws SemanticException {
@@ -127,9 +127,9 @@ public static void validateLateralView(ASTNode lateralView) throws SemanticExcep
           " children.");
     }
     ASTNode next = (ASTNode) lateralView.getChild(1);
-    if (!TABLE_ALIAS_TOKEN_TYPES.contains(next.getToken().getType()) &&
-          HiveParser.TOK_LATERAL_VIEW != next.getToken().getType()) {
-        throw new SemanticException(ASTErrorUtils.getMsg(
+    if (!TABLE_ALIAS_TOKEN_TYPES.contains(next.getToken().getType()) && 
+        !SemanticAnalyzer.isASTNodeLateralViewOrOuter(next)) {
+      throw new SemanticException(ASTErrorUtils.getMsg(
             ErrorMsg.LATERAL_VIEW_INVALID_CHILD.getMsg(), lateralView));
     }
   }

ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q

@@ -0,0 +1,16 @@
+CREATE TABLE test (id string, items array<string>);
+INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array());
+
+CREATE VIEW v AS
+SELECT test.id AS id, item
+FROM test
+LATERAL VIEW OUTER explode(test.items) lv AS item;
+
+-- CBO plan should contain `outer=[true]` in HiveTableFunctionScan node.
+EXPLAIN CBO
+SELECT id, item FROM v ORDER BY id, item;
+-- Explain plan should contain `outer lateral view: true` in the UDTF Operator
+EXPLAIN
+SELECT id, item FROM v ORDER BY id, item;
+-- One of the output row should be ('D', NULL) since it's an outer lateral view.
+SELECT id, item FROM v ORDER BY id, item;