[CBO] Make hints process many trees. (ydb-platform#8554)

Author: pashandor789

ydb/core/kqp/ut/join/data/queries/join_order_hints_complex.sql

@@ -1,6 +1,10 @@
 PRAGMA TablePathPrefix='/Root';
 
-PRAGMA ydb.OptJoinOrderHints='[ ["R", "S"], ["T", "U"] ]';
+PRAGMA ydb.OptJoinOrderHints=
+    '[ 
+        [["R", "S"], ["T", "U"]]
+    ]';
+
 PRAGMA ydb.OptCardinalityHints = 
     '[
         {"labels":["R"], "op":"#", "value":10e8},

ydb/core/kqp/ut/join/data/queries/join_order_hints_many_hint_trees.sql

@@ -0,0 +1,22 @@
+PRAGMA TablePathPrefix='/Root';
+
+PRAGMA ydb.OptJoinOrderHints=
+    '[ 
+        ["R", "S"],
+        ["T", "U"] 
+    ]';
+PRAGMA ydb.OptCardinalityHints = 
+    '[
+        {"labels":["R"], "op":"#", "value":10e8},
+        {"labels":["T"], "op":"#", "value":1},
+        {"labels":["R", "T"], "op":"#", "value":1},
+        {"labels":["R", "S"], "op":"#", "value":10e8},
+        {"labels":["T", "U"], "op":"#", "value":10e8},
+        {"labels":["V"], "op":"#", "value":1}
+    ]';
+
+SELECT * FROM 
+    R   INNER JOIN  S   on  R.id = S.id
+        INNER JOIN  T   on  R.id = T.id
+        INNER JOIN  U   on  T.id = U.id
+        INNER JOIN  V   on  U.id = V.id;

ydb/core/kqp/ut/join/data/queries/join_order_hints_simple.sql

@@ -8,7 +8,10 @@ PRAGMA ydb.OptCardinalityHints =
         {"labels":["R", "T"], "op":"#", "value":1},
         {"labels":["R", "S"], "op":"#", "value":10e8}
     ]';
-PRAGMA ydb.OptJoinOrderHints='[ "T", ["R", "S"] ]';
+PRAGMA ydb.OptJoinOrderHints=
+    '[
+        ["T", ["R", "S"]]
+    ]';
 
 SELECT * FROM 
     R   INNER JOIN  S   on  R.id = S.id

ydb/core/kqp/ut/join/kqp_join_order_ut.cpp

@@ -377,6 +377,39 @@ Y_UNIT_TEST_SUITE(KqpJoinOrder) {
         UNIT_ASSERT_VALUES_EQUAL(GetJoinOrder(plan).GetStringRobust(), R"([[["R","S"],["T","U"]],"V"])") ;
     }
 
+    void TraverseJoinTree(
+        const NJson::TJsonValue& tree, 
+        std::function<void(const NJson::TJsonValue&, const NJson::TJsonValue&)> joinVisitFunc
+    ) {
+        if (tree.IsArray()) {
+            auto join = tree.GetArray();
+            Y_ENSURE(join.size() == 2, Sprintf("Incorrect join tree piece, expected 2 children for join, got : %s", tree.GetStringRobust().c_str()));
+            joinVisitFunc(join[0], join[1]);
+            TraverseJoinTree(join[0], joinVisitFunc);
+            TraverseJoinTree(join[1], joinVisitFunc);
+        } else if (tree.IsString()) {
+        } else {
+            Y_ENSURE(false, Sprintf("Incorrect join tree piece: %s", tree.GetStringRobust().c_str()));
+        }
+    }
+
+    Y_UNIT_TEST_XOR_OR_BOTH_FALSE(TestJoinOrderHintsManyHintTrees, StreamLookupJoin, ColumnStore) {
+        auto plan = ExecuteJoinOrderTestDataQueryWithStats("queries/join_order_hints_many_hint_trees.sql", "stats/basic.json", StreamLookupJoin, ColumnStore); 
+        auto joinOrder = GetJoinOrder(plan);
+
+        THashSet<TString> expectedJoins = { "R, S", "T, U" };
+        TraverseJoinTree(
+            joinOrder, 
+            [&expectedJoins](const NJson::TJsonValue& joinLhs, const NJson::TJsonValue& joinRhs){ 
+                if (joinLhs.IsString() && joinRhs.IsString()) {
+                    expectedJoins.erase(Sprintf("%s, %s", joinLhs.GetString().c_str(), joinRhs.GetString().c_str()));
+                }
+            }
+        );
+
+        UNIT_ASSERT_C(expectedJoins.empty(), Sprintf("%s wasn't found in '%s' join order!", JoinSeq(", ", expectedJoins).c_str(), joinOrder.GetStringRobust().c_str()));
+    }
+
     void JoinOrderTestWithOverridenStats(const TString& queryPath, const TString& statsPath, TString correctJoinOrderPath, bool useStreamLookupJoin, bool useColumnStore
     ) {
         auto kikimr = GetKikimrWithJoinSettings(useStreamLookupJoin, GetStatic(statsPath));

ydb/library/yql/core/cbo/cbo_optimizer_new.cpp

@@ -330,9 +330,16 @@ std::shared_ptr<IBaseOptimizerNode> MakeJoinTreeFromJson(const NJson::TJsonValue
 }
 
 TJoinOrderHints::TJoinOrderHints(const TString& json) {
+    const static TString PARSING_FORMAT_ERROR = 
+        R"(Join order hints parsing failed. The example of the format: [ ["A", "B"], ["B", "C"] ])";
+
     NJson::TJsonValue jsonTree;
     NJson::ReadJsonTree(json, &jsonTree, true);
-    HintsTree = MakeJoinTreeFromJson(jsonTree);
+
+    Y_ENSURE(jsonTree.IsArray(), PARSING_FORMAT_ERROR);
+    for (const auto& hintTreeJson: jsonTree.GetArray()) {
+        HintTrees.push_back(MakeJoinTreeFromJson(hintTreeJson));
+    }
 }
 
 TJoinAlgoHints::TJoinAlgoHints(const TString& json) {

ydb/library/yql/core/cbo/cbo_optimizer_new.h

@@ -109,7 +109,7 @@ struct TJoinAlgoHints {
 };
 
 struct TJoinOrderHints {
-    std::shared_ptr<IBaseOptimizerNode> HintsTree;
+    TVector<std::shared_ptr<IBaseOptimizerNode>> HintTrees;
 
     TJoinOrderHints() {}
     TJoinOrderHints(const TString& json);

ydb/library/yql/dq/opt/dq_opt_join_hypergraph.h

@@ -2,6 +2,7 @@
 
 
 #include <numeric>
+#include <util/string/join.h>
 #include <util/string/printf.h>
 #include "bitset.h"
 
@@ -272,21 +273,23 @@ class TJoinOrderHintsApplier {
     {}
 
     void Apply(const TJoinOrderHints& hints) {
-        auto labels = ApplyHintsToSubgraph(hints.HintsTree);
-        auto nodes = Graph_.GetNodesByRelNames(labels);
-        
-        for (size_t i = 0; i < Graph_.GetEdges().size(); ++i) {
-            TNodeSet newLeft = Graph_.GetEdge(i).Left;
-            if (Overlaps(Graph_.GetEdge(i).Left, nodes) && !IsSubset(Graph_.GetEdge(i).Right, nodes)) {
-                newLeft |= nodes;
-            }
+        for (const auto& hintTree: hints.HintTrees) {
+            auto labels = ApplyHintsToSubgraph(hintTree);
+            auto nodes = Graph_.GetNodesByRelNames(labels);
+            
+            for (size_t i = 0; i < Graph_.GetEdges().size(); ++i) {
+                TNodeSet newLeft = Graph_.GetEdge(i).Left;
+                if (Overlaps(Graph_.GetEdge(i).Left, nodes) && !IsSubset(Graph_.GetEdge(i).Right, nodes)) {
+                    newLeft |= nodes;
+                }
 
-            TNodeSet newRight = Graph_.GetEdge(i).Right;
-            if (Overlaps(Graph_.GetEdge(i).Right, nodes) && !IsSubset(Graph_.GetEdge(i).Left, nodes)) {
-                newRight |= nodes;
-            }
+                TNodeSet newRight = Graph_.GetEdge(i).Right;
+                if (Overlaps(Graph_.GetEdge(i).Right, nodes) && !IsSubset(Graph_.GetEdge(i).Left, nodes)) {
+                    newRight |= nodes;
+                }
 
-            Graph_.UpdateEdgeSides(i, newLeft, newRight);
+                Graph_.UpdateEdgeSides(i, newLeft, newRight);
+            }
         }
     }
 
@@ -302,14 +305,8 @@ class TJoinOrderHintsApplier {
 
             auto* maybeEdge = Graph_.FindEdgeBetween(lhs, rhs);
             if (maybeEdge == nullptr) {
-                auto str = [](const TVector<TString>& v) -> TString {
-                    TString s;
-                    for (auto& el : v) { s += (el + ", "); }
-                    return s.empty()? s: s.substr(0, s.length() - 2);
-                };
-
                 const char* errStr = "There is no edge between {%s}, {%s}. The graf: %s";
-                Y_ENSURE(false, Sprintf(errStr, str(lhsLabels).c_str(), str(rhsLabels).c_str(), Graph_.String().c_str()));            
+                Y_ENSURE(false, Sprintf(errStr, JoinSeq(", ", lhsLabels).c_str(), JoinSeq(", ", rhsLabels).c_str(), Graph_.String().c_str()));            
             }
 
             size_t revEdgeIdx = maybeEdge->ReversedEdgeId;

ydb/library/yql/dq/opt/dq_opt_make_join_hypergraph.h

@@ -100,7 +100,7 @@ TJoinHypergraph<TNodeSet> MakeJoinHypergraph(
         YQL_CLOG(TRACE, CoreDq) << graph.String();
     }
 
-    if (hints.HintsTree != nullptr) {
+    if (!hints.HintTrees.empty()) { 
         TJoinOrderHintsApplier joinHints(graph);
         joinHints.Apply(hints);
         if (NYql::NLog::YqlLogger().NeedToLog(NYql::NLog::EComponent::CoreDq, NYql::NLog::ELevel::TRACE)) {