Merge 528badc into 7182df1

skywalker-jpg · web-flow · commit 999d4ceb89af · 2024-05-29T10:05:41.000Z
diff --git a/ydb/core/external_sources/external_source_factory.cpp b/ydb/core/external_sources/external_source_factory.cpp
@@ -54,8 +54,11 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
         {
             ToString(NYql::EDatabaseType::YT),
             CreateExternalDataSource(TString{NYql::YtProviderName}, {"NONE", "TOKEN"}, {}, hostnamePatternsRegEx)
-        }
-    });
+        },
+        {
+            ToString(NYql::EDatabaseType::Greenplum),
+            CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "mdb_cluster_id", "use_tls", "schema"}, hostnamePatternsRegEx)
+        }});
 }
 
 }
diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h
@@ -14,6 +14,7 @@ enum class EDatabaseType {
     DataStreams,
     ObjectStorage,
     PostgreSQL,
+    Greenplum,
     YT
 };
 
@@ -25,6 +26,8 @@ inline EDatabaseType DatabaseTypeFromDataSourceKind(NConnector::NApi::EDataSourc
             return EDatabaseType::ClickHouse;
         case NConnector::NApi::EDataSourceKind::YDB:
             return EDatabaseType::Ydb;
+        case NConnector::NApi::EDataSourceKind::GREENPLUM:
+            return EDatabaseType::Greenplum;
         default:
             ythrow yexception() << "Unknown data source kind: " << NConnector::NApi::EDataSourceKind_Name(dataSourceKind);
     }
@@ -38,6 +41,8 @@ inline NConnector::NApi::EDataSourceKind DatabaseTypeToDataSourceKind(EDatabaseT
             return  NConnector::NApi::EDataSourceKind::CLICKHOUSE;
         case EDatabaseType::Ydb:
             return  NConnector::NApi::EDataSourceKind::YDB;
+        case EDatabaseType::Greenplum:
+            return  NConnector::NApi::EDataSourceKind::GREENPLUM;
         default:
             ythrow yexception() << "Unknown database type: " << ToString(databaseType);
     }
diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_provider_factories.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_provider_factories.cpp
@@ -31,7 +31,7 @@ namespace NYql::NDq {
                 args.MaxKeysInRequest);
         };
 
-        for (auto& name : {"ClickHouseGeneric", "PostgreSqlGeneric", "YdbGeneric"}) {
+        for (auto& name : {"ClickHouseGeneric", "PostgreSqlGeneric", "YdbGeneric", "GreenplumGeneric"}) {
             factory.RegisterSource<Generic::TSource>(name, readActorFactory);
             factory.RegisterLookupSource<Generic::TLookupSource>(name, lookupActorFactory);
         }
diff --git a/ydb/library/yql/providers/generic/connector/api/common/data_source.proto b/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
@@ -50,6 +50,12 @@ message TPostgreSQLDataSourceOptions {
     string schema = 1;
 }
 
+// TGreenplumDataSourceOptions represents settings specific to Greenplum
+message TGreenplumDataSourceOptions {
+    // Greenplum schema
+    string schema = 1;
+}
+
 // TClickhouseDataSourceOptions represents settings specific to Clickhouse
 message TClickhouseDataSourceOptions {
 }
@@ -83,5 +89,6 @@ message TDataSourceInstance {
         TPostgreSQLDataSourceOptions pg_options = 7;
         TClickhouseDataSourceOptions ch_options = 8;
         TS3DataSourceOptions s3_options = 9;
+        TGreenplumDataSourceOptions gp_options = 10;
     }
 }
diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp
@@ -192,7 +192,7 @@ namespace NYql {
                        NYql::TGenericClusterConfig& clusterConfig) {
         using namespace NConnector::NApi;
 
-        if (clusterConfig.GetKind() == EDataSourceKind::YDB) {
+        if (clusterConfig.GetKind() == EDataSourceKind::YDB || clusterConfig.GetKind() == EDataSourceKind::GREENPLUM) {
             clusterConfig.SetProtocol(EProtocol::NATIVE);
             return;
         }
diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp
@@ -173,6 +173,9 @@ namespace NYql {
                         case NYql::NConnector::NApi::YDB:
                             sourceType = "YdbGeneric";
                             break;
+                        case NYql::NConnector::NApi::GREENPLUM:
+                            sourceType = "GreenplumGeneric";
+                            break;
                         default:
                             ythrow yexception() << "Data source kind is unknown or not specified";
                             break;
@@ -207,6 +210,9 @@ namespace NYql {
                         case NConnector::NApi::YDB:
                             properties["SourceType"] = "Ydb";
                             break;
+                        case NConnector::NApi::GREENPLUM:
+                            properties["SourceType"] = "Greenplum";
+                            break;
                         case NConnector::NApi::DATA_SOURCE_KIND_UNSPECIFIED:
                             break;
                         default:
diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp
@@ -313,26 +313,34 @@ namespace NYql {
             *dsi->mutable_credentials()->mutable_token()->mutable_type() = "IAM";
         }
 
+        template<typename T>
+        void SetSchema(T& request, const TGenericClusterConfig& clusterConfig) {
+            TString schema;
+            const auto it = clusterConfig.GetDataSourceOptions().find("schema");
+            if (it != clusterConfig.GetDataSourceOptions().end()) {
+                schema = it->second;
+            }
+            if (!schema) {
+                schema = "public";
+            }
+
+            request.set_schema(schema);
+        }
+
         void FillDataSourceOptions(NConnector::NApi::TDescribeTableRequest& request, const TGenericClusterConfig& clusterConfig) {
             const auto dataSourceKind = clusterConfig.GetKind();
             switch (dataSourceKind) {
                 case NYql::NConnector::NApi::CLICKHOUSE:
                     break;
                 case NYql::NConnector::NApi::YDB:
                     break;
+                case NYql::NConnector::NApi::GREENPLUM: {
+                    auto* options = request.mutable_data_source_instance()->mutable_gp_options();
+                    SetSchema(*options, clusterConfig);
+                } break;
                 case NYql::NConnector::NApi::POSTGRESQL: {
-                    // for backward compability set schema "public" by default
-                    // TODO: simplify during https://st.yandex-team.ru/YQ-2494
-                    TString schema;
-                    const auto it = clusterConfig.GetDataSourceOptions().find("schema");
-                    if (it != clusterConfig.GetDataSourceOptions().end()) {
-                        schema = it->second;
-                    }
-                    if (!schema) {
-                        schema = "public";
-                    }
-
-                    request.mutable_data_source_instance()->mutable_pg_options()->set_schema(schema);
+                    auto* options = request.mutable_data_source_instance()->mutable_pg_options();
+                    SetSchema(*options, clusterConfig);
                 } break;
 
                 default:

Original file line number	Diff line number	Diff line change
`@@ -54,8 +54,11 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri`
`54`	`54`	`{`
`55`	`55`	`ToString(NYql::EDatabaseType::YT),`
`56`	`56`	`CreateExternalDataSource(TString{NYql::YtProviderName}, {"NONE", "TOKEN"}, {}, hostnamePatternsRegEx)`
`57`		`- }`
`58`		`- });`
	`57`	`+ },`
	`58`	`+ {`
	`59`	`+ ToString(NYql::EDatabaseType::Greenplum),`
	`60`	`+ CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "mdb_cluster_id", "use_tls", "schema"}, hostnamePatternsRegEx)`
	`61`	`+ }});`
`59`	`62`	`}`
`60`	`63`
`61`	`64`	`}`
Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ namespace NYql::NDq {`
`31`	`31`	`args.MaxKeysInRequest);`
`32`	`32`	`};`
`33`	`33`
`34`		`- for (auto& name : {"ClickHouseGeneric", "PostgreSqlGeneric", "YdbGeneric"}) {`
	`34`	`+ for (auto& name : {"ClickHouseGeneric", "PostgreSqlGeneric", "YdbGeneric", "GreenplumGeneric"}) {`
`35`	`35`	`factory.RegisterSource<Generic::TSource>(name, readActorFactory);`
`36`	`36`	`factory.RegisterLookupSource<Generic::TLookupSource>(name, lookupActorFactory);`
`37`	`37`	`}`
Original file line number	Diff line number	Diff line change
`@@ -50,6 +50,12 @@ message TPostgreSQLDataSourceOptions {`
`50`	`50`	`string schema = 1;`
`51`	`51`	`}`
`52`	`52`
	`53`	`+// TGreenplumDataSourceOptions represents settings specific to Greenplum`
	`54`	`+message TGreenplumDataSourceOptions {`
	`55`	`+ // Greenplum schema`
	`56`	`+ string schema = 1;`
	`57`	`+}`
	`58`	`+`
`53`	`59`	`// TClickhouseDataSourceOptions represents settings specific to Clickhouse`
`54`	`60`	`message TClickhouseDataSourceOptions {`
`55`	`61`	`}`
`@@ -83,5 +89,6 @@ message TDataSourceInstance {`
`83`	`89`	`TPostgreSQLDataSourceOptions pg_options = 7;`
`84`	`90`	`TClickhouseDataSourceOptions ch_options = 8;`
`85`	`91`	`TS3DataSourceOptions s3_options = 9;`
	`92`	`+ TGreenplumDataSourceOptions gp_options = 10;`
`86`	`93`	`}`
`87`	`94`	`}`
Original file line number	Diff line number	Diff line change
`@@ -192,7 +192,7 @@ namespace NYql {`
`192`	`192`	`NYql::TGenericClusterConfig& clusterConfig) {`
`193`	`193`	`using namespace NConnector::NApi;`
`194`	`194`
`195`		`- if (clusterConfig.GetKind() == EDataSourceKind::YDB) {`
	`195`	`+ if (clusterConfig.GetKind() == EDataSourceKind::YDB \|\| clusterConfig.GetKind() == EDataSourceKind::GREENPLUM) {`
`196`	`196`	`clusterConfig.SetProtocol(EProtocol::NATIVE);`
`197`	`197`	`return;`
`198`	`198`	`}`