From a904692eaaee680ee692dcd95bf50a19d6113709 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 28 May 2024 20:25:16 +0000 Subject: [PATCH 1/2] add table creation in incremental backup --- ...ard__operation_alter_continuous_backup.cpp | 28 +++++++++++++++++++ .../schemeshard__operation_create_table.cpp | 2 +- .../ut_continuous_backup.cpp | 6 ++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_continuous_backup.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_continuous_backup.cpp index da3d631bd538..032309532579 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_continuous_backup.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_continuous_backup.cpp @@ -35,6 +35,23 @@ void DoAlterPqPart(const TOperationId& opId, const TPath& topicPath, TTopicInfo: result.push_back(CreateAlterPQ(NextPartId(opId, result), outTx)); } +void DoCreateIncBackupTable(const TOperationId& opId, const TPath& dst, NKikimrSchemeOp::TTableDescription tableDesc, TVector& result) { + auto outTx = TransactionTemplate(dst.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable); + // outTx.SetFailOnExist(!acceptExisted); + + outTx.SetAllowAccessToPrivatePaths(true); + + auto& desc = *outTx.MutableCreateTable(); + desc.CopyFrom(tableDesc); + desc.SetName(dst.LeafName()); + + auto col = desc.AddColumns(); + col->SetName("__incrBackupImpl_deleted"); + col->SetType("Bool"); + + result.push_back(CreateNewTable(NextPartId(opId, result), outTx)); +} + TVector CreateAlterContinuousBackup(TOperationId opId, const TTxTransaction& tx, TOperationContext& context) { Y_ABORT_UNLESS(tx.GetOperationType() == NKikimrSchemeOp::EOperationType::ESchemeOpAlterContinuousBackup); @@ -48,9 +65,19 @@ TVector CreateAlterContinuousBackup(TOperationId opId, cons } const auto [tablePath, streamPath] = std::get(checksResult); + TTableInfo::TPtr table = context.SS->Tables.at(tablePath.Base()->PathId); + const auto topicPath = streamPath.Child("streamImpl"); TTopicInfo::TPtr topic = context.SS->Topics.at(topicPath.Base()->PathId); + const auto backupTablePath = streamPath.Child("incBackupImpl"); + + const NScheme::TTypeRegistry* typeRegistry = AppData(context.Ctx)->TypeRegistry; + + NKikimrSchemeOp::TTableDescription schema; + context.SS->DescribeTable(table, typeRegistry, true, false, &schema); + schema.MutablePartitionConfig()->CopyFrom(table->TableDescription.GetPartitionConfig()); + TString errStr; if (!context.SS->CheckApplyIf(tx, errStr)) { return {CreateReject(opId, NKikimrScheme::StatusPreconditionFailed, errStr)}; @@ -79,6 +106,7 @@ TVector CreateAlterContinuousBackup(TOperationId opId, cons NCdc::DoAlterStream(alterCdcStreamOp, opId, workingDirPath, tablePath, result); if (cbOp.GetActionCase() == NKikimrSchemeOp::TAlterContinuousBackup::kTakeIncrementalBackup) { + DoCreateIncBackupTable(opId, backupTablePath, schema, result); DoAlterPqPart(opId, topicPath, topic, result); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index a43acf432e5d..66165c598e3b 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -449,7 +449,7 @@ class TCreateTable: public TSubOperation { checks.IsInsideTableIndexPath() .IsUnderCreating(NKikimrScheme::StatusNameConflict) .IsUnderTheSameOperation(OperationId.GetTxId()); //allow only as part of creating base table - } else { + } else if (!Transaction.GetAllowAccessToPrivatePaths()) { checks.IsCommonSensePath() .IsLikeDirectory(); } diff --git a/ydb/core/tx/schemeshard/ut_continuous_backup/ut_continuous_backup.cpp b/ydb/core/tx/schemeshard/ut_continuous_backup/ut_continuous_backup.cpp index 1e164b563a53..d35c827c7ef3 100644 --- a/ydb/core/tx/schemeshard/ut_continuous_backup/ut_continuous_backup.cpp +++ b/ydb/core/tx/schemeshard/ut_continuous_backup/ut_continuous_backup.cpp @@ -107,5 +107,11 @@ Y_UNIT_TEST_SUITE(TContinuousBackupTests) { NLs::PathExist, NLs::HasOffloadConfig, }); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/continuousBackupImpl/incBackupImpl"), { + NLs::PathExist, + NLs::IsTable, + NLs::CheckColumns("incBackupImpl", {"key", "value", "__incrBackupImpl_deleted"}, {}, {"key"}), + }); } } // TCdcStreamWithInitialScanTests From 1bbeeb70a1169145ad7dc4d7a67d4c26b26ad01a Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 29 May 2024 10:07:24 +0000 Subject: [PATCH 2/2] move incrBackupImplTable one level above --- .../schemeshard__operation_alter_continuous_backup.cpp | 2 +- ydb/core/tx/schemeshard/schemeshard_impl.cpp | 5 +++++ ydb/core/tx/schemeshard/schemeshard_path_describer.cpp | 3 +++ .../ut_continuous_backup/ut_continuous_backup.cpp | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_continuous_backup.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_continuous_backup.cpp index 032309532579..bd236cae9cc3 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_continuous_backup.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_continuous_backup.cpp @@ -70,7 +70,7 @@ TVector CreateAlterContinuousBackup(TOperationId opId, cons const auto topicPath = streamPath.Child("streamImpl"); TTopicInfo::TPtr topic = context.SS->Topics.at(topicPath.Base()->PathId); - const auto backupTablePath = streamPath.Child("incBackupImpl"); + const auto backupTablePath = tablePath.Child("incBackupImpl"); const NScheme::TTypeRegistry* typeRegistry = AppData(context.Ctx)->TypeRegistry; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index f45f03ad914e..b251c1c5be3f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -6693,6 +6693,11 @@ void TSchemeShard::FillTableDescriptionForShardIdx( break; } + case NKikimrSchemeOp::EPathTypeTable: { + // TODO: move BackupImplTable under special scheme element + break; + } + default: Y_FAIL_S("Unexpected table's child" << ": tableId# " << tableId diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index c14bf4926247..a3283ed699dc 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -372,6 +372,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa case NKikimrSchemeOp::EPathTypeSequence: Self->DescribeSequence(childPathId, childName, *entry->AddSequences(), returnSetVal); break; + case NKikimrSchemeOp::EPathTypeTable: + // TODO: move BackupImplTable under special scheme element + break; default: Y_FAIL_S("Unexpected table's child" << ": tableId# " << pathId diff --git a/ydb/core/tx/schemeshard/ut_continuous_backup/ut_continuous_backup.cpp b/ydb/core/tx/schemeshard/ut_continuous_backup/ut_continuous_backup.cpp index d35c827c7ef3..6edb921125a5 100644 --- a/ydb/core/tx/schemeshard/ut_continuous_backup/ut_continuous_backup.cpp +++ b/ydb/core/tx/schemeshard/ut_continuous_backup/ut_continuous_backup.cpp @@ -108,7 +108,7 @@ Y_UNIT_TEST_SUITE(TContinuousBackupTests) { NLs::HasOffloadConfig, }); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/continuousBackupImpl/incBackupImpl"), { + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/incBackupImpl"), { NLs::PathExist, NLs::IsTable, NLs::CheckColumns("incBackupImpl", {"key", "value", "__incrBackupImpl_deleted"}, {}, {"key"}),