Skip to content

observability for tablet starts #6584

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vporyadke merged 4 commits into from
Jul 17, 2024
Merged

observability for tablet starts #6584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
67f049b
dirty commit
vporyadke Jul 11, 2024
f004472
observability for tablet starts
vporyadke Jul 11, 2024
8e5bc0a
fix labels
vporyadke Jul 12, 2024
31b5078
change ranges
vporyadke Jul 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ydb/core/mind/hive/hive_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1686,6 +1686,14 @@ void THive::UpdateCounterNodesConnected(i64 nodesConnectedDiff) {
}
}

void THive::UpdateCounterTabletsStarting(i64 tabletsStartingDiff) {
if (TabletCounters != nullptr) {
auto& counter = TabletCounters->Simple()[NHive::COUNTER_TABLETS_STARTING];
auto newValue = counter.Get() + tabletsStartingDiff;
counter.Set(newValue);
}
}

void THive::RecordTabletMove(const TTabletMoveInfo& moveInfo) {
TabletMoveHistory.PushBack(moveInfo);
TabletCounters->Cumulative()[NHive::COUNTER_TABLETS_MOVED].Increment(1);
Expand Down
1 change: 1 addition & 0 deletions ydb/core/mind/hive/hive_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,7 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId
void UpdateCounterBootQueueSize(ui64 bootQueueSize);
void UpdateCounterEventQueueSize(i64 eventQueueSizeDiff);
void UpdateCounterNodesConnected(i64 nodesConnectedDiff);
void UpdateCounterTabletsStarting(i64 tabletsStartingDiff);
void RecordTabletMove(const TTabletMoveInfo& info);
bool DomainHasNodes(const TSubDomainKey &domainKey) const;
void ProcessBootQueue();
Expand Down
1 change: 1 addition & 0 deletions ydb/core/mind/hive/tablet_info.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ struct TTabletInfo {
TInstant PostponedStart;
EBalancerPolicy BalancerPolicy;
TNodeId FailedNodeId = 0; // last time we tried to start the tablet, we failed on this node
TInstant BootTime;

TTabletInfo(ETabletRole role, THive& hive);
TTabletInfo(const TTabletInfo&) = delete;
Expand Down
8 changes: 8 additions & 0 deletions ydb/core/mind/hive/tx__start_tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ class TTxStartTablet : public TTransactionBase<THive> {
ui64 Cookie;
bool External;
TSideEffects SideEffects;
bool Success;

public:
TTxStartTablet(TFullTabletId tabletId, const TActorId& local, ui64 cookie, bool external, THive *hive)
Expand All @@ -23,10 +24,12 @@ class TTxStartTablet : public TTransactionBase<THive> {
TTxType GetTxType() const override { return NHive::TXTYPE_START_TABLET; }

bool Execute(TTransactionContext& txc, const TActorContext&) override {
Success = false;
SideEffects.Reset(Self->SelfId());
BLOG_D("THive::TTxStartTablet::Execute Tablet " << TabletId);
TTabletInfo* tablet = Self->FindTablet(TabletId);
if (tablet != nullptr) {
tablet->BootTime = TActivationContext::Now();
// finish fast-move operation
if (tablet->LastNodeId != 0 && tablet->LastNodeId != Local.NodeId()) {
TNodeInfo* lastNode = Self->FindNode(tablet->LastNodeId);
Expand Down Expand Up @@ -65,6 +68,7 @@ class TTxStartTablet : public TTransactionBase<THive> {
new TEvLocal::TEvBootTablet(*leader.TabletStorageInfo, promotableFollowerId, leader.KnownGeneration),
IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
Cookie);
Success = true;
return true;
} else {
BLOG_W("THive::TTxStartTablet::Execute, ignoring TEvBootTablet(" << leader.ToString() << ") - wrong state or node");
Expand All @@ -79,6 +83,7 @@ class TTxStartTablet : public TTransactionBase<THive> {
new TEvLocal::TEvBootTablet(*follower.LeaderTablet.TabletStorageInfo, follower.Id),
IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession,
Cookie);
Success = true;
return true;
} else {
BLOG_W("THive::TTxStartTablet::Execute, ignoring TEvBootTablet(" << follower.ToString() << ") - wrong state or node");
Expand Down Expand Up @@ -108,6 +113,9 @@ class TTxStartTablet : public TTransactionBase<THive> {
void Complete(const TActorContext& ctx) override {
BLOG_D("THive::TTxStartTablet::Complete Tablet " << TabletId << " SideEffects: " << SideEffects);
SideEffects.Complete(ctx);
if (Success) {
Self->UpdateCounterTabletsStarting(+1);
}
}
};

Expand Down
8 changes: 8 additions & 0 deletions ydb/core/mind/hive/tx__update_tablet_status.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,14 @@ class TTxUpdateTabletStatus : public TTransactionBase<THive> {
if (Status == TEvLocal::TEvTabletStatus::StatusOk) {
tablet->Statistics.AddRestartTimestamp(now.MilliSeconds());
tablet->ActualizeTabletStatistics(now);
if (tablet->BootTime != TInstant()) {
TDuration startTime = now - tablet->BootTime;
if (startTime > TDuration::Seconds(30)) {
BLOG_W("Tablet " << tablet->GetFullTabletId() << " was starting for " << startTime.Seconds() << " seconds");
}
Self->TabletCounters->Percentile()[NHive::COUNTER_TABLETS_START_TIME].IncrementFor(startTime.MilliSeconds());
Self->UpdateCounterTabletsStarting(-1);
}
TNodeInfo* node = Self->FindNode(Local.NodeId());
if (node == nullptr) {
// event from IC about disconnection of the node could overtake events from the node itself because of Pipe Server
Expand Down
16 changes: 16 additions & 0 deletions ydb/core/protos/counters_hive.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ enum ESimpleCounters {
COUNTER_IMBALANCED_OBJECTS = 19 [(CounterOpts) = {Name: "ImbalancedObjects"}];
COUNTER_WORST_OBJECT_VARIANCE = 20 [(CounterOpts) = {Name: "WorstObjectVariance"}];
COUNTER_STORAGE_SCATTER = 21 [(CounterOpts) = {Name: "StorageScatter"}];
COUNTER_TABLETS_STARTING = 22 [(CounterOpts) = {Name: "TabletsStarting"}];
}

enum ECumulativeCounters {
Expand Down Expand Up @@ -75,6 +76,21 @@ enum EPercentileCounters {
Ranges: { Value: 95 Name: "95%" },
Ranges: { Value: 100 Name: "100%" },
}];

COUNTER_TABLETS_START_TIME = 2 [(CounterOpts) = {
Name: "TabletsStartTimeMs",
Ranges: { Value: 1 }
Ranges: { Value: 5 }
Ranges: { Value: 10 }
Ranges: { Value: 50 }
Ranges: { Value: 100 }
Ranges: { Value: 500 }
Ranges: { Value: 1000 }
Ranges: { Value: 5000 }
Ranges: { Value: 10000 }
Ranges: { Value: 30000 }
Ranges: { Value: 60000 }
}];
}

enum ETxTypes {
Expand Down