Skip to content

Commit 75413d1

Browse files
ubytenshestakov
ubyte
authored and
nshestakov
committed
sqs access rights: allow tokenless requests if EnforceUserTokenRequirement doesn't specified (ydb-platform#34552)
LOGBROKER-10209
1 parent 1eec552 commit 75413d1

14 files changed

Lines changed: 436 additions & 61 deletions

File tree

‎ydb/core/http_proxy/http_req.cpp‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1089,6 +1089,9 @@ namespace NKikimr::NHttpProxy {
10891089
, ProtoCall(protoCall)
10901090
, Method(method)
10911091
{
1092+
if (Signature && Signature->Empty()) {
1093+
Signature.Reset();
1094+
}
10921095
}
10931096

10941097
TStringBuilder LogPrefix() const {
@@ -1385,10 +1388,17 @@ namespace NKikimr::NHttpProxy {
13851388
"stream '" << ExtractStreamName<TProtoRequest>(Request) << "'");
13861389

13871390
ReportInputCounters(ctx);
1388-
if (HttpContext.IamToken.empty() && Signature) {
1391+
if (!HttpContext.IamToken.empty() || Signature) {
13891392
AuthActor = ctx.Register(AppData(ctx)->DataStreamsAuthFactory->CreateAuthActor(
13901393
ctx.SelfID, HttpContext, std::move(Signature)));
13911394
} else {
1395+
if (AppData(ctx)->EnforceUserTokenRequirement || AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
1396+
return ReplyWithMessageQueueError(
1397+
ctx,
1398+
NSQS::NErrors::INCOMPLETE_SIGNATURE.HttpStatusCode,
1399+
NSQS::NErrors::INCOMPLETE_SIGNATURE.ErrorCode,
1400+
NSQS::NErrors::INCOMPLETE_SIGNATURE.DefaultMessage);
1401+
}
13921402
SendGrpcRequestNoDriver(ctx);
13931403
}
13941404

‎ydb/core/http_proxy/ut/datastreams_fixture/datastreams_fixture.cpp‎

Lines changed: 89 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -11,29 +11,44 @@ THttpProxyTestMock::THttpProxyTestMock() = default;
1111
THttpProxyTestMock::~THttpProxyTestMock() = default;
1212

1313
void THttpProxyTestMock::TearDown(NUnitTest::TTestContext&) {
14-
Monitoring->Stop();
15-
GRpcServer->Stop();
14+
if (Monitoring) {
15+
Monitoring->Stop();
16+
}
17+
if (GRpcServer) {
18+
GRpcServer->Stop();
19+
}
1620
}
1721

1822
void THttpProxyTestMock::SetUp(NUnitTest::TTestContext&) {
19-
InitAll();
23+
InitAll(TInitParameters{});
2024
}
2125

22-
void THttpProxyTestMock::InitAll(bool yandexCloudMode, bool enableMetering, bool enableSqsTopic) {
26+
void THttpProxyTestMock::InitAll(const TInitParameters initParameters) {
2327
AccessServicePort = PortManager.GetPort(8443);
2428
AccessServiceEndpoint = "127.0.0.1:" + ToString(AccessServicePort);
25-
InitKikimr(yandexCloudMode, enableMetering);
29+
InitKikimr(initParameters.YandexCloudMode, initParameters.EnableMetering, initParameters.EnforceUserTokenRequirement);
2630
InitAccessServiceService();
27-
InitHttpServer(yandexCloudMode, enableSqsTopic);
31+
InitHttpServer(initParameters.YandexCloudMode, initParameters.EnableSqsTopic);
2832
}
2933

30-
TString THttpProxyTestMock::FormAuthorizationStr(const TString& region) {
34+
TString THttpProxyTestMock::FormAuthorizationStr(const TString& region) const {
35+
if (!SendAuthorizationStr) {
36+
return "";
37+
}
3138
return TStringBuilder() <<
3239
"Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/" << region <<
3340
"/service/aws4_request, SignedHeaders=host;x-amz-date, Signature="
3441
"5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b)__";
3542
}
3643

44+
void THttpProxyTestMock::EnableAuthorization() {
45+
SendAuthorizationStr = true;
46+
}
47+
48+
void THttpProxyTestMock::DisableAuthorization() {
49+
SendAuthorizationStr = false;
50+
}
51+
3752
NJson::TJsonValue THttpProxyTestMock::CreateCreateStreamRequest() {
3853
NJson::TJsonValue record;
3954
record["StreamName"] = "testtopic";
@@ -319,6 +334,7 @@ TMaybe<NYdb::TResultSet> THttpProxyTestMock::RunYqlDataQuery(TString query) {
319334
TString endpoint = TStringBuilder() << "localhost:" << KikimrGrpcPort;
320335
auto driverConfig = NYdb::TDriverConfig()
321336
.SetEndpoint(endpoint)
337+
.SetAuthToken("root@builtin")
322338
.SetLog(std::unique_ptr<TLogBackend>(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release()));
323339
NYdb::TDriver driver(driverConfig);
324340
auto tableClient = NYdb::NTable::TTableClient(driver);
@@ -343,7 +359,7 @@ TMaybe<NYdb::TResultSet> THttpProxyTestMock::RunYqlDataQuery(TString query) {
343359
return resultSet;
344360
}
345361

346-
void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
362+
void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering, bool enforceUserTokenRequirement) {
347363
AuthFactory = std::make_shared<NKikimr::NHttpProxy::TIamAuthFactory>();
348364
NKikimrConfig::TAppConfig appConfig;
349365
appConfig.MutablePQConfig()->SetTopicsAreFirstClassCitizen(true);
@@ -354,6 +370,10 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
354370
appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetEnabled(true);
355371

356372
appConfig.MutableFeatureFlags()->SetEnableTopicMessageLevelParallelism(true);
373+
if (enforceUserTokenRequirement) {
374+
auto* securityConfig = appConfig.MutableDomainsConfig()->MutableSecurityConfig();
375+
securityConfig->SetEnforceUserTokenRequirement(true);
376+
}
357377

358378
appConfig.MutableSqsConfig()->SetEnableSqs(true);
359379
appConfig.MutableSqsConfig()->SetYandexCloudMode(yandexCloudMode);
@@ -423,12 +443,12 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
423443
UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
424444
client.AlterUserAttributes("/", "Root", {{"folder_id", "folder4"},
425445
{"cloud_id", "cloud4"},
426-
{"database_id", "database4"}}));
446+
{"database_id", "database4"}}, {}, {}, "root@builtin"));
427447
NACLib::TDiffACL acl;
428448
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::GenericFull, "Service1_id@as");
429449
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::GenericFull, "proxy_sa@as");
430450

431-
client.ModifyACL("/", "Root", acl.SerializeAsString());
451+
client.ModifyACL("/", "Root", acl.SerializeAsString(), "root@builtin");
432452

433453
client.MkDir("/Root", "SQS");
434454

@@ -450,7 +470,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
450470
"Columns { Name: \"DlqName\" Type: \"Utf8\"}"
451471
"Columns { Name: \"TablesFormat\" Type: \"Uint32\"}"
452472
"Columns { Name: \"Tags\" Type: \"Utf8\"}"
453-
"KeyColumnNames: [\"Account\", \"QueueName\"]"
473+
"KeyColumnNames: [\"Account\", \"QueueName\"]",
474+
TDuration::Seconds(5000),
475+
"root@builtin"
454476
);
455477

456478
client.CreateTable("/Root/SQS",
@@ -466,7 +488,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
466488
"Columns { Name: \"TablesFormat\" Type: \"Uint32\"}"
467489
"Columns { Name: \"StartProcessTimestamp\" Type: \"Uint64\"}"
468490
"Columns { Name: \"NodeProcess\" Type: \"Uint32\"}"
469-
"KeyColumnNames: [\"RemoveTimestamp\", \"QueueIdNumber\"]"
491+
"KeyColumnNames: [\"RemoveTimestamp\", \"QueueIdNumber\"]",
492+
TDuration::Seconds(5000),
493+
"root@builtin"
470494
);
471495

472496
client.MkDir("/Root/SQS", ".STD");
@@ -479,7 +503,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
479503
"Columns { Name: \"RandomId\" Type: \"Uint64\"}"
480504
"Columns { Name: \"SentTimestamp\" Type: \"Uint64\"}"
481505
"Columns { Name: \"DelayDeadline\" Type: \"Uint64\"}"
482-
"KeyColumnNames: [\"QueueIdNumberAndShardHash\", \"QueueIdNumber\", \"Shard\", \"Offset\"]"
506+
"KeyColumnNames: [\"QueueIdNumberAndShardHash\", \"QueueIdNumber\", \"Shard\", \"Offset\"]",
507+
TDuration::Seconds(5000),
508+
"root@builtin"
483509
);
484510

485511
client.MkDir("/Root/SQS", ".FIFO");
@@ -495,7 +521,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
495521
"Columns { Name: \"ReceiveCount\" Type: \"Uint32\"}"
496522
"Columns { Name: \"FirstReceiveTimestamp\" Type: \"Uint64\"}"
497523
"Columns { Name: \"SentTimestamp\" Type: \"Uint64\"}"
498-
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"Offset\"]"
524+
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"Offset\"]",
525+
TDuration::Seconds(5000),
526+
"root@builtin"
499527
);
500528

501529
client.CreateTable("/Root/SQS/.FIFO",
@@ -506,7 +534,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
506534
"Columns { Name: \"Deadline\" Type: \"Uint64\"}"
507535
"Columns { Name: \"Offset\" Type: \"Uint64\"}"
508536
"Columns { Name: \"MessageId\" Type: \"String\"}"
509-
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"DedupId\"]"
537+
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"DedupId\"]",
538+
TDuration::Seconds(5000),
539+
"root@builtin"
510540
);
511541

512542
client.CreateTable("/Root/SQS/.FIFO",
@@ -520,7 +550,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
520550
"Columns { Name: \"Tail\" Type: \"Uint64\"}"
521551
"Columns { Name: \"ReceiveAttemptId\" Type: \"Utf8\"}"
522552
"Columns { Name: \"LockTimestamp\" Type: \"Uint64\"}"
523-
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"GroupId\"]"
553+
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"GroupId\"]",
554+
TDuration::Seconds(5000),
555+
"root@builtin"
524556
);
525557

526558
client.CreateTable("/Root/SQS/.FIFO",
@@ -534,7 +566,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
534566
"Columns { Name: \"Attributes\" Type: \"String\"}"
535567
"Columns { Name: \"Data\" Type: \"String\"}"
536568
"Columns { Name: \"MessageId\" Type: \"String\"}"
537-
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"RandomId\", \"Offset\"]"
569+
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"RandomId\", \"Offset\"]",
570+
TDuration::Seconds(5000),
571+
"root@builtin"
538572
);
539573

540574
client.CreateTable("/Root/SQS/.FIFO",
@@ -543,22 +577,28 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
543577
"Columns { Name: \"QueueIdNumber\" Type: \"Uint64\"}"
544578
"Columns { Name: \"ReceiveAttemptId\" Type: \"Utf8\"}"
545579
"Columns { Name: \"Deadline\" Type: \"Uint64\"}"
546-
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"ReceiveAttemptId\"]"
580+
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"ReceiveAttemptId\"]",
581+
TDuration::Seconds(5000),
582+
"root@builtin"
547583
);
548584

549585
client.CreateTable("/Root/SQS",
550586
"Name: \".Settings\""
551587
"Columns { Name: \"Account\" Type: \"Utf8\"}"
552588
"Columns { Name: \"Name\" Type: \"Utf8\"}"
553589
"Columns { Name: \"Value\" Type: \"Utf8\"}"
554-
"KeyColumnNames: [\"Account\", \"Name\"]"
590+
"KeyColumnNames: [\"Account\", \"Name\"]",
591+
TDuration::Seconds(5000),
592+
"root@builtin"
555593
);
556594

557595
client.CreateTable("/Root/SQS",
558596
"Name: \".AtomicCounter\""
559597
"Columns { Name: \"counter_key\" Type: \"Uint64\"}"
560598
"Columns { Name: \"value\" Type: \"Uint64\"}"
561-
"KeyColumnNames: [\"counter_key\"]"
599+
"KeyColumnNames: [\"counter_key\"]",
600+
TDuration::Seconds(5000),
601+
"root@builtin"
562602
);
563603
RunYqlDataQuery("INSERT INTO `/Root/SQS/.AtomicCounter` (counter_key, value) VALUES (0, 0)");
564604

@@ -577,8 +617,14 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
577617
"Columns { Name: \"MaxReceiveCount\" Type: \"Uint64\"}"
578618
"Columns { Name: \"ShowDetailedCountersDeadline\" Type: \"Uint64\"}"
579619
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\"]";
580-
client.CreateTable("/Root/SQS/.STD", attributesTable);
581-
client.CreateTable("/Root/SQS/.FIFO", attributesTable);
620+
client.CreateTable("/Root/SQS/.STD",
621+
attributesTable,
622+
TDuration::Seconds(5000),
623+
"root@builtin");
624+
client.CreateTable("/Root/SQS/.FIFO",
625+
attributesTable,
626+
TDuration::Seconds(5000),
627+
"root@builtin");
582628

583629
client.CreateTable("/Root/SQS",
584630
"Name: \".Events\""
@@ -589,7 +635,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
589635
"Columns { Name: \"EventTimestamp\" Type: \"Uint64\"}"
590636
"Columns { Name: \"FolderId\" Type: \"Utf8\"}"
591637
"Columns { Name: \"Labels\" Type: \"Utf8\"}"
592-
"KeyColumnNames: [\"Account\", \"QueueName\", \"EventType\"]"
638+
"KeyColumnNames: [\"Account\", \"QueueName\", \"EventType\"]",
639+
TDuration::Seconds(5000),
640+
"root@builtin"
593641
);
594642

595643
auto stateTableCommon =
@@ -610,12 +658,16 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
610658
TStringBuilder()
611659
<< stateTableCommon
612660
<< "Columns { Name: \"Shard\" Type: \"Uint32\"}"
613-
<< "KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"Shard\"]"
661+
<< "KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"Shard\"]",
662+
TDuration::Seconds(5000),
663+
"root@builtin"
614664
);
615665
client.CreateTable("/Root/SQS/.FIFO",
616666
TStringBuilder()
617667
<< stateTableCommon
618-
<< "KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\"]"
668+
<< "KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\"]",
669+
TDuration::Seconds(5000),
670+
"root@builtin"
619671
);
620672

621673

@@ -633,7 +685,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
633685
"Columns { Name: \"SentTimestamp\" Type: \"Uint64\"}"
634686
"Columns { Name: \"VisibilityDeadline\" Type: \"Uint64\"}"
635687
"Columns { Name: \"DelayDeadline\" Type: \"Uint64\"}"
636-
"KeyColumnNames: [\"QueueIdNumberAndShardHash\", \"QueueIdNumber\", \"Shard\", \"Offset\"]"
688+
"KeyColumnNames: [\"QueueIdNumberAndShardHash\", \"QueueIdNumber\", \"Shard\", \"Offset\"]",
689+
TDuration::Seconds(5000),
690+
"root@builtin"
637691
);
638692

639693
auto sentTimestampIdxCommonColumns=
@@ -649,7 +703,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
649703
TStringBuilder()
650704
<< "Name: \"SentTimestampIdx\""
651705
<< sentTimestampIdxCommonColumns
652-
<< sendTimestampIdsKeys
706+
<< sendTimestampIdsKeys,
707+
TDuration::Seconds(5000),
708+
"root@builtin"
653709
);
654710

655711
client.CreateTable("/Root/SQS/.FIFO",
@@ -661,7 +717,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
661717
"Columns { Name: \"GroupId\" Type: \"String\"}"
662718
"Columns { Name: \"RandomId\" Type: \"Uint64\"}"
663719
"Columns { Name: \"DelayDeadline\" Type: \"Uint64\"}"
664-
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"SentTimestamp\", \"Offset\"]"
720+
"KeyColumnNames: [\"QueueIdNumberHash\", \"QueueIdNumber\", \"SentTimestamp\", \"Offset\"]",
721+
TDuration::Seconds(5000),
722+
"root@builtin"
665723
);
666724

667725
client.CreateTable("/Root/SQS/.STD",
@@ -675,7 +733,9 @@ void THttpProxyTestMock::InitKikimr(bool yandexCloudMode, bool enableMetering) {
675733
"Columns { Name: \"Data\" Type: \"String\"}"
676734
"Columns { Name: \"MessageId\" Type: \"String\"}"
677735
"Columns { Name: \"SenderId\" Type: \"String\"}"
678-
"KeyColumnNames: [\"QueueIdNumberAndShardHash\", \"QueueIdNumber\", \"Shard\", \"RandomId\", \"Offset\"]"
736+
"KeyColumnNames: [\"QueueIdNumberAndShardHash\", \"QueueIdNumber\", \"Shard\", \"RandomId\", \"Offset\"]",
737+
TDuration::Seconds(5000),
738+
"root@builtin"
679739
);
680740
}
681741

‎ydb/core/http_proxy/ut/datastreams_fixture/datastreams_fixture.h‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,19 @@ class THttpProxyTestMock : public NUnitTest::TBaseFixture {
7575

7676
void SetUp(NUnitTest::TTestContext&) override;
7777

78-
void InitAll(bool yandexCloudMode = true, bool enableMetering = false, bool extendedQueueUrl = false);
78+
struct TInitParameters {
79+
bool YandexCloudMode : 1 = true;
80+
bool EnableMetering : 1 = false;
81+
bool EnableSqsTopic : 1 = false;
82+
bool EnforceUserTokenRequirement : 1 = false;
83+
};
7984

80-
static TString FormAuthorizationStr(const TString& region);
85+
void InitAll(const TInitParameters initParameters);
86+
87+
TString FormAuthorizationStr(const TString& region) const;
88+
89+
void EnableAuthorization();
90+
void DisableAuthorization();
8191

8292
static NJson::TJsonValue CreateCreateStreamRequest();
8393

@@ -223,7 +233,7 @@ class THttpProxyTestMock : public NUnitTest::TBaseFixture {
223233
private:
224234
TMaybe<NYdb::TResultSet> RunYqlDataQuery(TString query);
225235

226-
void InitKikimr(bool yandexCloudMode, bool enableMetering);
236+
void InitKikimr(bool yandexCloudMode, bool enableMetering, bool enforceUserTokenRequirement);
227237

228238
void InitAccessServiceService();
229239

@@ -251,22 +261,32 @@ class THttpProxyTestMock : public NUnitTest::TBaseFixture {
251261
ui16 MonPort = 0;
252262
ui16 KikimrGrpcPort = 0;
253263
bool SqsTopicMode = false;
264+
bool SendAuthorizationStr = true;
254265
};
255266

256267
class THttpProxyTestMockForSQS : public THttpProxyTestMock {
268+
public:
257269
void SetUp(NUnitTest::TTestContext&) override {
258-
InitAll(false);
270+
InitAll(TInitParameters{
271+
.YandexCloudMode = false,
272+
});
259273
}
260274
};
261275

262276
class THttpProxyTestMockWithMetering : public THttpProxyTestMock {
277+
public:
263278
void SetUp(NUnitTest::TTestContext&) override {
264-
InitAll(true, true);
279+
InitAll(TInitParameters{
280+
.EnableMetering = true,
281+
});
265282
}
266283
};
267284

268285
class THttpProxyTestMockForSQSTopic : public THttpProxyTestMock {
286+
public:
269287
void SetUp(NUnitTest::TTestContext&) override {
270-
InitAll(true, false, true);
288+
InitAll(TInitParameters{
289+
.EnableSqsTopic = true,
290+
});
271291
}
272292
};

0 commit comments

Comments
 (0)