Skip to content

Commit f1729e0

Browse files
committed
Addressed comments
1 parent 028ed62 commit f1729e0

4 files changed

Lines changed: 45 additions & 19 deletions

File tree

pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2924,9 +2924,9 @@ public double getLoadBalancerBandwidthOutResourceWeight() {
29242924
@FieldContext(
29252925
dynamic = true,
29262926
category = CATEGORY_LOAD_BALANCER,
2927-
doc = "The namespaces skip for load shedding"
2927+
doc = "The namespaces to be excluded from load shedding"
29282928
)
2929-
private Set<String> loadBalancerSheddingExcludedNamespaces = new TreeSet<>();
2929+
private Set<String> loadBalancerSheddingExcludedNamespaces = new HashSet<>();
29302930

29312931
@FieldContext(
29322932
category = CATEGORY_LOAD_BALANCER,

pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager, BrokerS
162162
@Getter
163163
private final BrokerSelectionStrategy brokerSelectionStrategy;
164164

165-
private final BrokerSelectionStrategy roundRobinBrokerSelectionStrategy;
165+
private final BrokerSelectionStrategy sheddingExcludedNamespaceSelectionStrategy;
166166

167167
@Getter
168168
private final List<BrokerFilter> brokerFilterPipeline;
@@ -257,7 +257,7 @@ public ExtensibleLoadManagerImpl() {
257257
this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter());
258258
this.brokerFilterPipeline.add(new BrokerVersionFilter());
259259
this.brokerSelectionStrategy = createBrokerSelectionStrategy();
260-
this.roundRobinBrokerSelectionStrategy = new RoundRobinBrokerSelectionStrategy();
260+
this.sheddingExcludedNamespaceSelectionStrategy = new RoundRobinBrokerSelectionStrategy();
261261
}
262262

263263
public static boolean isLoadManagerExtensionEnabled(PulsarService pulsar) {
@@ -640,20 +640,33 @@ public CompletableFuture<Optional<String>> selectAsync(ServiceUnitId bundle,
640640
return Optional.empty();
641641
}
642642
Set<String> candidateBrokers = availableBrokerCandidates.keySet();
643-
Set<String> sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces();
644-
645-
var namespace = NamespaceBundle.getBundleNamespace(bundle.toString());
646-
if (sheddingExcludedNamespaces.contains(namespace)) {
647-
if (debug(conf, log)) {
648-
log.info("Use round robin broker selector for {}", bundle);
649-
}
650-
return roundRobinBrokerSelectionStrategy.select(candidateBrokers, bundle, context);
651-
}
652-
return getBrokerSelectionStrategy().select(candidateBrokers, bundle, context);
643+
return getBrokerSelectionStrategy(bundle).select(candidateBrokers, bundle, context);
653644
});
654645
});
655646
}
656647

648+
/**
649+
* For shedding excluded namespaces, use RoundRobinBrokerSelector to assign the ownership,
650+
* it can make the assignment more average because these will not automatically rebalance to
651+
* another broker unless manually unloaded it.
652+
*
653+
* @param bundle the bundle to assign
654+
* @return the broker selection strategy
655+
*/
656+
private BrokerSelectionStrategy getBrokerSelectionStrategy(ServiceUnitId bundle) {
657+
658+
Set<String> sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces();
659+
660+
var namespace = NamespaceBundle.getBundleNamespace(bundle.toString());
661+
if (sheddingExcludedNamespaces.contains(namespace)) {
662+
if (debug(conf, log)) {
663+
log.info("Use round robin broker selector for {}", bundle);
664+
}
665+
return sheddingExcludedNamespaceSelectionStrategy;
666+
}
667+
return brokerSelectionStrategy;
668+
}
669+
657670
@Override
658671
public CompletableFuture<Boolean> checkOwnershipAsync(Optional<ServiceUnitId> topic, ServiceUnitId bundleUnit) {
659672
return getOwnershipAsync(topic, bundleUnit)

pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -504,7 +504,8 @@ public Set<UnloadDecision> findBundlesForUnloading(LoadManagerContext context,
504504
final String namespaceName = NamespaceBundle.getBundleNamespace(bundle);
505505
if (sheddingExcludedNamespaces.contains(namespaceName)) {
506506
if (debugMode) {
507-
log.info("Skipping load shedding for namespace {}", namespaceName);
507+
log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
508+
+ " Bundle namespace has been found in sheddingExcludedNamespaces", bundle));
508509
}
509510
continue;
510511
}

pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
151151
// Strategy used to determine where new topics should be placed.
152152
private ModularLoadManagerStrategy placementStrategy;
153153

154-
private ModularLoadManagerStrategy roundRobinBrokerSelector;
154+
private ModularLoadManagerStrategy sheddingExcludedNamespaceSelectionStrategy;
155155

156156
// Policies used to determine which brokers are available for particular namespaces.
157157
private SimpleResourceAllocationPolicies policies;
@@ -253,7 +253,7 @@ public void initialize(final PulsarService pulsar) {
253253
defaultStats.msgRateOut = DEFAULT_MESSAGE_RATE;
254254

255255
placementStrategy = ModularLoadManagerStrategy.create(conf);
256-
roundRobinBrokerSelector = new RoundRobinBrokerSelector();
256+
sheddingExcludedNamespaceSelectionStrategy = new RoundRobinBrokerSelector();
257257
policies = new SimpleResourceAllocationPolicies(pulsar);
258258
filterPipeline.add(new BrokerLoadManagerClassFilter());
259259
filterPipeline.add(new BrokerVersionFilter());
@@ -926,13 +926,17 @@ Optional<String> selectBroker(final ServiceUnitId serviceUnit) {
926926
}
927927

928928
Optional<String> broker;
929+
// For shedding excluded namespaces, use RoundRobinBrokerSelector to assign the ownership,
930+
// it can make the assignment more average because these will not automatically rebalance to
931+
// another broker unless manually unloaded it.
929932
Set<String> sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces();
930933
String namespaceNameFromBundleName = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
931934
if (sheddingExcludedNamespaces.contains(namespaceNameFromBundleName)) {
932935
if (log.isDebugEnabled()) {
933936
log.debug("Use round robin broker selector for {}", bundle);
934937
}
935-
broker = roundRobinBrokerSelector.selectBroker(brokerCandidateCache, data, loadData, conf);
938+
broker = sheddingExcludedNamespaceSelectionStrategy
939+
.selectBroker(brokerCandidateCache, data, loadData, conf);
936940
} else {
937941
// Choose a broker among the potentially smaller filtered list, when possible
938942
broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf);
@@ -1143,7 +1147,15 @@ public void writeBrokerDataOnZooKeeper(boolean force) {
11431147
*/
11441148
private int selectTopKBundle() {
11451149
bundleArr.clear();
1146-
bundleArr.addAll(loadData.getBundleData().entrySet());
1150+
Set<String> sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces();
1151+
for (Map.Entry<String, BundleData> entry : loadData.getBundleData().entrySet()) {
1152+
String bundle = entry.getKey();
1153+
String namespace = NamespaceBundle.getBundleNamespace(bundle);
1154+
if (sheddingExcludedNamespaces.contains(namespace)) {
1155+
continue;
1156+
}
1157+
bundleArr.add(entry);
1158+
}
11471159

11481160
int maxNumberOfBundlesInBundleLoadReport = pulsar.getConfiguration()
11491161
.getLoadBalancerMaxNumberOfBundlesInBundleLoadReport();

0 commit comments

Comments
 (0)