Skip to content

Commit a4a52c9

Browse files
sureshanapartisureshanaparti
committed
Merge branch '4.22'
2 parents 72b99a3 + 4359198 commit a4a52c9

47 files changed

Lines changed: 516 additions & 461 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎api/src/main/java/com/cloud/ha/Investigator.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,19 @@ public interface Investigator extends Adapter {
2626
* Returns if the vm is still alive.
2727
*
2828
* @param vm to work on.
29+
* @return true if vm is alive, otherwise false
2930
*/
30-
public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM;
31+
boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM;
3132

32-
public Status isAgentAlive(Host agent);
33+
/**
34+
* Returns the agent status of the host.
35+
*
36+
* @param host
37+
* @return status of the host agent
38+
*/
39+
Status getHostAgentStatus(Host host);
3340

3441
class UnknownVM extends Exception {
35-
36-
/**
37-
*
38-
*/
3942
private static final long serialVersionUID = 1L;
40-
4143
};
4244
}

‎api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ private void setupResponse(final boolean result, final String resourceUuid) {
8787
final HostHAResponse response = new HostHAResponse();
8888
response.setId(resourceUuid);
8989
response.setProvider(getHaProvider().toLowerCase());
90+
response.setStatus(result);
9091
response.setResponseName(getCommandName());
9192
setResponseObject(response);
9293
}

‎core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ public CheckOnHostAnswer(CheckOnHostCommand cmd, Boolean alive, String details)
3838

3939
public CheckOnHostAnswer(CheckOnHostCommand cmd, String details) {
4040
super(cmd, false, details);
41+
determined = false;
42+
alive = false;
4143
}
4244

4345
public boolean isDetermined() {
@@ -47,5 +49,4 @@ public boolean isDetermined() {
4749
public boolean isAlive() {
4850
return alive;
4951
}
50-
5152
}

‎core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525
public class CheckOnHostCommand extends Command {
2626
HostTO host;
27-
boolean reportCheckFailureIfOneStorageIsDown;
27+
boolean reportIfHeartBeatFailedForOneStoragePool;
2828

2929
protected CheckOnHostCommand() {
3030
}
@@ -34,17 +34,17 @@ public CheckOnHostCommand(Host host) {
3434
setWait(20);
3535
}
3636

37-
public CheckOnHostCommand(Host host, boolean reportCheckFailureIfOneStorageIsDown) {
37+
public CheckOnHostCommand(Host host, boolean reportIfHeartBeatFailedForOneStoragePool) {
3838
this(host);
39-
this.reportCheckFailureIfOneStorageIsDown = reportCheckFailureIfOneStorageIsDown;
39+
this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool;
4040
}
4141

4242
public HostTO getHost() {
4343
return host;
4444
}
4545

46-
public boolean isCheckFailedOnOneStorage() {
47-
return reportCheckFailureIfOneStorageIsDown;
46+
public boolean shouldReportIfHeartBeatFailedForOneStoragePool() {
47+
return reportIfHeartBeatFailedForOneStoragePool;
4848
}
4949

5050
@Override

‎engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,10 @@ public interface HighAvailabilityManager extends Manager {
7575
+ " which are registered for the HA event that were successful and are now ready to be purged.",
7676
true, Cluster);
7777

78-
public static final ConfigKey<Boolean> KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false",
78+
ConfigKey<Boolean> KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false",
7979
"Proceed fencing the host even the heartbeat failed for only one storage pool", false, ConfigKey.Scope.Zone);
8080

81-
public enum WorkType {
81+
enum WorkType {
8282
Migration, // Migrating VMs off of a host.
8383
Stop, // Stops a VM for storage pool migration purposes. This should be obsolete now.
8484
CheckStop, // Checks if a VM has been stopped.

‎engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -395,9 +395,9 @@ public AsyncCallFuture<VolumeApiResult> expungeVolumeAsync(VolumeInfo volume) {
395395
}
396396

397397
// Find out if the volume is at state of download_in_progress on secondary storage
398-
VolumeDataStoreVO volumeStore = _volumeStoreDao.findByVolume(volume.getId());
399-
if (volumeStore != null) {
400-
if (volumeStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) {
398+
VolumeDataStoreVO volumeOnImageStore = _volumeStoreDao.findByVolume(volume.getId());
399+
if (volumeOnImageStore != null) {
400+
if (volumeOnImageStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) {
401401
String msg = String.format("Volume: %s is currently being uploaded; can't delete it.", volume);
402402
logger.debug(msg);
403403
result.setSuccess(false);
@@ -416,10 +416,10 @@ public AsyncCallFuture<VolumeApiResult> expungeVolumeAsync(VolumeInfo volume) {
416416

417417
if (!volumeExistsOnPrimary(vol)) {
418418
// not created on primary store
419-
if (volumeStore == null) {
419+
if (volumeOnImageStore == null) {
420420
// also not created on secondary store
421421
if (logger.isDebugEnabled()) {
422-
logger.debug("Marking volume that was never created as destroyed: " + vol);
422+
logger.debug("Marking volume that was never created as destroyed: {}", vol);
423423
}
424424
VMTemplateVO template = templateDao.findById(vol.getTemplateId());
425425
if (template != null && !template.isDeployAsIs()) {
@@ -435,11 +435,21 @@ public AsyncCallFuture<VolumeApiResult> expungeVolumeAsync(VolumeInfo volume) {
435435
if (volume.getDataStore().getRole() == DataStoreRole.Image) {
436436
// no need to change state in volumes table
437437
volume.processEventOnly(Event.DestroyRequested);
438+
if (volumeOnImageStore == null) {
439+
logger.debug("Volume {} doesn't exist on image store, no need to delete", vol);
440+
future.complete(result);
441+
return future;
442+
}
438443
} else if (volume.getDataStore().getRole() == DataStoreRole.Primary) {
439444
if (vol.getState() == Volume.State.Expunging) {
440445
logger.info("Volume {} is already in Expunging, retrying", volume);
441446
}
442447
volume.processEvent(Event.ExpungeRequested);
448+
if (!volumeExistsOnPrimary(vol)) {
449+
logger.debug("Volume {} doesn't exist on primary storage, no need to delete", vol);
450+
future.complete(result);
451+
return future;
452+
}
443453
}
444454

445455
DeleteVolumeContext<VolumeApiResult> context = new DeleteVolumeContext<>(null, vo, future);
@@ -460,13 +470,11 @@ public void ensureVolumeIsExpungeReady(long volumeId) {
460470

461471
private boolean volumeExistsOnPrimary(VolumeVO vol) {
462472
Long poolId = vol.getPoolId();
463-
464473
if (poolId == null) {
465474
return false;
466475
}
467476

468477
PrimaryDataStore primaryStore = dataStoreMgr.getPrimaryDataStore(poolId);
469-
470478
if (primaryStore == null) {
471479
return false;
472480
}
@@ -476,8 +484,7 @@ private boolean volumeExistsOnPrimary(VolumeVO vol) {
476484
}
477485

478486
String volumePath = vol.getPath();
479-
480-
if (volumePath == null || volumePath.trim().isEmpty()) {
487+
if (StringUtils.isBlank(volumePath)) {
481488
return false;
482489
}
483490

‎framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1663,14 +1663,14 @@ public Extension getExtensionForCluster(long clusterId) {
16631663
public List<String> getExtensionReservedResourceDetails(long extensionId) {
16641664
ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(extensionId,
16651665
ApiConstants.RESERVED_RESOURCE_DETAILS);
1666-
if (detailsVO == null || !StringUtils.isNotBlank(detailsVO.getValue())) {
1667-
return Collections.emptyList();
1668-
}
16691666
List<String> reservedDetails = new ArrayList<>();
1670-
String[] parts = detailsVO.getValue().split(",");
1671-
for (String part : parts) {
1672-
if (StringUtils.isNotBlank(part)) {
1673-
reservedDetails.add(part.trim());
1667+
if (detailsVO != null && StringUtils.isNotBlank(detailsVO.getValue())) {
1668+
String[] parts = detailsVO.getValue().split(",");
1669+
for (String part : parts) {
1670+
String trimmedPart = part.trim();
1671+
if (StringUtils.isNotBlank(trimmedPart)) {
1672+
reservedDetails.add(trimmedPart);
1673+
}
16741674
}
16751675
}
16761676
addInbuiltExtensionReservedResourceDetails(extensionId, reservedDetails);

‎plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,15 @@ public class HypervInvestigator extends AdapterBase implements Investigator {
4141

4242
@Override
4343
public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM {
44-
Status status = isAgentAlive(host);
44+
Status status = getHostAgentStatus(host);
4545
if (status == null) {
4646
throw new UnknownVM();
4747
}
48-
return status == Status.Up ? true : null;
48+
return status == Status.Up;
4949
}
5050

5151
@Override
52-
public Status isAgentAlive(Host agent) {
52+
public Status getHostAgentStatus(Host agent) {
5353
if (agent.getHypervisorType() != Hypervisor.HypervisorType.Hyperv) {
5454
return null;
5555
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java‎

Lines changed: 21 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,7 @@
1919
package com.cloud.ha;
2020

2121
import com.cloud.agent.AgentManager;
22-
import com.cloud.agent.api.Answer;
23-
import com.cloud.agent.api.CheckOnHostCommand;
2422
import com.cloud.host.Host;
25-
import com.cloud.host.HostVO;
2623
import com.cloud.host.Status;
2724
import com.cloud.host.dao.HostDao;
2825
import com.cloud.hypervisor.Hypervisor;
@@ -34,11 +31,12 @@
3431
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
3532
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
3633
import org.apache.cloudstack.ha.HAManager;
34+
import org.apache.cloudstack.kvm.ha.KVMHostActivityChecker;
3735
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
3836
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
3937

4038
import javax.inject.Inject;
41-
import java.util.Arrays;
39+
import java.util.Collections;
4240
import java.util.List;
4341

4442
public class KVMInvestigator extends AdapterBase implements Investigator {
@@ -54,13 +52,15 @@ public class KVMInvestigator extends AdapterBase implements Investigator {
5452
private HAManager haManager;
5553
@Inject
5654
private DataStoreProviderManager dataStoreProviderMgr;
55+
@Inject
56+
private KVMHostActivityChecker hostActivityChecker;
5757

5858
@Override
5959
public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM {
6060
if (haManager.isHAEligible(host)) {
6161
return haManager.isVMAliveOnHost(host);
6262
}
63-
Status status = isAgentAlive(host);
63+
Status status = getHostAgentStatus(host);
6464
logger.debug("HA: HOST is ineligible legacy state {} for host {}", status, host);
6565
if (status == null) {
6666
throw new UnknownVM();
@@ -73,86 +73,41 @@ public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws Unkno
7373
}
7474

7575
@Override
76-
public Status isAgentAlive(Host agent) {
77-
if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
76+
public Status getHostAgentStatus(Host host) {
77+
if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
7878
return null;
7979
}
8080

81-
if (haManager.isHAEligible(agent)) {
82-
return haManager.getHostStatus(agent);
81+
if (haManager.isHAEligible(host)) {
82+
return haManager.getHostStatusFromHAConfig(host);
8383
}
8484

85-
List<StoragePoolVO> clusterPools = _storagePoolDao.findPoolsInClusters(Arrays.asList(agent.getClusterId()), null);
86-
boolean storageSupportHA = storageSupportHa(clusterPools);
87-
if (!storageSupportHA) {
88-
List<StoragePoolVO> zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(agent.getDataCenterId(), agent.getHypervisorType());
89-
storageSupportHA = storageSupportHa(zonePools);
85+
List<StoragePoolVO> clusterPools = _storagePoolDao.findPoolsInClusters(Collections.singletonList(host.getClusterId()), null);
86+
boolean storageSupportsHA = storageSupportsHA(clusterPools);
87+
if (!storageSupportsHA) {
88+
List<StoragePoolVO> zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(host.getDataCenterId(), host.getHypervisorType());
89+
storageSupportsHA = storageSupportsHA(zonePools);
9090
}
91-
if (!storageSupportHA) {
92-
logger.warn("Agent investigation was requested on host {}, but host does not support investigation because it has no NFS storage. Skipping investigation.", agent);
91+
if (!storageSupportsHA) {
92+
logger.warn("Agent investigation was requested on host {}, but host does not support investigation" +
93+
" because it has no HA supported storage. Skipping investigation.", host);
9394
return null;
9495
}
9596

96-
Status hostStatus = null;
97-
Status neighbourStatus = null;
98-
boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value();
99-
CheckOnHostCommand cmd = new CheckOnHostCommand(agent, reportFailureIfOneStorageIsDown);
100-
101-
try {
102-
Answer answer = _agentMgr.easySend(agent.getId(), cmd);
103-
if (answer != null) {
104-
hostStatus = answer.getResult() ? Status.Down : Status.Up;
105-
}
106-
} catch (Exception e) {
107-
logger.debug("Failed to send command to host: {}", agent);
108-
}
109-
if (hostStatus == null) {
110-
hostStatus = Status.Disconnected;
111-
}
112-
113-
List<HostVO> neighbors = _resourceMgr.listHostsInClusterByStatus(agent.getClusterId(), Status.Up);
114-
for (HostVO neighbor : neighbors) {
115-
if (neighbor.getId() == agent.getId()
116-
|| (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) {
117-
continue;
118-
}
119-
logger.debug("Investigating host:{} via neighbouring host:{}", agent, neighbor);
120-
try {
121-
Answer answer = _agentMgr.easySend(neighbor.getId(), cmd);
122-
if (answer != null) {
123-
neighbourStatus = answer.getResult() ? Status.Down : Status.Up;
124-
logger.debug("Neighbouring host:{} returned status:{} for the investigated host:{}", neighbor, neighbourStatus, agent);
125-
if (neighbourStatus == Status.Up) {
126-
break;
127-
}
128-
}
129-
} catch (Exception e) {
130-
logger.debug("Failed to send command to host: {}", neighbor);
131-
}
132-
}
133-
if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) {
134-
hostStatus = Status.Disconnected;
135-
}
136-
if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) {
137-
hostStatus = Status.Down;
138-
}
139-
logger.debug("HA: HOST is ineligible legacy state {} for host {}", hostStatus, agent);
140-
return hostStatus;
97+
return hostActivityChecker.getHostAgentStatus(host);
14198
}
14299

143-
private boolean storageSupportHa(List<StoragePoolVO> pools) {
144-
boolean storageSupportHA = false;
100+
private boolean storageSupportsHA(List<StoragePoolVO> pools) {
145101
for (StoragePoolVO pool : pools) {
146102
DataStoreProvider storeProvider = dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
147103
DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
148104
if (storeDriver instanceof PrimaryDataStoreDriver) {
149105
PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver)storeDriver;
150106
if (primaryStoreDriver.isStorageSupportHA(pool.getPoolType())) {
151-
storageSupportHA = true;
152-
break;
107+
return true;
153108
}
154109
}
155110
}
156-
return storageSupportHA;
111+
return false;
157112
}
158113
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,9 @@
3535
public class KVMHABase {
3636
protected Logger logger = LogManager.getLogger(getClass());
3737
private long _timeout = 60000; /* 1 minutes */
38-
protected long _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT);
39-
protected long _heartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY);
38+
protected long _heartBeatUpdateFreqInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY);
4039
protected long _heartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES);
41-
protected long _heartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
40+
protected long _heartBeatUpdateRetrySleepInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
4241

4342
public static enum PoolType {
4443
PrimaryStorage, SecondaryStorage
@@ -138,7 +137,7 @@ protected String checkingMountPoint(HAStoragePool pool, String poolName) {
138137
/* Can't find the mount point? */
139138
/* we need to mount it under poolName */
140139
if (poolName != null) {
141-
Script mount = new Script("/bin/bash", 60000);
140+
Script mount = new Script("/bin/bash", _timeout);
142141
mount.add("-c");
143142
mount.add("mount " + mountSource + " " + destPath);
144143
String result = mount.execute();
@@ -154,7 +153,6 @@ protected String checkingMountPoint(HAStoragePool pool, String poolName) {
154153
}
155154

156155
protected String getMountPoint(HAStoragePool storagePool) {
157-
158156
StoragePool pool = null;
159157
String poolName = null;
160158
try {
@@ -171,7 +169,6 @@ protected String getMountPoint(HAStoragePool storagePool) {
171169
}
172170
poolName = pool.getName();
173171
}
174-
175172
} catch (LibvirtException e) {
176173
logger.debug("Ignoring libvirt error.", e);
177174
} finally {
@@ -234,7 +231,7 @@ protected String runScriptRetry(String cmdString, OutputInterpreter interpreter)
234231
return result;
235232
}
236233

237-
public Boolean checkingHeartBeat() {
234+
public Boolean hasHeartBeat() {
238235
// TODO Auto-generated method stub
239236
return null;
240237
}

0 commit comments

Comments
 (0)