From 201ffc751ee78387bb267a83e94d4df9983c2ada Mon Sep 17 00:00:00 2001 From: Brad <100990646+Dogface2k@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:34:09 +0100 Subject: [PATCH] VMware: wait for NSX DV port group visibility Poll vCenter for named NSX backing port groups using a validated timeout, defaulting to 60 seconds and capped by the VMware operation timeout. Follow each completed sleep with a lookup, including the final sleep, and preserve lookup errors and interruption. Convert typed configuration values before validation. Cover timeout routing, configuration updates and readiness behavior in the existing configuration and VMware test suites. Co-authored-by: dahn --- .../framework/config/ValidatedConfigKey.java | 2 +- .../framework/config/ConfigKeyTest.java | 23 +++ .../vmware/manager/VmwareManager.java | 12 ++ .../vmware/manager/VmwareManagerImpl.java | 2 +- .../vmware/resource/VmwareResource.java | 16 +- .../vmware/resource/VmwareResourceTest.java | 108 ++++++++++++++ .../vmware/mo/HypervisorHostHelper.java | 57 +++++-- .../vmware/mo/HypervisorHostHelperTest.java | 139 ++++++++++++++++++ 8 files changed, 342 insertions(+), 17 deletions(-) diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ValidatedConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ValidatedConfigKey.java index 4fcbe4151d30..61d64a15967f 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ValidatedConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ValidatedConfigKey.java @@ -32,7 +32,7 @@ public Consumer getValidator() { public void validateValue(String value) { if (validator != null) { - validator.accept((T) value); + validator.accept(valueOf(value)); } } } diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java index 50be7200d56f..9d6715b41dda 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.framework.config; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Assert; import org.junit.Test; @@ -76,4 +77,26 @@ public void testGetDescendants() { Assert.assertTrue(descendant == Scope.Cluster || descendant == Scope.StoragePool || descendant == Scope.ImageStore); } } + + @Test + public void testValidatedNumericValue() { + AtomicReference received = new AtomicReference<>(); + ValidatedConfigKey key = new ValidatedConfigKey<>("Advanced", Long.class, "test.validated", "0", + "Test configuration", true, Scope.Global, null, received::set); + key.validateValue("60000"); + Assert.assertEquals(Long.valueOf(60000), received.get()); + Assert.assertThrows(NumberFormatException.class, () -> key.validateValue("invalid")); + Assert.assertEquals(Long.valueOf(60000), received.get()); + } + + @Test + public void testValidatedStringValueIsUnchanged() { + AtomicReference received = new AtomicReference<>(); + ValidatedConfigKey key = new ValidatedConfigKey<>("Advanced", String.class, "test.validated", null, + "Test configuration", true, Scope.Global, null, received::set); + key.validateValue(" value "); + Assert.assertEquals(" value ", received.get()); + key.validateValue(null); + Assert.assertNull(received.get()); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java index 8be881b94940..e003b876d87f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java @@ -21,7 +21,9 @@ import java.util.Map; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.ValidatedConfigKey; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.util.VmwareContext; @@ -34,6 +36,16 @@ public interface VmwareManager { public static final ConfigKey s_vmwareNicHotplugWaitTimeout = new ConfigKey("Advanced", Long.class, "vmware.nic.hotplug.wait.timeout", "15000", "Wait timeout (milli seconds) for hot plugged NIC of VM to be detected by guest OS.", false, ConfigKey.Scope.Global); + ConfigKey VmwareNsxDvPortGroupWaitTimeout = new ValidatedConfigKey<>("Advanced", Long.class, + "vmware.nsx.dvportgroup.wait.timeout", "60000", + "Maximum polling time (milliseconds) per network preparation for an NSX-backed distributed virtual port group to appear in vCenter. " + + "Set to 0 for a single lookup. Must not be negative. Capped by the VMware operation timeout; vCenter API request timeouts are separate.", + true, ConfigKey.Scope.Global, null, timeout -> { + if (timeout == null || timeout < 0) { + throw new InvalidParameterValueException("vmware.nsx.dvportgroup.wait.timeout must be a non-negative number of milliseconds"); + } + }); + public static final ConfigKey templateCleanupInterval = new ConfigKey("Advanced", Long.class, "vmware.full.clone.template.cleanup.period", "0", "period (in minutes) between the start of template cleanup jobs for VMWare full cloned Templates.", false, ConfigKey.Scope.Global); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 8052317817ad..5c477a5a6923 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -301,7 +301,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder, s_vmwareOVAPackageTimeout, s_vmwareCleanupPortGroups, VMWARE_STATS_TIME_WINDOW, VmwareUserVmNicDeviceType}; + return new ConfigKey[] {s_vmwareNicHotplugWaitTimeout, VmwareNsxDvPortGroupWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder, s_vmwareOVAPackageTimeout, s_vmwareCleanupPortGroups, VMWARE_STATS_TIME_WINDOW, VmwareUserVmNicDeviceType}; } @Override public boolean configure(String name, Map params) throws ConfigurationException { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 7b9f258920c5..e3d909a30760 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4039,7 +4039,7 @@ private String getVlanInfo(NicTO nicTo, String defaultVlan) { return defaultVlan; } - private Pair prepareNetworkFromNicInfo(HostMO hostMo, NicTO nicTo, boolean configureVServiceInNexus, String networkName, VirtualMachine.Type vmType) + Pair prepareNetworkFromNicInfo(HostMO hostMo, NicTO nicTo, boolean configureVServiceInNexus, String networkName, VirtualMachine.Type vmType) throws Exception { Ternary switchDetails = getTargetSwitch(nicTo); @@ -4067,14 +4067,26 @@ private Pair prepareNetworkFromNicInfo(HostMO ho // plumb this network to the isolated vlan. svlanId = getPvlanInfo(nicTo); } + long networkTimeoutMs = _opsTimeout; + if (switchType == VirtualSwitchType.VMwareDistributedVirtualSwitch && nicTo.getBroadcastType() == BroadcastDomainType.NSX && networkName != null) { + long nsxTimeoutMs = getNsxDvPortGroupWaitTimeout(); + if (nsxTimeoutMs < 0) { + throw new IllegalArgumentException(VmwareManager.VmwareNsxDvPortGroupWaitTimeout.key() + " must not be negative"); + } + networkTimeoutMs = Math.min(_opsTimeout, nsxTimeoutMs); + } networkInfo = HypervisorHostHelper.prepareNetwork(switchName, namePrefix, hostMo, vlanId, svlanId, - nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(), _opsTimeout, switchType, + nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(), networkTimeoutMs, switchType, _portsPerDvPortGroup, nicTo.getGateway(), configureVServiceInNexus, nicTo.getBroadcastType(), _vsmCredentials, nicTo.getDetails(), networkName); } return networkInfo; } + protected long getNsxDvPortGroupWaitTimeout() { + return VmwareManager.VmwareNsxDvPortGroupWaitTimeout.value(); + } + // return Ternary private Ternary getTargetSwitch(NicTO nicTo) throws CloudException { TrafficType[] supportedTrafficTypes = new TrafficType[]{TrafficType.Guest, TrafficType.Public, TrafficType.Control, TrafficType.Management, TrafficType.Storage}; diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java index be7ee46f6dda..489ea26c1093 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; @@ -40,6 +41,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import com.cloud.agent.api.Command; import com.cloud.agent.api.ScaleVmAnswer; @@ -67,6 +69,12 @@ import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.springframework.test.util.ReflectionTestUtils; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckGuestOsMappingAnswer; @@ -106,6 +114,15 @@ import com.vmware.vim25.VirtualDeviceConfigSpec; import com.vmware.vim25.VirtualMachineConfigSpec; import com.vmware.vim25.VirtualMachineVideoCard; +import com.cloud.configuration.ConfigurationManagerImpl; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.vmware.manager.VmwareManager; +import com.cloud.hypervisor.vmware.manager.VmwareManagerImpl; +import com.cloud.hypervisor.vmware.mo.VirtualSwitchType; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.TrafficType; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.vm.VirtualMachine; @RunWith(MockitoJUnitRunner.class) public class VmwareResourceTest { @@ -874,4 +891,95 @@ public void testRemoveVirtualTPMDevice() throws Exception { Mockito.verify(vmwareResource, Mockito.times(1)).removeVirtualTPMDevice(vmConfigSpec, tpm); Mockito.verify(deviceChanges, Mockito.times(1)).add(any(VirtualDeviceConfigSpec.class)); } + + private void assertNetworkTimeout(VirtualSwitchType switchType, BroadcastDomainType broadcast, String name, + long configured, long expected) throws Exception { + doReturn(configured).when(_resource).getNsxDvPortGroupWaitTimeout(); + Mockito.clearInvocations(_resource); + NicTO nic = new NicTO(); + nic.setName("NSX-VDS,untagged," + switchType); + nic.setType(TrafficType.Guest); + nic.setBroadcastType(broadcast); + nic.setBroadcastUri(broadcast.toUri(broadcast == BroadcastDomainType.Vlan ? "123" : "nsx")); + try (MockedStatic helper = mockStatic(HypervisorHostHelper.class)) { + _resource.prepareNetworkFromNicInfo(host, nic, false, name, VirtualMachine.Type.User); + if (switchType == VirtualSwitchType.StandardVirtualSwitch) { + helper.verify(() -> HypervisorHostHelper.prepareNetwork(any(), any(), eq(host), any(), any(), any(), eq(expected), + eq(true), eq(broadcast), any(), any())); + } else { + helper.verify(() -> HypervisorHostHelper.prepareNetwork(any(), any(), eq(host), any(), any(), any(), any(), eq(expected), + eq(switchType), Mockito.anyInt(), any(), eq(false), eq(broadcast), any(), any(), eq(name))); + } + helper.verifyNoMoreInteractions(); + if (broadcast != BroadcastDomainType.NSX || switchType != VirtualSwitchType.VMwareDistributedVirtualSwitch || name == null) { + verify(_resource, never()).getNsxDvPortGroupWaitTimeout(); + } + } + } + + @Test + public void testNsxVisibilityTimeoutRouting() throws Exception { + assertEquals(60000L, new VmwareResource().getNsxDvPortGroupWaitTimeout()); + assertTrue(Arrays.asList(new VmwareManagerImpl().getConfigKeys()).contains(VmwareManager.VmwareNsxDvPortGroupWaitTimeout)); + for (long timeout : new long[] {0, 2500, Long.MAX_VALUE}) { + assertNetworkTimeout(VirtualSwitchType.VMwareDistributedVirtualSwitch, BroadcastDomainType.NSX, "nsx", timeout, Math.min(900000, timeout)); + } + } + + @Test + public void testOtherNetworkTimeoutsAreUnchanged() throws Exception { + assertNetworkTimeout(VirtualSwitchType.VMwareDistributedVirtualSwitch, BroadcastDomainType.Vlan, "network", -1, 900000); + assertNetworkTimeout(VirtualSwitchType.VMwareDistributedVirtualSwitch, BroadcastDomainType.Lswitch, "network", -1, 900000); + assertNetworkTimeout(VirtualSwitchType.VMwareDistributedVirtualSwitch, BroadcastDomainType.NSX, null, -1, 900000); + assertNetworkTimeout(VirtualSwitchType.NexusDistributedVirtualSwitch, BroadcastDomainType.NSX, "network", -1, 900000); + assertNetworkTimeout(VirtualSwitchType.StandardVirtualSwitch, BroadcastDomainType.Vlan, "network", -1, 900000); + } + + @Test + public void testNsxTimeoutConfigurationIsValidatedBeforeSaving() { + String key = VmwareManager.VmwareNsxDvPortGroupWaitTimeout.key(); + ConfigurationDao dao = mock(ConfigurationDao.class); + ConfigDepotImpl depot = mock(ConfigDepotImpl.class); + ConfigurationVO row = mock(ConfigurationVO.class); + MessageBus bus = mock(MessageBus.class); + AtomicReference stored = new AtomicReference<>("60000"); + when(dao.findByName(key)).thenReturn(row); + when(row.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); + doReturn(VmwareManager.VmwareNsxDvPortGroupWaitTimeout).when(depot).get(key); + when(dao.getValue(key)).thenAnswer(ignored -> stored.get()); + when(dao.update(eq(key), anyString(), anyString())).thenAnswer(invocation -> { + stored.set(invocation.getArgument(2)); + return true; + }); + when(depot.getConfigStringValue(key, ConfigKey.Scope.Global, null)).thenAnswer(ignored -> stored.get()); + ConfigurationManagerImpl manager = new ConfigurationManagerImpl(); + ReflectionTestUtils.setField(manager, "_configDao", dao); + ReflectionTestUtils.setField(manager, "_configDepot", depot); + ReflectionTestUtils.setField(manager, "messageBus", bus); + ConfigDepotImpl previous = (ConfigDepotImpl) ReflectionTestUtils.getField(ConfigKey.class, "s_depot"); + ConfigKey.init(depot); + try (MockedStatic transactions = mockStatic(TransactionLegacy.class)) { + for (String value : List.of("-1", "", "invalid", "1.5", "9223372036854775808")) { + assertThrows(InvalidParameterValueException.class, + () -> manager.updateConfiguration(1, key, "Advanced", value, ConfigKey.Scope.Global, null)); + } + transactions.verifyNoInteractions(); + verify(dao, never()).update(anyString(), anyString(), anyString()); + verify(depot, never()).invalidateConfigCache(key, ConfigKey.Scope.Global, null); + Mockito.verifyNoInteractions(bus); + assertEquals("60000", stored.get()); + assertEquals(60000L, new VmwareResource().getNsxDvPortGroupWaitTimeout()); + TransactionLegacy transaction = mock(TransactionLegacy.class); + transactions.when(TransactionLegacy::currentTxn).thenReturn(transaction); + for (String value : List.of("0", "60000", "2500", String.valueOf(Long.MAX_VALUE))) { + assertEquals(value, manager.updateConfiguration(1, key, "Advanced", value, ConfigKey.Scope.Global, null)); + assertEquals(Long.parseLong(value), new VmwareResource().getNsxDvPortGroupWaitTimeout()); + } + verify(transaction, times(4)).start(); + verify(transaction, times(4)).commit(); + verify(depot, times(4)).invalidateConfigCache(key, ConfigKey.Scope.Global, null); + } finally { + ConfigKey.init(previous); + } + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index a079081b847e..21019aff2845 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -30,6 +30,8 @@ import java.util.Map; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -649,7 +651,7 @@ public static Pair prepareNetwork(String physica bWaitPortGroupReady = false; } else if (BroadcastDomainType.NSX == broadcastDomainType && Objects.nonNull(netName)){ networkName = netName; - bWaitPortGroupReady = false; + bWaitPortGroupReady = true; } else { boolean dvSwitchSupportNewPolicies = (isFeatureSupportedInVcenterApiVersion(vcApiVersion, MINIMUM_VCENTER_API_VERSION_WITH_DVS_NEW_POLICIES_SUPPORT) && isVersionEqualOrHigher(dvSwitchVersion, MINIMUM_DVS_VERSION_WITH_NEW_POLICIES_SUPPORT)); @@ -719,6 +721,9 @@ public static Pair prepareNetwork(String physica morNetwork = dataCenterMo.getDvPortGroupMor(networkName); if (morNetwork == null) { String msg = "Failed to create guest network " + networkName; + if (vSwitchType == VirtualSwitchType.VMwareDistributedVirtualSwitch && broadcastDomainType == BroadcastDomainType.NSX && netName != null) { + msg = String.format("NSX DV port group %s was not visible in vCenter (polling timeout: %d ms)", networkName, timeOutMs); + } LOGGER.error(msg); throw new Exception(msg); } @@ -1031,21 +1036,47 @@ public static boolean isSpecMatch(DVPortgroupConfigInfo currentDvPortgroupInfo, } public static ManagedObjectReference waitForDvPortGroupReady(DatacenterMO dataCenterMo, String dvPortGroupName, long timeOutMs) throws Exception { - ManagedObjectReference morDvPortGroup = null; + return waitForDvPortGroupReady(dataCenterMo, dvPortGroupName, timeOutMs, System::nanoTime, TimeUnit.NANOSECONDS::sleep); + } - // if DvPortGroup is just created, we may fail to retrieve it, we - // need to retry - long startTick = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTick <= timeOutMs) { - morDvPortGroup = dataCenterMo.getDvPortGroupMor(dvPortGroupName); - if (morDvPortGroup != null) { - break; - } + @FunctionalInterface + interface NanoSleeper { + void sleep(long nanos) throws InterruptedException; + } - LOGGER.info("Waiting for dvPortGroup " + dvPortGroupName + " to be ready"); - Thread.sleep(1000); + static ManagedObjectReference waitForDvPortGroupReady(DatacenterMO dataCenterMo, String dvPortGroupName, long timeOutMs, + LongSupplier nanoTime, NanoSleeper sleeper) throws Exception { + if (timeOutMs < 0) { + throw new IllegalArgumentException("DV port group readiness timeout must not be negative"); + } + long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeOutMs); + long startTick = nanoTime.getAsLong(); + boolean firstLookup = true; + try { + // Always check after sleeping, even if the last sleep reaches or passes the deadline. + while (true) { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("Interrupted while waiting for dvPortGroup " + dvPortGroupName); + } + ManagedObjectReference morDvPortGroup = dataCenterMo.getDvPortGroupMor(dvPortGroupName); + if (morDvPortGroup != null) { + return morDvPortGroup; + } + if (firstLookup) { + LOGGER.info("Waiting up to {} ms for dvPortGroup {} to be visible in vCenter", timeOutMs, dvPortGroupName); + firstLookup = false; + } + long elapsedNanos = nanoTime.getAsLong() - startTick; + if (elapsedNanos >= timeoutNanos) { + break; + } + sleeper.sleep(Math.min(TimeUnit.SECONDS.toNanos(1), timeoutNanos - elapsedNanos)); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw e; } - return morDvPortGroup; + return null; } public static boolean isSpecMatch(DVPortgroupConfigInfo configInfo, Integer vid, DVSTrafficShapingPolicy shapingPolicy) { diff --git a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java index ae4a05380c14..3b67068bd6e7 100644 --- a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java +++ b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelperTest.java @@ -20,17 +20,25 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoMoreInteractions; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.TimeUnit; import com.cloud.hypervisor.vmware.util.VmwareClient; import com.cloud.network.Networks; @@ -1041,4 +1049,135 @@ public void testPrepareNetwork() throws Exception { null, networkName); assertEquals(morNet.second(), networkName); } + + @Test + public void testPrepareNetworkWaitsForNsxDvPortGroup() throws Exception { + String networkName = "D1-A2-Z2-V8-S4"; + DynamicProperty property = new DynamicProperty(); + property.setVal(networkName); + + when(hostMO.getHyperHostDatacenter()).thenReturn(mor); + when(datacenterMO.getDvSwitchMor(any(String.class))).thenReturn(mor); + when(vmwareClient.getDecendentMoRef(nullable(ManagedObjectReference.class), any(String.class), any(String.class))).thenReturn(mor); + when(vimService.retrieveProperties(any(), anyList())).thenReturn(List.of(), List.of(ocs)); + when(ocs.getPropSet()).thenReturn(List.of(property)); + when(ocs.getObj()).thenReturn(mor); + + Pair morNet = HypervisorHostHelper.prepareNetwork("NSX-VDS", "cloud.guest", hostMO, null, null, + 200, null, 900000, VirtualSwitchType.VMwareDistributedVirtualSwitch, 1, null, + false, Networks.BroadcastDomainType.NSX, null, + null, networkName); + assertEquals(networkName, morNet.second()); + verify(vimService, times(2)).retrieveProperties(any(), anyList()); + } + + @Test + public void testDvPortGroupWaitBudget() throws Exception { + for (long timeout : new long[] {0, 25, 2500}) { + for (long start : new long[] {0, Long.MAX_VALUE - TimeUnit.MILLISECONDS.toNanos(500)}) { + DatacenterMO dc = mock(DatacenterMO.class); + AtomicLong clock = new AtomicLong(start); + List sleeps = new ArrayList<>(); + assertNull(HypervisorHostHelper.waitForDvPortGroupReady(dc, "nsx", timeout, clock::get, nanos -> { + sleeps.add(nanos); + clock.addAndGet(nanos); + })); + assertEquals(TimeUnit.MILLISECONDS.toNanos(timeout), clock.get() - start); + List expected = timeout == 0 ? List.of() : timeout == 25 ? List.of(25000000L) + : List.of(1000000000L, 1000000000L, 500000000L); + assertEquals(expected, sleeps); + verify(dc, times(expected.size() + 1)).getDvPortGroupMor("nsx"); + } + } + } + + @Test + public void testDvPortGroupWaitFindsGroupDuringFinalSleep() throws Exception { + for (long timeout : new long[] {25, 2500}) { + for (long wakeupDelay : new long[] {0, TimeUnit.MILLISECONDS.toNanos(1)}) { + DatacenterMO dc = mock(DatacenterMO.class); + AtomicLong clock = new AtomicLong(); + long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeout); + long visibleAtNanos = timeoutNanos - TimeUnit.MILLISECONDS.toNanos(1); + when(dc.getDvPortGroupMor("nsx")).thenAnswer(ignored -> clock.get() >= visibleAtNanos ? mor : null); + + assertSame(mor, HypervisorHostHelper.waitForDvPortGroupReady(dc, "nsx", timeout, clock::get, + nanos -> clock.addAndGet(nanos + wakeupDelay))); + + assertEquals(timeoutNanos + wakeupDelay, clock.get()); + verify(dc, times(timeout == 25 ? 2 : 4)).getDvPortGroupMor("nsx"); + } + } + } + + @Test + public void testDvPortGroupWaitChecksOnceWithZeroBudget() throws Exception { + when(datacenterMO.getDvPortGroupMor("nsx")).thenReturn(mor); + assertSame(mor, HypervisorHostHelper.waitForDvPortGroupReady(datacenterMO, "nsx", 0, () -> 0, + nanos -> { throw new AssertionError("Unexpected sleep"); })); + assertThrows(IllegalArgumentException.class, + () -> HypervisorHostHelper.waitForDvPortGroupReady(datacenterMO, "nsx", -1)); + verify(datacenterMO).getDvPortGroupMor("nsx"); + } + + @Test + public void testDvPortGroupWaitAccountsForSlowLookupAndLateWakeup() throws Exception { + AtomicLong clock = new AtomicLong(); + when(datacenterMO.getDvPortGroupMor("nsx")).thenAnswer(ignored -> { + clock.addAndGet(TimeUnit.SECONDS.toNanos(2)); + return null; + }); + assertNull(HypervisorHostHelper.waitForDvPortGroupReady(datacenterMO, "nsx", 1000, clock::get, + nanos -> { throw new AssertionError("Lookup already exhausted the budget"); })); + verify(datacenterMO).getDvPortGroupMor("nsx"); + DatacenterMO dc = mock(DatacenterMO.class); + assertNull(HypervisorHostHelper.waitForDvPortGroupReady(dc, "nsx", 1000, clock::get, + nanos -> clock.addAndGet(TimeUnit.SECONDS.toNanos(2)))); + verify(dc, times(2)).getDvPortGroupMor("nsx"); + } + + @Test + public void testDvPortGroupWaitPropagatesLookupFailure() throws Exception { + Exception failure = new Exception("vCenter unavailable"); + when(datacenterMO.getDvPortGroupMor("nsx")).thenThrow(failure); + assertSame(failure, assertThrows(Exception.class, + () -> HypervisorHostHelper.waitForDvPortGroupReady(datacenterMO, "nsx", 60000))); + verify(datacenterMO).getDvPortGroupMor("nsx"); + } + + @Test + public void testDvPortGroupWaitPreservesInterrupt() throws Exception { + for (boolean duringLookup : new boolean[] {true, false}) { + DatacenterMO dc = mock(DatacenterMO.class); + InterruptedException interrupted = new InterruptedException("cancelled"); + if (duringLookup) { + when(dc.getDvPortGroupMor("nsx")).thenThrow(interrupted); + } + try { + assertSame(interrupted, assertThrows(InterruptedException.class, + () -> HypervisorHostHelper.waitForDvPortGroupReady(dc, "nsx", 60000, () -> 0, + nanos -> { throw interrupted; }))); + assertTrue(Thread.currentThread().isInterrupted()); + verify(dc).getDvPortGroupMor("nsx"); + assertThrows(InterruptedException.class, + () -> HypervisorHostHelper.waitForDvPortGroupReady(dc, "nsx", 60000)); + verify(dc).getDvPortGroupMor("nsx"); + } finally { + Thread.interrupted(); + } + } + } + + @Test + public void testPrepareNetworkFailsForMissingNsxPortGroup() throws Exception { + when(hostMO.getHyperHostDatacenter()).thenReturn(mor); + when(vmwareClient.getDecendentMoRef(nullable(ManagedObjectReference.class), any(String.class), any(String.class))).thenReturn(mor); + when(vimService.retrieveProperties(any(), anyList())).thenReturn(List.of()); + Exception failure = assertThrows(Exception.class, () -> HypervisorHostHelper.prepareNetwork("NSX-VDS", "cloud.guest", hostMO, + null, null, 200, null, 0, VirtualSwitchType.VMwareDistributedVirtualSwitch, 1, null, + false, Networks.BroadcastDomainType.NSX, null, null, "nsx")); + assertEquals("NSX DV port group nsx was not visible in vCenter (polling timeout: 0 ms)", failure.getMessage()); + verify(vimService).retrieveProperties(any(), anyList()); + verifyNoMoreInteractions(vimService); + } }