From 88d09d7c26840984e943a51f7931911e06b2e1cc Mon Sep 17 00:00:00 2001 From: "Locharla, Sandeep" Date: Tue, 19 May 2026 07:36:58 +0530 Subject: [PATCH 1/6] CSTACKEX-189: Cloudstack StoragePool Operations automation tests --- test/integration/plugins/ontap/__init__.py | 16 + test/integration/plugins/ontap/ontap.cfg | 117 +++ .../plugins/ontap/ontap_test_base.py | 313 ++++++++ ...test_ontap_create_primary_storage_iscsi.py | 710 ++++++++++++++++++ .../test_ontap_create_primary_storage_nfs3.py | 404 ++++++++++ 5 files changed, 1560 insertions(+) create mode 100644 test/integration/plugins/ontap/__init__.py create mode 100644 test/integration/plugins/ontap/ontap.cfg create mode 100644 test/integration/plugins/ontap/ontap_test_base.py create mode 100644 test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py create mode 100644 test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py diff --git a/test/integration/plugins/ontap/__init__.py b/test/integration/plugins/ontap/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/ontap.cfg b/test/integration/plugins/ontap/ontap.cfg new file mode 100644 index 000000000000..9d0e2bec06e1 --- /dev/null +++ b/test/integration/plugins/ontap/ontap.cfg @@ -0,0 +1,117 @@ +{ + "zones": [ + { + "name": "Zone-ONTAP", + "localstorageenabled": true, + "dns1": "8.8.8.8", + "internal_dns1": "10.192.0.250", + "guestcidraddress": "10.1.1.0/24", + "physical_networks": [ + { + "broadcastdomainrange": "Zone", + "name": "physical_network", + "vlan": "100-300", + "traffictypes": [ + {"typ": "Guest"}, + {"typ": "Management"}, + {"typ": "Public"} + ], + "providers": [ + { + "broadcastdomainrange": "ZONE", + "name": "VirtualRouter" + } + ] + } + ], + "secondaryStorages": [ + { + "url": "nfs://10.193.56.61/exports/secondary", + "provider": "NFS", + "tags": "secondary-nfs" + } + ], + "ipranges": [ + { + "gateway": "10.193.56.1", + "startip": "10.193.56.70", + "endip": "10.193.56.79", + "netmask": "255.255.252.0", + "vlan": "untagged" + } + ], + "pods": [ + { + "name": "Pod-ONTAP", + "gateway": "10.193.56.1", + "startip": "10.193.56.80", + "endip": "10.193.56.89", + "netmask": "255.255.252.0", + "clusters": [ + { + "clustername": "KVM-Cluster-ONTAP", + "hypervisor": "KVM", + "clustertype": "CloudManaged", + "hosts": [ + { + "url": "http://10.193.56.61", + "username": "root", + "password": "netapp1!", + "hosttags": "kvmHostONTAP" + } + ], + "primaryStorages": [ + { + "name": "primary-nfs-ontap", + "url": "nfs://10.193.56.61/exports/primary", + "scope": "CLUSTER", + "provider": "DefaultPrimary", + "tags": "primary-nfs" + } + ] + } + ] + } + ] + } + ], + "dbSvr": { + "dbSvr": "10.193.56.61", + "passwd": "cloud", + "db": "cloud", + "port": 3306, + "user": "cloud" + }, + "logger": { + "LogFolderPath": "/tmp/" + }, + "TestData": { + "Path": "test/integration/plugins/ontap/ontap.cfg" + }, + "mgtSvr": [ + { + "mgtSvrIp": "10.193.56.61", + "port": 8096, + "user": "admin", + "passwd": "password", + "hypervisor": "kvm", + "timeout": 600 + } + ], + "ontap": { + "storageIP": "10.196.35.107", + "svmName": "vs0", + "username": "admin", + "password": "netapp1!", + "protocol": "NFS3", + "storagePoolScope": "CLUSTER", + "storagePoolProvider": "NetApp ONTAP", + "storagePoolTags": "ontap-nfs3", + "capacitybytes": 3355443200 + }, + "cloudstack": { + "zoneName": "Zone1", + "clusterName": "Cluster1", + "domainName": "ROOT" + } +} diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py new file mode 100644 index 000000000000..5d17d37df814 --- /dev/null +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -0,0 +1,313 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Shared base class and helper utilities for NetApp ONTAP Marvin integration tests. + +Provides: + OntapRestClient - thin wrapper around the ONTAP REST API (NFS + iSCSI methods) + _parse_pool_details - converts a StoragePool details attribute to a plain dict + OntapTestBase - base cloudstackTestCase with common tearDownClass, + _poll_pool_state, _create_volume, and _delete_pool +""" + +import logging +import random +import requests +import time +import urllib3 + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createVolume as createVolumeAPI, + deleteStoragePool as deleteStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + listDiskOfferings as listDiskOfferingsAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackAPI import listHosts as listHostsAPI +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.base import Account +from marvin.lib.common import get_domain, get_zone, list_clusters, list_storage_pools +from marvin.lib.utils import cleanup_resources + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +logger = logging.getLogger("OntapTestBase") + + +# --------------------------------------------------------------------------- +# Pool detail helper +# --------------------------------------------------------------------------- + +def _parse_pool_details(pool): + """ + Convert a StoragePool object's ``details`` attribute to a plain Python dict, + regardless of how Marvin chose to represent it. + + Note: listStoragePools only returns a subset of detail keys + (volumeUUID, exportPolicyName, exportPolicyId). For the full set + use the pool object returned directly by createStoragePool. + """ + details_raw = getattr(pool, "details", None) + if not details_raw: + return {} + if isinstance(details_raw, dict): + return details_raw + if isinstance(details_raw, list): + return {d.name: d.value for d in details_raw} + return { + k: v for k, v in vars(details_raw).items() + if not k.startswith("_") and k != "typeInfo" + } + + +# --------------------------------------------------------------------------- +# ONTAP REST helper +# --------------------------------------------------------------------------- + +class OntapRestClient: + """Thin wrapper around the ONTAP REST API for backend validation.""" + + def __init__(self, storage_ip, username, password, port=443): + self._base = "https://%s:%d/api" % (storage_ip, port) + self._auth = (username, password) + + def _get(self, path, params=None): + url = self._base + path + resp = requests.get(url, auth=self._auth, params=params, + verify=False, timeout=30) + resp.raise_for_status() + return resp.json() + + def get_volume(self, name): + """Return the ONTAP FlexVol record for the given name, or None.""" + data = self._get("/storage/volumes", params={"name": name}) + records = data.get("records", []) + if not records: + return None + uuid = records[0].get("uuid") + if uuid: + return self._get("/storage/volumes/%s" % uuid, + params={"fields": "name,uuid,state,space"}) + return records[0] + + # -- NFS helpers --------------------------------------------------------- + + def get_export_policy(self, name): + """Return the ONTAP NFS export policy record for the given name, or None.""" + data = self._get("/protocols/nfs/export-policies", params={"name": name}) + records = data.get("records", []) + if not records: + return None + policy_id = records[0].get("id") + if policy_id: + return self._get( + "/protocols/nfs/export-policies/%s" % policy_id, + params={"fields": "name,svm,rules"} + ) + return records[0] + + def get_data_lifs(self, svm_name): + """Return a list of NFS data LIF IP addresses for the given SVM.""" + data = self._get( + "/network/ip/interfaces", + params={"svm.name": svm_name, "services": "data-nfs", + "fields": "ip,name"} + ) + records = data.get("records", []) + return [r.get("ip", {}).get("address") + for r in records if r.get("ip", {}).get("address")] + + # -- iSCSI helpers ------------------------------------------------------- + + def get_igroup(self, svm_name, igroup_name): + """Return the ONTAP igroup record, or None if not found.""" + data = self._get("/protocols/san/igroups", + params={"svm.name": svm_name, "name": igroup_name, + "fields": "name,uuid,initiators"}) + records = data.get("records", []) + return records[0] if records else None + + def get_lun(self, svm_name, lun_path): + """Return the ONTAP LUN record for the given full path, or None.""" + data = self._get("/storage/luns", + params={"svm.name": svm_name, "name": lun_path, + "fields": "name,uuid,enabled,status"}) + records = data.get("records", []) + return records[0] if records else None + + def list_luns_in_volume(self, svm_name, vol_name): + """Return all LUN records whose path starts with /vol/{vol_name}/.""" + prefix = "/vol/%s/" % vol_name + data = self._get("/storage/luns", + params={"svm.name": svm_name, + "fields": "name,uuid,enabled,status"}) + return [r for r in data.get("records", []) + if r.get("name", "").startswith(prefix)] + + +# --------------------------------------------------------------------------- +# Base test class +# --------------------------------------------------------------------------- + +class OntapTestBase(cloudstackTestCase): + """ + Shared base for sequential ONTAP primary-storage workflow tests. + + Subclasses must: + - Set ``_vol_name_prefix`` to distinguish volume names per protocol. + - Define ``setUpClass`` that builds ``cls.testdata``, creates + ``cls.ontap`` and ``cls.svm_name``, then calls + ``cls._setup_cloudstack_resources(config, account_testdata)``. + - Define ``_create_pool`` (protocol-specific URL scheme and name). + """ + + # ---- shared state (set/cleared by individual tests) ---------------- + pool = None + volume = None + pool2 = None + volume2 = None + disk_offering_id = None + svm_name = None + cluster_hosts = None + ontap = None + testdata = None + zone = None + cluster = None + domain = None + account = None + _cleanup = [] + + # Subclass sets this to distinguish volume names, e.g. "OntapNFS3Vol" + _vol_name_prefix = "OntapVol" + + # ---- shared setup helper ------------------------------------------- + + @classmethod + def _setup_cloudstack_resources(cls, config, account_testdata): + """ + Resolve zone, cluster, domain, account, cluster hosts, and disk + offering from the Marvin config. Call this from subclass setUpClass + after ``cls.ontap`` and ``cls.svm_name`` have been assigned. + """ + cs_cfg = config.get("cloudstack", {}) + zone_name = cs_cfg.get("zoneName", None) + cluster_name = cs_cfg.get("clusterName", None) + domain_name = cs_cfg.get("domainName", "ROOT") + + cls.zone = get_zone(cls.apiClient, zone_name=zone_name) + clusters = (list_clusters(cls.apiClient, name=cluster_name) + if cluster_name else list_clusters(cls.apiClient)) + cls.cluster = clusters[0] + cls.domain = get_domain(cls.apiClient, domain_name=domain_name) + + cls.account = Account.create(cls.apiClient, account_testdata, admin=1) + cls._cleanup = [cls.account] + + list_hosts_cmd = listHostsAPI.listHostsCmd() + list_hosts_cmd.clusterid = cls.cluster.id + list_hosts_cmd.type = "Routing" + cls.cluster_hosts = cls.apiClient.listHosts(list_hosts_cmd) or [] + + list_do_cmd = listDiskOfferingsAPI.listDiskOfferingsCmd() + list_do_cmd.listall = True + offerings = cls.apiClient.listDiskOfferings(list_do_cmd) + cls.disk_offering_id = offerings[0].id if offerings else None + + # ---- shared teardown ----------------------------------------------- + + @classmethod + def tearDownClass(cls): + """Best-effort cleanup of any resources left behind by a failed run.""" + for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + cls.apiClient.deleteVolume(cmd) + except Exception as e: + logger.warning("tearDownClass: could not delete volume %s: %s" + % (vol.id, e)) + + for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: + try: + try: + cc = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cc.id = pool.id + cls.apiClient.cancelStorageMaintenance(cc) + time.sleep(5) + except Exception: + pass + try: + ec = updateStoragePoolAPI.updateStoragePoolCmd() + ec.id = pool.id + ec.enabled = True + cls.apiClient.updateStoragePool(ec) + time.sleep(3) + except Exception: + pass + dc = deleteStoragePoolAPI.deleteStoragePoolCmd() + dc.id = pool.id + dc.forced = True + cls.apiClient.deleteStoragePool(dc) + except Exception as e: + logger.warning("tearDownClass: could not delete pool %s: %s" + % (pool.id, e)) + + try: + cleanup_resources(cls.apiClient, cls._cleanup) + except Exception as e: + logger.debug("tearDownClass cleanup_resources: %s" % e) + + # No per-test tearDown — state intentionally persists between steps. + + # ---- shared helpers ------------------------------------------------ + + def _poll_pool_state(self, pool_id, target_state, timeout=120, interval=5): + """Poll listStoragePools until the pool reaches target_state or timeout.""" + deadline = time.time() + timeout + current_state = "unknown" + while time.time() < deadline: + pools = list_storage_pools(self.apiClient, id=pool_id) + if pools: + current_state = pools[0].state + if current_state == target_state: + return pools[0] + time.sleep(interval) + self.fail( + "Pool %s did not reach state '%s' within %ds (last: '%s')" + % (pool_id, target_state, timeout, current_state) + ) + + def _create_volume(self, pool_id): + """Create a data volume on the given pool; uses _vol_name_prefix.""" + cmd = createVolumeAPI.createVolumeCmd() + cmd.name = "%s_%d" % (self._vol_name_prefix, random.randint(0, 99999)) + cmd.diskofferingid = self.disk_offering_id + cmd.zoneid = self.zone.id + cmd.storageid = pool_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + return self.apiClient.createVolume(cmd) + + def _delete_pool(self, pool_id, forced=False): + """Issue deleteStoragePool for the given pool id.""" + cmd = deleteStoragePoolAPI.deleteStoragePoolCmd() + cmd.id = pool_id + if forced: + cmd.forced = True + self.apiClient.deleteStoragePool(cmd) diff --git a/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py b/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py new file mode 100644 index 000000000000..acdd4fe33ee8 --- /dev/null +++ b/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py @@ -0,0 +1,710 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage pool. + +Tests are numbered test_01 ... test_11 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create primary storage pool + 02 Disable storage pool + 03 Enable storage pool + 04 Enter maintenance mode + 05 Cancel maintenance mode + 06 Create a data volume on the pool + 07 Enter maintenance mode (pool has a volume) + 08 Cancel maintenance mode (pool has a volume) + 09 Delete the data volume + 10 Enter maintenance mode and delete the storage pool + 11 Create a second pool, attach a volume, enter maintenance, + then force-delete the pool (volume still present) + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py -v +""" + +import base64 +import logging +import random + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + cancelStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIWorkflow") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-wf@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-WF", + "username": "ontap_iscsi_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSI_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + import re + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +def _lun_path(vol_name, lun_name): + """Mirror OntapStorageUtils.getLunName: /vol/{volName}/{lunName}""" + return "/vol/%s/%s" % (vol_name, lun_name) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIPrimaryStorageWorkflow(OntapTestBase): + + # ---- iSCSI-specific state (set/cleared by individual tests) -------- + _vol_name_prefix = "OntapISCSIVol" + lun_path = None # ONTAP LUN path of cls.volume + lun_path2 = None # ONTAP LUN path of cls.volume2 + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIPrimaryStorageWorkflow, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + scope = ontap_cfg.get("storagePoolScope", "CLUSTER") + provider = ontap_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = ontap_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = ontap_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSI_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ------------------------------------------------------------------ + # Step 01 - Create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_01_create_primary_storage_pool(self): + """ + Create an iSCSI primary storage pool and verify: + - CloudStack state is Up, type is Iscsi + - ONTAP: FlexVol exists and is online + - ONTAP: one igroup per cluster host exists with the correct IQN initiator + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroup must exist for each cluster host that has an IQN + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue # host not iSCSI-enabled; skip igroup check for it + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s'" % (igroup_name, host.name) + ) + initiator_names = [ + i.get("name", "") for i in igroup.get("initiators", []) + ] + self.assertIn( + iqn, initiator_names, + "Host IQN '%s' not in igroup '%s' initiators: %s" + % (iqn, igroup_name, initiator_names) + ) + + # ------------------------------------------------------------------ + # Step 02 - Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_02_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online (disable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + # ONTAP: disable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 03 - Enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_03_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online (enable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + # ONTAP: enable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 04 - Enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_04_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + # ONTAP: maintenance must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 05 - Cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_05_cancel_maintenance_mode(self): + """ + Cancel maintenance and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 06 - Create a data volume on the pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_06_create_volume(self): + """ + Allocate a data volume on the iSCSI pool and verify: + - CloudStack returns a volume id + - ONTAP: a LUN is created inside the FlexVol at /vol/{poolName}/{volName} + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + if not self.disk_offering_id: + self.skipTest("No disk offering available - skipping volume steps") + + try: + vol = self._create_volume(self.__class__.pool.id) + except Exception as e: + self.skipTest("createVolume failed (iSCSI may require an attached VM): %s" % e) + + self.__class__.volume = vol + vol_id = getattr(vol, "id", None) + self.assertIsNotNone(vol_id, "Volume creation returned no id") + + # ONTAP: a LUN must exist inside the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume creation" + % self.__class__.pool.name + ) + self.__class__.lun_path = luns[0].get("name") # cache for later steps + + # ------------------------------------------------------------------ + # Step 07 - Enter maintenance mode (pool has a volume) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_07_enter_maintenance_mode_with_volume(self): + """ + Enter maintenance mode while the pool holds a data volume and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol still online and LUN still present (maintenance + does not affect ONTAP data plane) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + # ONTAP: FlexVol and LUN must be untouched + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared during maintenance (with volume)") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' during maintenance, got '%s'" + % ontap_vol.get("state") + ) + if getattr(self.__class__, "lun_path", None): + lun = self.ontap.get_lun(self.svm_name, self.__class__.lun_path) + self.assertIsNotNone( + lun, + "ONTAP LUN '%s' disappeared during maintenance" % self.__class__.lun_path + ) + + # ------------------------------------------------------------------ + # Step 08 - Cancel maintenance mode (pool has a volume) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_08_cancel_maintenance_mode_with_volume(self): + """ + Cancel maintenance mode while the pool still holds the volume and verify: + - CloudStack reports Up + - ONTAP: FlexVol online and LUN still present + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance (with volume)") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + if getattr(self.__class__, "lun_path", None): + lun = self.ontap.get_lun(self.svm_name, self.__class__.lun_path) + self.assertIsNotNone( + lun, + "ONTAP LUN '%s' disappeared after cancel maintenance" % self.__class__.lun_path + ) + + # ------------------------------------------------------------------ + # Step 09 - Delete the volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_09_delete_volume(self): + """ + Delete the data volume and verify: + - ONTAP: the LUN is removed from the FlexVol + - ONTAP: the FlexVol itself is still online (only the LUN is gone) + """ + if self.__class__.volume is None: + self.skipTest("No volume from test_06 - skipping") + + vol_id = self.__class__.volume.id + lun_path = getattr(self.__class__, "lun_path", None) + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol_id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + self.__class__.lun_path = None + + logger.info("Volume %s deleted" % vol_id) + + # ONTAP: LUN must be gone + if lun_path: + lun = self.ontap.get_lun(self.svm_name, lun_path) + self.assertIsNone( + lun, + "ONTAP LUN '%s' still exists after volume deletion" % lun_path + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after volume deletion") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 10 - Enter maintenance mode and delete the storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_10_enter_maintenance_and_delete_pool(self): + """ + Enter maintenance mode then delete the pool. + Verifies the pool is removed from CloudStack and the backing ONTAP + FlexVol is deleted. + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + pool = self.__class__.pool + pool_name = pool.name + + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) + + # ------------------------------------------------------------------ + # Step 11 - Create pool + volume, enter maintenance, force-delete + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_11_create_pool_volume_maintenance_force_delete(self): + """ + Validates the forced=True behaviour of deleteStoragePool. + + CloudStack distinguishes two volume categories on a pool: + - non-destroyed (Allocated/Ready): active volumes + - destroyed (Destroy state) : soft-deleted, awaiting GC expunge + + forced=False → fails if ANY volume record exists on the pool (any state) + forced=True → fails only if non-destroyed volumes exist; + if only destroyed volumes remain CloudStack force-expunges + them and removes the pool. + + This test covers the two reliable halves of that contract: + + Step 1 Create pool + allocate a data volume (non-destroyed). + Step 2 Enter maintenance mode. + Step 3 forced=False delete MUST FAIL — non-destroyed volume present. + Step 4 Soft-delete the volume (deleteVolume API). + Step 5 forced=True delete MUST SUCCEED — handles any remaining state + (immediately-expunged or still Destroyed — both pass). + Step 6 Assert pool is gone from CloudStack and ONTAP. + + Note: The Destroyed-only scenario (forced=True succeeds where forced=False + would still fail) requires a VM lifecycle to produce Destroyed volumes and + is covered by higher-level system tests rather than this FT suite. + """ + if not self.disk_offering_id: + self.skipTest( + "No disk offering available; force-delete test requires a volume " + "to be present on the pool." + ) + + pool2 = self._create_pool() + self.__class__.pool2 = pool2 + self.assertEqual( + pool2.state, "Up", + "Pool2 state should be 'Up', got '%s'" % pool2.state + ) + + # Step 1: allocate a data volume — must succeed for this test to be valid + try: + vol2 = self._create_volume(pool2.id) + self.__class__.volume2 = vol2 + except Exception as e: + self.skipTest( + "createVolume failed (iSCSI may require an attached VM): %s" % e + ) + self.assertIsNotNone(getattr(vol2, "id", None), + "Volume2 creation returned no id") + + # ONTAP: LUN must be created in pool2's FlexVol + luns2 = self.ontap.list_luns_in_volume(self.svm_name, pool2.name) + self.assertTrue( + len(luns2) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume2 creation" % pool2.name + ) + self.__class__.lun_path2 = luns2[0].get("name") + + # Step 2: enter maintenance + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool2.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool2.id, "Maintenance", timeout=120) + + # ONTAP: LUN still present during maintenance + if self.__class__.lun_path2: + lun2 = self.ontap.get_lun(self.svm_name, self.__class__.lun_path2) + self.assertIsNotNone( + lun2, + "ONTAP LUN '%s' disappeared during maintenance (pool2)" % self.__class__.lun_path2 + ) + + # Step 3: forced=False must FAIL — active (non-destroyed) volume present + from marvin.cloudstackException import CloudstackAPIException + with self.assertRaises(CloudstackAPIException, + msg="deleteStoragePool (forced=False) should fail " + "when a non-destroyed volume is on the pool"): + self._delete_pool(pool2.id, forced=False) + + # Step 4: soft-delete the volume via the deleteVolume API + del_vol_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_vol_cmd.id = self.__class__.volume2.id + self.apiClient.deleteVolume(del_vol_cmd) + self.__class__.volume2 = None + + # ONTAP: LUN must be gone after deleteVolume + if self.__class__.lun_path2: + lun2 = self.ontap.get_lun(self.svm_name, self.__class__.lun_path2) + self.assertIsNone( + lun2, + "ONTAP LUN '%s' still exists after deleteVolume (pool2)" % self.__class__.lun_path2 + ) + self.__class__.lun_path2 = None + + # Step 5: forced=True must SUCCEED — handles any remaining volume state + self._delete_pool(pool2.id, forced=True) + self.__class__.pool2 = None + + # Step 6: assert CloudStack and ONTAP cleaned up + try: + remaining = list_storage_pools(self.apiClient, id=pool2.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool2 still listed in CloudStack after force-deletion") + + # ONTAP: FlexVol and igroups must be deleted + ontap_vol = self.ontap.get_volume(pool2.name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after force-deletion" % pool2.name + ) + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool2 force-deletion" % igroup_name + ) diff --git a/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py b/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py new file mode 100644 index 000000000000..adbc61579292 --- /dev/null +++ b/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py @@ -0,0 +1,404 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. + +Tests are numbered test_01 ... test_04 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create primary storage pool + 02 Disable storage pool + 03 Enable storage pool + 04 Enter maintenance mode + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py -v + +Note: Tests 01-04 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapNFS3Workflow") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + DETAIL_VOLUME_UUID = "volumeUUID" + DETAIL_VOLUME_NAME = "volumeName" + DETAIL_DATA_LIF = "dataLIF" + DETAIL_NFS_MOUNT_OPTS = "nfsmountopts" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-wf@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-WF", + "username": "ontap_nfs3_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): + + # ---- NFS3-specific shared state ------------------------------------ + pool_ep_name = None # NFS export policy name for pool + cluster_host_ips = None + + _vol_name_prefix = "OntapNFS3Vol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapNFS3PrimaryStorageWorkflow, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + protocol = ontap_cfg.get("protocol", "NFS3") + scope = ontap_cfg.get("storagePoolScope", "CLUSTER") + provider = ontap_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = ontap_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = ontap_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Resolve cluster host IPs for export policy rule assertions + cls.cluster_host_ips = [ + h.ipaddress for h in cls.cluster_hosts + if getattr(h, "ipaddress", None) + ] + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + # Fallback: plugin typically uses cs-{svmName}-{poolName} + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_export_policy_has_host_ips(self, ep_name): + """Assert that the export policy exists and its rules include each cluster host IP.""" + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP" % ep_name + ) + if not self.cluster_host_ips: + return # no host IPs registered; skip rule-level check + all_clients = [] + for rule in policy.get("rules", []): + for client in rule.get("clients", []): + all_clients.append(client.get("match", "")) + for ip in self.cluster_host_ips: + self.assertTrue( + any(ip in c for c in all_clients), + "Host IP '%s' not found in export policy '%s' rules: %s" + % (ip, ep_name, all_clients) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_01_create_primary_storage_pool(self): + """ + Create an NFS3 primary storage pool and verify: + - CloudStack state is Up, type is NetworkFilesystem + - nfsmountopts contains 'vers=3' + - ONTAP: FlexVol exists and is online + - ONTAP: NFS export policy exists with cluster host IP rules + - ONTAP: at least one NFS data LIF is present on the SVM + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "NetworkFilesystem", + "Pool type should be 'NetworkFilesystem', got '%s'" % pool.type + ) + + # Verify nfsmountopts via listStoragePools + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone(listed, "listStoragePools returned None for pool %s" % pool.id) + nfs_opts = getattr(listed[0], "nfsmountopts", "") + self.assertIn( + "vers=3", nfs_opts, + "nfsmountopts should contain 'vers=3', got '%s'" % nfs_opts + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist with host IP rules + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + self._assert_export_policy_has_host_ips(ep_name) + + # ONTAP: at least one NFS data LIF must be present + lifs = self.ontap.get_data_lifs(self.svm_name) + self.assertTrue( + len(lifs) > 0, + "No NFS data LIFs found on SVM '%s'" % self.svm_name + ) + + # ------------------------------------------------------------------ + # Step 02 — Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_02_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_03_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_04_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online and export policy unchanged + (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist during maintenance" + % self.__class__.pool_ep_name + ) + + + From e364ce30045f3453a554cf31a8f65db7b9350c69 Mon Sep 17 00:00:00 2001 From: "Locharla, Sandeep" Date: Tue, 23 Jun 2026 17:36:05 +0530 Subject: [PATCH 2/6] NFS3 and iSCSI automation tests; Fix for NFS Cancel Maintenance --- .../kvm/storage/KVMStoragePoolManager.java | 138 ++- .../kvm/storage/LibvirtStorageAdaptor.java | 536 ++++++----- .../plugins/ontap/iscsi/__init__.py | 16 + .../plugins/ontap/iscsi/instance/__init__.py | 16 + .../iscsi/instance/test_vm_volume_attach.py | 826 +++++++++++++++++ .../plugins/ontap/iscsi/pool/__init__.py | 16 + .../ontap/iscsi/pool/test_pool_lifecycle.py | 630 +++++++++++++ .../iscsi/pool/test_pool_with_volumes.py | 706 +++++++++++++++ .../ontap/iscsi/pool/test_zone_scoped_pool.py | 386 ++++++++ .../plugins/ontap/iscsi/volume/__init__.py | 16 + .../iscsi/volume/test_volume_lifecycle.py | 416 +++++++++ .../plugins/ontap/manual_cancel_maint_test.py | 0 .../plugins/ontap/nfs3/__init__.py | 16 + .../plugins/ontap/nfs3/instance/__init__.py | 16 + .../nfs3/instance/test_vm_volume_attach.py | 832 ++++++++++++++++++ .../plugins/ontap/nfs3/pool/__init__.py | 16 + .../ontap/nfs3/pool/test_pool_lifecycle.py | 709 +++++++++++++++ .../ontap/nfs3/pool/test_pool_with_volumes.py | 767 ++++++++++++++++ .../ontap/nfs3/pool/test_zone_scoped_pool.py | 439 +++++++++ .../plugins/ontap/nfs3/volume/__init__.py | 16 + .../nfs3/volume/test_volume_lifecycle.py | 470 ++++++++++ test/integration/plugins/ontap/ontap.cfg | 72 +- .../plugins/ontap/ontap_test_base.py | 237 ++++- test/integration/plugins/ontap/probe_test.py | 35 + test/integration/plugins/ontap/run_tests.sh | 73 ++ 25 files changed, 7068 insertions(+), 332 deletions(-) create mode 100644 test/integration/plugins/ontap/iscsi/__init__.py create mode 100644 test/integration/plugins/ontap/iscsi/instance/__init__.py create mode 100644 test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py create mode 100644 test/integration/plugins/ontap/iscsi/pool/__init__.py create mode 100644 test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py create mode 100644 test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py create mode 100644 test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py create mode 100644 test/integration/plugins/ontap/iscsi/volume/__init__.py create mode 100644 test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py create mode 100644 test/integration/plugins/ontap/manual_cancel_maint_test.py create mode 100644 test/integration/plugins/ontap/nfs3/__init__.py create mode 100644 test/integration/plugins/ontap/nfs3/instance/__init__.py create mode 100644 test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py create mode 100644 test/integration/plugins/ontap/nfs3/pool/__init__.py create mode 100644 test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py create mode 100644 test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py create mode 100644 test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py create mode 100644 test/integration/plugins/ontap/nfs3/volume/__init__.py create mode 100644 test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py create mode 100644 test/integration/plugins/ontap/probe_test.py create mode 100644 test/integration/plugins/ontap/run_tests.sh diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 35cc864268c3..61c755b1a9ab 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -83,7 +83,8 @@ public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) { this._storageMapper.put("libvirt", new LibvirtStorageAdaptor(storagelayer)); // add other storage adaptors manually here - // add any adaptors that wish to register themselves via call to adaptor.getStoragePoolType() + // add any adaptors that wish to register themselves via call to + // adaptor.getStoragePoolType() Reflections reflections = new Reflections("com.cloud.hypervisor.kvm.storage"); Set> storageAdaptorClasses = reflections.getSubTypesOf(StorageAdaptor.class); for (Class storageAdaptorClass : storageAdaptorClasses) { @@ -112,7 +113,8 @@ public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) { StoragePoolType storagePoolType = adaptor.getStoragePoolType(); if (storagePoolType != null) { if (this._storageMapper.containsKey(storagePoolType.toString())) { - logger.warn(String.format("Duplicate StorageAdaptor type %s, not loading %s", storagePoolType, storageAdaptorClass.getName())); + logger.warn(String.format("Duplicate StorageAdaptor type %s, not loading %s", storagePoolType, + storageAdaptorClass.getName())); } else { logger.info(String.format("Adding storage adaptor for %s", storageAdaptorClass.getName())); this._storageMapper.put(storagePoolType.toString(), adaptor); @@ -135,7 +137,8 @@ public boolean supportsPhysicalDiskCopy(StoragePoolType type) { return getStorageAdaptor(type).supportsPhysicalDiskCopy(type); } - public boolean connectPhysicalDisk(StoragePoolType type, String poolUuid, String volPath, Map details) { + public boolean connectPhysicalDisk(StoragePoolType type, String poolUuid, String volPath, + Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); KVMStoragePool pool = adaptor.getStoragePool(poolUuid); @@ -155,8 +158,8 @@ public boolean connectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec, boolean is continue; } - VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); - PrimaryDataStoreTO store = (PrimaryDataStoreTO)vol.getDataStore(); + VolumeObjectTO vol = (VolumeObjectTO) disk.getData(); + PrimaryDataStoreTO store = (PrimaryDataStoreTO) vol.getDataStore(); if (!store.isManaged() && VirtualMachine.State.Migrating.equals(vmSpec.getState())) { result = true; continue; @@ -168,7 +171,8 @@ public boolean connectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec, boolean is result = adaptor.connectPhysicalDisk(vol.getPath(), pool, disk.getDetails(), isVMMigrate); if (!result) { - logger.error("Failed to connect disks via Instance spec for Instance: " + vmName + " volume:" + vol.toString()); + logger.error("Failed to connect disks via Instance spec for Instance: " + vmName + " volume:" + + vol.toString()); return result; } } @@ -186,18 +190,22 @@ public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { String poolType = volumeToDisconnect.get(DiskTO.PROTOCOL_TYPE); StorageAdaptor adaptor = _storageMapper.get(poolType); if (adaptor != null) { - logger.info(String.format("Disconnecting physical disk using the storage adaptor found for pool type: %s", poolType)); + logger.info(String.format( + "Disconnecting physical disk using the storage adaptor found for pool type: %s", poolType)); return adaptor.disconnectPhysicalDisk(volumeToDisconnect); } - logger.debug(String.format("Couldn't find the storage adaptor for pool type: %s to disconnect the physical disk, trying with others", poolType)); + logger.debug(String.format( + "Couldn't find the storage adaptor for pool type: %s to disconnect the physical disk, trying with others", + poolType)); } for (Map.Entry set : _storageMapper.entrySet()) { StorageAdaptor adaptor = set.getValue(); if (adaptor.disconnectPhysicalDisk(volumeToDisconnect)) { - logger.debug(String.format("Disconnected physical disk using the storage adaptor for pool type: %s", set.getKey())); + logger.debug(String.format("Disconnected physical disk using the storage adaptor for pool type: %s", + set.getKey())); return true; } } @@ -211,7 +219,9 @@ public boolean disconnectPhysicalDiskByPath(String path) { StorageAdaptor adaptor = set.getValue(); if (adaptor.disconnectPhysicalDiskByPath(path)) { - logger.debug(String.format("Disconnected physical disk by local path: %s, using the storage adaptor for pool type: %s", path, set.getKey())); + logger.debug(String.format( + "Disconnected physical disk by local path: %s, using the storage adaptor for pool type: %s", + path, set.getKey())); return true; } } @@ -221,10 +231,15 @@ public boolean disconnectPhysicalDiskByPath(String path) { public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { if (vmSpec == null) { - /* CloudStack often tries to stop VMs that shouldn't be running, to ensure a known state, - for example if we lose communication with the agent and the VM is brought up elsewhere. - We may not know about these yet. This might mean that we can't use the vmspec map, because - when we restart the agent we lose all of the info about running VMs. */ + /* + * CloudStack often tries to stop VMs that shouldn't be running, to ensure a + * known state, + * for example if we lose communication with the agent and the VM is brought up + * elsewhere. + * We may not know about these yet. This might mean that we can't use the vmspec + * map, because + * when we restart the agent we lose all of the info about running VMs. + */ logger.debug("disconnectPhysicalDiskViaVmSpec: Attempted to stop a VM that is not yet in our hash map"); @@ -241,13 +256,14 @@ public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { if (disk.getType() != Volume.Type.ISO) { logger.debug("Disconnecting disk " + disk.getPath()); - VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); - PrimaryDataStoreTO store = (PrimaryDataStoreTO)vol.getDataStore(); + VolumeObjectTO vol = (VolumeObjectTO) disk.getData(); + PrimaryDataStoreTO store = (PrimaryDataStoreTO) vol.getDataStore(); KVMStoragePool pool = getStoragePool(store.getPoolType(), store.getUuid()); if (pool == null) { - logger.error("Pool " + store.getUuid() + " of type " + store.getPoolType() + " was not found, skipping disconnect logic"); + logger.error("Pool " + store.getUuid() + " of type " + store.getPoolType() + + " was not found, skipping disconnect logic"); continue; } @@ -258,7 +274,8 @@ public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { boolean subResult = adaptor.disconnectPhysicalDisk(vol.getPath(), pool); if (!subResult) { - logger.error("Failed to disconnect disks via Instance spec for Instance: " + vmName + " volume:" + vol.toString()); + logger.error("Failed to disconnect disks via Instance spec for Instance: " + vmName + " volume:" + + vol.toString()); result = false; } @@ -281,9 +298,11 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean } catch (Exception e) { StoragePoolInformation info = _storagePools.get(uuid); if (info != null) { - pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType()); + pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), + info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType()); } else { - throw new CloudRuntimeException("Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage()); + throw new CloudRuntimeException( + "Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage()); } } @@ -296,8 +315,11 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean } /** - * As the class {@link LibvirtStoragePool} is constrained to the {@link org.libvirt.StoragePool} class, there is no way of saving a generic parameter such as the details, hence, - * this method was created to always make available the details of libvirt primary storages for when they are needed. + * As the class {@link LibvirtStoragePool} is constrained to the + * {@link org.libvirt.StoragePool} class, there is no way of saving a generic + * parameter such as the details, hence, + * this method was created to always make available the details of libvirt + * primary storages for when they are needed. */ private void addPoolDetails(String uuid, LibvirtStoragePool pool) { StoragePoolInformation storagePoolInformation = _storagePools.get(uuid); @@ -333,7 +355,7 @@ public KVMStoragePool getStoragePoolByURI(String uri) { sourcePath = sourcePath.replace("//", "/"); sourceHost = storageUri.getHost(); uuid = UuidUtils.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString(); - protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem; + protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem : StoragePoolType.NetworkFilesystem; // storage registers itself through here return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocol, null, false); @@ -343,8 +365,9 @@ public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, St int cnt = 0; int retries = 100; KVMPhysicalDisk vol = null; - //harden get volume, try cnt times to get volume, in case volume is created on other host - //Poll more frequently and return immediately once disk is found + // harden get volume, try cnt times to get volume, in case volume is created on + // other host + // Poll more frequently and return immediately once disk is found String errMsg = ""; while (cnt < retries) { try { @@ -375,7 +398,8 @@ public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, St } } - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, + StoragePoolType type) { // primary storage registers itself through here return createStoragePool(name, host, port, path, userInfo, type, null, true); } @@ -383,24 +407,30 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri /** * Primary Storage registers itself through here */ - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, + StoragePoolType type, Map details) { return createStoragePool(name, host, port, path, userInfo, type, details, true); } - //Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in parallel, so need to be synced. - private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { + // Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in + // parallel, so need to be synced. + private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, + String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); - KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, primaryStorage); + KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, + primaryStorage); if (pool instanceof LibvirtStoragePool) { ((LibvirtStoragePool) pool).setType(type); } // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { - KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, PoolType.PrimaryStorage); + KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, + PoolType.PrimaryStorage); _haMonitor.addStoragePool(storagePool); } - StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, details, primaryStorage); + StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, details, + primaryStorage); addStoragePool(pool.getUuid(), info); return pool; } @@ -417,7 +447,8 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) { if (type == StoragePoolType.NetworkFilesystem) { _haMonitor.removeStoragePool(uuid); } - boolean deleteStatus = adaptor.deleteStoragePool(uuid);; + boolean deleteStatus = adaptor.deleteStoragePool(uuid); + ; synchronized (_storagePools) { _storagePools.remove(uuid); } @@ -426,23 +457,29 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) { public boolean deleteStoragePool(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); + logger.debug("[deleteStoragePool] calling adaptor.deleteStoragePool for pool {} (type={})", uuid, type); + boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); + logger.debug("[deleteStoragePool] adaptor.deleteStoragePool returned {} for pool {}", deleteStatus, uuid); if (type == StoragePoolType.NetworkFilesystem) { + logger.debug("[deleteStoragePool] calling haMonitor.removeStoragePool for NFS pool {}", uuid); _haMonitor.removeStoragePool(uuid); } - boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); synchronized (_storagePools) { _storagePools.remove(uuid); } return deleteStatus; } - public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, Storage.ProvisioningType provisioningType, - KVMStoragePool destPool, int timeout, byte[] passphrase) { - return createDiskFromTemplate(template, name, provisioningType, destPool, template.getSize(), timeout, passphrase); + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, + Storage.ProvisioningType provisioningType, + KVMStoragePool destPool, int timeout, byte[] passphrase) { + return createDiskFromTemplate(template, name, provisioningType, destPool, template.getSize(), timeout, + passphrase); } - public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, Storage.ProvisioningType provisioningType, - KVMStoragePool destPool, long size, int timeout, byte[] passphrase) { + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, + Storage.ProvisioningType provisioningType, + KVMStoragePool destPool, long size, int timeout, byte[] passphrase) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); // LibvirtStorageAdaptor-specific statement @@ -469,7 +506,8 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String n } } - public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, + long size, KVMStoragePool destPool) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); return adaptor.createTemplateFromDisk(disk, name, format, size, destPool); } @@ -479,28 +517,34 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt return adaptor.copyPhysicalDisk(disk, name, destPool, timeout, null, null, null); } - public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); return adaptor.copyPhysicalDisk(disk, name, destPool, timeout, srcPassphrase, dstPassphrase, provisioningType); } - public KVMPhysicalDisk createDiskWithTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, - KVMStoragePool destPool, int timeout, byte[] passphrase) { + public KVMPhysicalDisk createDiskWithTemplateBacking(KVMPhysicalDisk template, String name, + PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); return adaptor.createDiskFromTemplateBacking(template, name, format, size, destPool, timeout, passphrase); } - public KVMPhysicalDisk createPhysicalDiskFromDirectDownloadTemplate(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + public KVMPhysicalDisk createPhysicalDiskFromDirectDownloadTemplate(String templateFilePath, + String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); - return adaptor.createTemplateFromDirectDownloadFile(templateFilePath, destTemplatePath, destPool, format, timeout); + return adaptor.createTemplateFromDirectDownloadFile(templateFilePath, destTemplatePath, destPool, format, + timeout); } - public Ternary, String> prepareStorageClient(StoragePoolType type, String uuid, Map details) { + public Ternary, String> prepareStorageClient(StoragePoolType type, String uuid, + Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); return adaptor.prepareStorageClient(uuid, details); } - public Pair unprepareStorageClient(StoragePoolType type, String uuid, Map details) { + public Pair unprepareStorageClient(StoragePoolType type, String uuid, + Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); return adaptor.unprepareStorageClient(uuid, details); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index a03daeb197bf..e2dfe754cac2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -94,11 +94,13 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private static final int RBD_FEATURE_OBJECT_MAP = 8; private static final int RBD_FEATURE_FAST_DIFF = 16; private static final int RBD_FEATURE_DEEP_FLATTEN = 32; - public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; + public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ - private static final Set poolTypesThatEnableCreateDiskFromTemplateBacking = new HashSet<>(Arrays.asList(StoragePoolType.NetworkFilesystem, - StoragePoolType.Filesystem)); + private static final Set poolTypesThatEnableCreateDiskFromTemplateBacking = new HashSet<>( + Arrays.asList(StoragePoolType.NetworkFilesystem, + StoragePoolType.Filesystem)); public LibvirtStorageAdaptor(StorageLayer storage) { _storageLayer = storage; @@ -115,8 +117,10 @@ public boolean createFolder(String uuid, String path, String localPath) { String mountPoint = _mountPoint + File.separator + uuid; if (localPath != null) { - logger.debug(String.format("Pool [%s] is of type local or shared mount point; therefore, we will use the local path [%s] to create the folder [%s] (if it does not" - + " exist).", uuid, localPath, path)); + logger.debug(String.format( + "Pool [%s] is of type local or shared mount point; therefore, we will use the local path [%s] to create the folder [%s] (if it does not" + + " exist).", + uuid, localPath, path)); mountPoint = localPath; } @@ -129,20 +133,25 @@ public boolean createFolder(String uuid, String path, String localPath) { } @Override - public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, - KVMStoragePool destPool, int timeout, byte[] passphrase) { - String volumeDesc = String.format("volume [%s], with template backing [%s], in pool [%s] (%s), with size [%s] and encryption is %s", name, template.getName(), destPool.getUuid(), - destPool.getType(), size, passphrase != null && passphrase.length > 0); + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, + PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { + String volumeDesc = String.format( + "volume [%s], with template backing [%s], in pool [%s] (%s), with size [%s] and encryption is %s", name, + template.getName(), destPool.getUuid(), + destPool.getType(), size, passphrase != null && passphrase.length > 0); if (!poolTypesThatEnableCreateDiskFromTemplateBacking.contains(destPool.getType())) { - logger.info(String.format("Skipping creation of %s due to pool type is none of the following types %s.", volumeDesc, poolTypesThatEnableCreateDiskFromTemplateBacking.stream() - .map(type -> type.toString()).collect(Collectors.joining(", ")))); + logger.info(String.format("Skipping creation of %s due to pool type is none of the following types %s.", + volumeDesc, poolTypesThatEnableCreateDiskFromTemplateBacking.stream() + .map(type -> type.toString()).collect(Collectors.joining(", ")))); return null; } if (format != PhysicalDiskFormat.QCOW2) { - logger.info(String.format("Skipping creation of %s due to format [%s] is not [%s].", volumeDesc, format, PhysicalDiskFormat.QCOW2)); + logger.info(String.format("Skipping creation of %s due to format [%s] is not [%s].", volumeDesc, format, + PhysicalDiskFormat.QCOW2)); return null; } @@ -159,15 +168,18 @@ public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, S QemuImgFile backingFile = new QemuImgFile(template.getPath(), template.getFormat()); if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, + keyFile.toString(), "sec0", options)); } logger.debug(String.format("Passphrase is staged to keyFile: %s", keyFile.isSet())); QemuImg qemu = new QemuImg(timeout); qemu.create(destFile, backingFile, options, passphraseObjects); } catch (QemuImgException | LibvirtException | IOException e) { - // why don't we throw an exception here? I guess we fail to find the volume later and that results in a failure returned? - logger.error(String.format("Failed to create %s in [%s] due to [%s].", volumeDesc, destPath, e.getMessage()), e); + // why don't we throw an exception here? I guess we fail to find the volume + // later and that results in a failure returned? + logger.error( + String.format("Failed to create %s in [%s] due to [%s].", volumeDesc, destPath, e.getMessage()), e); } return null; @@ -176,17 +188,21 @@ public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, S /** * Extract downloaded template into installPath, remove compressed file */ - public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { - String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile); + public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, + String destinationFile) { + String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, + destinationFile); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + downloadedTemplateFile); } @Override - public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, + KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { File sourceFile = new File(templateFilePath); if (!sourceFile.exists()) { - throw new CloudRuntimeException("Direct download template file " + sourceFile + " does not exist on this host"); + throw new CloudRuntimeException( + "Direct download template file " + sourceFile + " does not exist on this host"); } String templateUuid = UUID.randomUUID().toString(); if (Storage.ImageFormat.ISO.equals(format)) { @@ -195,8 +211,9 @@ public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFileP String destinationFile = destPool.getLocalPath() + File.separator + templateUuid; if (destPool.getType() == StoragePoolType.NetworkFilesystem || destPool.getType() == StoragePoolType.Filesystem - || destPool.getType() == StoragePoolType.SharedMountPoint) { - if (!Storage.ImageFormat.ISO.equals(format) && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { + || destPool.getType() == StoragePoolType.SharedMountPoint) { + if (!Storage.ImageFormat.ISO.equals(format) + && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { extractDownloadedTemplate(templateFilePath, destPool, destinationFile); } else { Script.runSimpleBashScript("mv " + templateFilePath + " " + destinationFile); @@ -209,14 +226,16 @@ public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFileP return destPool.getPhysicalDisk(templateUuid); } - private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePath, String templateUuid, KVMStoragePool destPool, int timeout) { + private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePath, String templateUuid, + KVMStoragePool destPool, int timeout) { try { QemuImg.PhysicalDiskFormat srcFileFormat = QemuImg.PhysicalDiskFormat.QCOW2; QemuImgFile srcFile = new QemuImgFile(srcTemplateFilePath, srcFileFormat); QemuImg qemu = new QemuImg(timeout); Map info = qemu.info(srcFile); Long virtualSize = Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); - KVMPhysicalDisk destDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + templateUuid, templateUuid, destPool); + KVMPhysicalDisk destDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + templateUuid, templateUuid, + destPool); destDisk.setFormat(PhysicalDiskFormat.RAW); destDisk.setSize(virtualSize); destDisk.setVirtualSize(virtualSize); @@ -224,7 +243,8 @@ private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePat destFile.setFormat(PhysicalDiskFormat.RAW); qemu.convert(srcFile, destFile); } catch (LibvirtException | QemuImgException e) { - String err = String.format("Error creating template from direct download file on pool %s: %s", destPool.getUuid(), e.getMessage()); + String err = String.format("Error creating template from direct download file on pool %s: %s", + destPool.getUuid(), e.getMessage()); logger.error(err, e); throw new CloudRuntimeException(err, e); } @@ -254,7 +274,8 @@ public StorageVol getVolume(StoragePool pool, String volName) { try { vol = pool.storageVolLookupByName(volName); - logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool"); + logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + + " after refreshing the pool"); } catch (LibvirtException e) { throw new CloudRuntimeException("Could not find volume " + volName + ": " + e.getMessage()); } @@ -263,8 +284,10 @@ public StorageVol getVolume(StoragePool pool, String volName) { return vol; } - public StorageVol createVolume(Connect conn, StoragePool pool, String uuid, long size, VolumeFormat format) throws LibvirtException { - LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID.randomUUID().toString(), size, format, null, null); + public StorageVol createVolume(Connect conn, StoragePool pool, String uuid, long size, VolumeFormat format) + throws LibvirtException { + LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID.randomUUID().toString(), size, format, null, + null); logger.debug(volDef.toString()); return pool.storageVolCreateXML(volDef.toString(), 0); @@ -290,7 +313,8 @@ private void checkNetfsStoragePoolMounted(String uuid) { } } - private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, List nfsMountOpts) throws LibvirtException { + private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, + List nfsMountOpts) throws LibvirtException { String targetPath = _mountPoint + File.separator + uuid; LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath, nfsMountOpts); _storageLayer.mkdir(targetPath); @@ -300,7 +324,7 @@ private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String // check whether the pool is already mounted int mountpointResult = Script.runSimpleBashScriptForExitValue("mountpoint -q " + targetPath); // if the pool is mounted, try to unmount it - if(mountpointResult == 0) { + if (mountpointResult == 0) { logger.info("Attempting to unmount old mount at " + targetPath); String result = Script.runSimpleBashScript("umount -l " + targetPath); if (result == null) { @@ -355,7 +379,8 @@ private StoragePool createCLVMStoragePool(Connect conn, String uuid, String host String volgroupName = path; volgroupName = volgroupName.replaceFirst("/", ""); - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.LOGICAL, volgroupName, uuid, host, volgroupPath, volgroupPath); + LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.LOGICAL, volgroupName, uuid, host, volgroupPath, + volgroupPath); StoragePool sp = null; try { logger.debug(spd.toString()); @@ -417,7 +442,8 @@ private boolean destroyStoragePoolOnNFSMountOptionsChange(StoragePool sp, Connec return false; } - private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, String path) { + private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, + String path) { LibvirtStoragePoolDef spd; StoragePool sp = null; @@ -445,7 +471,8 @@ private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, } return null; } - spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, userInfoTemp[0], AuthenticationType.CEPH, uuid); + spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, userInfoTemp[0], + AuthenticationType.CEPH, uuid); } else { spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, ""); } @@ -484,7 +511,8 @@ private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, } } - public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destVol, StorageVol srcVol, int timeout) throws LibvirtException { + public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destVol, StorageVol srcVol, int timeout) + throws LibvirtException { StorageVol vol = destPool.storageVolCreateXML(destVol.toString(), 0); String srcPath = srcVol.getKey(); String destPath = vol.getKey(); @@ -492,12 +520,14 @@ public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destV return vol; } - public boolean copyVolume(String srcPath, String destPath, String volumeName, int timeout) throws InternalErrorException { + public boolean copyVolume(String srcPath, String destPath, String volumeName, int timeout) + throws InternalErrorException { _storageLayer.mkdirs(destPath); if (!_storageLayer.exists(srcPath)) { throw new InternalErrorException("volume:" + srcPath + " is not exits"); } - String result = Script.runSimpleBashScript("cp " + srcPath + " " + destPath + File.separator + volumeName, timeout); + String result = Script.runSimpleBashScript("cp " + srcPath + " " + destPath + File.separator + volumeName, + timeout); return result == null; } @@ -516,7 +546,7 @@ public LibvirtStorageVolumeDef getStorageVolumeDef(Connect conn, StorageVol vol) @Override public StoragePoolType getStoragePoolType() { // This is mapped manually in KVMStoragePoolManager - return null; + return null; } @Override @@ -532,30 +562,28 @@ protected void updateLocalPoolIops(LibvirtStoragePool pool) { // Run script to get data List commands = new ArrayList<>(); - commands.add(new String[]{ + commands.add(new String[] { Script.getExecutableAbsolutePath("bash"), "-c", String.format( "%s %s | %s 'NR==2 {print $1}'", Script.getExecutableAbsolutePath("df"), pool.getLocalPath(), - Script.getExecutableAbsolutePath("awk") - ) + Script.getExecutableAbsolutePath("awk")) }); String result = Script.executePipedCommands(commands, 1000).second(); if (StringUtils.isBlank(result)) { return; } result = result.trim(); - commands.add(new String[]{ + commands.add(new String[] { Script.getExecutableAbsolutePath("bash"), "-c", String.format( "%s -z %s 1 2 | %s 'NR==7 {print $2}'", Script.getExecutableAbsolutePath("iostat"), result, - Script.getExecutableAbsolutePath("awk") - ) + Script.getExecutableAbsolutePath("awk")) }); result = Script.executePipedCommands(commands, 10000).second(); logger.trace("Pool used IOPS result: {}", result); @@ -618,7 +646,8 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { String authUsername = spd.getAuthUserName(); if (authUsername != null) { Secret secret = conn.secretLookupByUUIDString(spd.getSecretUUID()); - String secretValue = new String(Base64.encodeBase64(secret.getByteValue()), Charset.defaultCharset()); + String secretValue = new String(Base64.encodeBase64(secret.getByteValue()), + Charset.defaultCharset()); pool.setAuthUsername(authUsername); pool.setAuthSecret(secretValue); } @@ -628,12 +657,15 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { * On large (RBD) storage pools it can take up to a couple of minutes * for libvirt to refresh the pool. * - * Refreshing a storage pool means that libvirt will have to iterate the whole pool + * Refreshing a storage pool means that libvirt will have to iterate the whole + * pool * and fetch information of each volume in there * - * It is not always required to refresh a pool. So we can control if we want to or not + * It is not always required to refresh a pool. So we can control if we want to + * or not * - * By default only the getStorageStats call in the LibvirtComputingResource will ask to + * By default only the getStorageStats call in the LibvirtComputingResource will + * ask to * refresh the pool */ if (refreshInfo) { @@ -646,9 +678,9 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { pool.setAvailable(storage.getInfo().available); logger.debug("Successfully refreshed pool " + uuid + - " Capacity: " + toHumanReadableSize(storage.getInfo().capacity) + - " Used: " + toHumanReadableSize(storage.getInfo().allocation) + - " Available: " + toHumanReadableSize(storage.getInfo().available)); + " Capacity: " + toHumanReadableSize(storage.getInfo().capacity) + + " Used: " + toHumanReadableSize(storage.getInfo().allocation) + + " Available: " + toHumanReadableSize(storage.getInfo().available)); return pool; } catch (LibvirtException e) { @@ -659,7 +691,7 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { @Override public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid); @@ -718,15 +750,20 @@ private int adjustStoragePoolRefCount(String uuid, int adjustment) { return refCount; } } + /** * Thread-safe increment storage pool usage refcount + * * @param uuid UUID of the storage pool to increment the count */ private void incStoragePoolRefCount(String uuid) { adjustStoragePoolRefCount(uuid, 1); } + /** - * Thread-safe decrement storage pool usage refcount for the given uuid and return if storage pool still in use. + * Thread-safe decrement storage pool usage refcount for the given uuid and + * return if storage pool still in use. + * * @param uuid UUID of the storage pool to decrement the count * @return true if the storage pool is still used, else false. */ @@ -735,7 +772,8 @@ private boolean decStoragePoolRefCount(String uuid) { } @Override - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean isPrimaryStorage) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, + StoragePoolType type, Map details, boolean isPrimaryStorage) { logger.info("Attempting to create storage pool {} ({}) in libvirt", name, type); StoragePool sp; Connect conn; @@ -771,7 +809,8 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri // if anyone is, undefine the pool so we can define it as requested. // This should be safe since a pool in use can't be removed, and no // volumes are affected by unregistering the pool with libvirt. - logger.info("Didn't find an existing storage pool " + name + " by UUID, checking for pools with duplicate paths"); + logger.info("Didn't find an existing storage pool " + name + + " by UUID, checking for pools with duplicate paths"); try { String[] poolnames = conn.listStoragePools(); @@ -780,7 +819,8 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri StoragePool p = conn.storagePoolLookupByName(poolname); LibvirtStoragePoolDef pdef = getStoragePoolDef(conn, p); if (pdef == null) { - throw new CloudRuntimeException("Unable to parse the storage pool definition for storage pool " + poolname); + throw new CloudRuntimeException( + "Unable to parse the storage pool definition for storage pool " + poolname); } String targetPath = pdef.getTargetPath(); @@ -796,13 +836,15 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri } } } catch (LibvirtException e) { - logger.error("Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" + e); + logger.error( + "Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" + + e); } } List nfsMountOpts = getNFSMountOptsFromDetails(type, details); if (sp != null && CollectionUtils.isNotEmpty(nfsMountOpts) && - destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) { + destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) { sp = null; } @@ -814,7 +856,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path, nfsMountOpts); } catch (LibvirtException e) { - logger.error("Failed to create netfs mount: " + host + ":" + path , e); + logger.error("Failed to create netfs mount: " + host + ":" + path, e); logger.error(e.getStackTrace()); throw new CloudRuntimeException(e.toString()); } @@ -822,7 +864,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path, null); } catch (LibvirtException e) { - logger.error("Failed to create glusterfs mount: " + host + ":" + path , e); + logger.error("Failed to create glusterfs mount: " + host + ":" + path, e); logger.error(e.getStackTrace()); throw new CloudRuntimeException(e.toString()); } @@ -841,7 +883,8 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { if (!isPrimaryStorage) { - // only ref count storage pools for secondary storage, as primary storage is assumed + // only ref count storage pools for secondary storage, as primary storage is + // assumed // to be always mounted, as long the primary storage isn't fully deleted. incStoragePoolRefCount(name); } @@ -861,7 +904,8 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri String error = e.toString(); if (error.contains("Storage source conflict")) { throw new CloudRuntimeException("A pool matching this location already exists in libvirt, " + - " but has a different UUID/Name. Cannot create new pool without first " + " removing it. Check for inactive pools via 'virsh pool-list --all'. " + + " but has a different UUID/Name. Cannot create new pool without first " + + " removing it. Check for inactive pools via 'virsh pool-list --all'. " + error); } else { throw new CloudRuntimeException(error); @@ -895,8 +939,7 @@ private boolean destroyStoragePool(Connect conn, String uuid) throws LibvirtExce } } - private boolean destroyStoragePoolHandleException(Connect conn, String uuid) - { + private boolean destroyStoragePoolHandleException(Connect conn, String uuid) { try { return destroyStoragePool(conn, uuid); } catch (LibvirtException e) { @@ -905,6 +948,13 @@ private boolean destroyStoragePoolHandleException(Connect conn, String uuid) return false; } + @Override + public boolean deleteStoragePool(String uuid, Map details) { + logger.debug("[deleteStoragePool] details overload called for pool {}, delegating to deleteStoragePool(uuid)", + uuid); + return deleteStoragePool(uuid); + } + @Override public boolean deleteStoragePool(String uuid) { logger.info("Attempting to remove storage pool " + uuid + " from libvirt"); @@ -948,8 +998,10 @@ public boolean deleteStoragePool(String uuid) { // handle ebusy error when pool is quickly destroyed if (e.toString().contains("exit status 16")) { String targetPath = _mountPoint + File.separator + uuid; - logger.error("deleteStoragePool removed pool from libvirt, but libvirt had trouble unmounting the pool. Trying umount location " + targetPath + - " again in a few seconds"); + logger.error( + "deleteStoragePool removed pool from libvirt, but libvirt had trouble unmounting the pool. Trying umount location " + + targetPath + + " again in a few seconds"); String result = Script.runSimpleBashScript("sleep 5 && umount " + targetPath); if (result == null) { logger.info("Succeeded in unmounting " + targetPath); @@ -965,51 +1017,61 @@ public boolean deleteStoragePool(String uuid) { /** * Creates a physical disk depending on the {@link StoragePoolType}: *
    - *
  • - * {@link StoragePoolType#RBD} - *
      - *
    • - * If it is an erasure code pool, utilizes QemuImg to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} - *
      • - *
    • - * Otherwise, utilize Libvirt to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
      • - *
    - *
    • - *
  • - * {@link StoragePoolType#NetworkFilesystem} and {@link StoragePoolType#Filesystem} - *
      - *
    • - * If the format is {@link PhysicalDiskFormat#QCOW2} or {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} - *
      • - *
    • - * If the format is {@link PhysicalDiskFormat#DIR} or {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
      • - *
    - *
    • - *
  • - * For the rest of the {@link StoragePoolType} types, utilizes the Libvirt method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
    • + *
  • + * {@link StoragePoolType#RBD} + *
      + *
    • + * If it is an erasure code pool, utilizes QemuImg to create the physical disk + * through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} + *
      • + *
    • + * Otherwise, utilize Libvirt to create the physical disk through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
      • + *
    + *
    • + *
  • + * {@link StoragePoolType#NetworkFilesystem} and + * {@link StoragePoolType#Filesystem} + *
      + *
    • + * If the format is {@link PhysicalDiskFormat#QCOW2} or + * {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk + * through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} + *
      • + *
    • + * If the format is {@link PhysicalDiskFormat#DIR} or + * {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk + * through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
      • + *
    + *
    • + *
  • + * For the rest of the {@link StoragePoolType} types, utilizes the Libvirt + * method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
    • *
*/ @Override public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { - logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), pool.getUuid(), toHumanReadableSize(size)); + logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), + pool.getUuid(), toHumanReadableSize(size)); StoragePoolType poolType = pool.getType(); if (StoragePoolType.RBD.equals(poolType)) { Map details = pool.getDetails(); String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); - return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : - createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase); + return (dataPool == null) + ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) + : createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, + passphrase); } else if (StoragePoolType.NetworkFilesystem.equals(poolType) || StoragePoolType.Filesystem.equals(poolType)) { switch (format) { case QCOW2: @@ -1057,9 +1119,9 @@ private KVMPhysicalDisk createPhysicalDiskByLibVirt(String name, KVMStoragePool return disk; } - - private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, - byte[] passphrase) { + private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, + Storage.ProvisioningType provisioningType, long size, + byte[] passphrase) { String volPath; String volName = name; long virtualSize = 0; @@ -1081,13 +1143,15 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool destFile.setSize(size); Map options = new HashMap(); if (List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem).contains(pool.getType())) { - options.put(QemuImg.PREALLOCATION, QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); + options.put(QemuImg.PREALLOCATION, + QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); } try (KeyFile keyFile = new KeyFile(passphrase)) { QemuImg qemu = new QemuImg(timeout); if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, + keyFile.toString(), "sec0", options)); // make room for encryption header on raw format, use LUKS if (format == PhysicalDiskFormat.RAW) { @@ -1102,7 +1166,8 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool virtualSize = Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); actualSize = new File(destFile.getFileName()).length(); } catch (QemuImgException | LibvirtException | IOException e) { - throw new CloudRuntimeException(String.format("Failed to create %s due to a failed execution of qemu-img", volPath), e); + throw new CloudRuntimeException( + String.format("Failed to create %s due to a failed execution of qemu-img", volPath), e); } KVMPhysicalDisk disk = new KVMPhysicalDisk(volPath, volName, pool); @@ -1114,7 +1179,8 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool } @Override - public boolean connectPhysicalDisk(String name, KVMStoragePool pool, Map details, boolean isVMMigrate) { + public boolean connectPhysicalDisk(String name, KVMStoragePool pool, Map details, + boolean isVMMigrate) { // this is for managed storage that needs to prep disks prior to use return true; } @@ -1177,7 +1243,8 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag */ if (pool.getType() == StoragePoolType.RBD) { try { - logger.info("Unprotecting and Removing RBD snapshots of image " + pool.getSourceDir() + "/" + uuid + " prior to removing the image"); + logger.info("Unprotecting and Removing RBD snapshots of image " + pool.getSourceDir() + "/" + uuid + + " prior to removing the image"); Rados r = new Rados(pool.getAuthUserName()); r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); @@ -1197,17 +1264,20 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag logger.debug("Unprotecting snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); image.snapUnprotect(snap.name); } else { - logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + " is not protected."); + logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + + " is not protected."); } logger.debug("Removing snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); image.snapRemove(snap.name); } - logger.info("Successfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of " - + pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image"); + logger.info( + "Successfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of " + + pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image"); } catch (RbdException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + throw new CloudRuntimeException( + e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } finally { logger.debug("Closing image and destroying context"); rbd.close(image); @@ -1215,20 +1285,20 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } } catch (RadosException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } catch (RbdException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } } - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), uuid); logger.debug("Instructing libvirt to remove volume " + uuid + " from pool " + pool.getUuid()); - if(Storage.ImageFormat.DIR.equals(format)){ + if (Storage.ImageFormat.DIR.equals(format)) { deleteDirVol(libvirtPool, vol); } else { deleteVol(libvirtPool, vol); @@ -1241,40 +1311,52 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } /** - * This function copies a physical disk from Secondary Storage to Primary Storage + * This function copies a physical disk from Secondary Storage to Primary + * Storage * or from Primary to Primary Storage * - * The first time a template is deployed in Primary Storage it will be copied from + * The first time a template is deployed in Primary Storage it will be copied + * from * Secondary to Primary. * - * If it has been created on Primary Storage, it will be copied on the Primary Storage + * If it has been created on Primary Storage, it will be copied on the Primary + * Storage */ @Override public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { - logger.info("Creating volume " + name + " from template " + template.getName() + " in pool " + destPool.getUuid() + - " (" + destPool.getType().toString() + ") with size " + toHumanReadableSize(size)); + logger.info( + "Creating volume " + name + " from template " + template.getName() + " in pool " + destPool.getUuid() + + " (" + destPool.getType().toString() + ") with size " + toHumanReadableSize(size)); KVMPhysicalDisk disk = null; if (destPool.getType() == StoragePoolType.RBD) { disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout); } else { - try (KeyFile keyFile = new KeyFile(passphrase)){ + try (KeyFile keyFile = new KeyFile(passphrase)) { String newUuid = name; List passphraseObjects = new ArrayList<>(); - disk = destPool.createPhysicalDisk(newUuid, format, provisioningType, template.getVirtualSize(), passphrase); + disk = destPool.createPhysicalDisk(newUuid, format, provisioningType, template.getVirtualSize(), + passphrase); if (disk == null) { throw new CloudRuntimeException("Failed to create disk from template " + template.getName()); } if (template.getFormat() == PhysicalDiskFormat.TAR) { - Script.runSimpleBashScript("tar -x -f " + template.getPath() + " -C " + disk.getPath(), timeout); // TO BE FIXED to aware provisioningType + Script.runSimpleBashScript("tar -x -f " + template.getPath() + " -C " + disk.getPath(), timeout); // TO + // BE + // FIXED + // to + // aware + // provisioningType } else if (template.getFormat() == PhysicalDiskFormat.DIR) { Script.runSimpleBashScript("mkdir -p " + disk.getPath()); Script.runSimpleBashScript("chmod 755 " + disk.getPath()); - Script.runSimpleBashScript("tar -x -f " + template.getPath() + "/*.tar -C " + disk.getPath(), timeout); + Script.runSimpleBashScript("tar -x -f " + template.getPath() + "/*.tar -C " + disk.getPath(), + timeout); } else if (format == PhysicalDiskFormat.QCOW2) { QemuImg qemu = new QemuImg(timeout); QemuImgFile destFile = new QemuImgFile(disk.getPath(), format); @@ -1284,31 +1366,34 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, destFile.setSize(template.getVirtualSize()); } Map options = new HashMap(); - options.put("preallocation", QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); - + options.put("preallocation", + QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, + keyFile.toString(), "sec0", options)); disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); } QemuImgFile srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - Boolean createFullClone = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CREATE_FULL_CLONE); - switch(provisioningType){ - case THIN: - logger.info("Creating volume [{}] {} backing file [{}] as the property [{}] is [{}].", destFile.getFileName(), createFullClone ? "without" : "with", - template.getPath(), AgentProperties.CREATE_FULL_CLONE.getName(), createFullClone); - if (createFullClone) { + Boolean createFullClone = AgentPropertiesFileHandler + .getPropertyValue(AgentProperties.CREATE_FULL_CLONE); + switch (provisioningType) { + case THIN: + logger.info("Creating volume [{}] {} backing file [{}] as the property [{}] is [{}].", + destFile.getFileName(), createFullClone ? "without" : "with", + template.getPath(), AgentProperties.CREATE_FULL_CLONE.getName(), createFullClone); + if (createFullClone) { + qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); + } else { + qemu.create(destFile, srcFile, options, passphraseObjects); + } + break; + case SPARSE: + case FAT: + srcFile = new QemuImgFile(template.getPath(), template.getFormat()); qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); - } else { - qemu.create(destFile, srcFile, options, passphraseObjects); - } - break; - case SPARSE: - case FAT: - srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); - break; + break; } } else if (format == PhysicalDiskFormat.RAW) { PhysicalDiskFormat destFormat = PhysicalDiskFormat.RAW; @@ -1317,7 +1402,8 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, if (keyFile.isSet()) { destFormat = PhysicalDiskFormat.LUKS; disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(destFormat, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(destFormat, + QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); } QemuImgFile sourceFile = new QemuImgFile(template.getPath(), template.getFormat()); @@ -1331,7 +1417,8 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, qemu.convert(sourceFile, destFile, options, passphraseObjects, null, false); } } catch (QemuImgException | LibvirtException | IOException e) { - throw new CloudRuntimeException(String.format("Failed to create %s due to a failed execution of qemu-img", name), e); + throw new CloudRuntimeException( + String.format("Failed to create %s due to a failed execution of qemu-img", name), e); } } @@ -1339,14 +1426,16 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, } private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout){ + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, + KVMStoragePool destPool, int timeout) { /* - With RBD you can't run qemu-img convert with an existing RBD image as destination - qemu-img will exit with the error that the destination already exists. - So for RBD we don't create the image, but let qemu-img do that for us. - - We then create a KVMPhysicalDisk object that we can return + * With RBD you can't run qemu-img convert with an existing RBD image as + * destination + * qemu-img will exit with the error that the destination already exists. + * So for RBD we don't create the image, but let qemu-img do that for us. + * + * We then create a KVMPhysicalDisk object that we can return */ KVMStoragePool srcPool = template.getPool(); @@ -1365,14 +1454,13 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, disk.setVirtualSize(disk.getSize()); } - QemuImgFile srcFile; QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath())); destFile.setFormat(format); if (srcPool.getType() != StoragePoolType.RBD) { srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - try{ + try { QemuImg qemu = new QemuImg(timeout); qemu.convert(srcFile, destFile); } catch (QemuImgException | LibvirtException e) { @@ -1390,9 +1478,14 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, */ try { - if ((srcPool.getSourceHost().equals(destPool.getSourceHost())) && (srcPool.getSourceDir().equals(destPool.getSourceDir()))) { - /* We are on the same Ceph cluster, but we require RBD format 2 on the source image */ - logger.debug("Trying to perform a RBD clone (layering) since we are operating in the same storage pool"); + if ((srcPool.getSourceHost().equals(destPool.getSourceHost())) + && (srcPool.getSourceDir().equals(destPool.getSourceDir()))) { + /* + * We are on the same Ceph cluster, but we require RBD format 2 on the source + * image + */ + logger.debug( + "Trying to perform a RBD clone (layering) since we are operating in the same storage pool"); Rados r = new Rados(srcPool.getAuthUserName()); r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); @@ -1408,15 +1501,18 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, if (srcImage.isOldFormat()) { /* The source image is RBD format 1, we have to do a regular copy */ logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() + - " is RBD format 1. We have to perform a regular copy (" + toHumanReadableSize(disk.getVirtualSize()) + " bytes)"); + " is RBD format 1. We have to perform a regular copy (" + + toHumanReadableSize(disk.getVirtualSize()) + " bytes)"); rbd.create(disk.getName(), disk.getVirtualSize(), RBD_FEATURES, rbdOrder); RbdImage destImage = rbd.open(disk.getName()); - logger.debug("Starting to copy " + srcImage.getName() + " to " + destImage.getName() + " in Ceph pool " + srcPool.getSourceDir()); + logger.debug("Starting to copy " + srcImage.getName() + " to " + destImage.getName() + + " in Ceph pool " + srcPool.getSourceDir()); rbd.copy(srcImage, destImage); - logger.debug("Finished copying " + srcImage.getName() + " to " + destImage.getName() + " in Ceph pool " + srcPool.getSourceDir()); + logger.debug("Finished copying " + srcImage.getName() + " to " + destImage.getName() + + " in Ceph pool " + srcPool.getSourceDir()); rbd.close(destImage); } else { logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() @@ -1424,12 +1520,12 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, + rbdTemplateSnapName); /* The source image is format 2, we can do a RBD snapshot+clone (layering) */ - logger.debug("Checking if RBD snapshot " + srcPool.getSourceDir() + "/" + template.getName() + "@" + rbdTemplateSnapName + " exists prior to attempting a clone operation."); List snaps = srcImage.snapList(); - logger.debug("Found " + snaps.size() + " snapshots on RBD image " + srcPool.getSourceDir() + "/" + template.getName()); + logger.debug("Found " + snaps.size() + " snapshots on RBD image " + srcPool.getSourceDir() + "/" + + template.getName()); boolean snapFound = false; for (RbdSnapInfo snap : snaps) { if (rbdTemplateSnapName.equals(snap.name)) { @@ -1448,13 +1544,18 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } rbd.clone(template.getName(), rbdTemplateSnapName, io, disk.getName(), RBD_FEATURES, rbdOrder); - logger.debug("Successfully cloned " + template.getName() + "@" + rbdTemplateSnapName + " to " + disk.getName()); - /* We also need to resize the image if the VM was deployed with a larger root disk size */ + logger.debug("Successfully cloned " + template.getName() + "@" + rbdTemplateSnapName + " to " + + disk.getName()); + /* + * We also need to resize the image if the VM was deployed with a larger root + * disk size + */ if (disk.getVirtualSize() > template.getVirtualSize()) { RbdImage diskImage = rbd.open(disk.getName()); diskImage.resize(disk.getVirtualSize()); rbd.close(diskImage); - logger.debug("Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); + logger.debug( + "Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); } } @@ -1462,8 +1563,12 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, rbd.close(srcImage); r.ioCtxDestroy(io); } else { - /* The source pool or host is not the same Ceph cluster, we do a simple copy with Qemu-Img */ - logger.debug("Both the source and destination are RBD, but not the same Ceph cluster. Performing a copy"); + /* + * The source pool or host is not the same Ceph cluster, we do a simple copy + * with Qemu-Img + */ + logger.debug( + "Both the source and destination are RBD, but not the same Ceph cluster. Performing a copy"); Rados rSrc = new Rados(srcPool.getAuthUserName()); rSrc.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); @@ -1485,14 +1590,16 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, IoCTX dIO = rDest.ioCtxCreate(destPool.getSourceDir()); Rbd dRbd = new Rbd(dIO); - logger.debug("Creating " + disk.getName() + " on the destination cluster " + rDest.confGet("mon_host") + " in pool " + + logger.debug("Creating " + disk.getName() + " on the destination cluster " + + rDest.confGet("mon_host") + " in pool " + destPool.getSourceDir()); dRbd.create(disk.getName(), disk.getVirtualSize(), RBD_FEATURES, rbdOrder); RbdImage srcImage = sRbd.open(template.getName()); RbdImage destImage = dRbd.open(disk.getName()); - logger.debug("Copying " + template.getName() + " from Ceph cluster " + rSrc.confGet("mon_host") + " to " + disk.getName() + logger.debug("Copying " + template.getName() + " from Ceph cluster " + rSrc.confGet("mon_host") + + " to " + disk.getName() + " on cluster " + rDest.confGet("mon_host")); sRbd.copy(srcImage, destImage); @@ -1514,13 +1621,14 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } @Override - public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, + long size, KVMStoragePool destPool) { return null; } @Override public List listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; StoragePool virtPool = libvirtPool.getPool(); List disks = new ArrayList(); try { @@ -1543,35 +1651,44 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt /** * This copies a volume from Primary Storage to Secondary Storage * - * In theory it could also do it the other way around, but the current implementation - * in ManagementServerImpl shows that the destPool is always a Secondary Storage Pool + * In theory it could also do it the other way around, but the current + * implementation + * in ManagementServerImpl shows that the destPool is always a Secondary Storage + * Pool */ @Override - public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { /** - With RBD you can't run qemu-img convert with an existing RBD image as destination - qemu-img will exit with the error that the destination already exists. - So for RBD we don't create the image, but let qemu-img do that for us. - - We then create a KVMPhysicalDisk object that we can return - - It is however very unlikely that the destPool will be RBD, since it isn't supported - for Secondary Storage + * With RBD you can't run qemu-img convert with an existing RBD image as + * destination + * qemu-img will exit with the error that the destination already exists. + * So for RBD we don't create the image, but let qemu-img do that for us. + * + * We then create a KVMPhysicalDisk object that we can return + * + * It is however very unlikely that the destPool will be RBD, since it isn't + * supported + * for Secondary Storage */ KVMStoragePool srcPool = disk.getPool(); - /* Linstor images are always stored as RAW, but Linstor uses qcow2 in DB, - to support snapshots(backuped) as qcow2 files. */ - PhysicalDiskFormat sourceFormat = srcPool.getType() != StoragePoolType.Linstor ? - disk.getFormat() : PhysicalDiskFormat.RAW; + /* + * Linstor images are always stored as RAW, but Linstor uses qcow2 in DB, + * to support snapshots(backuped) as qcow2 files. + */ + PhysicalDiskFormat sourceFormat = srcPool.getType() != StoragePoolType.Linstor ? disk.getFormat() + : PhysicalDiskFormat.RAW; String sourcePath = disk.getPath(); KVMPhysicalDisk newDisk; - logger.debug("copyPhysicalDisk: disk size:{}, virtualsize:{} format:{}", toHumanReadableSize(disk.getSize()), toHumanReadableSize(disk.getVirtualSize()), disk.getFormat()); + logger.debug("copyPhysicalDisk: disk size:{}, virtualsize:{} format:{}", toHumanReadableSize(disk.getSize()), + toHumanReadableSize(disk.getVirtualSize()), disk.getFormat()); if (destPool.getType() != StoragePoolType.RBD) { if (disk.getFormat() == PhysicalDiskFormat.TAR) { - newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, disk.getVirtualSize(), null); + newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, + disk.getVirtualSize(), null); } else { newDisk = destPool.createPhysicalDisk(name, Storage.ProvisioningType.THIN, disk.getVirtualSize(), null); } @@ -1589,15 +1706,15 @@ to support snapshots(backuped) as qcow2 files. */ try { qemu = new QemuImg(timeout); - } catch (QemuImgException | LibvirtException ex ) { + } catch (QemuImgException | LibvirtException ex) { throw new CloudRuntimeException("Failed to create qemu-img command", ex); } QemuImgFile srcFile = null; QemuImgFile destFile = null; if ((srcPool.getType() != StoragePoolType.RBD) && (destPool.getType() != StoragePoolType.RBD)) { - if(sourceFormat == PhysicalDiskFormat.TAR && destFormat == PhysicalDiskFormat.DIR) { //LXC template - Script.runSimpleBashScript("cp "+ sourcePath + " " + destPath); + if (sourceFormat == PhysicalDiskFormat.TAR && destFormat == PhysicalDiskFormat.DIR) { // LXC template + Script.runSimpleBashScript("cp " + sourcePath + " " + destPath); } else if (sourceFormat == PhysicalDiskFormat.TAR) { Script.runSimpleBashScript("tar -x -f " + sourcePath + " -C " + destPath, timeout); } else if (sourceFormat == PhysicalDiskFormat.DIR) { @@ -1619,26 +1736,30 @@ to support snapshots(backuped) as qcow2 files. */ destFile = new QemuImgFile(destPath, destFormat); try { boolean isQCOW2 = PhysicalDiskFormat.QCOW2.equals(sourceFormat); - qemu.convert(srcFile, destFile, null, null, new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), + qemu.convert(srcFile, destFile, null, null, + new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), null, false, isQCOW2); Map destInfo = qemu.info(destFile); Long virtualSize = Long.parseLong(destInfo.get(QemuImg.VIRTUAL_SIZE)); newDisk.setVirtualSize(virtualSize); newDisk.setSize(virtualSize); } catch (QemuImgException e) { - logger.error("Failed to convert [{}] to [{}] due to: [{}].", srcFile.getFileName(), destFile.getFileName(), e.getMessage(), e); + logger.error("Failed to convert [{}] to [{}] due to: [{}].", srcFile.getFileName(), + destFile.getFileName(), e.getMessage(), e); newDisk = null; } } } catch (QemuImgException e) { - logger.error("Failed to fetch the information of file " + srcFile.getFileName() + " the error was: " + e.getMessage()); + logger.error("Failed to fetch the information of file " + srcFile.getFileName() + " the error was: " + + e.getMessage()); newDisk = null; } } } else if ((srcPool.getType() != StoragePoolType.RBD) && (destPool.getType() == StoragePoolType.RBD)) { /** * Using qemu-img we copy the QCOW2 disk to RAW (on RBD) directly. - * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph Dumpling) + * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph + * Dumpling) */ logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2"); try { @@ -1647,9 +1768,11 @@ to support snapshots(backuped) as qcow2 files. */ String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath); destFile = new QemuImgFile(rbdDestFile, destFormat); - logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); + logger.debug( + "Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); qemu.convert(srcFile, destFile); - logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); + logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + + rbdDestPath); /* We have to stat the RBD image to see how big it became afterwards */ Rados r = new Rados(destPool.getAuthUserName()); @@ -1666,26 +1789,32 @@ to support snapshots(backuped) as qcow2 files. */ RbdImageInfo rbdInfo = image.stat(); newDisk.setSize(rbdInfo.size); newDisk.setVirtualSize(rbdInfo.size); - logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + toHumanReadableSize(rbdInfo.size) + " bytes long"); + logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + + toHumanReadableSize(rbdInfo.size) + " bytes long"); rbd.close(image); r.ioCtxDestroy(io); } catch (QemuImgException | LibvirtException e) { String srcFilename = srcFile != null ? srcFile.getFileName() : null; String destFilename = destFile != null ? destFile.getFileName() : null; - logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, destFilename, e.getMessage())); + logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, + destFilename, e.getMessage())); newDisk = null; } catch (RadosException e) { - logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); + logger.error( + "A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; } catch (RbdException e) { - logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); + logger.error( + "A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; } } else { /** - We let Qemu-Img do the work here. Although we could work with librbd and have that do the cloning - it doesn't benefit us. It's better to keep the current code in place which works + * We let Qemu-Img do the work here. Although we could work with librbd and have + * that do the cloning + * it doesn't benefit us. It's better to keep the current code in place which + * works */ srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, sourcePath)); srcFile.setFormat(sourceFormat); @@ -1699,7 +1828,8 @@ to support snapshots(backuped) as qcow2 files. */ try { qemu.convert(srcFile, destFile); } catch (QemuImgException | LibvirtException e) { - logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage()); + logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + + " the error was: " + e.getMessage()); newDisk = null; } } @@ -1713,7 +1843,7 @@ to support snapshots(backuped) as qcow2 files. */ @Override public boolean refresh(KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; StoragePool virtPool = libvirtPool.getPool(); try { refreshPool(virtPool); diff --git a/test/integration/plugins/ontap/iscsi/__init__.py b/test/integration/plugins/ontap/iscsi/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/instance/__init__.py b/test/integration/plugins/ontap/iscsi/instance/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/instance/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py new file mode 100644 index 000000000000..1dd55049f76e --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py @@ -0,0 +1,826 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI data volume +lifecycle with a running virtual machine. + +Covers TDS (section 10) iSCSI VM volume scenarios: + + TDS Approach-1 SN 27 — Create CS Volume and allocate it to an Instance (iSCSI) + (attach data volume to running VM; verify LUN-map) + TDS VM Stop (iSCSI) — Stop running VM; verify LUN-maps are removed + TDS VM Start (iSCSI) — Start stopped VM; verify LUN-maps are re-created + TDS Detach (iSCSI) — Detach data volume; verify LUN-map removed + +Key iSCSI behaviour verified at each step via ONTAP REST API: + - createVolume → LUN is created inside the pool's FlexVol + - attachVolume → LUN-map is created linking the LUN to the host's igroup + - stopVirtualMachine → LUN-map is removed (LUN stays; just unmapped) + - startVirtualMachine → LUN-map is re-created + - detachVolume → LUN-map is removed + +Tests are numbered test_01 ... test_08 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create iSCSI primary storage pool on ONTAP + 02 Create a CloudStack data volume on the iSCSI pool (LUN on ONTAP) + 03 Deploy a VM using any available KVM template + 04 Attach iSCSI data volume to running VM (LUN-map created) + 05 Stop VM — LUN-map for attached volume is removed from ONTAP + 06 Start VM — LUN-map is re-created on ONTAP + 07 Detach data volume from running VM — LUN-map removed + 08 Destroy VM, delete data volume, delete pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI initiator configured (iqn.* IQN) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + - At least one KVM template must be fully downloaded and ready (isready=True) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_vm_volume_attach_iscsi.py -v +""" + +import base64 +import logging +import random +import re +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, + createNetwork as createNetworkAPI, + createStoragePool as createStoragePoolAPI, + deleteNetwork as deleteNetworkAPI, + deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + detachVolume as detachVolumeAPI, + enableStorageMaintenance, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listServiceOfferings as listServiceOfferingsAPI, + listTemplates as listTemplatesAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + startVirtualMachine as startVirtualMachineAPI, + stopVirtualMachine as stopVirtualMachineAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapVMVolumeAttachISCSI") + + +# --------------------------------------------------------------------------- +# Utility functions +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + cmd.listall = True + return cmd + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=300, + interval=10): + """Block until the VM reaches target_state or timeout expires.""" + deadline = time.time() + timeout + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms and vms[0].state.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + return None + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-vm@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-VM", + "username": "ontap_iscsi_vm_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIVM_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapVMVolumeAttachISCSI(OntapTestBase): + """ + Tests iSCSI ONTAP data volume lifecycle with a running CloudStack VM. + All tests are sequential — state is carried on class attributes. + """ + + # ---- extra shared state beyond OntapTestBase ----------------------- + vm = None + template_id = None + service_offering_id = None + network_id = None + _created_network_id = None # network created by this suite for Advanced zones + + _vol_name_prefix = "OntapISCSIVM" + + # ---- setup --------------------------------------------------------- + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapVMVolumeAttachISCSI, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Discover a ready user KVM template (exclude SYSTEM type) + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = cls.zone.id + templates = cls.apiClient.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + cls.template_id = kvm_ready[0].id if kvm_ready else None + if cls.template_id is None: + logger.warning( + "No ready KVM user template found — VM tests will be skipped." + ) + + # Smallest service offering + so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() + offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] + assert offerings, "No service offerings available in CloudStack" + offerings.sort(key=lambda s: getattr(s, "memory", 9999)) + cls.service_offering_id = offerings[0].id + + # Network ID for VM deployment + cls.network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + # Find a network already accessible to the test account + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + # Create an Isolated guest network for the test account + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + offerings[0] if offerings else None + ) + if snat_offering: + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "ontap-iscsi-vm-net-%d" % random.randint( + 0, 9999) + cn_cmd.displaytext = "ONTAP iSCSI VM test network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def tearDownClass(cls): + """ + Safety-net cleanup: destroy VM if still alive, delete the guest + network created for Advanced zones (if not already deleted by test_08), + then delegate pool/volume/account cleanup to the base class. + """ + if cls.vm is not None: + try: + vms = cls.apiClient.listVirtualMachines( + _list_vms_cmd(cls.vm.id)) + state = vms[0].state if vms else "unknown" + if state.lower() not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = cls.vm.id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, cls.vm.id, + "Stopped", timeout=120) + except Exception as e: + logger.warning("tearDownClass: could not stop VM %s: %s" + % (cls.vm.id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = cls.vm.id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as e: + logger.warning("tearDownClass: could not destroy VM %s: %s" + % (cls.vm.id, e)) + + # Delete the guest network created for this account in Advanced zones. + # test_08 deletes it on the happy path; this is the fallback for + # mid-suite failures. + if cls._created_network_id is not None: + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + cls._created_network_id = None + except Exception as e: + logger.warning( + "tearDownClass: could not delete network %s: %s" + % (cls._created_network_id, e)) + + super(TestOntapVMVolumeAttachISCSI, cls).tearDownClass() + + # ---- helpers ------------------------------------------------------- + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIVM_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _poll_vm_state(self, vm_id, target_state, timeout=300, interval=10): + deadline = time.time() + timeout + current_state = "unknown" + while time.time() < deadline: + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms: + current_state = vms[0].state + if current_state.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + self.fail("VM %s did not reach '%s' within %ds (last: '%s')" + % (vm_id, target_state, timeout, current_state)) + + def _poll_volume_field(self, vol_id, field, target, timeout=120, + interval=5): + """Poll a volume field until it matches target; return the volume.""" + deadline = time.time() + timeout + while time.time() < deadline: + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) + if vols: + val = getattr(vols[0], field, None) + if val == target: + return vols[0] + time.sleep(interval) + return None + + def _lun_maps(self): + """Return current LUN-maps for the pool's FlexVol.""" + if self.__class__.pool is None: + return [] + return self.ontap.list_lun_maps_for_volume( + self.svm_name, self.__class__.pool.name) + + # ================================================================== + # Test steps + # ================================================================== + + # ------------------------------------------------------------------ + # Step 01 — Create iSCSI ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_01_create_iscsi_pool(self): + """ + Create an iSCSI primary storage pool on ONTAP. + Verifies: + - Pool reaches 'Up' state; type is 'Iscsi' + - ONTAP: FlexVol is online + - ONTAP: igroup exists for every host in the cluster that has an IQN + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual(pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state) + self.assertEqual(pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name) + self.assertEqual(ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online'") + + # ------------------------------------------------------------------ + # Step 02 — Create iSCSI data volume (LUN on ONTAP) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_02_create_ontap_data_volume(self): + """ + Allocate a CloudStack data volume on the iSCSI ONTAP pool. + Verifies: + - createVolume returns a volume object + - ONTAP: at least one LUN is created inside the pool's FlexVol + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + luns = self.ontap.list_luns_in_volume( + self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "Expected ≥1 LUN in ONTAP FlexVol '%s' after volume creation, " + "found 0" % self.__class__.pool.name + ) + + # ------------------------------------------------------------------ + # Step 03 — Deploy a VM + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_03_deploy_vm(self): + """ + Deploy a VM using the first available ready KVM template. + Verifies: + - VM reaches Running state + - ONTAP: the iSCSI data volume's LUN is NOT yet mapped (no VM + attachment has been performed yet) + """ + if self.__class__.template_id is None: + self.skipTest( + "No ready KVM user template available — waiting for template " + "download to complete" + ) + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.__class__.vm = vm + + result = self._poll_vm_state(vm.id, "Running", timeout=300) + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after deploy, got '%s'" % result.state + ) + + # Data volume LUN-map must not exist yet (volume not yet attached) + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 LUN-maps before volume attach, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ------------------------------------------------------------------ + # Step 04 — Attach iSCSI volume to running VM (TDS SN 27 iSCSI) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_04_attach_volume_to_vm(self): + """ + Attach the iSCSI data volume to the running VM. + Covers TDS Approach-1 SN 27 (iSCSI): + - attachVolume completes successfully + - CloudStack: volume shows virtualmachineid set + - ONTAP: a LUN-map is created linking the data LUN to the host's + igroup (the LUN is now accessible to the VM's KVM host) + """ + if self.__class__.vm is None: + self.skipTest( + "VM not deployed — test_03 was skipped (no ready template)" + ) + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + cmd = attachVolumeAPI.attachVolumeCmd() + cmd.id = self.__class__.volume.id + cmd.virtualmachineid = self.__class__.vm.id + self.apiClient.attachVolume(cmd) + + # Poll until virtualmachineid is set on the volume + result = self._poll_volume_field( + self.__class__.volume.id, "virtualmachineid", + self.__class__.vm.id, timeout=120) + self.assertIsNotNone( + result, + "Volume virtualmachineid was not set after attachVolume" + ) + + # ONTAP: at least one LUN-map must exist for the pool's FlexVol + lun_maps = self._lun_maps() + self.assertGreater( + len(lun_maps), 0, + "Expected ≥1 LUN-map after volume attach, found 0 — " + "LUN is not accessible to the VM's host" + ) + + # ------------------------------------------------------------------ + # Step 05 — Stop VM — LUN-maps should be removed + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_05_stop_vm_lun_unmapped(self): + """ + Stop the running VM while the iSCSI data volume is still attached. + Covers TDS VM Stop (iSCSI): 'Luns for the volumes under this VM + should be unmapped.' + Verifies: + - VM reaches Stopped state + - ONTAP: LUN-map is removed (LUN itself stays in the FlexVol) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + + cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + cmd.id = self.__class__.vm.id + self.apiClient.stopVirtualMachine(cmd) + + result = self._poll_vm_state(self.__class__.vm.id, "Stopped", + timeout=300) + self.assertEqual( + result.state, "Stopped", + "VM should be 'Stopped', got '%s'" % result.state + ) + + # ONTAP: LUN-map must be removed once VM is stopped + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 LUN-maps after VM stop, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ONTAP: LUN itself must still exist in the FlexVol + luns = self.ontap.list_luns_in_volume( + self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "LUN should still exist in ONTAP FlexVol after VM stop" + ) + + # ------------------------------------------------------------------ + # Step 06 — Start VM — LUN-maps should be re-created + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_06_start_vm_lun_remapped(self): + """ + Start the stopped VM. + Covers TDS VM Start (iSCSI): 'luns should be re-mapped again to + provide access.' + Verifies: + - VM reaches Running state + - ONTAP: LUN-map is re-created (LUN accessible to VM's host) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + + cmd = startVirtualMachineAPI.startVirtualMachineCmd() + cmd.id = self.__class__.vm.id + self.apiClient.startVirtualMachine(cmd) + + result = self._poll_vm_state(self.__class__.vm.id, "Running", + timeout=300) + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after start, got '%s'" % result.state + ) + + # ONTAP: LUN-map must be re-created after VM starts + lun_maps = self._lun_maps() + self.assertGreater( + len(lun_maps), 0, + "Expected ≥1 LUN-map after VM start (re-map), found 0" + ) + + # ------------------------------------------------------------------ + # Step 07 — Detach volume from running VM + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_07_detach_volume_from_vm(self): + """ + Detach the iSCSI data volume from the running VM. + Verifies: + - detachVolume completes successfully + - CloudStack: volume virtualmachineid cleared + - ONTAP: LUN-map is removed (LUN stays in FlexVol) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + # Allow the guest OS to fully initialize the iSCSI device after VM + # start before requesting a hot-detach. Without this pause, the + # libvirt device-removal handshake can time out because the guest + # hasn't finished its early-boot device scan. + time.sleep(20) + + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = self.__class__.volume.id + self.apiClient.detachVolume(cmd) + + # Poll until virtualmachineid is cleared + result = self._poll_volume_field( + self.__class__.volume.id, "virtualmachineid", None, timeout=120) + self.assertIsNotNone( + result, + "Volume virtualmachineid was not cleared after detachVolume" + ) + + # ONTAP: LUN-map must be removed after detach + lun_maps = self._lun_maps() + self.assertEqual( + len(lun_maps), 0, + "Expected 0 LUN-maps after volume detach, found %d: %s" + % (len(lun_maps), lun_maps) + ) + + # ONTAP: LUN still exists in FlexVol + luns = self.ontap.list_luns_in_volume( + self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "LUN should still exist in ONTAP FlexVol after detach" + ) + + # ------------------------------------------------------------------ + # Step 08 — Destroy VM and clean up pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_08_destroy_vm_and_cleanup(self): + """ + Destroy the VM (with expunge), delete the data volume, force-delete + the ONTAP pool, and delete the guest network created for this suite. + This test leaves no entities behind in either CloudStack or ONTAP. + Verifies: + - VM is destroyed and expunged from CloudStack + - deleteVolume removes the LUN from the ONTAP FlexVol + - deleteStoragePool(forced=True) removes the pool from CS + - ONTAP: FlexVol deleted + - ONTAP: all per-host igroups deleted + - CloudStack: guest network deleted (Advanced zones only) + """ + pool = self.__class__.pool + vol = self.__class__.volume + + if self.__class__.vm is not None: + # Ensure VM is stopped before destroying + vms = self.apiClient.listVirtualMachines( + _list_vms_cmd(self.__class__.vm.id)) + current_state = vms[0].state if vms else "unknown" + if current_state.lower() not in ("stopped", "destroyed"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = self.__class__.vm.id + stop_cmd.forced = True + self.apiClient.stopVirtualMachine(stop_cmd) + self._poll_vm_state(self.__class__.vm.id, "Stopped", + timeout=120) + + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = self.__class__.vm.id + dest_cmd.expunge = True + self.apiClient.destroyVirtualMachine(dest_cmd) + self.__class__.vm = None + + if vol is not None and pool is not None: + pool_name = pool.name + + # Delete the data volume + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + + # ONTAP: LUN must be removed after volume deletion + luns = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertEqual( + len(luns), 0, + "Expected 0 LUNs in FlexVol '%s' after volume delete, " + "found %d" % (pool_name, len(luns)) + ) + + # Enter maintenance and force-delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after force deletion" + % pool_name + ) + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool force deletion" + % pool_name + ) + + # ONTAP: per-host igroups must be deleted by the pool force-delete + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) + if not iqn or not iqn.startswith("iqn."): + continue + short = host.name.split(".")[0] + igroup_name = "cs_%s_%s" % ( + self.svm_name, + re.sub(r"[^a-zA-Z0-9_-]", "_", short), + ) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool force deletion" + % igroup_name + ) + + # Delete the guest network created by setUpClass for Advanced zones. + # Doing this inside the test (rather than only in tearDownClass) makes + # the full sequence self-contained when all tests pass. + if self.__class__._created_network_id is not None: + net_id = self.__class__._created_network_id + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = net_id + # The management server may briefly drop the connection after the + # heavy teardown above; retry deleteNetwork up to 3× with 15s gaps. + last_net_exc = None + for attempt in range(3): + try: + self.apiClient.deleteNetwork(dn_cmd) + last_net_exc = None + break + except Exception as exc: + last_net_exc = exc + if attempt < 2: + time.sleep(15) + if last_net_exc is not None: + raise last_net_exc + self.__class__._created_network_id = None + self.__class__.network_id = None + + # CloudStack: guest network must be gone + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.id = net_id + net_cmd.listall = True + remaining_nets = self.apiClient.listNetworks(net_cmd) or [] + self.assertFalse( + remaining_nets, + "Guest network %s still listed in CloudStack after deletion" + % net_id + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/__init__.py b/test/integration/plugins/ontap/iscsi/pool/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py new file mode 100644 index 000000000000..8a8584b0725f --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py @@ -0,0 +1,630 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage +pool lifecycle (no volumes). + +Tests are numbered test_01 ... test_08 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create primary storage pool + 02 Disable storage pool + 03 Enable storage pool + 04 Enter maintenance mode + 05 Cancel maintenance mode + 06 Enter maintenance mode and delete the storage pool + 07 Create a new pool and allocate a CloudStack data volume (LUN created) + 08 Delete the volume (LUN removed), enter maintenance, force-delete pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/pool/ -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIPoolLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-wf@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-WF", + "username": "ontap_iscsi_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSI_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIPoolLifecycle(OntapTestBase): + + # ---- iSCSI-specific state (set/cleared by individual tests) -------- + _vol_name_prefix = "OntapISCSIVol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIPoolLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSI_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 - Create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_01_create_primary_storage_pool(self): + """ + Create an iSCSI primary storage pool and verify: + - CloudStack state is Up, type is Iscsi + - ONTAP: FlexVol exists and is online + - ONTAP: one igroup per cluster host exists with the correct IQN initiator + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroup must exist for each cluster host that has an IQN + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue # host not iSCSI-enabled; skip igroup check for it + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s'" % (igroup_name, host.name) + ) + initiator_names = [ + i.get("name", "") for i in igroup.get("initiators", []) + ] + self.assertIn( + iqn, initiator_names, + "Host IQN '%s' not in igroup '%s' initiators: %s" + % (iqn, igroup_name, initiator_names) + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "pool-created") + + # ------------------------------------------------------------------ + # Step 02 - Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_02_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online (disable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + # ONTAP: disable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 03 - Enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_03_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online (enable is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + # ONTAP: enable must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 04 - Enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_04_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + # ONTAP: maintenance must not touch the FlexVol + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 05 - Cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_05_cancel_maintenance_mode(self): + """ + Cancel maintenance and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 06 - Enter maintenance mode and delete the storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_06_enter_maintenance_and_delete_pool(self): + """ + Enter maintenance mode then delete the pool. + Verifies the pool is removed from CloudStack and the backing ONTAP + FlexVol is deleted. + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + pool = self.__class__.pool + pool_name = pool.name + + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) + + # ------------------------------------------------------------------ + # Step 07 - Create fresh pool and allocate a CloudStack volume (LUN) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_07_create_volume_on_pool(self): + """ + Create a new iSCSI pool and allocate a CloudStack data volume. + For iSCSI, createAsync creates a LUN inside the pool's ONTAP FlexVol. + Verifies: + - pool.state is Up, type is Iscsi + - createVolume returns a non-None volume object + - ONTAP: FlexVol is still online + - ONTAP: at least one LUN is present in the FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: FlexVol must still be online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: at least one LUN must be present in the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume creation" % pool.name + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 08 - Delete volume (LUN) then force-delete the pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_08_delete_volume_and_pool(self): + """ + Delete the volume from test_07, enter maintenance, then force-delete + the pool. + Verifies: + - deleteVolume removes the LUN from ONTAP + - Pool transitions to Maintenance + - Pool is removed from CloudStack after force deletion + - ONTAP: FlexVol deleted + - ONTAP: igroups for all cluster hosts deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume — LUN is removed from ONTAP + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertEqual( + len(luns), 0, + "Expected 0 LUNs in FlexVol '%s' after volume deletion, found %d" + % (pool_name, len(luns)) + ) + + # ONTAP: FlexVol must still be online (pool not yet deleted) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after volume deletion" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion" + ) + + # Capacity reporting: capacity stable after volume deletion + self._assert_pool_capacity(pool, "volume-deleted") + + # Enter maintenance then force-delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py new file mode 100644 index 000000000000..ef799b644ce8 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py @@ -0,0 +1,706 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +iSCSI pool lifecycle tests with a CloudStack data volume present throughout. + +Covers the TDS (section 10) scenarios that require a data volume to already +exist on the pool during pool state transitions — the iSCSI variants of those +scenarios: + + TDS Approach-1 SN 11 — Disable iSCSI pool WITH volumes + TDS Approach-1 SN 15 — Enable iSCSI pool WITH volumes + TDS Approach-1 SN 19 — Enter maintenance WITH volumes + TDS Approach-1 SN 23 — Cancel maintenance WITH volumes + TDS Negative SN 5 — Delete iSCSI pool that has volumes; forced=False rejected + TDS Approach-1 SN 7 — Force-delete iSCSI pool (volume deleted first from + Maintenance — allowed on iSCSI unlike NFS3) + +Key iSCSI difference from NFS3: cancelStorageMaintenance works on iSCSI because +the KVM agent can unmount/remount iSCSI LUNs correctly. This allows the full +maintenance-cancel-maintenance lifecycle and proper volume cleanup while pool +is in Maintenance state. + +Tests are numbered test_01 ... test_07 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create iSCSI pool and allocate a CloudStack data volume (LUN on ONTAP) + 02 Disable pool — volume survives; ONTAP LUN still exists (SN 11) + 03 Re-enable pool — volume intact; ONTAP LUN accessible (SN 15) + 04 Enter maintenance with volume — pool Maintenance; LUN exists (SN 19) + 05 Cancel maintenance with volume — pool Up; LUN accessible (SN 23) + 06 Re-enter maintenance; forced=False delete rejected (Neg SN 5) + 07 Delete volume from Maintenance, then force-delete pool (SN 7) + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI initiator configured + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_iscsi_pool_with_volumes.py -v +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIPoolWithVolumes") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-wv@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-WV", + "username": "ontap_iscsi_wv_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIWV_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIPoolWithVolumes(OntapTestBase): + """ + iSCSI pool lifecycle tests with a CloudStack data volume present throughout. + All 7 tests are sequential and share class-level state. + """ + + _vol_name_prefix = "OntapISCSIWV" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIPoolWithVolumes, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIWV_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return len(vols) > 0 + + def _assert_lun_exists(self, pool_name, msg_context=""): + """Assert that at least one LUN exists in the pool's ONTAP FlexVol.""" + luns = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertTrue( + len(luns) > 0, + "Expected ≥1 LUN in ONTAP FlexVol '%s'%s, found 0" + % (pool_name, " (%s)" % msg_context if msg_context else "") + ) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create pool and allocate a data volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create an iSCSI primary storage pool and allocate a CloudStack data + volume on it. + Verifies: + - Pool state is Up; pool type is Iscsi + - ONTAP: FlexVol is online + - ONTAP: at least one igroup exists (one per cluster host with IQN) + - ONTAP: after createVolume, a LUN exists in the FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroup must exist for each cluster host that has an IQN + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s'" + % (igroup_name, host.name) + ) + + # Allocate a CloudStack data volume on this pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: a LUN must exist in the FlexVol after volume creation + self._assert_lun_exists(pool.name, "after volume creation") + + # Capacity reporting: LUN allocated but FlexVol size unchanged + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 02 — Disable pool with volume present (TDS SN 11) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_02_disable_pool_volume_survives(self): + """ + Disable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 11 (iSCSI): + - Pool transitions to Disabled + - Existing CS volume still listed + - ONTAP: FlexVol remains online; LUN still exists + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual( + result.state, "Disabled", + "Pool should be 'Disabled', got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool disable" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after pool disable" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after pool disable") + + # ------------------------------------------------------------------ + # Step 03 — Re-enable pool with volume present (TDS SN 15) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_03_enable_pool_volume_intact(self): + """ + Re-enable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 15 (iSCSI): + - Pool transitions back to Up + - CS volume still listed + - ONTAP: FlexVol online; LUN still exists + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after re-enable, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool re-enable" + ) + + # ONTAP: FlexVol online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool re-enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after pool re-enable" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after pool re-enable") + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance with volume present (TDS SN 19) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_04_enter_maintenance_volume_present(self): + """ + Enter maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 19 (iSCSI): + - Pool transitions to Maintenance + - CS volume still listed (not destroyed) + - ONTAP: FlexVol remains online (maintenance is a CloudStack state) + - ONTAP: LUN still exists in the FlexVol + + Note: the TDS additionally expects VMs using this pool to stop and their + LUN maps to be removed. This suite uses a standalone data volume (not + attached to any VM), so the VM stop behaviour is not exercised here — it + is covered by the VM lifecycle test suite. + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual( + result.state, "Maintenance", + "Pool should be 'Maintenance', got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool entered Maintenance" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after entering Maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' in Maintenance" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after entering Maintenance") + + # ------------------------------------------------------------------ + # Step 05 — Cancel maintenance with volume present (TDS SN 23) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_05_cancel_maintenance_volume_present(self): + """ + Cancel maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 23 (iSCSI): + - cancelStorageMaintenance works on iSCSI (unlike the NFS3 variant) + - Pool transitions back to Up + - CS volume still listed + - ONTAP: FlexVol online; LUN still present in FlexVol + + Note: when VMs are attached to volumes on this pool, ONTAP would + re-create the LUN-maps (igroup bindings) at cancel-maintenance time. + This suite has no VMs attached, so LUN-map re-creation is not verified + here; it is covered by the VM lifecycle test suite. + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after cancel maintenance" + ) + + # ONTAP: FlexVol online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance" + ) + + # ONTAP: LUN still exists + self._assert_lun_exists(self.__class__.pool.name, "after cancel maintenance") + + # ------------------------------------------------------------------ + # Step 06 — forced=False delete rejected (negative) (TDS Neg SN 5) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_06_forced_false_delete_rejected(self): + """ + Enter maintenance then attempt deleteStoragePool(forced=False) while + a CloudStack volume exists on the pool. The operation must be rejected. + Covers TDS Negative Scenario SN 5 (iSCSI): + - CloudstackAPIException is raised + - Pool remains in Maintenance state + - CS volume still exists + - ONTAP: FlexVol and LUN unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + # Re-enter Maintenance (pool is Up from test_05) + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + + # Attempt forced=False delete — must raise + with self.assertRaises(Exception, + msg="deleteStoragePool(forced=False) with a live " + "volume should raise an exception"): + self._delete_pool(self.__class__.pool.id, forced=False) + + # Pool must still be listed (in Maintenance) + try: + remaining = list_storage_pools(self.apiClient, id=self.__class__.pool.id) + except Exception: + remaining = None + self.assertTrue( + remaining, + "Pool was deleted even though forced=False delete should have failed" + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume was deleted after rejected pool deletion" + ) + + # ONTAP: FlexVol still online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol should still exist after rejected pool deletion" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after rejected deletion" + ) + + # ------------------------------------------------------------------ + # Step 07 — Delete volume from Maintenance, then force-delete pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_07_delete_volume_and_force_delete_pool(self): + """ + Delete the CloudStack volume (while pool is in Maintenance) then + force-delete the pool. + Covers TDS Approach-1 SN 7 (iSCSI): + - On iSCSI, deleteVolume succeeds even when pool is in Maintenance + (unlike NFS3 where the KVM agent raises NPE) + - After volume deletion, the LUN is removed from the ONTAP FlexVol + - force-delete pool removes pool, FlexVol, and all igroups + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume while pool is in Maintenance + # (this works on iSCSI — no KVM NPE unlike NFS3) + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol after volume deletion + luns_after = self.ontap.list_luns_in_volume(self.svm_name, pool_name) + self.assertEqual( + len(luns_after), 0, + "Expected 0 LUNs in ONTAP FlexVol '%s' after volume deletion, " + "found %d: %s" % (pool_name, len(luns_after), luns_after) + ) + + # ONTAP: FlexVol must still be online (pool deletion removes the FlexVol) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after CS volume deletion" + % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after CS volume deletion" + ) + + # Capacity reporting: capacity fields stable after LUN removal + self._assert_pool_capacity(pool, "volume-deleted") + + # Force-delete the pool (no live volumes remain; pool is in Maintenance) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after force deletion" % pool_name + ) + + # ONTAP: FlexVol must be deleted + ontap_vol_after = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol_after, + "ONTAP FlexVol '%s' still exists after pool force deletion" % pool_name + ) + + # ONTAP: igroups for all cluster hosts must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool force deletion" + % igroup_name + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py new file mode 100644 index 000000000000..a6b57de5573f --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py @@ -0,0 +1,386 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Zone-scoped primary storage lifecycle tests for NetApp ONTAP (iSCSI). + +Creates a zone-scoped pool (scope=ZONE, no clusterid/podid). CloudStack calls +OntapPrimaryDatastoreLifecycle.attachZone(), which connects all eligible KVM +hosts in the zone and creates igroups for each host's IQN. + +Workflow: + 01 Create zone-scoped iSCSI pool — pool.state Up; ONTAP FlexVol online; + igroup present for each cluster host IQN + 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 04 Delete zone-scoped pool — pool gone; FlexVol deleted; igroups deleted + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM hosts with iSCSI registered in the zone + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py -v + +Note: Tests 01-04 share class-level state (sequential). Always run the full +suite. +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIZoneScopedPool") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + provider="NetApp ONTAP", tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-zone@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-Zone", + "username": "ontap_iscsi_zone_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapZoneISCSI_%d" % random.randint(0, 9999), + TestData.scope: "ZONE", + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIZoneScopedPool(OntapTestBase): + + _vol_name_prefix = "OntapISCSIZoneVol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIZoneScopedPool, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + provider=provider, tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_zone_pool(self): + """Create a zone-scoped iSCSI pool (no clusterid / podid).""" + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapZoneISCSI_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + # Intentionally omit clusterid and podid — zone-scoped pool + cmd.scope = "ZONE" + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _assert_igroups_for_hosts(self, expect_present): + """Assert igroups are present (or absent) for each cluster host IQN.""" + for host in self.cluster_hosts: + iqn = (getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None)) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + if expect_present: + self.assertIsNotNone( + igroup, + "ONTAP igroup '%s' not found for host '%s' after pool creation" + % (igroup_name, host.name) + ) + initiator_names = [ + i.get("name", "") for i in igroup.get("initiators", []) + ] + self.assertIn( + iqn, initiator_names, + "Host IQN '%s' not in igroup '%s' initiators: %s" + % (iqn, igroup_name, initiator_names) + ) + else: + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after pool deletion" % igroup_name + ) + + # ------------------------------------------------------------------ + # Step 01 — Create zone-scoped iSCSI pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_01_create_zone_scoped_pool(self): + """ + Create a zone-scoped iSCSI primary storage pool (no clusterid/podid). + CloudStack calls attachZone(), which connects all eligible KVM hosts + in the zone and creates igroups for each host's IQN. + Verifies: + - pool.state is Up, type is Iscsi + - ONTAP: FlexVol is online + - ONTAP: igroup exists for each cluster host with the correct IQN + """ + pool = self._create_zone_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "Iscsi", + "Pool type should be 'Iscsi', got '%s'" % pool.type + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: igroups must exist for each cluster host with IQN + self._assert_igroups_for_hosts(expect_present=True) + + # ------------------------------------------------------------------ + # Step 02 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_02_disable_zone_scoped_pool(self): + """ + Disable the zone-scoped iSCSI pool. + Verifies: + - pool.state is Disabled + - ONTAP: FlexVol still online; igroups unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable" + ) + + # igroups must still be present after a simple disable + self._assert_igroups_for_hosts(expect_present=True) + + # ------------------------------------------------------------------ + # Step 03 — Enable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_03_enable_zone_scoped_pool(self): + """ + Re-enable the zone-scoped iSCSI pool. + Verifies: + - pool.state is Up + - ONTAP: FlexVol still online; igroups unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable" + ) + + # igroups must still be present after re-enable + self._assert_igroups_for_hosts(expect_present=True) + + # ------------------------------------------------------------------ + # Step 04 — Delete zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_04_delete_zone_scoped_pool(self): + """ + Enter maintenance then delete the zone-scoped iSCSI pool. + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol deleted + - ONTAP: igroups deleted for all cluster hosts + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + self._assert_igroups_for_hosts(expect_present=False) diff --git a/test/integration/plugins/ontap/iscsi/volume/__init__.py b/test/integration/plugins/ontap/iscsi/volume/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/volume/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py b/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py new file mode 100644 index 000000000000..7f02f9f29903 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py @@ -0,0 +1,416 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP iSCSI data volume +lifecycle (LUN create / delete / negative-delete / force-delete). + +Tests are numbered test_01 ... test_05 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create iSCSI primary storage pool (infrastructure) and allocate a + CloudStack data volume — LUN is created inside the pool's ONTAP FlexVol + 02 Delete the volume — LUN is removed from the FlexVol + 03 Recreate volume — LUN is present again (setup for negative delete tests) + 04 Put pool in Maintenance; attempt forced=False deleteStoragePool — must be + rejected because volumes exist; pool stays in Maintenance + 05 Delete volume from Maintenance; forced=True deleteStoragePool — FlexVol, + igroups, and all LUNs are removed from ONTAP + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) + - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/iscsi/volume/ -v + +Note: Tests share class-level state (sequential). Always run the full suite. +The pool is cleaned up in test_05 on the happy path; OntapTestBase tearDownClass +provides a safety net for mid-run failures. +""" + +import base64 +import logging +import random +import re +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase + +logger = logging.getLogger("TestOntapISCSIVolumeLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-iscsi", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-iscsi-vol@test.com", + "firstname": "ONTAP", + "lastname": "iSCSI-Vol", + "username": "ontap_iscsi_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapISCSIVol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "ISCSI", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# iSCSI path helpers +# --------------------------------------------------------------------------- + +def _igroup_name(svm_name, host_name): + """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" + short = host_name.split(".")[0] + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) + return "cs_%s_%s" % (svm_name, sanitized) + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapISCSIVolumeLifecycle(OntapTestBase): + + _vol_name_prefix = "OntapISCSIVol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapISCSIVolumeLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + iscsi_cfg = pool_cfg.get("protocols", {}).get("iscsi", {}) + if not iscsi_cfg.get("enabled", True): + raise unittest.SkipTest( + "iSCSI tests disabled in ontap.cfg " + "(set protocols.iscsi.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapISCSIVol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "iscsi://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ------------------------------------------------------------------ + # Step 01 - Create pool (infrastructure) and allocate a volume + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create a new iSCSI pool and allocate a CloudStack data volume on it. + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: at least one LUN exists in the pool's FlexVol + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: at least one LUN must be present in the pool FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume creation" % pool.name + ) + + # ------------------------------------------------------------------ + # Step 02 - Delete volume; LUN must be removed from ONTAP + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_02_delete_volume(self): + """ + Delete the volume created in test_01. + Verifies: + - deleteVolume completes without error + - ONTAP: LUN is removed from the pool's FlexVol + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_01 must pass first") + + pool = self.__class__.pool + vol = self.__class__.volume + + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: LUN must be gone from the FlexVol + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertEqual( + len(luns), 0, + "Expected 0 LUNs in FlexVol '%s' after volume deletion, found %d: %s" + % (pool.name, len(luns), luns) + ) + + # ------------------------------------------------------------------ + # Step 03 - Recreate volume for negative delete tests + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_03_recreate_volume_for_delete_tests(self): + """ + Recreate a volume on the existing pool (setup for tests 04-05). + Verifies: + - volume created successfully + - ONTAP: LUN present in pool FlexVol + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + luns = self.ontap.list_luns_in_volume(self.svm_name, self.__class__.pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs found in ONTAP FlexVol '%s' after volume re-creation" + % self.__class__.pool.name + ) + + # ------------------------------------------------------------------ + # Step 04 - Forced=False delete with live volume must fail + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_04_forced_false_delete_with_volume_fails(self): + """ + Put pool in Maintenance then attempt deleteStoragePool(forced=False). + With a live volume present CloudStack must reject the request. + Verifies: + - CloudstackAPIException is raised + - Pool is still listed in CloudStack (in Maintenance state) + - ONTAP: FlexVol still exists and is online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + + # Enter maintenance mode + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # Attempt forced=False delete — must raise exception because volumes exist + with self.assertRaises(Exception): + self._delete_pool(pool.id, forced=False) + + # Pool must still be listed in CloudStack + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + listed, + "Pool should still exist in CloudStack after failed forced=False delete" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after failed delete" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 05 - Delete volume then force-delete pool from Maintenance + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_volume"], required_hardware=True) + def test_05_delete_volume_and_force_delete_pool(self): + """ + Delete the live volume then force-delete the pool while it is still + in Maintenance state (pool is in Maintenance from test_04). + Verifies: + - Volume can be deleted while pool is in Maintenance + - Pool is removed from CloudStack using forced=True from Maintenance + - ONTAP: FlexVol deleted + - ONTAP: igroups deleted for all cluster hosts + """ + self.assertIsNotNone( + self.__class__.pool, + "Pool absent - test_04 must not have cleaned up the pool" + ) + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + vol = self.__class__.volume + + # Delete the volume first (pool is in Maintenance — volume deletion is allowed) + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # Force-delete the pool from Maintenance (no live volumes remaining) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after force deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after force deletion" % pool_name + ) + + # ONTAP: igroups for each cluster host must be deleted + for host in self.cluster_hosts: + iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) + if not iqn or not iqn.startswith("iqn."): + continue + igroup_name = _igroup_name(self.svm_name, host.name) + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self.assertIsNone( + igroup, + "ONTAP igroup '%s' still exists after force deletion" % igroup_name + ) diff --git a/test/integration/plugins/ontap/manual_cancel_maint_test.py b/test/integration/plugins/ontap/manual_cancel_maint_test.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/integration/plugins/ontap/nfs3/__init__.py b/test/integration/plugins/ontap/nfs3/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/instance/__init__.py b/test/integration/plugins/ontap/nfs3/instance/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/instance/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py new file mode 100644 index 000000000000..461ee104cbf9 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py @@ -0,0 +1,832 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP data volume lifecycle +with a running virtual machine. + +Tests are numbered test_01 ... test_08 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create NFS3 primary storage pool on ONTAP + 02 Create a CloudStack data volume on the ONTAP pool + 03 Deploy a VM (template and service offering discovered at setup time) + 04 Attach the ONTAP data volume to the running VM + 05 Stop the VM — export policy stays; volume remains attached in CS + 06 Start the VM — VM Running; volume still attached; FlexVol online + 07 Detach the ONTAP data volume from the VM + 08 Destroy VM; delete ONTAP volume; enter maintenance; delete pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack with at least one executable template + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_vm_volume_attach.py -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + attachVolume as attachVolumeAPI, + createNetwork as createNetworkAPI, + createStoragePool as createStoragePoolAPI, + deleteNetwork as deleteNetworkAPI, + deleteVolume as deleteVolumeAPI, + deployVirtualMachine as deployVirtualMachineAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + detachVolume as detachVolumeAPI, + enableStorageMaintenance, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listServiceOfferings as listServiceOfferingsAPI, + listTemplates as listTemplatesAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + startVirtualMachine as startVirtualMachineAPI, + stopVirtualMachine as stopVirtualMachineAPI, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapVMVolumeAttach") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-vm-vol@test.com", + "firstname": "ONTAP", + "lastname": "VMVol", + "username": "ontap_vm_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapVMVol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapVMVolumeAttach(OntapTestBase): + """ + Tests ONTAP data volume lifecycle with a running CloudStack VM. + + All tests are sequential — state is carried on class attributes. + """ + + # ---- extra shared state beyond OntapTestBase ----------------------- + vm = None # running VirtualMachine + template_id = None # KVM template ID discovered at setup + service_offering_id = None + network_id = None # None for Basic zones + _created_network_id = None # network created by this suite for Advanced zones + + _vol_name_prefix = "OntapVMVol" + + # ---- setup --------------------------------------------------------- + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapVMVolumeAttach, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Discover a suitable user KVM template in the zone (must be fully + # downloaded; system-type templates are excluded as they cannot be + # deployed as user VMs). + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = cls.zone.id + templates = cls.apiClient.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + if kvm_ready: + cls.template_id = kvm_ready[0].id + else: + logger.warning( + "No ready user KVM template found in zone '%s'. " + "Tests that deploy VMs will be skipped until a template " + "finishes downloading." % cls.zone.name + ) + cls.template_id = None + + # Discover the smallest service offering + so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() + offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] + assert offerings, "No service offerings available in CloudStack" + offerings.sort(key=lambda s: getattr(s, "memory", 9999)) + cls.service_offering_id = offerings[0].id + + # Detect zone type; resolve network ID for Advanced zones + cls.network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + # Find a network already accessible to the test account + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + # Create an Isolated guest network for the test account + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.specifyvlan = "false" + no_offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + snat_offering = next( + (o for o in no_offerings + if "SourceNat" in o.name and "Vpc" not in o.name + and "NSX" not in o.name and "Netris" not in o.name), + no_offerings[0] if no_offerings else None + ) + if snat_offering: + cn_cmd = createNetworkAPI.createNetworkCmd() + cn_cmd.zoneid = cls.zone.id + cn_cmd.networkofferingid = snat_offering.id + cn_cmd.name = "ontap-nfs3-vm-net-%d" % random.randint( + 0, 9999) + cn_cmd.displaytext = "ONTAP NFS3 VM test network" + cn_cmd.account = cls.account.name + cn_cmd.domainid = cls.domain.id + net = cls.apiClient.createNetwork(cn_cmd) + cls.network_id = net.id + cls._created_network_id = net.id + + @classmethod + def tearDownClass(cls): + """Destroy the VM first, then delegate pool/volume cleanup to super.""" + if cls.vm is not None: + try: + # Ensure VM is stopped before destroying + vms = cls.apiClient.listVirtualMachines( + _list_vms_cmd(cls.vm.id)) + current_state = vms[0].state if vms else "unknown" + if current_state.lower() not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = cls.vm.id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, cls.vm.id, "Stopped", + timeout=120) + except Exception as e: + logger.warning("tearDownClass: could not stop VM %s: %s" + % (cls.vm.id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = cls.vm.id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as e: + logger.warning("tearDownClass: could not destroy VM %s: %s" + % (cls.vm.id, e)) + + # Delete the guest network created for this account in Advanced zones. + if cls._created_network_id is not None: + try: + dn_cmd = deleteNetworkAPI.deleteNetworkCmd() + dn_cmd.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn_cmd) + cls._created_network_id = None + except Exception as e: + logger.warning( + "tearDownClass: could not delete network %s: %s" + % (cls._created_network_id, e)) + + super(TestOntapVMVolumeAttach, cls).tearDownClass() + + # ---- pool creation helper ----------------------------------------- + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapVMVol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + # ---- VM state helpers ---------------------------------------------- + + def _poll_vm_state(self, vm_id, target_state, timeout=300, interval=10): + """Poll listVirtualMachines until the VM reaches target_state.""" + deadline = time.time() + timeout + current_state = "unknown" + while time.time() < deadline: + vms = self.apiClient.listVirtualMachines( + _list_vms_cmd(vm_id)) + if vms: + current_state = vms[0].state + if current_state.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + self.fail( + "VM %s did not reach state '%s' within %ds (last: '%s')" + % (vm_id, target_state, timeout, current_state) + ) + + def _volume_state(self, vol_id): + """Return the current CloudStack state string for a volume.""" + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + vols = self.apiClient.listVolumes(cmd) + return vols[0].state if vols else "unknown" + + # ================================================================== + # Test steps + # ================================================================== + + # ------------------------------------------------------------------ + # Step 01 - Create NFS3 ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_01_create_nfs3_pool(self): + """ + Create an NFS3 ONTAP primary storage pool. + Verifies: + - Pool reaches 'Up' state in CloudStack + - ONTAP: FlexVol is created and online + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not created for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" + % ontap_vol.get("state") + ) + + # ------------------------------------------------------------------ + # Step 02 - Create CloudStack data volume on ONTAP pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_02_create_ontap_data_volume(self): + """ + Allocate a CloudStack data volume on the ONTAP NFS3 pool. + Verifies: + - Volume is created and in 'Allocated' or 'Ready' state + - ONTAP: FlexVol remains online + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + pool = self.__class__.pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + vol_state = self._volume_state(vol.id) + self.assertIn( + vol_state.lower(), ("allocated", "ready"), + "Volume should be 'Allocated' or 'Ready', got '%s'" % vol_state + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after data volume creation" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume creation" + ) + + # ------------------------------------------------------------------ + # Step 03 - Deploy a VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_03_deploy_vm(self): + """ + Deploy a VM using the first available KVM template and smallest + service offering discovered at setup time. + Verifies: + - VM reaches 'Running' state in CloudStack + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + if self.__class__.template_id is None: + self.skipTest( + "No ready user KVM template available — " + "waiting for template download to complete" + ) + self.assertIsNotNone(self.__class__.service_offering_id, + "No service offering available — check setup") + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + self.__class__.vm = vm + + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=600) + self.assertEqual( + vm_obj.state, "Running", + "VM should be 'Running', got '%s'" % vm_obj.state + ) + + # ------------------------------------------------------------------ + # Step 04 - Attach ONTAP data volume to the running VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_04_attach_volume_to_vm(self): + """ + Attach the ONTAP data volume to the running VM. + Verifies: + - Volume virtualmachineid is set to the VM's ID in CloudStack + (Note: on ONTAP/NFS shared storage the volume state remains 'Ready'; + attachment is signalled by virtualmachineid being populated) + - ONTAP: FlexVol remains online + - VM remains 'Running' + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = attachVolumeAPI.attachVolumeCmd() + cmd.id = vol.id + cmd.virtualmachineid = vm.id + attached = self.apiClient.attachVolume(cmd) + self.assertIsNotNone(attached, "attachVolume returned None") + + # On ONTAP/NFS shared storage CloudStack does not transition the volume + # state to 'In Use' — attachment is indicated by virtualmachineid being + # set on the volume record. Poll on that field instead of state. + deadline = time.time() + 120 + vol_vmid = None + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol.id)) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if vol_vmid: + break + time.sleep(5) + + self.assertEqual( + vol_vmid, vm.id, + "Volume should be attached to VM %s after attach, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + # VM must still be Running + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=30) + self.assertEqual(vm_obj.state, "Running", + "VM should still be 'Running' after volume attach") + + # ONTAP FlexVol must remain online + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol not found after attach") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after attach" + ) + + # ------------------------------------------------------------------ + # Step 05 - Stop VM — export policy must be retained + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_05_stop_vm_export_retained(self): + """ + Stop the running VM while the NFS3 data volume is still attached. + Unlike iSCSI (where LUN-maps are removed on VM stop), NFS3 export + policies are not torn down when a VM stops — the FlexVol stays + accessible on the same mount. + Verifies: + - VM reaches Stopped state + - ONTAP: FlexVol still online + - CloudStack: volume virtualmachineid still set (volume stays attached) + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + cmd.id = vm.id + self.apiClient.stopVirtualMachine(cmd) + + result = self._poll_vm_state(vm.id, "Stopped", timeout=300) + self.assertEqual( + result.state, "Stopped", + "VM should be 'Stopped', got '%s'" % result.state + ) + + # ONTAP: FlexVol must remain online — NFS export is not torn down on VM stop + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' disappeared after VM stop" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after VM stop, " + "got '%s'" % ontap_vol.get("state") + ) + + # CloudStack: volume must still be attached (virtualmachineid set) + cmd_list = listVolumesAPI.listVolumesCmd() + cmd_list.id = vol.id + vols = self.apiClient.listVolumes(cmd_list) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + self.assertEqual( + vol_vmid, vm.id, + "Volume should still be attached to VM %s after stop, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + # ------------------------------------------------------------------ + # Step 06 - Start VM — volume accessible; FlexVol online + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_06_start_vm_volume_accessible(self): + """ + Start the stopped VM. + Verifies: + - VM reaches Running state + - ONTAP: FlexVol still online + - CloudStack: volume virtualmachineid still set (volume remains attached) + - VM remains 'Running' after start + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = startVirtualMachineAPI.startVirtualMachineCmd() + cmd.id = vm.id + self.apiClient.startVirtualMachine(cmd) + + result = self._poll_vm_state(vm.id, "Running", timeout=300) + self.assertEqual( + result.state, "Running", + "VM should be 'Running' after start, got '%s'" % result.state + ) + + # ONTAP: FlexVol must remain online after VM start + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after VM start" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after VM start, " + "got '%s'" % ontap_vol.get("state") + ) + + # CloudStack: volume must still be attached to the VM + cmd_list = listVolumesAPI.listVolumesCmd() + cmd_list.id = vol.id + vols = self.apiClient.listVolumes(cmd_list) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + self.assertEqual( + vol_vmid, vm.id, + "Volume should still be attached to VM %s after start, " + "got virtualmachineid=%s" % (vm.id, vol_vmid) + ) + + # ------------------------------------------------------------------ + # Step 07 - Detach ONTAP data volume from the VM + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_07_detach_volume_from_vm(self): + """ + Detach the ONTAP data volume from the running VM. + Verifies: + - Volume state returns to 'Ready' in CloudStack + - Volume no longer lists the VM's ID + - VM remains 'Running' + - ONTAP: FlexVol remains online + """ + if self.__class__.vm is None: + self.skipTest("VM not deployed — test_03 was skipped (no ready template)") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_02 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + + cmd = detachVolumeAPI.detachVolumeCmd() + cmd.id = vol.id + # The hypervisor may briefly mark the device as busy; retry up to 3×. + last_exc = None + for attempt in range(3): + try: + self.apiClient.detachVolume(cmd) + last_exc = None + break + except Exception as exc: + last_exc = exc + if attempt < 2: + time.sleep(30) + if last_exc is not None: + raise last_exc + + # On ONTAP/NFS shared storage the volume state stays 'Ready' throughout. + # Poll until virtualmachineid is cleared instead. + deadline = time.time() + 120 + vol_vmid = "pending" + while time.time() < deadline: + vols = self.apiClient.listVolumes(_list_vols_cmd(vol.id)) + vol_vmid = getattr(vols[0], "virtualmachineid", None) if vols else None + if not vol_vmid: + break + time.sleep(5) + + self.assertIsNone( + vol_vmid, + "Volume should have no virtualmachineid after detach, got '%s'" + % vol_vmid + ) + + # VM must still be Running + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=30) + self.assertEqual(vm_obj.state, "Running", + "VM should still be 'Running' after volume detach") + + # ONTAP FlexVol must remain online + pool = self.__class__.pool + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol not found after detach") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after detach" + ) + + # ------------------------------------------------------------------ + # Step 08 - Destroy VM, delete volume, delete pool + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_08_destroy_vm_and_cleanup(self): + """ + Destroy the VM, delete the ONTAP data volume, enter maintenance, + then delete the pool. + Verifies: + - VM is destroyed/expunged from CloudStack + - Volume is deleted from CloudStack + - Pool is removed from CloudStack + - ONTAP: FlexVol is deleted after pool removal + - ONTAP: Export policy is removed after pool removal + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + vm = self.__class__.vm + vol = self.__class__.volume + pool = self.__class__.pool + pool_name = pool.name + + # Stop VM if still running + if vm is not None: + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm.id)) + current_state = vms[0].state.lower() if vms else "unknown" + if current_state not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm.id + self.apiClient.stopVirtualMachine(stop_cmd) + self._poll_vm_state(vm.id, "Stopped", timeout=300) + + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm.id + dest_cmd.expunge = True + self.apiClient.destroyVirtualMachine(dest_cmd) + self.__class__.vm = None + + # Delete the ONTAP data volume + if vol is not None: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # Enter maintenance then delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id) + self.__class__.pool = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, + "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: Export policy must be removed + ep_name = "cs-%s-%s" % (self.svm_name, pool_name) + ep = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + ep, + "ONTAP export policy '%s' still exists after pool deletion" + % ep_name + ) + + +# --------------------------------------------------------------------------- +# Module-level helpers (used in tearDownClass and test helpers) +# --------------------------------------------------------------------------- + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + return cmd + + +def _list_vols_cmd(vol_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + return cmd + + +def _wait_for_vm_state(api_client, vm_id, target_state, timeout=120, + interval=5): + """Blocking wait for a VM to reach target_state (used in tearDownClass).""" + deadline = time.time() + timeout + while time.time() < deadline: + vms = api_client.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms and vms[0].state.lower() == target_state.lower(): + return + time.sleep(interval) diff --git a/test/integration/plugins/ontap/nfs3/pool/__init__.py b/test/integration/plugins/ontap/nfs3/pool/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py new file mode 100644 index 000000000000..f4f7ac7bb039 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py @@ -0,0 +1,709 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. + +Tests are numbered test_01 ... test_08 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create primary storage pool + 02 Disable storage pool + 03 Enable storage pool + 04 Enter maintenance mode + 05 Cancel maintenance mode + 06 Delete the storage pool (enters Maintenance first, then deletes) + 07 Create fresh pool and allocate a CloudStack volume + 08 Delete volume then force-delete the pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py -v + +Note: Tests 01-06 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapNFS3Workflow") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + DETAIL_VOLUME_UUID = "volumeUUID" + DETAIL_VOLUME_NAME = "volumeName" + DETAIL_DATA_LIF = "dataLIF" + DETAIL_NFS_MOUNT_OPTS = "nfsmountopts" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-wf@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-WF", + "username": "ontap_nfs3_wf_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): + + # ---- NFS3-specific shared state ------------------------------------ + pool_ep_name = None # NFS export policy name for pool + cluster_host_ips = None + + _vol_name_prefix = "OntapNFS3Vol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapNFS3PrimaryStorageWorkflow, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Resolve cluster host IPs for export policy rule assertions + cls.cluster_host_ips = [ + h.ipaddress for h in cls.cluster_hosts + if getattr(h, "ipaddress", None) + ] + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + # Fallback: plugin typically uses cs-{svmName}-{poolName} + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_export_policy_has_host_ips(self, ep_name): + """Assert that the export policy exists and its rules include each cluster host IP.""" + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP" % ep_name + ) + if not self.cluster_host_ips: + return # no host IPs registered; skip rule-level check + all_clients = [] + for rule in policy.get("rules", []): + for client in rule.get("clients", []): + all_clients.append(client.get("match", "")) + for ip in self.cluster_host_ips: + self.assertTrue( + any(ip in c for c in all_clients), + "Host IP '%s' not found in export policy '%s' rules: %s" + % (ip, ep_name, all_clients) + ) + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create primary storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_01_create_primary_storage_pool(self): + """ + Create an NFS3 primary storage pool and verify: + - CloudStack state is Up, type is NetworkFilesystem + - nfsmountopts contains 'vers=3' + - ONTAP: FlexVol exists and is online + - ONTAP: NFS export policy exists with cluster host IP rules + - ONTAP: at least one NFS data LIF is present on the SVM + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + self.assertEqual( + pool.type, "NetworkFilesystem", + "Pool type should be 'NetworkFilesystem', got '%s'" % pool.type + ) + + # Verify nfsmountopts via listStoragePools + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone(listed, "listStoragePools returned None for pool %s" % pool.id) + nfs_opts = getattr(listed[0], "nfsmountopts", "") + self.assertIn( + "vers=3", nfs_opts, + "nfsmountopts should contain 'vers=3', got '%s'" % nfs_opts + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist with host IP rules + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + self._assert_export_policy_has_host_ips(ep_name) + + # ONTAP: at least one NFS data LIF must be present + lifs = self.ontap.get_data_lifs(self.svm_name) + self.assertTrue( + len(lifs) > 0, + "No NFS data LIFs found on SVM '%s'" % self.svm_name + ) + + # Capacity reporting + self._assert_pool_capacity(pool, "pool-created") + + # ------------------------------------------------------------------ + # Step 02 — Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_02_disable_storage_pool(self): + """ + Disable the pool and verify: + - CloudStack reports Disabled + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Enable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_03_enable_storage_pool(self): + """ + Re-enable the pool and verify: + - CloudStack reports Up + - ONTAP: FlexVol is still online and export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_04_enter_maintenance_mode(self): + """ + Put the pool into maintenance mode and verify: + - CloudStack reports Maintenance + - ONTAP: FlexVol is still online and export policy unchanged + (maintenance is a CS-only state change) + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual(result.state, "Maintenance") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist during maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 — Cancel maintenance mode + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_05_cancel_maintenance_mode(self): + """ + Cancel maintenance mode and verify the pool returns to Up. + + cancelStorageMaintenance sends ModifyStoragePoolCommand(add=True) to the + KVM agent, which calls createStoragePool() with details that include + nfsMountOptions=vers=3. The agent rebuilds the libvirt pool XML with the + xmlns:fs namespace extension and mounts the NFS share with vers=3. + + Fix confirmed — LibvirtStorageAdaptor now correctly handles the case + where a stale-active libvirt pool entry lingers at the mount point after + sp.destroy() during enter-maintenance. The fix: + 1. Detects a stale-active pool (isActive==1 but mountpoint -q fails) + and destroys it before re-creating. + 2. Retries createNetfsStoragePool once after 5 s on LibvirtException. + + Verifies: + - CloudStack reports pool state Up + - ONTAP: FlexVol is still online + - ONTAP: NFS export policy still present + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state( + self.__class__.pool.id, "Up", timeout=120 + ) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" + % result.state + ) + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after cancel maintenance" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy( + self.__class__.pool_ep_name + ) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after cancel maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 06 — Delete the storage pool (already in Maintenance) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_06_delete_pool_from_maintenance(self): + """ + Enter maintenance mode then delete the storage pool. + + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol is deleted + - ONTAP: NFS export policy is deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Pool is Up after test_05 succeeded; must enter Maintenance before deletion. + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 07 - Create fresh pool and allocate a CloudStack volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_07_create_volume_on_pool(self): + """ + Create a new NFS3 pool and allocate a CloudStack data volume. + For NFS3, createAsync is a no-op on ONTAP (volume is a CloudStack record + only — no new ONTAP object is created). + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: FlexVol is still online and export policy still present + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: FlexVol must still be online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after volume creation" % ep_name + ) + + # Capacity reporting: FlexVol size and reported capacity unchanged after volume allocation + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 08 - Delete volume then force-delete the pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_08_delete_volume_and_pool(self): + """ + Delete the volume from test_07, enter maintenance, then force-delete + the pool. + Verifies: + - deleteVolume completes without error + - Pool transitions to Maintenance + - Pool is removed from CloudStack after force deletion + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + # Delete the volume + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: FlexVol must still be online (volume deletion does not affect NFS FlexVol) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after volume deletion" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion" + ) + + # Capacity reporting: capacity fields stable after volume deletion + self._assert_pool_capacity(pool, "volume-deleted") + + # Enter maintenance then force-delete the pool + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py new file mode 100644 index 000000000000..d0238295c918 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py @@ -0,0 +1,767 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +NFS3 pool lifecycle tests with a CloudStack volume present throughout. + +Covers the TDS (section 10) scenarios that require a data volume to already +exist on the pool during pool state transitions: + + TDS Approach-1 SN 11-12 — Disable pool WITH volumes + TDS Approach-1 SN 15-16 — Enable pool WITH volumes + TDS Approach-1 SN 19-20 — Enter maintenance WITH volumes + TDS Approach-1 SN 21-22 — Cancel maintenance WITH volumes (fix confirmed) + TDS Negative SN 5-6 — Delete pool that has volumes; forced=False rejected + +Note: TDS SN 7-8 (force-delete NFS3 pool after manual volume deletion) is +already covered by test_ontap_create_primary_storage_nfs3.py test_07/test_08. + +Tests are numbered test_01 ... test_07 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create NFS3 pool and allocate a CloudStack data volume + 02 Disable pool — volume still exists in CloudStack; ONTAP FlexVol online + 03 Re-enable pool — volume still accessible; FlexVol online + 04 Enter maintenance with volume present — Maintenance state; FlexVol online + 05 Cancel maintenance with volume present — pool returns to Up (fix confirmed) + 06 Forced=False delete rejected — pool stays in Maintenance (negative) + 07 Cleanup — cancel maintenance, delete volume, force-delete pool + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster registered in CloudStack + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values (protocol=NFS3) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_nfs3_pool_with_volumes.py -v + +Note: Tests 01-05 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. + +Post-run ONTAP cleanup: The suite ends with the pool in Maintenance state (from +test_04) and a CS volume present (test_05 negative test leaves both intact). The +OntapTestBase teardown exits Maintenance via cancelStorageMaintenance (which +transitions CS pool state even though KVM remount fails on NFS3), deletes the +volume, re-enters Maintenance, and force-deletes the pool. In rare cases where +CS pool state does not transition, one orphaned ONTAP FlexVol and export policy +may be left behind. Clean these up manually: + + curl -sk -u : \\ + "https:///api/storage/volumes?name=OntapNFS3WV_*&fields=name,state" + # Offline + DELETE each orphan, then DELETE the matching export policy.""" + +import base64 +import logging +import random +import time +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + cancelStorageMaintenance, + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapNFS3PoolWithVolumes") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-wv@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-WV", + "username": "ontap_nfs3_wv_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3WV_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3PoolWithVolumes(OntapTestBase): + """ + NFS3 pool lifecycle tests with a CloudStack data volume present throughout. + All tests are sequential and share class-level state. + """ + + pool_ep_name = None # NFS export policy name extracted at pool creation + + _vol_name_prefix = "OntapNFS3WV" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapNFS3PoolWithVolumes, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, scope=scope, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3WV_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the NFS export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return len(vols) > 0 + + def _assert_pool_capacity(self, pool, label): + """Assert CloudStack capacity fields and ONTAP FlexVol size are consistent. + + Logs configured bytes, reported capacity, used bytes, and ONTAP + FlexVol space.size at each check point. Asserts: + - listStoragePools.capacitybytes >= 90% of configured value + - listStoragePools.disksizeused >= 0 (ONTAP reports actual used bytes; + even a fresh FlexVol has metadata overhead so a non-zero value is + expected and is not an error) + - ONTAP FlexVol space.size >= 90% of configured value + """ + configured = self.testdata[TestData.primaryStorage]["capacitybytes"] + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertIsNotNone( + listed, + "[capacity/%s] listStoragePools returned None for pool %s" + % (label, pool.id) + ) + lp = listed[0] + reported = getattr(lp, "capacitybytes", 0) or 0 + used = getattr(lp, "disksizeused", 0) or 0 + min_expected = int(configured * 0.90) + + logger.info( + "[capacity/%s] configured=%d B reported=%d B used=%d B", + label, configured, reported, used + ) + self.assertGreaterEqual( + reported, min_expected, + "[capacity/%s] capacitybytes %d is >10%% below configured %d" + % (label, reported, configured) + ) + self.assertGreaterEqual( + used, 0, + "[capacity/%s] disksizeused must not be negative, got %d" + % (label, used) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + if ontap_vol: + ontap_size = ontap_vol.get("space", {}).get("size", 0) + logger.info( + "[capacity/%s] ONTAP FlexVol space.size=%d B", + label, ontap_size + ) + self.assertGreaterEqual( + ontap_size, min_expected, + "[capacity/%s] ONTAP FlexVol space.size %d is >10%% below configured %d" + % (label, ontap_size, configured) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create pool and allocate a data volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create an NFS3 primary storage pool and allocate a CloudStack data + volume on it. + Verifies: + - Pool state is Up; ONTAP FlexVol is online + - NFS export policy exists + - createVolume returns a volume object (NFS3 data vols are CS records + backed by a qcow2 file inside the FlexVol) + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP after pool creation" % ep_name + ) + + # Allocate a CloudStack data volume on this pool + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # Capacity reporting: volume allocated on FlexVol + self._assert_pool_capacity(pool, "volume-allocated") + + # ------------------------------------------------------------------ + # Step 02 — Disable pool with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_02_disable_pool_volume_survives(self): + """ + Disable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 11 (iSCSI) and SN 12 (NFS3): + - Pool should no longer be available for scheduling new CS volumes + - The existing CS volume should continue to exist (not deleted) + - ONTAP: FlexVol remains online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual( + result.state, "Disabled", + "Pool should be 'Disabled', got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool disable" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after pool disable, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after pool disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Re-enable pool with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_03_enable_pool_volume_intact(self): + """ + Re-enable the pool while a CloudStack data volume exists on it. + Covers TDS Approach-1 SN 15 (iSCSI) and SN 16 (NFS3): + - Pool state transitions back to Up + - The existing CS volume is still accessible + - ONTAP: FlexVol remains online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after re-enable, got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool re-enable" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after pool re-enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after pool re-enable, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after pool re-enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Enter maintenance mode with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_04_enter_maintenance_volume_present(self): + """ + Enter maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 19 (iSCSI) and SN 20 (NFS3): + - Pool transitions to Maintenance state + - Existing CS volume remains in CloudStack + - ONTAP: FlexVol stays online (maintenance is a CS-only state) + - ONTAP: export policy is unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + self.assertEqual( + result.state, "Maintenance", + "Pool should be 'Maintenance', got '%s'" % result.state + ) + + # Volume must still exist in CloudStack + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after pool entered Maintenance" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol disappeared after entering Maintenance") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' in Maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist during Maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 — Cancel maintenance mode with volume present + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_05_cancel_maintenance_with_volume(self): + """ + Cancel maintenance mode while a CloudStack data volume exists on the pool. + Covers TDS Approach-1 SN 21 (iSCSI) and SN 22 (NFS3): + - cancelStorageMaintenance succeeds (KVM/NFS3 fix confirmed) + - Pool returns to Up state + - Existing CS volume is still present in CloudStack + - ONTAP: FlexVol is still online + - ONTAP: NFS export policy is unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = self.__class__.pool.id + self.apiClient.cancelStorageMaintenance(cmd) + + result = self._poll_pool_state( + self.__class__.pool.id, "Up", timeout=120 + ) + self.assertEqual( + result.state, "Up", + "Pool should be 'Up' after cancel maintenance, got '%s'" % result.state + ) + + # CS volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume disappeared after cancel maintenance" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol disappeared after cancel maintenance" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" + % ontap_vol.get("state") + ) + + # ONTAP: export policy must still exist + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after cancel maintenance" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 06 — forced=False delete rejected when volume present (negative) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_06_forced_false_delete_rejected(self): + """ + Enter maintenance mode then attempt to delete the pool (forced=False) + while a CloudStack volume still exists on it. The operation must be + rejected. + Covers TDS Negative Scenarios SN 5 (iSCSI) and SN 6 (NFS3): + - CloudstackAPIException is raised with an appropriate error + - Pool remains in Maintenance state + - CS volume still exists + - ONTAP: FlexVol and export policy are unchanged + """ + self.assertIsNotNone(self.__class__.pool, + "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, + "Volume absent — test_01 must pass first") + + # Pool is Up after test_05 (cancel maintenance); re-enter Maintenance + # before attempting the delete so it reaches the forced=False gate. + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = self.__class__.pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) + + with self.assertRaises(CloudstackAPIException, + msg="deleteStoragePool(forced=False) with a live " + "volume should raise CloudstackAPIException"): + self._delete_pool(self.__class__.pool.id, forced=False) + + # Pool must still be in Maintenance (not deleted) + try: + remaining = list_storage_pools( + self.apiClient, id=self.__class__.pool.id) + except CloudstackAPIException: + remaining = None + self.assertTrue( + remaining, + "Pool was deleted even though forced=False delete should have failed" + ) + + # Volume must still exist + self.assertTrue( + self._volume_exists_in_cs(self.__class__.volume.id), + "CS volume was deleted even though pool deletion was rejected" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol should still exist after rejected pool deletion" + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should remain 'online' after rejected deletion" + ) + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_07_force_delete_pool_and_cleanup(self): + """ + Explicit cleanup after the test_06 negative test. + + The pool is in Maintenance with a CS volume still present. + Cleanup sequence: + 1. Try cancelStorageMaintenance. + 2. If still in Maintenance: try updateStoragePool(enabled=True) as a + fallback exit path (works on KVM/NFS3 even when cancel fails). + 3. Once pool exits Maintenance: delete volume, re-enter Maintenance. + 4. Force-delete the pool. + 5. Verify CS pool is gone. + 6. Verify ONTAP FlexVol and export policy are removed. + If the CS force-delete fails (volume couldn't be removed), the + ONTAP FlexVol and export policy are cleaned up directly via REST + so the storage array is never left with orphans. The CS pool + record is left for tearDownClass in that edge case only. + """ + pool = self.__class__.pool + vol = self.__class__.volume + self.assertIsNotNone(pool, "No pool from test_06 to clean up") + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Step 1: Try cancelStorageMaintenance + pool_state = "Maintenance" + try: + cm = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cm.id = pool.id + self.apiClient.cancelStorageMaintenance(cm) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(self.apiClient, id=pool.id) + if ps and ps[0].state != "Maintenance": + pool_state = ps[0].state + break + time.sleep(5) + except Exception: + pass # falls through to step 2 + + # Step 2: If still in Maintenance, try updateStoragePool(enabled=True). + # On KVM/NFS3 this succeeds in moving the pool to Disabled/Up even + # when cancelStorageMaintenance fails. + if pool_state == "Maintenance": + try: + ec = updateStoragePoolAPI.updateStoragePoolCmd() + ec.id = pool.id + ec.enabled = True + self.apiClient.updateStoragePool(ec) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(self.apiClient, id=pool.id) + if ps and ps[0].state != "Maintenance": + pool_state = ps[0].state + break + time.sleep(5) + except Exception: + pass + + # Step 3: If pool exited Maintenance, delete the CS volume and + # re-enter Maintenance so the pool can be force-deleted. + if pool_state != "Maintenance" and vol is not None: + if self._volume_exists_in_cs(vol.id): + try: + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.__class__.volume = None + vol = None + except Exception: + pass + else: + self.__class__.volume = None + vol = None + try: + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + self.apiClient.enableStorageMaintenance(mc) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(self.apiClient, id=pool.id) + if ps and ps[0].state == "Maintenance": + pool_state = "Maintenance" + break + time.sleep(5) + except Exception: + pass + + # Step 4: Force-delete the CS pool. + cs_pool_deleted = False + if vol is None or not self._volume_exists_in_cs(vol.id): + # Volume is gone — safe to force-delete + self.__class__.volume = None + vol = None + try: + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + cs_pool_deleted = True + except CloudstackAPIException: + pass + + # Step 5: Assert CS pool is gone (only when deletion was attempted) + if cs_pool_deleted: + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' should have been deleted with forced=True" % pool_name + ) + + # Step 6: ONTAP FlexVol must be gone. + # If the CS pool could not be deleted (volume still present — an + # NFS3/KVM platform edge case), delete the ONTAP FlexVol and export + # policy directly via REST so the storage array is always clean. + # The orphaned CS pool record is left for tearDownClass. + ontap_vol = self.ontap.get_volume(pool_name) + if ontap_vol is not None: + self.ontap.delete_volume(pool_name) + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' should be gone after cleanup" % pool_name + ) + + policy = self.ontap.get_export_policy(ep_name) + if policy is not None: + self.ontap.delete_export_policy(ep_name) + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "NFS export policy '%s' should be removed after cleanup" % ep_name + ) diff --git a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py new file mode 100644 index 000000000000..4509ce41cb22 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py @@ -0,0 +1,439 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Zone-scoped primary storage lifecycle tests for NetApp ONTAP (NFS3). + +Creates a zone-scoped pool (scope=ZONE, no clusterid/podid). CloudStack calls +OntapPrimaryDatastoreLifecycle.attachZone(), which connects all eligible KVM +hosts in the zone to the pool and creates an NFS export policy covering their +IPs. + +Workflow: + 01 Create zone-scoped NFS3 pool — pool.state Up; ONTAP FlexVol online; + export policy has all cluster host IPs + 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 04 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM hosts registered in the zone + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values (protocol=NFS3) + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/test_ontap_zone_scoped_pool.py -v + +Note: Tests 01-04 share class-level state (sequential). Running a single test +with -m "test_NN" will invoke setUpClass but the guard assertion will fail +immediately if earlier steps have not yet run. Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapZoneScopedPool") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-zone@test.com", + "firstname": "ONTAP", + "lastname": "Zone", + "username": "ontap_zone_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapZoneNFS3_%d" % random.randint(0, 9999), + TestData.scope: "ZONE", + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: protocol, + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapZoneScopedPool(OntapTestBase): + + # ---- zone-pool-specific shared state -------------------------------- + pool_ep_name = None + cluster_host_ips = None + + _vol_name_prefix = "OntapZoneVol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapZoneScopedPool, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + protocol = "NFS3" + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + protocol=protocol, provider=provider, + tags=tags, capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # Collect host IPs for export policy assertions + cls.cluster_host_ips = [ + h.ipaddress for h in cls.cluster_hosts + if getattr(h, "ipaddress", None) + ] + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_zone_pool(self): + """Create a zone-scoped NFS3 pool (no clusterid / podid).""" + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapZoneNFS3_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + # Intentionally omit clusterid and podid — zone-scoped pool + cmd.scope = "ZONE" + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + def _assert_export_policy_has_host_ips(self, ep_name): + """Assert export policy exists and contains each cluster host IP.""" + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' not found on ONTAP" % ep_name + ) + if not self.cluster_host_ips: + return + all_clients = [] + for rule in policy.get("rules", []): + for client in rule.get("clients", []): + all_clients.append(client.get("match", "")) + for ip in self.cluster_host_ips: + self.assertTrue( + any(ip in c for c in all_clients), + "Host IP '%s' not found in export policy '%s' rules: %s" + % (ip, ep_name, all_clients) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_01_create_zone_scoped_pool(self): + """ + Create a zone-scoped NFS3 primary storage pool (no clusterid/podid). + CloudStack calls attachZone(), which connects all eligible KVM hosts + in the zone and creates an NFS export policy. + Verifies: + - pool.state is Up + - ONTAP: FlexVol is online + - ONTAP: export policy exists and contains cluster host IPs + - ONTAP: at least one NFS data LIF is present on the SVM + """ + pool = self._create_zone_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + # ONTAP: FlexVol must be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol not found for pool '%s'" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must exist with cluster host IPs + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + self._assert_export_policy_has_host_ips(ep_name) + + # ONTAP: at least one NFS data LIF must be present + lifs = self.ontap.get_data_lifs(self.svm_name) + self.assertTrue( + len(lifs) > 0, + "No NFS data LIFs found on SVM '%s'" % self.svm_name + ) + + # ------------------------------------------------------------------ + # Step 02 — Disable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_02_disable_zone_scoped_pool(self): + """ + Disable the zone-scoped pool. + Verifies: + - pool.state is Disabled + - ONTAP: FlexVol still online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = False + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) + self.assertEqual(result.state, "Disabled") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after disable" + ) + + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after disable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 — Enable zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_03_enable_zone_scoped_pool(self): + """ + Re-enable the zone-scoped pool. + Verifies: + - pool.state is Up + - ONTAP: FlexVol still online; export policy unchanged + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = self.__class__.pool.id + cmd.enabled = True + self.apiClient.updateStoragePool(cmd) + + result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) + self.assertEqual(result.state, "Up") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after enable" + ) + + if self.__class__.pool_ep_name: + policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after enable" + % self.__class__.pool_ep_name + ) + + # ------------------------------------------------------------------ + # Step 04 — Delete zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_04_delete_zone_scoped_pool(self): + """ + Enter maintenance then delete the zone-scoped pool. + Verifies: + - Pool is removed from CloudStack + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # Unmount the NFS on each KVM host BEFORE deleteStoragePool removes + # the ONTAP export. Without this, the mount becomes stale and + # KVMHAMonitor will fail its heartbeat 5 times then reboot the host + # via `echo b > /proc/sysrq-trigger`. + self._cleanup_kvm_storage_pool_mounts(pool.id) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) + + # ------------------------------------------------------------------ + # Class-level teardown + # ------------------------------------------------------------------ + + @classmethod + def tearDownClass(cls): + """ + Clean up any lingering zone-scoped pool NFS mounts on KVM hosts + before the base-class teardown deletes the ONTAP FlexVol. Without + this, a failed test_04 leaves a stale NFS mount that will cause + KVMHAMonitor to reboot the host. + """ + for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: + try: + cls._cleanup_kvm_storage_pool_mounts(pool.id) + except Exception as e: + logger.warning( + "tearDownClass: KVM NFS cleanup failed for pool %s: %s" + % (pool.id, e) + ) + super(TestOntapZoneScopedPool, cls).tearDownClass() diff --git a/test/integration/plugins/ontap/nfs3/volume/__init__.py b/test/integration/plugins/ontap/nfs3/volume/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/volume/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py b/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py new file mode 100644 index 000000000000..981ea5156cbf --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py @@ -0,0 +1,470 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Sequential workflow integration tests for NetApp ONTAP NFS3 data volume +lifecycle (volume create / delete / negative-delete / force-delete). + +For NFS3, a CloudStack data volume is a metadata record only — no new ONTAP +object is created per volume (the pool's single FlexVol serves all volumes). +Volume deletion likewise removes the CloudStack record while leaving the +FlexVol intact. + +Tests are numbered test_01 ... test_05 and must run in that order. Each step +builds on the shared state established by the previous step. + +Workflow: + 01 Create NFS3 primary storage pool and allocate a CloudStack data volume + 02 Delete the volume — CloudStack record removed; FlexVol stays online + 03 Recreate volume — CS record back; FlexVol stays online (setup for 04-05) + 04 Put pool in Maintenance; attempt forced=False deleteStoragePool — must be + rejected because volumes exist; pool stays in Maintenance + 05 Delete volume from Maintenance; forced=True deleteStoragePool — FlexVol + and export policy are removed from ONTAP + +Prerequisites: + - CloudStack management server with the NetApp ONTAP plugin deployed + - KVM cluster where every host has NFS configured + - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF + - ontap.cfg populated with real values + +Running: + nosetests --with-marvin \\ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ + test/integration/plugins/ontap/nfs3/volume/ -v + +Note: Tests share class-level state (sequential). Always run the full suite. +""" + +import base64 +import logging +import random +import unittest + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteVolume as deleteVolumeAPI, + enableStorageMaintenance, + updateStoragePool as updateStoragePoolAPI, +) +from marvin.lib.base import StoragePool +from marvin.lib.common import list_storage_pools + +from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details + +logger = logging.getLogger("TestOntapNFS3VolumeLifecycle") + + +# --------------------------------------------------------------------------- +# Test data +# --------------------------------------------------------------------------- + +class TestData: + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + ONTAP_MIN_VOLUME_SIZE = 1677721600 + + def __init__(self, storage_ip, svm_name, username, password, + scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-nfs3", capacitybytes=None): + if capacitybytes is None: + capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 + encoded_password = base64.b64encode(password.encode()).decode() + self.testdata = { + TestData.ontap: { + TestData.DETAIL_STORAGE_IP: storage_ip, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: password, + }, + TestData.account: { + "email": "ontap-nfs3-vol@test.com", + "firstname": "ONTAP", + "lastname": "NFS3-Vol", + "username": "ontap_nfs3_vol_%d" % random.randint(0, 9999), + "password": "password", + }, + TestData.primaryStorage: { + "name": "OntapNFS3Vol_%d" % random.randint(0, 9999), + TestData.scope: scope, + TestData.provider: provider, + TestData.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + TestData.DETAIL_USERNAME: username, + TestData.DETAIL_PASSWORD: encoded_password, + TestData.DETAIL_SVM_NAME: svm_name, + TestData.DETAIL_PROTOCOL: "NFS3", + TestData.DETAIL_STORAGE_IP: storage_ip, + }, + }, + } + + +# --------------------------------------------------------------------------- +# Sequential workflow test class +# --------------------------------------------------------------------------- + +class TestOntapNFS3VolumeLifecycle(OntapTestBase): + + _vol_name_prefix = "OntapNFS3Vol" + + @classmethod + def setUpClass(cls): + testclient = super( + TestOntapNFS3VolumeLifecycle, cls + ).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = testclient.getParsedTestDataConfig() + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + nfs3_cfg = pool_cfg.get("protocols", {}).get("nfs3", {}) + if not nfs3_cfg.get("enabled", True): + raise unittest.SkipTest( + "NFS3 tests disabled in ontap.cfg " + "(set protocols.nfs3.enabled=true to enable)" + ) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") + capacitybytes = pool_cfg.get("capacitybytes", None) + + cls.testdata = TestData( + storage_ip, svm_name, username, password, + scope=scope, provider=provider, tags=tags, + capacitybytes=capacitybytes, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + + cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + + # No per-test tearDown — state intentionally persists between steps. + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _create_pool(self): + ps = self.testdata[TestData.primaryStorage] + storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] + pool_name = "OntapNFS3Vol_%d" % random.randint(0, 99999) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "nfs://%s/ontap" % storage_ip + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TestData.scope] + cmd.provider = ps[TestData.provider] + cmd.tags = ps[TestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + return StoragePool(response.__dict__) + + def _get_export_policy_name(self, pool): + """Extract the export policy name from pool creation response details.""" + details = _parse_pool_details(pool) + ep_name = details.get("exportPolicyName") + if not ep_name: + ep_name = "cs-%s-%s" % (self.svm_name, pool.name) + return ep_name + + # ------------------------------------------------------------------ + # Step 01 - Create pool (infrastructure) and allocate a volume + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_01_create_pool_and_volume(self): + """ + Create a new NFS3 pool and allocate a CloudStack data volume on it. + For NFS3, volume creation is a CloudStack metadata record only — no + new ONTAP object is created (the pool's FlexVol serves all volumes). + Verifies: + - pool.state is Up + - createVolume returns a non-None volume object + - ONTAP: FlexVol remains online after volume allocation + - ONTAP: export policy still present + """ + pool = self._create_pool() + self.__class__.pool = pool + + self.assertEqual( + pool.state, "Up", + "Pool state should be 'Up', got '%s'" % pool.state + ) + + ep_name = self._get_export_policy_name(pool) + self.__class__.pool_ep_name = ep_name + + vol = self._create_volume(pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + # ONTAP: FlexVol must remain online after volume allocation + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume creation" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should exist after volume creation" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 02 - Delete volume; FlexVol must remain untouched + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_02_delete_volume(self): + """ + Delete the volume created in test_01. + For NFS3, volume deletion removes only the CloudStack record. + Verifies: + - deleteVolume completes without error + - ONTAP: FlexVol is still online (unaffected by volume deletion) + - ONTAP: export policy still present + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_01 must pass first") + + pool = self.__class__.pool + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + self.__class__.volume = None + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after volume deletion" % pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online' after volume deletion, " + "got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after volume deletion" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 03 - Recreate volume for negative delete tests + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_03_recreate_volume_for_delete_tests(self): + """ + Recreate a volume on the existing pool (setup for tests 04-05). + Verifies: + - volume created successfully + - ONTAP: FlexVol still online + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + + vol = self._create_volume(self.__class__.pool.id) + self.__class__.volume = vol + self.assertIsNotNone(vol, "createVolume returned None") + + ontap_vol = self.ontap.get_volume(self.__class__.pool.name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' not found after volume re-creation" + % self.__class__.pool.name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should be 'online' after volume re-creation" + ) + + # ------------------------------------------------------------------ + # Step 04 - Forced=False delete with live volume must fail + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_04_forced_false_delete_with_volume_fails(self): + """ + Put pool in Maintenance then attempt deleteStoragePool(forced=False). + With a live volume present CloudStack must reject the request. + Verifies: + - Exception is raised (CloudStack rejects the delete) + - Pool is still listed in CloudStack (in Maintenance state) + - ONTAP: FlexVol still exists and is online + - ONTAP: export policy still present + """ + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + + # Enter maintenance mode + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + # Attempt forced=False delete — must raise exception because volumes exist + with self.assertRaises(Exception): + self._delete_pool(pool.id, forced=False) + + # Pool must still be listed in CloudStack + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue( + listed, + "Pool should still exist in CloudStack after failed forced=False delete" + ) + + # ONTAP: FlexVol must still be online + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNotNone( + ontap_vol, + "ONTAP FlexVol '%s' should still exist after failed delete" % pool_name + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "ONTAP FlexVol should still be 'online', got '%s'" % ontap_vol.get("state") + ) + + # ONTAP: export policy must still be present + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNotNone( + policy, + "Export policy '%s' should still exist after failed delete" % ep_name + ) + + # ------------------------------------------------------------------ + # Step 05 - Delete volume then force-delete pool from Maintenance + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_volume"], required_hardware=True) + def test_05_delete_volume_and_force_delete_pool(self): + """ + Delete the live volume then force-delete the pool while it is still + in Maintenance state (pool is in Maintenance from test_04). + Verifies: + - Volume can be deleted while pool is in Maintenance + - Pool is removed from CloudStack using forced=True from Maintenance + - ONTAP: FlexVol deleted + - ONTAP: export policy deleted + """ + self.assertIsNotNone( + self.__class__.pool, + "Pool absent - test_04 must not have cleaned up the pool" + ) + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_03 must pass first") + + pool = self.__class__.pool + pool_name = pool.name + ep_name = self.__class__.pool_ep_name + vol = self.__class__.volume + + # Delete the volume first (pool is in Maintenance — volume deletion is + # allowed). For NFS3 a forced=False delete attempt in test_04 may have + # already destroyed the libvirt NFS pool representation on the host; + # if so deleteVolume raises "Storage pool not found". The CS metadata + # record will be cleaned up by the subsequent force-delete of the pool, + # so we treat that specific error as a no-op here. + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + if "Storage pool not found" in str(exc) or "storage pool" in str(exc).lower(): + logger.warning( + "deleteVolume raised expected NFS3 libvirt pool-not-found " + "error; proceeding to force-delete pool: %s", exc + ) + else: + raise + self.__class__.volume = None + + # Force-delete the pool from Maintenance (no live volumes remaining) + self._delete_pool(pool.id, forced=True) + self.__class__.pool = None + self.__class__.pool_ep_name = None + + # CloudStack: pool must be gone + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse(remaining, "Pool still listed in CloudStack after force deletion") + + # ONTAP: FlexVol must be deleted + ontap_vol = self.ontap.get_volume(pool_name) + self.assertIsNone( + ontap_vol, + "ONTAP FlexVol '%s' still exists after force deletion" % pool_name + ) + + # ONTAP: export policy must be deleted + if ep_name: + policy = self.ontap.get_export_policy(ep_name) + self.assertIsNone( + policy, + "Export policy '%s' still exists after pool deletion" % ep_name + ) diff --git a/test/integration/plugins/ontap/ontap.cfg b/test/integration/plugins/ontap/ontap.cfg index 9d0e2bec06e1..4bf9d17dc499 100644 --- a/test/integration/plugins/ontap/ontap.cfg +++ b/test/integration/plugins/ontap/ontap.cfg @@ -2,19 +2,17 @@ "zones": [ { "name": "Zone-ONTAP", - "localstorageenabled": true, "dns1": "8.8.8.8", - "internal_dns1": "10.192.0.250", + "internal_dns1": "8.8.8.8", "guestcidraddress": "10.1.1.0/24", "physical_networks": [ { "broadcastdomainrange": "Zone", "name": "physical_network", - "vlan": "100-300", "traffictypes": [ {"typ": "Guest"}, {"typ": "Management"}, - {"typ": "Public"} + {"typ": "Storage"} ], "providers": [ { @@ -24,29 +22,13 @@ ] } ], - "secondaryStorages": [ - { - "url": "nfs://10.193.56.61/exports/secondary", - "provider": "NFS", - "tags": "secondary-nfs" - } - ], - "ipranges": [ - { - "gateway": "10.193.56.1", - "startip": "10.193.56.70", - "endip": "10.193.56.79", - "netmask": "255.255.252.0", - "vlan": "untagged" - } - ], "pods": [ { "name": "Pod-ONTAP", "gateway": "10.193.56.1", - "startip": "10.193.56.80", - "endip": "10.193.56.89", - "netmask": "255.255.252.0", + "startip": "10.193.56.10", + "endip": "10.193.56.50", + "netmask": "255.255.255.128", "clusters": [ { "clustername": "KVM-Cluster-ONTAP", @@ -54,21 +36,12 @@ "clustertype": "CloudManaged", "hosts": [ { - "url": "http://10.193.56.61", + "url": "http://10.193.56.65", "username": "root", - "password": "netapp1!", - "hosttags": "kvmHostONTAP" + "password": "netapp1!" } ], - "primaryStorages": [ - { - "name": "primary-nfs-ontap", - "url": "nfs://10.193.56.61/exports/primary", - "scope": "CLUSTER", - "provider": "DefaultPrimary", - "tags": "primary-nfs" - } - ] + "primaryStorages": [] } ] } @@ -76,42 +49,31 @@ } ], "dbSvr": { - "dbSvr": "10.193.56.61", - "passwd": "cloud", + "dbSvr": "10.193.56.65", + "passwd": "", "db": "cloud", "port": 3306, - "user": "cloud" + "user": "root" }, "logger": { "LogFolderPath": "/tmp/" }, - "TestData": { - "Path": "test/integration/plugins/ontap/ontap.cfg" - }, "mgtSvr": [ { - "mgtSvrIp": "10.193.56.61", + "mgtSvrIp": "10.193.56.65", "port": 8096, "user": "admin", "passwd": "password", - "hypervisor": "kvm", - "timeout": 600 + "hypervisor": "kvm" } ], "ontap": { - "storageIP": "10.196.35.107", + "storageIP": "10.196.38.187", "svmName": "vs0", "username": "admin", - "password": "netapp1!", - "protocol": "NFS3", - "storagePoolScope": "CLUSTER", - "storagePoolProvider": "NetApp ONTAP", - "storagePoolTags": "ontap-nfs3", - "capacitybytes": 3355443200 + "password": "netapp1!" }, - "cloudstack": { - "zoneName": "Zone1", - "clusterName": "Cluster1", - "domainName": "ROOT" + "TestData": { + "Path": "test/integration/plugins/ontap/ontap.cfg" } } diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index 5d17d37df814..e40dc22283e5 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -30,6 +30,7 @@ import requests import time import urllib3 +from urllib.parse import urlparse from marvin.cloudstackAPI import ( cancelStorageMaintenance, @@ -41,7 +42,8 @@ ) from marvin.cloudstackAPI import listHosts as listHostsAPI from marvin.cloudstackTestCase import cloudstackTestCase -from marvin.lib.base import Account +from marvin.lib.base import Account, DiskOffering +from marvin.sshClient import SshClient from marvin.lib.common import get_domain, get_zone, list_clusters, list_storage_pools from marvin.lib.utils import cleanup_resources @@ -94,6 +96,32 @@ def _get(self, path, params=None): resp.raise_for_status() return resp.json() + def _delete(self, path, params=None): + url = self._base + path + resp = requests.delete(url, auth=self._auth, params=params, + verify=False, timeout=30) + resp.raise_for_status() + + def delete_volume(self, name): + """Delete the ONTAP FlexVol with the given name. No-op if not found.""" + data = self._get("/storage/volumes", params={"name": name}) + records = data.get("records", []) + if not records: + return + uuid = records[0].get("uuid") + if uuid: + self._delete("/storage/volumes/%s" % uuid) + + def delete_export_policy(self, name): + """Delete the NFS export policy with the given name. No-op if not found.""" + data = self._get("/protocols/nfs/export-policies", params={"name": name}) + records = data.get("records", []) + if not records: + return + policy_id = records[0].get("id") + if policy_id: + self._delete("/protocols/nfs/export-policies/%s" % policy_id) + def get_volume(self, name): """Return the ONTAP FlexVol record for the given name, or None.""" data = self._get("/storage/volumes", params={"name": name}) @@ -160,6 +188,15 @@ def list_luns_in_volume(self, svm_name, vol_name): return [r for r in data.get("records", []) if r.get("name", "").startswith(prefix)] + def list_lun_maps_for_volume(self, svm_name, vol_name): + """Return all LUN-map records for LUNs residing in the given FlexVol.""" + prefix = "/vol/%s/" % vol_name + data = self._get("/protocols/san/lun-maps", + params={"svm.name": svm_name, + "fields": "lun.name,igroup.name"}) + return [r for r in data.get("records", []) + if r.get("lun", {}).get("name", "").startswith(prefix)] + # --------------------------------------------------------------------------- # Base test class @@ -185,6 +222,7 @@ class OntapTestBase(cloudstackTestCase): disk_offering_id = None svm_name = None cluster_hosts = None + kvm_hosts_ssh_creds = [] # [{'host': '10.x.x.x', 'user': 'root', 'password': '...'}] ontap = None testdata = None zone = None @@ -225,41 +263,151 @@ def _setup_cloudstack_resources(cls, config, account_testdata): cls.cluster_hosts = cls.apiClient.listHosts(list_hosts_cmd) or [] list_do_cmd = listDiskOfferingsAPI.listDiskOfferingsCmd() - list_do_cmd.listall = True + list_do_cmd.domainid = cls.domain.id offerings = cls.apiClient.listDiskOfferings(list_do_cmd) - cls.disk_offering_id = offerings[0].id if offerings else None + if offerings: + cls.disk_offering_id = offerings[0].id + else: + # No disk offerings exist yet — create a minimal one for tests + do = DiskOffering.create( + cls.apiClient, + {"name": "ontap-test-do", "displaytext": "ONTAP test disk offering", "disksize": 2}, + ) + cls._cleanup.append(do) + cls.disk_offering_id = do.id + + # Parse KVM host SSH credentials from zones/pods/clusters/hosts config. + # Used by _cleanup_kvm_storage_pool_mounts to unmount stale NFS pools. + cls.kvm_hosts_ssh_creds = [] + try: + for zone in config.get("zones", []): + for pod in zone.get("pods", []): + for cluster in pod.get("clusters", []): + for host_cfg in cluster.get("hosts", []): + host_ip = urlparse( + host_cfg.get("url", "") + ).hostname or "" + if host_ip: + cls.kvm_hosts_ssh_creds.append({ + "host": host_ip, + "user": host_cfg.get("username", "root"), + "password": host_cfg.get("password", ""), + }) + except Exception as parse_ex: + logger.warning( + "_setup_cloudstack_resources: could not parse KVM SSH creds: %s" + % parse_ex + ) + + # ---- KVM storage cleanup helper ------------------------------------ + + @classmethod + def _cleanup_kvm_storage_pool_mounts(cls, pool_uuid): + """ + SSH to each KVM host and unmount the NFS storage pool mount for + *pool_uuid*, then destroy and undefine the libvirt storage pool. + + Must be called BEFORE the ONTAP FlexVol is deleted (i.e., before + deleteStoragePool) so that the unmount completes while the NFS + export is still reachable. Prevents stale NFS mounts from + triggering KVMHAMonitor heartbeat failures that reboot the host + via ``echo b > /proc/sysrq-trigger``. + """ + for creds in cls.kvm_hosts_ssh_creds: + host_ip = creds["host"] + try: + ssh = SshClient( + host_ip, 22, + creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0, + ) + for cmd in [ + "umount -f -l /mnt/{u} 2>/dev/null; true".format( + u=pool_uuid), + "virsh pool-destroy {u} 2>/dev/null; true".format( + u=pool_uuid), + "virsh pool-undefine {u} 2>/dev/null; true".format( + u=pool_uuid), + ]: + try: + ssh.execute(cmd) + except Exception as cmd_ex: + logger.warning( + "_cleanup_kvm_storage_pool_mounts: cmd '%s' " + "failed on %s: %s" % (cmd, host_ip, cmd_ex) + ) + except Exception as ex: + logger.warning( + "_cleanup_kvm_storage_pool_mounts: SSH to %s failed: %s" + % (host_ip, ex) + ) # ---- shared teardown ----------------------------------------------- @classmethod def tearDownClass(cls): """Best-effort cleanup of any resources left behind by a failed run.""" - for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: - try: - cmd = deleteVolumeAPI.deleteVolumeCmd() - cmd.id = vol.id - cls.apiClient.deleteVolume(cmd) - except Exception as e: - logger.warning("tearDownClass: could not delete volume %s: %s" - % (vol.id, e)) - for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: try: - try: - cc = cancelStorageMaintenance.cancelStorageMaintenanceCmd() - cc.id = pool.id - cls.apiClient.cancelStorageMaintenance(cc) - time.sleep(5) - except Exception: - pass - try: - ec = updateStoragePoolAPI.updateStoragePoolCmd() - ec.id = pool.id - ec.enabled = True - cls.apiClient.updateStoragePool(ec) - time.sleep(3) - except Exception: - pass + # Step 1: Check current pool state + pools = list_storage_pools(cls.apiClient, id=pool.id) + if not pools: + continue # already deleted + pool_state = pools[0].state + + # Step 2: If in Maintenance, attempt to exit it + if pool_state == "Maintenance": + try: + cc = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cc.id = pool.id + cls.apiClient.cancelStorageMaintenance(cc) + time.sleep(5) + except Exception: + pass + try: + ec = updateStoragePoolAPI.updateStoragePoolCmd() + ec.id = pool.id + ec.enabled = True + cls.apiClient.updateStoragePool(ec) + time.sleep(3) + except Exception: + pass + pools = list_storage_pools(cls.apiClient, id=pool.id) + if pools: + pool_state = pools[0].state + + # Step 3: Delete volumes — always attempt regardless of pool + # state. For iSCSI this works even in Maintenance; for NFS3/KVM + # it may fail with NPE ("storagePoolInformation is null") when + # pool is in Maintenance — that exception is caught below. + for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + cls.apiClient.deleteVolume(cmd) + except Exception as ve: + logger.warning( + "tearDownClass: could not delete volume %s: %s" + % (vol.id, ve)) + + # Re-enter Maintenance only if pool was Up/Disabled (avoid + # double-entering when cancel maintenance above already left it + # in Maintenance) + if pool_state in ("Up", "Disabled"): + try: + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + cls.apiClient.enableStorageMaintenance(mc) + deadline = time.time() + 60 + while time.time() < deadline: + ps = list_storage_pools(cls.apiClient, id=pool.id) + if ps and ps[0].state == "Maintenance": + break + time.sleep(5) + except Exception: + pass + + # Step 4: Force-delete the pool dc = deleteStoragePoolAPI.deleteStoragePoolCmd() dc.id = pool.id dc.forced = True @@ -267,6 +415,41 @@ def tearDownClass(cls): except Exception as e: logger.warning("tearDownClass: could not delete pool %s: %s" % (pool.id, e)) + # Last resort: delete ONTAP FlexVol and export policy directly + # so that ONTAP is never left with orphaned volumes even when + # the CloudStack pool record cannot be removed. + if hasattr(cls, "ontap") and cls.ontap is not None: + try: + cls.ontap.delete_volume(pool.name) + logger.warning( + "tearDownClass: deleted ONTAP FlexVol '%s' directly" + % pool.name) + except Exception as oe: + logger.warning( + "tearDownClass: ONTAP direct volume delete '%s' " + "failed: %s" % (pool.name, oe)) + try: + # For NFS3 pools also remove the export policy + ep_name = getattr(cls, "pool_ep_name", None) + if ep_name is None: + ep_name = "cs-%s-%s" % ( + getattr(cls, "svm_name", ""), pool.name) + cls.ontap.delete_export_policy(ep_name) + logger.warning( + "tearDownClass: deleted export policy '%s' directly" + % ep_name) + except Exception: + pass + + # Clean up volumes that may not have been handled with pool teardown + for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + cls.apiClient.deleteVolume(cmd) + except Exception as e: + logger.warning("tearDownClass: could not delete volume %s: %s" + % (vol.id, e)) try: cleanup_resources(cls.apiClient, cls._cleanup) diff --git a/test/integration/plugins/ontap/probe_test.py b/test/integration/plugins/ontap/probe_test.py new file mode 100644 index 000000000000..8ed1ca03da29 --- /dev/null +++ b/test/integration/plugins/ontap/probe_test.py @@ -0,0 +1,35 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +import json +from marvin.cloudstackAPI import listTemplates, listServiceOfferings, listNetworks +from marvin.cloudstackTestCase import cloudstackTestCase + +class ProbeResources(cloudstackTestCase): + @classmethod + def setUpClass(cls): + tc = super(ProbeResources, cls).getClsTestClient() + cls.api = tc.getApiClient() + + def test_01_probe(self): + out = {} + cmd = listTemplates.listTemplatesCmd() + cmd.templatefilter = "executable" + resp = self.api.listTemplates(cmd) + out["templates"] = [{"id": t.id, "name": t.name, "hypervisor": getattr(t,"hypervisor","?"), "status": getattr(t,"status","?")} for t in (resp or [])] + cmd2 = listServiceOfferings.listServiceOfferingsCmd() + resp2 = self.api.listServiceOfferings(cmd2) + out["offerings"] = [{"id": s.id, "name": s.name, "cpu": getattr(s,"cpunumber","?"), "mem": getattr(s,"memory","?")} for s in (resp2 or [])] + cmd3 = listNetworks.listNetworksCmd() + cmd3.listall = True + resp3 = self.api.listNetworks(cmd3) + out["networks"] = [{"id": n.id, "name": n.name, "type": getattr(n,"type","?"), "state": getattr(n,"state","?")} for n in (resp3 or [])] + with open("/tmp/cs_probe_out.json","w") as f: + json.dump(out, f, indent=2) + self.assertTrue(True) diff --git a/test/integration/plugins/ontap/run_tests.sh b/test/integration/plugins/ontap/run_tests.sh new file mode 100644 index 000000000000..d6d561d14cb0 --- /dev/null +++ b/test/integration/plugins/ontap/run_tests.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Run the full ONTAP Marvin integration test suite by tag. +# Each test file is run individually so sequential test state is preserved. +# +# Usage (from cloudstack root): +# bash test/integration/plugins/ontap/run_tests.sh +# +# Optional: limit to a specific group by passing the tag as an argument: +# bash test/integration/plugins/ontap/run_tests.sh nfs3_workflow + +CFG=test/integration/plugins/ontap/ontap.cfg +export PYTHONPATH=test/integration/plugins/ontap:${PYTHONPATH:-} +FILTER="${1:-all}" + +PASS=0 +FAIL=0 + +run_group() { + local label="$1" + local tag="$2" + local file="$3" + + if [[ "$FILTER" != "all" && "$FILTER" != "$tag" ]]; then + return + fi + + echo "" + echo "================================================================" + echo " ${label} (tag: ${tag})" + echo "================================================================" + + local out + out=$(python3 -m nose --with-marvin --marvin-config="$CFG" "$file" -a "tags=${tag}" -v 2>&1) + + # Resolve the log folder (handle /tmp -> /private/tmp symlink on macOS) + local log_folder + log_folder=$(echo "$out" | grep "Final results are now copied to" | sed 's/.*copied to: //; s/ ===.*//' | tr -d '[:space:]') + log_folder=$(python3 -c "import os; print(os.path.realpath('$log_folder'))" 2>/dev/null || echo "") + + if [[ -n "$log_folder" && -f "${log_folder}/results.txt" ]]; then + local suite_pass suite_fail + while IFS= read -r line; do + echo " $line" + done < <(grep "TestName.*Status" "${log_folder}/results.txt" | grep -v "^===") + suite_pass=$(grep -c "Status : SUCCESS" "${log_folder}/results.txt" 2>/dev/null | tr -d '[:space:]' || echo 0) + suite_fail=$(grep "Status : FAIL\|Status : EXCEPTION" "${log_folder}/results.txt" 2>/dev/null | wc -l | tr -d '[:space:]' || echo 0) + PASS=$((PASS + suite_pass)) + FAIL=$((FAIL + suite_fail)) + echo " -> ${suite_pass} passed, ${suite_fail} failed" + else + echo "$out" | grep -E "ERROR|Exception|failed" | head -5 + echo " [could not read results — log folder: ${log_folder:-not found}]" + FAIL=$((FAIL + 1)) + fi +} + +run_group "NFS3 pool lifecycle" nfs3_workflow test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py +run_group "NFS3 pool with volumes" nfs3_with_volumes test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py +run_group "NFS3 zone-scoped pool" zone_pool test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py +run_group "NFS3 volume lifecycle" nfs3_volume test/integration/plugins/ontap/nfs3/volume/test_volume_lifecycle.py +run_group "NFS3 VM volume attach" vm_volume_workflow test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py +run_group "iSCSI pool lifecycle" iscsi_workflow test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py +run_group "iSCSI pool with volumes" iscsi_with_volumes test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py +run_group "iSCSI zone-scoped pool" iscsi_zone_pool test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py +run_group "iSCSI volume lifecycle" iscsi_volume test/integration/plugins/ontap/iscsi/volume/test_volume_lifecycle.py +run_group "iSCSI VM volume workflow" iscsi_vm_workflow test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py + +echo "" +echo "================================================================" +echo " TOTAL: ${PASS} passed, ${FAIL} failed" +echo "================================================================" + +[[ "$FAIL" -eq 0 ]] From d73717fa817ded4c6e13d7cc52323b94b593c0fa Mon Sep 17 00:00:00 2001 From: "Locharla, Sandeep" Date: Wed, 24 Jun 2026 14:27:07 +0530 Subject: [PATCH 3/6] refactored few test cases --- .../kvm/storage/KVMStoragePoolManager.java | 3 -- .../kvm/storage/LibvirtStorageAdaptor.java | 2 - .../nfs3/instance/test_vm_volume_attach.py | 39 ++++++++++++++++++- .../ontap/nfs3/pool/test_pool_lifecycle.py | 14 +------ .../ontap/nfs3/pool/test_pool_with_volumes.py | 27 +++---------- .../plugins/ontap/ontap_test_base.py | 33 ++++++++++++++++ 6 files changed, 77 insertions(+), 41 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 61c755b1a9ab..b503643c96db 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -457,11 +457,8 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) { public boolean deleteStoragePool(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); - logger.debug("[deleteStoragePool] calling adaptor.deleteStoragePool for pool {} (type={})", uuid, type); boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); - logger.debug("[deleteStoragePool] adaptor.deleteStoragePool returned {} for pool {}", deleteStatus, uuid); if (type == StoragePoolType.NetworkFilesystem) { - logger.debug("[deleteStoragePool] calling haMonitor.removeStoragePool for NFS pool {}", uuid); _haMonitor.removeStoragePool(uuid); } synchronized (_storagePools) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index e2dfe754cac2..46b98bebb934 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -950,8 +950,6 @@ private boolean destroyStoragePoolHandleException(Connect conn, String uuid) { @Override public boolean deleteStoragePool(String uuid, Map details) { - logger.debug("[deleteStoragePool] details overload called for pool {}, delegating to deleteStoragePool(uuid)", - uuid); return deleteStoragePool(uuid); } diff --git a/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py index 461ee104cbf9..a3f944cdbf8d 100644 --- a/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py +++ b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py @@ -491,6 +491,7 @@ def test_04_attach_volume_to_vm(self): (Note: on ONTAP/NFS shared storage the volume state remains 'Ready'; attachment is signalled by virtualmachineid being populated) - ONTAP: FlexVol remains online + - ONTAP: NFS3 volume data file created in FlexVol after attach (lazy creation) - VM remains 'Running' """ if self.__class__.vm is None: @@ -539,6 +540,17 @@ def test_04_attach_volume_to_vm(self): "ONTAP FlexVol should be 'online' after attach" ) + # ONTAP: NFS3 uses lazy file creation — the volume data file is + # materialised on the FlexVol only when CloudStack calls createAsync + # during attachVolume. Verify that the file now exists. + files = self.ontap.list_files_in_volume(pool.name) + vol_file = next((f for f in files if vol.id in f), None) + self.assertIsNotNone( + vol_file, + "No data file matching volume UUID '%s' found in FlexVol '%s' " + "after attach; files present: %s" % (vol.id, pool.name, files) + ) + # ------------------------------------------------------------------ # Step 05 - Stop VM — export policy must be retained # ------------------------------------------------------------------ @@ -666,6 +678,7 @@ def test_07_detach_volume_from_vm(self): - Volume no longer lists the VM's ID - VM remains 'Running' - ONTAP: FlexVol remains online + - ONTAP: NFS3 volume data file persists in FlexVol after detach """ if self.__class__.vm is None: self.skipTest("VM not deployed — test_03 was skipped (no ready template)") @@ -723,6 +736,16 @@ def test_07_detach_volume_from_vm(self): "ONTAP FlexVol should be 'online' after detach" ) + # ONTAP: NFS3 volume data file must still exist after detach — the file + # is only removed when deleteVolume is called, not on detach. + files = self.ontap.list_files_in_volume(pool.name) + vol_file = next((f for f in files if vol.id in f), None) + self.assertIsNotNone( + vol_file, + "Volume data file for '%s' should persist in FlexVol '%s' after " + "detach; files present: %s" % (vol.id, pool.name, files) + ) + # ------------------------------------------------------------------ # Step 08 - Destroy VM, delete volume, delete pool # ------------------------------------------------------------------ @@ -735,6 +758,7 @@ def test_08_destroy_vm_and_cleanup(self): Verifies: - VM is destroyed/expunged from CloudStack - Volume is deleted from CloudStack + - ONTAP: NFS3 volume data file removed from FlexVol after deleteVolume - Pool is removed from CloudStack - ONTAP: FlexVol is deleted after pool removal - ONTAP: Export policy is removed after pool removal @@ -766,11 +790,24 @@ def test_08_destroy_vm_and_cleanup(self): # Delete the ONTAP data volume if vol is not None: + vol_id = vol.id cmd = deleteVolumeAPI.deleteVolumeCmd() - cmd.id = vol.id + cmd.id = vol_id self.apiClient.deleteVolume(cmd) self.__class__.volume = None + # ONTAP: NFS3 volume data file must be removed from the FlexVol + # after deleteVolume (CloudStack/libvirt deletes the file from the + # NFS mount as part of the destroy workflow). + files = self.ontap.list_files_in_volume(pool_name) + vol_file = next((f for f in files if vol_id in f), None) + self.assertIsNone( + vol_file, + "Volume data file for '%s' should be gone from FlexVol '%s' " + "after deleteVolume; files still present: %s" + % (vol_id, pool_name, files) + ) + # Enter maintenance then delete the pool maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() maint_cmd.id = pool.id diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py index f4f7ac7bb039..c043fa741431 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py @@ -40,7 +40,7 @@ Running: nosetests --with-marvin \\ --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ - test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py -v + test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py -v Note: Tests 01-06 share class-level state (sequential). Running a single test with -m "test_NN" will invoke setUpClass but the guard assertion will fail @@ -477,18 +477,6 @@ def test_05_cancel_maintenance_mode(self): """ Cancel maintenance mode and verify the pool returns to Up. - cancelStorageMaintenance sends ModifyStoragePoolCommand(add=True) to the - KVM agent, which calls createStoragePool() with details that include - nfsMountOptions=vers=3. The agent rebuilds the libvirt pool XML with the - xmlns:fs namespace extension and mounts the NFS share with vers=3. - - Fix confirmed — LibvirtStorageAdaptor now correctly handles the case - where a stale-active libvirt pool entry lingers at the mount point after - sp.destroy() during enter-maintenance. The fix: - 1. Detects a stale-active pool (isActive==1 but mountpoint -q fails) - and destroys it before re-creating. - 2. Retries createNetfsStoragePool once after 5 s on LibvirtException. - Verifies: - CloudStack reports pool state Up - ONTAP: FlexVol is still online diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py index d0238295c918..2e8ded580283 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py @@ -18,18 +18,6 @@ """ NFS3 pool lifecycle tests with a CloudStack volume present throughout. -Covers the TDS (section 10) scenarios that require a data volume to already -exist on the pool during pool state transitions: - - TDS Approach-1 SN 11-12 — Disable pool WITH volumes - TDS Approach-1 SN 15-16 — Enable pool WITH volumes - TDS Approach-1 SN 19-20 — Enter maintenance WITH volumes - TDS Approach-1 SN 21-22 — Cancel maintenance WITH volumes (fix confirmed) - TDS Negative SN 5-6 — Delete pool that has volumes; forced=False rejected - -Note: TDS SN 7-8 (force-delete NFS3 pool after manual volume deletion) is -already covered by test_ontap_create_primary_storage_nfs3.py test_07/test_08. - Tests are numbered test_01 ... test_07 and must run in that order. Each step builds on the shared state established by the previous step. @@ -360,8 +348,7 @@ def test_01_create_pool_and_volume(self): @attr(tags=["nfs3_with_volumes"], required_hardware=True) def test_02_disable_pool_volume_survives(self): """ - Disable the pool while a CloudStack data volume exists on it. - Covers TDS Approach-1 SN 11 (iSCSI) and SN 12 (NFS3): + Disable the pool while a CloudStack data volume exists on it: - Pool should no longer be available for scheduling new CS volumes - The existing CS volume should continue to exist (not deleted) - ONTAP: FlexVol remains online; export policy unchanged @@ -412,8 +399,7 @@ def test_02_disable_pool_volume_survives(self): @attr(tags=["nfs3_with_volumes"], required_hardware=True) def test_03_enable_pool_volume_intact(self): """ - Re-enable the pool while a CloudStack data volume exists on it. - Covers TDS Approach-1 SN 15 (iSCSI) and SN 16 (NFS3): + Re-enable the pool while a CloudStack data volume exists on it: - Pool state transitions back to Up - The existing CS volume is still accessible - ONTAP: FlexVol remains online; export policy unchanged @@ -464,8 +450,7 @@ def test_03_enable_pool_volume_intact(self): @attr(tags=["nfs3_with_volumes"], required_hardware=True) def test_04_enter_maintenance_volume_present(self): """ - Enter maintenance mode while a CloudStack data volume exists on the pool. - Covers TDS Approach-1 SN 19 (iSCSI) and SN 20 (NFS3): + Enter maintenance mode while a CloudStack data volume exists on the pool: - Pool transitions to Maintenance state - Existing CS volume remains in CloudStack - ONTAP: FlexVol stays online (maintenance is a CS-only state) @@ -517,8 +502,7 @@ def test_04_enter_maintenance_volume_present(self): @attr(tags=["nfs3_with_volumes"], required_hardware=True) def test_05_cancel_maintenance_with_volume(self): """ - Cancel maintenance mode while a CloudStack data volume exists on the pool. - Covers TDS Approach-1 SN 21 (iSCSI) and SN 22 (NFS3): + Cancel maintenance mode while a CloudStack data volume exists on the pool: - cancelStorageMaintenance succeeds (KVM/NFS3 fix confirmed) - Pool returns to Up state - Existing CS volume is still present in CloudStack @@ -578,8 +562,7 @@ def test_06_forced_false_delete_rejected(self): """ Enter maintenance mode then attempt to delete the pool (forced=False) while a CloudStack volume still exists on it. The operation must be - rejected. - Covers TDS Negative Scenarios SN 5 (iSCSI) and SN 6 (NFS3): + rejected: - CloudstackAPIException is raised with an appropriate error - Pool remains in Maintenance state - CS volume still exists diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index e40dc22283e5..74e02067aa45 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -197,6 +197,39 @@ def list_lun_maps_for_volume(self, svm_name, vol_name): return [r for r in data.get("records", []) if r.get("lun", {}).get("name", "").startswith(prefix)] + # -- NFS file helpers ---------------------------------------------------- + + def list_files_in_volume(self, vol_name, path="/"): + """Return a list of file names at ``path`` inside the named FlexVol. + + Uses the ONTAP REST file-system API: + GET /api/storage/volumes/{uuid}/files/{url_encoded_path} + + The path must appear in the URL (not as a query parameter). The root + directory is represented as ``%2F``. + + Returns an empty list if the volume does not exist, the path is empty, + or the request fails. + """ + vol = self.get_volume(vol_name) + if not vol: + return [] + vol_uuid = vol.get("uuid", "") + if not vol_uuid: + return [] + # URL-encode the path component (/ → %2F) and embed it in the URL. + from urllib.parse import quote + encoded_path = quote(path, safe="") + try: + resp = self._get( + "/storage/volumes/%s/files/%s" % (vol_uuid, encoded_path), + params={"fields": "name,type", "max_records": "500"} + ) + except Exception: + return [] + return [r.get("name", "") for r in resp.get("records", []) + if r.get("name") not in (".", "..")] + # --------------------------------------------------------------------------- # Base test class From 0c243042a3ae618b5c5c0825ecdb2710a5e84729 Mon Sep 17 00:00:00 2001 From: "Locharla, Sandeep" Date: Wed, 24 Jun 2026 22:46:39 +0530 Subject: [PATCH 4/6] removed some unnecessary formatting changes --- .../kvm/storage/KVMStoragePoolManager.java | 2 +- .../kvm/storage/LibvirtStorageAdaptor.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index b503643c96db..497130e378f0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -545,4 +545,4 @@ public Pair unprepareStorageClient(StoragePoolType type, String StorageAdaptor adaptor = getStorageAdaptor(type); return adaptor.unprepareStorageClient(uuid, details); } -} +} \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 46b98bebb934..357bd19d80cd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -753,7 +753,7 @@ private int adjustStoragePoolRefCount(String uuid, int adjustment) { /** * Thread-safe increment storage pool usage refcount - * + * * @param uuid UUID of the storage pool to increment the count */ private void incStoragePoolRefCount(String uuid) { @@ -763,7 +763,7 @@ private void incStoragePoolRefCount(String uuid) { /** * Thread-safe decrement storage pool usage refcount for the given uuid and * return if storage pool still in use. - * + * * @param uuid UUID of the storage pool to decrement the count * @return true if the storage pool is still used, else false. */ @@ -1432,7 +1432,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, * destination * qemu-img will exit with the error that the destination already exists. * So for RBD we don't create the image, but let qemu-img do that for us. - * + * * We then create a KVMPhysicalDisk object that we can return */ @@ -1663,9 +1663,9 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt * destination * qemu-img will exit with the error that the destination already exists. * So for RBD we don't create the image, but let qemu-img do that for us. - * + * * We then create a KVMPhysicalDisk object that we can return - * + * * It is however very unlikely that the destPool will be RBD, since it isn't * supported * for Secondary Storage @@ -1868,4 +1868,4 @@ private void deleteVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtEx private void deleteDirVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException { Script.runSimpleBashScript("rm -r --interactive=never " + vol.getPath()); } -} +} \ No newline at end of file From 44a2b33fa462cfa5da33420041f4bc76375c7529 Mon Sep 17 00:00:00 2001 From: "Locharla, Sandeep" Date: Tue, 30 Jun 2026 15:56:38 +0530 Subject: [PATCH 5/6] Reverted the fix for NFS3 Cancel Maintenance failure --- .../cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java | 2 +- .../cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 497130e378f0..10d58be52757 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -457,10 +457,10 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) { public boolean deleteStoragePool(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); - boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); if (type == StoragePoolType.NetworkFilesystem) { _haMonitor.removeStoragePool(uuid); } + boolean deleteStatus = adaptor.deleteStoragePool(uuid, details); synchronized (_storagePools) { _storagePools.remove(uuid); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 357bd19d80cd..dcdf3fc50f42 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -948,11 +948,6 @@ private boolean destroyStoragePoolHandleException(Connect conn, String uuid) { return false; } - @Override - public boolean deleteStoragePool(String uuid, Map details) { - return deleteStoragePool(uuid); - } - @Override public boolean deleteStoragePool(String uuid) { logger.info("Attempting to remove storage pool " + uuid + " from libvirt"); From a2cd65edb9715ad8eb01637955724136af23bce3 Mon Sep 17 00:00:00 2001 From: "Locharla, Sandeep" Date: Wed, 15 Jul 2026 15:50:52 +0530 Subject: [PATCH 6/6] Addressed comments --- .../kvm/storage/KVMStoragePoolManager.java | 135 +- .../kvm/storage/LibvirtStorageAdaptor.java | 531 ++--- test/integration/plugins/ontap/OVERVIEW.html | 1969 ++++++++++++++++ test/integration/plugins/ontap/README.md | 277 ++- .../integration/plugins/ontap/TEST_CASES.html | 2046 +++++++++++++++++ test/integration/plugins/ontap/TEST_CASES.md | 221 ++ .../plugins/ontap/manual_cancel_maint_test.py | 0 test/integration/plugins/ontap/ontap.cfg | 57 +- .../plugins/ontap/ontap_test_base.py | 18 - test/integration/plugins/ontap/probe_test.py | 35 - ...test_ontap_create_primary_storage_iscsi.py | 710 ------ .../test_ontap_create_primary_storage_nfs3.py | 404 ---- 12 files changed, 4782 insertions(+), 1621 deletions(-) create mode 100644 test/integration/plugins/ontap/OVERVIEW.html create mode 100644 test/integration/plugins/ontap/TEST_CASES.html create mode 100644 test/integration/plugins/ontap/TEST_CASES.md delete mode 100644 test/integration/plugins/ontap/manual_cancel_maint_test.py delete mode 100644 test/integration/plugins/ontap/probe_test.py delete mode 100644 test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py delete mode 100644 test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 10d58be52757..35cc864268c3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -83,8 +83,7 @@ public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) { this._storageMapper.put("libvirt", new LibvirtStorageAdaptor(storagelayer)); // add other storage adaptors manually here - // add any adaptors that wish to register themselves via call to - // adaptor.getStoragePoolType() + // add any adaptors that wish to register themselves via call to adaptor.getStoragePoolType() Reflections reflections = new Reflections("com.cloud.hypervisor.kvm.storage"); Set> storageAdaptorClasses = reflections.getSubTypesOf(StorageAdaptor.class); for (Class storageAdaptorClass : storageAdaptorClasses) { @@ -113,8 +112,7 @@ public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) { StoragePoolType storagePoolType = adaptor.getStoragePoolType(); if (storagePoolType != null) { if (this._storageMapper.containsKey(storagePoolType.toString())) { - logger.warn(String.format("Duplicate StorageAdaptor type %s, not loading %s", storagePoolType, - storageAdaptorClass.getName())); + logger.warn(String.format("Duplicate StorageAdaptor type %s, not loading %s", storagePoolType, storageAdaptorClass.getName())); } else { logger.info(String.format("Adding storage adaptor for %s", storageAdaptorClass.getName())); this._storageMapper.put(storagePoolType.toString(), adaptor); @@ -137,8 +135,7 @@ public boolean supportsPhysicalDiskCopy(StoragePoolType type) { return getStorageAdaptor(type).supportsPhysicalDiskCopy(type); } - public boolean connectPhysicalDisk(StoragePoolType type, String poolUuid, String volPath, - Map details) { + public boolean connectPhysicalDisk(StoragePoolType type, String poolUuid, String volPath, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); KVMStoragePool pool = adaptor.getStoragePool(poolUuid); @@ -158,8 +155,8 @@ public boolean connectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec, boolean is continue; } - VolumeObjectTO vol = (VolumeObjectTO) disk.getData(); - PrimaryDataStoreTO store = (PrimaryDataStoreTO) vol.getDataStore(); + VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); + PrimaryDataStoreTO store = (PrimaryDataStoreTO)vol.getDataStore(); if (!store.isManaged() && VirtualMachine.State.Migrating.equals(vmSpec.getState())) { result = true; continue; @@ -171,8 +168,7 @@ public boolean connectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec, boolean is result = adaptor.connectPhysicalDisk(vol.getPath(), pool, disk.getDetails(), isVMMigrate); if (!result) { - logger.error("Failed to connect disks via Instance spec for Instance: " + vmName + " volume:" - + vol.toString()); + logger.error("Failed to connect disks via Instance spec for Instance: " + vmName + " volume:" + vol.toString()); return result; } } @@ -190,22 +186,18 @@ public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { String poolType = volumeToDisconnect.get(DiskTO.PROTOCOL_TYPE); StorageAdaptor adaptor = _storageMapper.get(poolType); if (adaptor != null) { - logger.info(String.format( - "Disconnecting physical disk using the storage adaptor found for pool type: %s", poolType)); + logger.info(String.format("Disconnecting physical disk using the storage adaptor found for pool type: %s", poolType)); return adaptor.disconnectPhysicalDisk(volumeToDisconnect); } - logger.debug(String.format( - "Couldn't find the storage adaptor for pool type: %s to disconnect the physical disk, trying with others", - poolType)); + logger.debug(String.format("Couldn't find the storage adaptor for pool type: %s to disconnect the physical disk, trying with others", poolType)); } for (Map.Entry set : _storageMapper.entrySet()) { StorageAdaptor adaptor = set.getValue(); if (adaptor.disconnectPhysicalDisk(volumeToDisconnect)) { - logger.debug(String.format("Disconnected physical disk using the storage adaptor for pool type: %s", - set.getKey())); + logger.debug(String.format("Disconnected physical disk using the storage adaptor for pool type: %s", set.getKey())); return true; } } @@ -219,9 +211,7 @@ public boolean disconnectPhysicalDiskByPath(String path) { StorageAdaptor adaptor = set.getValue(); if (adaptor.disconnectPhysicalDiskByPath(path)) { - logger.debug(String.format( - "Disconnected physical disk by local path: %s, using the storage adaptor for pool type: %s", - path, set.getKey())); + logger.debug(String.format("Disconnected physical disk by local path: %s, using the storage adaptor for pool type: %s", path, set.getKey())); return true; } } @@ -231,15 +221,10 @@ public boolean disconnectPhysicalDiskByPath(String path) { public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { if (vmSpec == null) { - /* - * CloudStack often tries to stop VMs that shouldn't be running, to ensure a - * known state, - * for example if we lose communication with the agent and the VM is brought up - * elsewhere. - * We may not know about these yet. This might mean that we can't use the vmspec - * map, because - * when we restart the agent we lose all of the info about running VMs. - */ + /* CloudStack often tries to stop VMs that shouldn't be running, to ensure a known state, + for example if we lose communication with the agent and the VM is brought up elsewhere. + We may not know about these yet. This might mean that we can't use the vmspec map, because + when we restart the agent we lose all of the info about running VMs. */ logger.debug("disconnectPhysicalDiskViaVmSpec: Attempted to stop a VM that is not yet in our hash map"); @@ -256,14 +241,13 @@ public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { if (disk.getType() != Volume.Type.ISO) { logger.debug("Disconnecting disk " + disk.getPath()); - VolumeObjectTO vol = (VolumeObjectTO) disk.getData(); - PrimaryDataStoreTO store = (PrimaryDataStoreTO) vol.getDataStore(); + VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); + PrimaryDataStoreTO store = (PrimaryDataStoreTO)vol.getDataStore(); KVMStoragePool pool = getStoragePool(store.getPoolType(), store.getUuid()); if (pool == null) { - logger.error("Pool " + store.getUuid() + " of type " + store.getPoolType() - + " was not found, skipping disconnect logic"); + logger.error("Pool " + store.getUuid() + " of type " + store.getPoolType() + " was not found, skipping disconnect logic"); continue; } @@ -274,8 +258,7 @@ public boolean disconnectPhysicalDisksViaVmSpec(VirtualMachineTO vmSpec) { boolean subResult = adaptor.disconnectPhysicalDisk(vol.getPath(), pool); if (!subResult) { - logger.error("Failed to disconnect disks via Instance spec for Instance: " + vmName + " volume:" - + vol.toString()); + logger.error("Failed to disconnect disks via Instance spec for Instance: " + vmName + " volume:" + vol.toString()); result = false; } @@ -298,11 +281,9 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean } catch (Exception e) { StoragePoolInformation info = _storagePools.get(uuid); if (info != null) { - pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), - info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType()); + pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType()); } else { - throw new CloudRuntimeException( - "Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage()); + throw new CloudRuntimeException("Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage()); } } @@ -315,11 +296,8 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean } /** - * As the class {@link LibvirtStoragePool} is constrained to the - * {@link org.libvirt.StoragePool} class, there is no way of saving a generic - * parameter such as the details, hence, - * this method was created to always make available the details of libvirt - * primary storages for when they are needed. + * As the class {@link LibvirtStoragePool} is constrained to the {@link org.libvirt.StoragePool} class, there is no way of saving a generic parameter such as the details, hence, + * this method was created to always make available the details of libvirt primary storages for when they are needed. */ private void addPoolDetails(String uuid, LibvirtStoragePool pool) { StoragePoolInformation storagePoolInformation = _storagePools.get(uuid); @@ -355,7 +333,7 @@ public KVMStoragePool getStoragePoolByURI(String uri) { sourcePath = sourcePath.replace("//", "/"); sourceHost = storageUri.getHost(); uuid = UuidUtils.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString(); - protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem : StoragePoolType.NetworkFilesystem; + protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem; // storage registers itself through here return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocol, null, false); @@ -365,9 +343,8 @@ public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, St int cnt = 0; int retries = 100; KVMPhysicalDisk vol = null; - // harden get volume, try cnt times to get volume, in case volume is created on - // other host - // Poll more frequently and return immediately once disk is found + //harden get volume, try cnt times to get volume, in case volume is created on other host + //Poll more frequently and return immediately once disk is found String errMsg = ""; while (cnt < retries) { try { @@ -398,8 +375,7 @@ public KVMPhysicalDisk getPhysicalDisk(StoragePoolType type, String poolUuid, St } } - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, - StoragePoolType type) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type) { // primary storage registers itself through here return createStoragePool(name, host, port, path, userInfo, type, null, true); } @@ -407,30 +383,24 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri /** * Primary Storage registers itself through here */ - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, - StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details) { return createStoragePool(name, host, port, path, userInfo, type, details, true); } - // Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in - // parallel, so need to be synced. - private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, - String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { + //Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in parallel, so need to be synced. + private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); - KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, - primaryStorage); + KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, primaryStorage); if (pool instanceof LibvirtStoragePool) { ((LibvirtStoragePool) pool).setType(type); } // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { - KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, - PoolType.PrimaryStorage); + KVMHABase.HAStoragePool storagePool = new KVMHABase.HAStoragePool(pool, host, path, PoolType.PrimaryStorage); _haMonitor.addStoragePool(storagePool); } - StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, details, - primaryStorage); + StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, details, primaryStorage); addStoragePool(pool.getUuid(), info); return pool; } @@ -447,8 +417,7 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) { if (type == StoragePoolType.NetworkFilesystem) { _haMonitor.removeStoragePool(uuid); } - boolean deleteStatus = adaptor.deleteStoragePool(uuid); - ; + boolean deleteStatus = adaptor.deleteStoragePool(uuid);; synchronized (_storagePools) { _storagePools.remove(uuid); } @@ -467,16 +436,13 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid, Map, String> prepareStorageClient(StoragePoolType type, String uuid, - Map details) { + public Ternary, String> prepareStorageClient(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); return adaptor.prepareStorageClient(uuid, details); } - public Pair unprepareStorageClient(StoragePoolType type, String uuid, - Map details) { + public Pair unprepareStorageClient(StoragePoolType type, String uuid, Map details) { StorageAdaptor adaptor = getStorageAdaptor(type); return adaptor.unprepareStorageClient(uuid, details); } -} \ No newline at end of file +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index dcdf3fc50f42..a03daeb197bf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -94,13 +94,11 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private static final int RBD_FEATURE_OBJECT_MAP = 8; private static final int RBD_FEATURE_FAST_DIFF = 16; private static final int RBD_FEATURE_DEEP_FLATTEN = 32; - public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP - + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; + public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ - private static final Set poolTypesThatEnableCreateDiskFromTemplateBacking = new HashSet<>( - Arrays.asList(StoragePoolType.NetworkFilesystem, - StoragePoolType.Filesystem)); + private static final Set poolTypesThatEnableCreateDiskFromTemplateBacking = new HashSet<>(Arrays.asList(StoragePoolType.NetworkFilesystem, + StoragePoolType.Filesystem)); public LibvirtStorageAdaptor(StorageLayer storage) { _storageLayer = storage; @@ -117,10 +115,8 @@ public boolean createFolder(String uuid, String path, String localPath) { String mountPoint = _mountPoint + File.separator + uuid; if (localPath != null) { - logger.debug(String.format( - "Pool [%s] is of type local or shared mount point; therefore, we will use the local path [%s] to create the folder [%s] (if it does not" - + " exist).", - uuid, localPath, path)); + logger.debug(String.format("Pool [%s] is of type local or shared mount point; therefore, we will use the local path [%s] to create the folder [%s] (if it does not" + + " exist).", uuid, localPath, path)); mountPoint = localPath; } @@ -133,25 +129,20 @@ public boolean createFolder(String uuid, String path, String localPath) { } @Override - public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, - PhysicalDiskFormat format, long size, - KVMStoragePool destPool, int timeout, byte[] passphrase) { - String volumeDesc = String.format( - "volume [%s], with template backing [%s], in pool [%s] (%s), with size [%s] and encryption is %s", name, - template.getName(), destPool.getUuid(), - destPool.getType(), size, passphrase != null && passphrase.length > 0); + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { + String volumeDesc = String.format("volume [%s], with template backing [%s], in pool [%s] (%s), with size [%s] and encryption is %s", name, template.getName(), destPool.getUuid(), + destPool.getType(), size, passphrase != null && passphrase.length > 0); if (!poolTypesThatEnableCreateDiskFromTemplateBacking.contains(destPool.getType())) { - logger.info(String.format("Skipping creation of %s due to pool type is none of the following types %s.", - volumeDesc, poolTypesThatEnableCreateDiskFromTemplateBacking.stream() - .map(type -> type.toString()).collect(Collectors.joining(", ")))); + logger.info(String.format("Skipping creation of %s due to pool type is none of the following types %s.", volumeDesc, poolTypesThatEnableCreateDiskFromTemplateBacking.stream() + .map(type -> type.toString()).collect(Collectors.joining(", ")))); return null; } if (format != PhysicalDiskFormat.QCOW2) { - logger.info(String.format("Skipping creation of %s due to format [%s] is not [%s].", volumeDesc, format, - PhysicalDiskFormat.QCOW2)); + logger.info(String.format("Skipping creation of %s due to format [%s] is not [%s].", volumeDesc, format, PhysicalDiskFormat.QCOW2)); return null; } @@ -168,18 +159,15 @@ public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, S QemuImgFile backingFile = new QemuImgFile(template.getPath(), template.getFormat()); if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, - keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); } logger.debug(String.format("Passphrase is staged to keyFile: %s", keyFile.isSet())); QemuImg qemu = new QemuImg(timeout); qemu.create(destFile, backingFile, options, passphraseObjects); } catch (QemuImgException | LibvirtException | IOException e) { - // why don't we throw an exception here? I guess we fail to find the volume - // later and that results in a failure returned? - logger.error( - String.format("Failed to create %s in [%s] due to [%s].", volumeDesc, destPath, e.getMessage()), e); + // why don't we throw an exception here? I guess we fail to find the volume later and that results in a failure returned? + logger.error(String.format("Failed to create %s in [%s] due to [%s].", volumeDesc, destPath, e.getMessage()), e); } return null; @@ -188,21 +176,17 @@ public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, S /** * Extract downloaded template into installPath, remove compressed file */ - public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, - String destinationFile) { - String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, - destinationFile); + public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { + String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + downloadedTemplateFile); } @Override - public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, - KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { File sourceFile = new File(templateFilePath); if (!sourceFile.exists()) { - throw new CloudRuntimeException( - "Direct download template file " + sourceFile + " does not exist on this host"); + throw new CloudRuntimeException("Direct download template file " + sourceFile + " does not exist on this host"); } String templateUuid = UUID.randomUUID().toString(); if (Storage.ImageFormat.ISO.equals(format)) { @@ -211,9 +195,8 @@ public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFileP String destinationFile = destPool.getLocalPath() + File.separator + templateUuid; if (destPool.getType() == StoragePoolType.NetworkFilesystem || destPool.getType() == StoragePoolType.Filesystem - || destPool.getType() == StoragePoolType.SharedMountPoint) { - if (!Storage.ImageFormat.ISO.equals(format) - && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { + || destPool.getType() == StoragePoolType.SharedMountPoint) { + if (!Storage.ImageFormat.ISO.equals(format) && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { extractDownloadedTemplate(templateFilePath, destPool, destinationFile); } else { Script.runSimpleBashScript("mv " + templateFilePath + " " + destinationFile); @@ -226,16 +209,14 @@ public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFileP return destPool.getPhysicalDisk(templateUuid); } - private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePath, String templateUuid, - KVMStoragePool destPool, int timeout) { + private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePath, String templateUuid, KVMStoragePool destPool, int timeout) { try { QemuImg.PhysicalDiskFormat srcFileFormat = QemuImg.PhysicalDiskFormat.QCOW2; QemuImgFile srcFile = new QemuImgFile(srcTemplateFilePath, srcFileFormat); QemuImg qemu = new QemuImg(timeout); Map info = qemu.info(srcFile); Long virtualSize = Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); - KVMPhysicalDisk destDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + templateUuid, templateUuid, - destPool); + KVMPhysicalDisk destDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + templateUuid, templateUuid, destPool); destDisk.setFormat(PhysicalDiskFormat.RAW); destDisk.setSize(virtualSize); destDisk.setVirtualSize(virtualSize); @@ -243,8 +224,7 @@ private void createTemplateOnRBDFromDirectDownloadFile(String srcTemplateFilePat destFile.setFormat(PhysicalDiskFormat.RAW); qemu.convert(srcFile, destFile); } catch (LibvirtException | QemuImgException e) { - String err = String.format("Error creating template from direct download file on pool %s: %s", - destPool.getUuid(), e.getMessage()); + String err = String.format("Error creating template from direct download file on pool %s: %s", destPool.getUuid(), e.getMessage()); logger.error(err, e); throw new CloudRuntimeException(err, e); } @@ -274,8 +254,7 @@ public StorageVol getVolume(StoragePool pool, String volName) { try { vol = pool.storageVolLookupByName(volName); - logger.debug("Found volume " + volName + " in storage pool " + pool.getName() - + " after refreshing the pool"); + logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool"); } catch (LibvirtException e) { throw new CloudRuntimeException("Could not find volume " + volName + ": " + e.getMessage()); } @@ -284,10 +263,8 @@ public StorageVol getVolume(StoragePool pool, String volName) { return vol; } - public StorageVol createVolume(Connect conn, StoragePool pool, String uuid, long size, VolumeFormat format) - throws LibvirtException { - LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID.randomUUID().toString(), size, format, null, - null); + public StorageVol createVolume(Connect conn, StoragePool pool, String uuid, long size, VolumeFormat format) throws LibvirtException { + LibvirtStorageVolumeDef volDef = new LibvirtStorageVolumeDef(UUID.randomUUID().toString(), size, format, null, null); logger.debug(volDef.toString()); return pool.storageVolCreateXML(volDef.toString(), 0); @@ -313,8 +290,7 @@ private void checkNetfsStoragePoolMounted(String uuid) { } } - private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, - List nfsMountOpts) throws LibvirtException { + private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, List nfsMountOpts) throws LibvirtException { String targetPath = _mountPoint + File.separator + uuid; LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath, nfsMountOpts); _storageLayer.mkdir(targetPath); @@ -324,7 +300,7 @@ private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String // check whether the pool is already mounted int mountpointResult = Script.runSimpleBashScriptForExitValue("mountpoint -q " + targetPath); // if the pool is mounted, try to unmount it - if (mountpointResult == 0) { + if(mountpointResult == 0) { logger.info("Attempting to unmount old mount at " + targetPath); String result = Script.runSimpleBashScript("umount -l " + targetPath); if (result == null) { @@ -379,8 +355,7 @@ private StoragePool createCLVMStoragePool(Connect conn, String uuid, String host String volgroupName = path; volgroupName = volgroupName.replaceFirst("/", ""); - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.LOGICAL, volgroupName, uuid, host, volgroupPath, - volgroupPath); + LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.LOGICAL, volgroupName, uuid, host, volgroupPath, volgroupPath); StoragePool sp = null; try { logger.debug(spd.toString()); @@ -442,8 +417,7 @@ private boolean destroyStoragePoolOnNFSMountOptionsChange(StoragePool sp, Connec return false; } - private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, - String path) { + private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, String path) { LibvirtStoragePoolDef spd; StoragePool sp = null; @@ -471,8 +445,7 @@ private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, } return null; } - spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, userInfoTemp[0], - AuthenticationType.CEPH, uuid); + spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, userInfoTemp[0], AuthenticationType.CEPH, uuid); } else { spd = new LibvirtStoragePoolDef(PoolType.RBD, uuid, uuid, host, port, path, ""); } @@ -511,8 +484,7 @@ private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, } } - public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destVol, StorageVol srcVol, int timeout) - throws LibvirtException { + public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destVol, StorageVol srcVol, int timeout) throws LibvirtException { StorageVol vol = destPool.storageVolCreateXML(destVol.toString(), 0); String srcPath = srcVol.getKey(); String destPath = vol.getKey(); @@ -520,14 +492,12 @@ public StorageVol copyVolume(StoragePool destPool, LibvirtStorageVolumeDef destV return vol; } - public boolean copyVolume(String srcPath, String destPath, String volumeName, int timeout) - throws InternalErrorException { + public boolean copyVolume(String srcPath, String destPath, String volumeName, int timeout) throws InternalErrorException { _storageLayer.mkdirs(destPath); if (!_storageLayer.exists(srcPath)) { throw new InternalErrorException("volume:" + srcPath + " is not exits"); } - String result = Script.runSimpleBashScript("cp " + srcPath + " " + destPath + File.separator + volumeName, - timeout); + String result = Script.runSimpleBashScript("cp " + srcPath + " " + destPath + File.separator + volumeName, timeout); return result == null; } @@ -546,7 +516,7 @@ public LibvirtStorageVolumeDef getStorageVolumeDef(Connect conn, StorageVol vol) @Override public StoragePoolType getStoragePoolType() { // This is mapped manually in KVMStoragePoolManager - return null; + return null; } @Override @@ -562,28 +532,30 @@ protected void updateLocalPoolIops(LibvirtStoragePool pool) { // Run script to get data List commands = new ArrayList<>(); - commands.add(new String[] { + commands.add(new String[]{ Script.getExecutableAbsolutePath("bash"), "-c", String.format( "%s %s | %s 'NR==2 {print $1}'", Script.getExecutableAbsolutePath("df"), pool.getLocalPath(), - Script.getExecutableAbsolutePath("awk")) + Script.getExecutableAbsolutePath("awk") + ) }); String result = Script.executePipedCommands(commands, 1000).second(); if (StringUtils.isBlank(result)) { return; } result = result.trim(); - commands.add(new String[] { + commands.add(new String[]{ Script.getExecutableAbsolutePath("bash"), "-c", String.format( "%s -z %s 1 2 | %s 'NR==7 {print $2}'", Script.getExecutableAbsolutePath("iostat"), result, - Script.getExecutableAbsolutePath("awk")) + Script.getExecutableAbsolutePath("awk") + ) }); result = Script.executePipedCommands(commands, 10000).second(); logger.trace("Pool used IOPS result: {}", result); @@ -646,8 +618,7 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { String authUsername = spd.getAuthUserName(); if (authUsername != null) { Secret secret = conn.secretLookupByUUIDString(spd.getSecretUUID()); - String secretValue = new String(Base64.encodeBase64(secret.getByteValue()), - Charset.defaultCharset()); + String secretValue = new String(Base64.encodeBase64(secret.getByteValue()), Charset.defaultCharset()); pool.setAuthUsername(authUsername); pool.setAuthSecret(secretValue); } @@ -657,15 +628,12 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { * On large (RBD) storage pools it can take up to a couple of minutes * for libvirt to refresh the pool. * - * Refreshing a storage pool means that libvirt will have to iterate the whole - * pool + * Refreshing a storage pool means that libvirt will have to iterate the whole pool * and fetch information of each volume in there * - * It is not always required to refresh a pool. So we can control if we want to - * or not + * It is not always required to refresh a pool. So we can control if we want to or not * - * By default only the getStorageStats call in the LibvirtComputingResource will - * ask to + * By default only the getStorageStats call in the LibvirtComputingResource will ask to * refresh the pool */ if (refreshInfo) { @@ -678,9 +646,9 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { pool.setAvailable(storage.getInfo().available); logger.debug("Successfully refreshed pool " + uuid + - " Capacity: " + toHumanReadableSize(storage.getInfo().capacity) + - " Used: " + toHumanReadableSize(storage.getInfo().allocation) + - " Available: " + toHumanReadableSize(storage.getInfo().available)); + " Capacity: " + toHumanReadableSize(storage.getInfo().capacity) + + " Used: " + toHumanReadableSize(storage.getInfo().allocation) + + " Available: " + toHumanReadableSize(storage.getInfo().available)); return pool; } catch (LibvirtException e) { @@ -691,7 +659,7 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { @Override public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid); @@ -750,20 +718,15 @@ private int adjustStoragePoolRefCount(String uuid, int adjustment) { return refCount; } } - /** * Thread-safe increment storage pool usage refcount - * * @param uuid UUID of the storage pool to increment the count */ private void incStoragePoolRefCount(String uuid) { adjustStoragePoolRefCount(uuid, 1); } - /** - * Thread-safe decrement storage pool usage refcount for the given uuid and - * return if storage pool still in use. - * + * Thread-safe decrement storage pool usage refcount for the given uuid and return if storage pool still in use. * @param uuid UUID of the storage pool to decrement the count * @return true if the storage pool is still used, else false. */ @@ -772,8 +735,7 @@ private boolean decStoragePoolRefCount(String uuid) { } @Override - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, - StoragePoolType type, Map details, boolean isPrimaryStorage) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean isPrimaryStorage) { logger.info("Attempting to create storage pool {} ({}) in libvirt", name, type); StoragePool sp; Connect conn; @@ -809,8 +771,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri // if anyone is, undefine the pool so we can define it as requested. // This should be safe since a pool in use can't be removed, and no // volumes are affected by unregistering the pool with libvirt. - logger.info("Didn't find an existing storage pool " + name - + " by UUID, checking for pools with duplicate paths"); + logger.info("Didn't find an existing storage pool " + name + " by UUID, checking for pools with duplicate paths"); try { String[] poolnames = conn.listStoragePools(); @@ -819,8 +780,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri StoragePool p = conn.storagePoolLookupByName(poolname); LibvirtStoragePoolDef pdef = getStoragePoolDef(conn, p); if (pdef == null) { - throw new CloudRuntimeException( - "Unable to parse the storage pool definition for storage pool " + poolname); + throw new CloudRuntimeException("Unable to parse the storage pool definition for storage pool " + poolname); } String targetPath = pdef.getTargetPath(); @@ -836,15 +796,13 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri } } } catch (LibvirtException e) { - logger.error( - "Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" - + e); + logger.error("Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" + e); } } List nfsMountOpts = getNFSMountOptsFromDetails(type, details); if (sp != null && CollectionUtils.isNotEmpty(nfsMountOpts) && - destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) { + destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) { sp = null; } @@ -856,7 +814,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path, nfsMountOpts); } catch (LibvirtException e) { - logger.error("Failed to create netfs mount: " + host + ":" + path, e); + logger.error("Failed to create netfs mount: " + host + ":" + path , e); logger.error(e.getStackTrace()); throw new CloudRuntimeException(e.toString()); } @@ -864,7 +822,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path, null); } catch (LibvirtException e) { - logger.error("Failed to create glusterfs mount: " + host + ":" + path, e); + logger.error("Failed to create glusterfs mount: " + host + ":" + path , e); logger.error(e.getStackTrace()); throw new CloudRuntimeException(e.toString()); } @@ -883,8 +841,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri try { if (!isPrimaryStorage) { - // only ref count storage pools for secondary storage, as primary storage is - // assumed + // only ref count storage pools for secondary storage, as primary storage is assumed // to be always mounted, as long the primary storage isn't fully deleted. incStoragePoolRefCount(name); } @@ -904,8 +861,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri String error = e.toString(); if (error.contains("Storage source conflict")) { throw new CloudRuntimeException("A pool matching this location already exists in libvirt, " + - " but has a different UUID/Name. Cannot create new pool without first " - + " removing it. Check for inactive pools via 'virsh pool-list --all'. " + + " but has a different UUID/Name. Cannot create new pool without first " + " removing it. Check for inactive pools via 'virsh pool-list --all'. " + error); } else { throw new CloudRuntimeException(error); @@ -939,7 +895,8 @@ private boolean destroyStoragePool(Connect conn, String uuid) throws LibvirtExce } } - private boolean destroyStoragePoolHandleException(Connect conn, String uuid) { + private boolean destroyStoragePoolHandleException(Connect conn, String uuid) + { try { return destroyStoragePool(conn, uuid); } catch (LibvirtException e) { @@ -991,10 +948,8 @@ public boolean deleteStoragePool(String uuid) { // handle ebusy error when pool is quickly destroyed if (e.toString().contains("exit status 16")) { String targetPath = _mountPoint + File.separator + uuid; - logger.error( - "deleteStoragePool removed pool from libvirt, but libvirt had trouble unmounting the pool. Trying umount location " - + targetPath + - " again in a few seconds"); + logger.error("deleteStoragePool removed pool from libvirt, but libvirt had trouble unmounting the pool. Trying umount location " + targetPath + + " again in a few seconds"); String result = Script.runSimpleBashScript("sleep 5 && umount " + targetPath); if (result == null) { logger.info("Succeeded in unmounting " + targetPath); @@ -1010,61 +965,51 @@ public boolean deleteStoragePool(String uuid) { /** * Creates a physical disk depending on the {@link StoragePoolType}: *
    - *
  • - * {@link StoragePoolType#RBD} - *
      - *
    • - * If it is an erasure code pool, utilizes QemuImg to create the physical disk - * through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} - *
      • - *
    • - * Otherwise, utilize Libvirt to create the physical disk through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
      • - *
    - *
    • - *
  • - * {@link StoragePoolType#NetworkFilesystem} and - * {@link StoragePoolType#Filesystem} - *
      - *
    • - * If the format is {@link PhysicalDiskFormat#QCOW2} or - * {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk - * through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} - *
      • - *
    • - * If the format is {@link PhysicalDiskFormat#DIR} or - * {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk - * through the method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
      • - *
    - *
    • - *
  • - * For the rest of the {@link StoragePoolType} types, utilizes the Libvirt - * method - * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} - *
    • + *
  • + * {@link StoragePoolType#RBD} + *
      + *
    • + * If it is an erasure code pool, utilizes QemuImg to create the physical disk through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} + *
      • + *
    • + * Otherwise, utilize Libvirt to create the physical disk through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
      • + *
    + *
    • + *
  • + * {@link StoragePoolType#NetworkFilesystem} and {@link StoragePoolType#Filesystem} + *
      + *
    • + * If the format is {@link PhysicalDiskFormat#QCOW2} or {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])} + *
      • + *
    • + * If the format is {@link PhysicalDiskFormat#DIR} or {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk through the method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
      • + *
    + *
    • + *
  • + * For the rest of the {@link StoragePoolType} types, utilizes the Libvirt method + * {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)} + *
    • *
*/ @Override public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { - logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), - pool.getUuid(), toHumanReadableSize(size)); + logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), pool.getUuid(), toHumanReadableSize(size)); StoragePoolType poolType = pool.getType(); if (StoragePoolType.RBD.equals(poolType)) { Map details = pool.getDetails(); String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); - return (dataPool == null) - ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) - : createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, - passphrase); + return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : + createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase); } else if (StoragePoolType.NetworkFilesystem.equals(poolType) || StoragePoolType.Filesystem.equals(poolType)) { switch (format) { case QCOW2: @@ -1112,9 +1057,9 @@ private KVMPhysicalDisk createPhysicalDiskByLibVirt(String name, KVMStoragePool return disk; } - private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, - Storage.ProvisioningType provisioningType, long size, - byte[] passphrase) { + + private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, + byte[] passphrase) { String volPath; String volName = name; long virtualSize = 0; @@ -1136,15 +1081,13 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool destFile.setSize(size); Map options = new HashMap(); if (List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem).contains(pool.getType())) { - options.put(QemuImg.PREALLOCATION, - QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); + options.put(QemuImg.PREALLOCATION, QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); } try (KeyFile keyFile = new KeyFile(passphrase)) { QemuImg qemu = new QemuImg(timeout); if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, - keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); // make room for encryption header on raw format, use LUKS if (format == PhysicalDiskFormat.RAW) { @@ -1159,8 +1102,7 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool virtualSize = Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); actualSize = new File(destFile.getFileName()).length(); } catch (QemuImgException | LibvirtException | IOException e) { - throw new CloudRuntimeException( - String.format("Failed to create %s due to a failed execution of qemu-img", volPath), e); + throw new CloudRuntimeException(String.format("Failed to create %s due to a failed execution of qemu-img", volPath), e); } KVMPhysicalDisk disk = new KVMPhysicalDisk(volPath, volName, pool); @@ -1172,8 +1114,7 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool } @Override - public boolean connectPhysicalDisk(String name, KVMStoragePool pool, Map details, - boolean isVMMigrate) { + public boolean connectPhysicalDisk(String name, KVMStoragePool pool, Map details, boolean isVMMigrate) { // this is for managed storage that needs to prep disks prior to use return true; } @@ -1236,8 +1177,7 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag */ if (pool.getType() == StoragePoolType.RBD) { try { - logger.info("Unprotecting and Removing RBD snapshots of image " + pool.getSourceDir() + "/" + uuid - + " prior to removing the image"); + logger.info("Unprotecting and Removing RBD snapshots of image " + pool.getSourceDir() + "/" + uuid + " prior to removing the image"); Rados r = new Rados(pool.getAuthUserName()); r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); @@ -1257,20 +1197,17 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag logger.debug("Unprotecting snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); image.snapUnprotect(snap.name); } else { - logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name - + " is not protected."); + logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + " is not protected."); } logger.debug("Removing snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); image.snapRemove(snap.name); } - logger.info( - "Successfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of " - + pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image"); + logger.info("Successfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of " + + pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image"); } catch (RbdException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException( - e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } finally { logger.debug("Closing image and destroying context"); rbd.close(image); @@ -1278,20 +1215,20 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } } catch (RadosException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } catch (RbdException e) { logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); + ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); } } - LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), uuid); logger.debug("Instructing libvirt to remove volume " + uuid + " from pool " + pool.getUuid()); - if (Storage.ImageFormat.DIR.equals(format)) { + if(Storage.ImageFormat.DIR.equals(format)){ deleteDirVol(libvirtPool, vol); } else { deleteVol(libvirtPool, vol); @@ -1304,52 +1241,40 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag } /** - * This function copies a physical disk from Secondary Storage to Primary - * Storage + * This function copies a physical disk from Secondary Storage to Primary Storage * or from Primary to Primary Storage * - * The first time a template is deployed in Primary Storage it will be copied - * from + * The first time a template is deployed in Primary Storage it will be copied from * Secondary to Primary. * - * If it has been created on Primary Storage, it will be copied on the Primary - * Storage + * If it has been created on Primary Storage, it will be copied on the Primary Storage */ @Override public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, - KVMStoragePool destPool, int timeout, byte[] passphrase) { + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { - logger.info( - "Creating volume " + name + " from template " + template.getName() + " in pool " + destPool.getUuid() + - " (" + destPool.getType().toString() + ") with size " + toHumanReadableSize(size)); + logger.info("Creating volume " + name + " from template " + template.getName() + " in pool " + destPool.getUuid() + + " (" + destPool.getType().toString() + ") with size " + toHumanReadableSize(size)); KVMPhysicalDisk disk = null; if (destPool.getType() == StoragePoolType.RBD) { disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout); } else { - try (KeyFile keyFile = new KeyFile(passphrase)) { + try (KeyFile keyFile = new KeyFile(passphrase)){ String newUuid = name; List passphraseObjects = new ArrayList<>(); - disk = destPool.createPhysicalDisk(newUuid, format, provisioningType, template.getVirtualSize(), - passphrase); + disk = destPool.createPhysicalDisk(newUuid, format, provisioningType, template.getVirtualSize(), passphrase); if (disk == null) { throw new CloudRuntimeException("Failed to create disk from template " + template.getName()); } if (template.getFormat() == PhysicalDiskFormat.TAR) { - Script.runSimpleBashScript("tar -x -f " + template.getPath() + " -C " + disk.getPath(), timeout); // TO - // BE - // FIXED - // to - // aware - // provisioningType + Script.runSimpleBashScript("tar -x -f " + template.getPath() + " -C " + disk.getPath(), timeout); // TO BE FIXED to aware provisioningType } else if (template.getFormat() == PhysicalDiskFormat.DIR) { Script.runSimpleBashScript("mkdir -p " + disk.getPath()); Script.runSimpleBashScript("chmod 755 " + disk.getPath()); - Script.runSimpleBashScript("tar -x -f " + template.getPath() + "/*.tar -C " + disk.getPath(), - timeout); + Script.runSimpleBashScript("tar -x -f " + template.getPath() + "/*.tar -C " + disk.getPath(), timeout); } else if (format == PhysicalDiskFormat.QCOW2) { QemuImg qemu = new QemuImg(timeout); QemuImgFile destFile = new QemuImgFile(disk.getPath(), format); @@ -1359,34 +1284,31 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, destFile.setSize(template.getVirtualSize()); } Map options = new HashMap(); - options.put("preallocation", - QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); + options.put("preallocation", QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); + if (keyFile.isSet()) { - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, - keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); } QemuImgFile srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - Boolean createFullClone = AgentPropertiesFileHandler - .getPropertyValue(AgentProperties.CREATE_FULL_CLONE); - switch (provisioningType) { - case THIN: - logger.info("Creating volume [{}] {} backing file [{}] as the property [{}] is [{}].", - destFile.getFileName(), createFullClone ? "without" : "with", - template.getPath(), AgentProperties.CREATE_FULL_CLONE.getName(), createFullClone); - if (createFullClone) { - qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); - } else { - qemu.create(destFile, srcFile, options, passphraseObjects); - } - break; - case SPARSE: - case FAT: - srcFile = new QemuImgFile(template.getPath(), template.getFormat()); + Boolean createFullClone = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CREATE_FULL_CLONE); + switch(provisioningType){ + case THIN: + logger.info("Creating volume [{}] {} backing file [{}] as the property [{}] is [{}].", destFile.getFileName(), createFullClone ? "without" : "with", + template.getPath(), AgentProperties.CREATE_FULL_CLONE.getName(), createFullClone); + if (createFullClone) { qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); - break; + } else { + qemu.create(destFile, srcFile, options, passphraseObjects); + } + break; + case SPARSE: + case FAT: + srcFile = new QemuImgFile(template.getPath(), template.getFormat()); + qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); + break; } } else if (format == PhysicalDiskFormat.RAW) { PhysicalDiskFormat destFormat = PhysicalDiskFormat.RAW; @@ -1395,8 +1317,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, if (keyFile.isSet()) { destFormat = PhysicalDiskFormat.LUKS; disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); - passphraseObjects.add(QemuObject.prepareSecretForQemuImg(destFormat, - QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); + passphraseObjects.add(QemuObject.prepareSecretForQemuImg(destFormat, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); } QemuImgFile sourceFile = new QemuImgFile(template.getPath(), template.getFormat()); @@ -1410,8 +1331,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, qemu.convert(sourceFile, destFile, options, passphraseObjects, null, false); } } catch (QemuImgException | LibvirtException | IOException e) { - throw new CloudRuntimeException( - String.format("Failed to create %s due to a failed execution of qemu-img", name), e); + throw new CloudRuntimeException(String.format("Failed to create %s due to a failed execution of qemu-img", name), e); } } @@ -1419,16 +1339,14 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, } private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, - KVMStoragePool destPool, int timeout) { + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout){ /* - * With RBD you can't run qemu-img convert with an existing RBD image as - * destination - * qemu-img will exit with the error that the destination already exists. - * So for RBD we don't create the image, but let qemu-img do that for us. - * - * We then create a KVMPhysicalDisk object that we can return + With RBD you can't run qemu-img convert with an existing RBD image as destination + qemu-img will exit with the error that the destination already exists. + So for RBD we don't create the image, but let qemu-img do that for us. + + We then create a KVMPhysicalDisk object that we can return */ KVMStoragePool srcPool = template.getPool(); @@ -1447,13 +1365,14 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, disk.setVirtualSize(disk.getSize()); } + QemuImgFile srcFile; QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath())); destFile.setFormat(format); if (srcPool.getType() != StoragePoolType.RBD) { srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - try { + try{ QemuImg qemu = new QemuImg(timeout); qemu.convert(srcFile, destFile); } catch (QemuImgException | LibvirtException e) { @@ -1471,14 +1390,9 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, */ try { - if ((srcPool.getSourceHost().equals(destPool.getSourceHost())) - && (srcPool.getSourceDir().equals(destPool.getSourceDir()))) { - /* - * We are on the same Ceph cluster, but we require RBD format 2 on the source - * image - */ - logger.debug( - "Trying to perform a RBD clone (layering) since we are operating in the same storage pool"); + if ((srcPool.getSourceHost().equals(destPool.getSourceHost())) && (srcPool.getSourceDir().equals(destPool.getSourceDir()))) { + /* We are on the same Ceph cluster, but we require RBD format 2 on the source image */ + logger.debug("Trying to perform a RBD clone (layering) since we are operating in the same storage pool"); Rados r = new Rados(srcPool.getAuthUserName()); r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); @@ -1494,18 +1408,15 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, if (srcImage.isOldFormat()) { /* The source image is RBD format 1, we have to do a regular copy */ logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() + - " is RBD format 1. We have to perform a regular copy (" - + toHumanReadableSize(disk.getVirtualSize()) + " bytes)"); + " is RBD format 1. We have to perform a regular copy (" + toHumanReadableSize(disk.getVirtualSize()) + " bytes)"); rbd.create(disk.getName(), disk.getVirtualSize(), RBD_FEATURES, rbdOrder); RbdImage destImage = rbd.open(disk.getName()); - logger.debug("Starting to copy " + srcImage.getName() + " to " + destImage.getName() - + " in Ceph pool " + srcPool.getSourceDir()); + logger.debug("Starting to copy " + srcImage.getName() + " to " + destImage.getName() + " in Ceph pool " + srcPool.getSourceDir()); rbd.copy(srcImage, destImage); - logger.debug("Finished copying " + srcImage.getName() + " to " + destImage.getName() - + " in Ceph pool " + srcPool.getSourceDir()); + logger.debug("Finished copying " + srcImage.getName() + " to " + destImage.getName() + " in Ceph pool " + srcPool.getSourceDir()); rbd.close(destImage); } else { logger.debug("The source image " + srcPool.getSourceDir() + "/" + template.getName() @@ -1513,12 +1424,12 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, + rbdTemplateSnapName); /* The source image is format 2, we can do a RBD snapshot+clone (layering) */ + logger.debug("Checking if RBD snapshot " + srcPool.getSourceDir() + "/" + template.getName() + "@" + rbdTemplateSnapName + " exists prior to attempting a clone operation."); List snaps = srcImage.snapList(); - logger.debug("Found " + snaps.size() + " snapshots on RBD image " + srcPool.getSourceDir() + "/" - + template.getName()); + logger.debug("Found " + snaps.size() + " snapshots on RBD image " + srcPool.getSourceDir() + "/" + template.getName()); boolean snapFound = false; for (RbdSnapInfo snap : snaps) { if (rbdTemplateSnapName.equals(snap.name)) { @@ -1537,18 +1448,13 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } rbd.clone(template.getName(), rbdTemplateSnapName, io, disk.getName(), RBD_FEATURES, rbdOrder); - logger.debug("Successfully cloned " + template.getName() + "@" + rbdTemplateSnapName + " to " - + disk.getName()); - /* - * We also need to resize the image if the VM was deployed with a larger root - * disk size - */ + logger.debug("Successfully cloned " + template.getName() + "@" + rbdTemplateSnapName + " to " + disk.getName()); + /* We also need to resize the image if the VM was deployed with a larger root disk size */ if (disk.getVirtualSize() > template.getVirtualSize()) { RbdImage diskImage = rbd.open(disk.getName()); diskImage.resize(disk.getVirtualSize()); rbd.close(diskImage); - logger.debug( - "Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); + logger.debug("Resized " + disk.getName() + " to " + toHumanReadableSize(disk.getVirtualSize())); } } @@ -1556,12 +1462,8 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, rbd.close(srcImage); r.ioCtxDestroy(io); } else { - /* - * The source pool or host is not the same Ceph cluster, we do a simple copy - * with Qemu-Img - */ - logger.debug( - "Both the source and destination are RBD, but not the same Ceph cluster. Performing a copy"); + /* The source pool or host is not the same Ceph cluster, we do a simple copy with Qemu-Img */ + logger.debug("Both the source and destination are RBD, but not the same Ceph cluster. Performing a copy"); Rados rSrc = new Rados(srcPool.getAuthUserName()); rSrc.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); @@ -1583,16 +1485,14 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, IoCTX dIO = rDest.ioCtxCreate(destPool.getSourceDir()); Rbd dRbd = new Rbd(dIO); - logger.debug("Creating " + disk.getName() + " on the destination cluster " - + rDest.confGet("mon_host") + " in pool " + + logger.debug("Creating " + disk.getName() + " on the destination cluster " + rDest.confGet("mon_host") + " in pool " + destPool.getSourceDir()); dRbd.create(disk.getName(), disk.getVirtualSize(), RBD_FEATURES, rbdOrder); RbdImage srcImage = sRbd.open(template.getName()); RbdImage destImage = dRbd.open(disk.getName()); - logger.debug("Copying " + template.getName() + " from Ceph cluster " + rSrc.confGet("mon_host") - + " to " + disk.getName() + logger.debug("Copying " + template.getName() + " from Ceph cluster " + rSrc.confGet("mon_host") + " to " + disk.getName() + " on cluster " + rDest.confGet("mon_host")); sRbd.copy(srcImage, destImage); @@ -1614,14 +1514,13 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } @Override - public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, - long size, KVMStoragePool destPool) { + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { return null; } @Override public List listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; StoragePool virtPool = libvirtPool.getPool(); List disks = new ArrayList(); try { @@ -1644,44 +1543,35 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt /** * This copies a volume from Primary Storage to Secondary Storage * - * In theory it could also do it the other way around, but the current - * implementation - * in ManagementServerImpl shows that the destPool is always a Secondary Storage - * Pool + * In theory it could also do it the other way around, but the current implementation + * in ManagementServerImpl shows that the destPool is always a Secondary Storage Pool */ @Override - public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, - byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { /** - * With RBD you can't run qemu-img convert with an existing RBD image as - * destination - * qemu-img will exit with the error that the destination already exists. - * So for RBD we don't create the image, but let qemu-img do that for us. - * - * We then create a KVMPhysicalDisk object that we can return - * - * It is however very unlikely that the destPool will be RBD, since it isn't - * supported - * for Secondary Storage + With RBD you can't run qemu-img convert with an existing RBD image as destination + qemu-img will exit with the error that the destination already exists. + So for RBD we don't create the image, but let qemu-img do that for us. + + We then create a KVMPhysicalDisk object that we can return + + It is however very unlikely that the destPool will be RBD, since it isn't supported + for Secondary Storage */ KVMStoragePool srcPool = disk.getPool(); - /* - * Linstor images are always stored as RAW, but Linstor uses qcow2 in DB, - * to support snapshots(backuped) as qcow2 files. - */ - PhysicalDiskFormat sourceFormat = srcPool.getType() != StoragePoolType.Linstor ? disk.getFormat() - : PhysicalDiskFormat.RAW; + /* Linstor images are always stored as RAW, but Linstor uses qcow2 in DB, + to support snapshots(backuped) as qcow2 files. */ + PhysicalDiskFormat sourceFormat = srcPool.getType() != StoragePoolType.Linstor ? + disk.getFormat() : PhysicalDiskFormat.RAW; String sourcePath = disk.getPath(); KVMPhysicalDisk newDisk; - logger.debug("copyPhysicalDisk: disk size:{}, virtualsize:{} format:{}", toHumanReadableSize(disk.getSize()), - toHumanReadableSize(disk.getVirtualSize()), disk.getFormat()); + logger.debug("copyPhysicalDisk: disk size:{}, virtualsize:{} format:{}", toHumanReadableSize(disk.getSize()), toHumanReadableSize(disk.getVirtualSize()), disk.getFormat()); if (destPool.getType() != StoragePoolType.RBD) { if (disk.getFormat() == PhysicalDiskFormat.TAR) { - newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, - disk.getVirtualSize(), null); + newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, disk.getVirtualSize(), null); } else { newDisk = destPool.createPhysicalDisk(name, Storage.ProvisioningType.THIN, disk.getVirtualSize(), null); } @@ -1699,15 +1589,15 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt try { qemu = new QemuImg(timeout); - } catch (QemuImgException | LibvirtException ex) { + } catch (QemuImgException | LibvirtException ex ) { throw new CloudRuntimeException("Failed to create qemu-img command", ex); } QemuImgFile srcFile = null; QemuImgFile destFile = null; if ((srcPool.getType() != StoragePoolType.RBD) && (destPool.getType() != StoragePoolType.RBD)) { - if (sourceFormat == PhysicalDiskFormat.TAR && destFormat == PhysicalDiskFormat.DIR) { // LXC template - Script.runSimpleBashScript("cp " + sourcePath + " " + destPath); + if(sourceFormat == PhysicalDiskFormat.TAR && destFormat == PhysicalDiskFormat.DIR) { //LXC template + Script.runSimpleBashScript("cp "+ sourcePath + " " + destPath); } else if (sourceFormat == PhysicalDiskFormat.TAR) { Script.runSimpleBashScript("tar -x -f " + sourcePath + " -C " + destPath, timeout); } else if (sourceFormat == PhysicalDiskFormat.DIR) { @@ -1729,30 +1619,26 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt destFile = new QemuImgFile(destPath, destFormat); try { boolean isQCOW2 = PhysicalDiskFormat.QCOW2.equals(sourceFormat); - qemu.convert(srcFile, destFile, null, null, - new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), + qemu.convert(srcFile, destFile, null, null, new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), null, false, isQCOW2); Map destInfo = qemu.info(destFile); Long virtualSize = Long.parseLong(destInfo.get(QemuImg.VIRTUAL_SIZE)); newDisk.setVirtualSize(virtualSize); newDisk.setSize(virtualSize); } catch (QemuImgException e) { - logger.error("Failed to convert [{}] to [{}] due to: [{}].", srcFile.getFileName(), - destFile.getFileName(), e.getMessage(), e); + logger.error("Failed to convert [{}] to [{}] due to: [{}].", srcFile.getFileName(), destFile.getFileName(), e.getMessage(), e); newDisk = null; } } } catch (QemuImgException e) { - logger.error("Failed to fetch the information of file " + srcFile.getFileName() + " the error was: " - + e.getMessage()); + logger.error("Failed to fetch the information of file " + srcFile.getFileName() + " the error was: " + e.getMessage()); newDisk = null; } } } else if ((srcPool.getType() != StoragePoolType.RBD) && (destPool.getType() == StoragePoolType.RBD)) { /** * Using qemu-img we copy the QCOW2 disk to RAW (on RBD) directly. - * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph - * Dumpling) + * To do so it's mandatory that librbd on the system is at least 0.67.7 (Ceph Dumpling) */ logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2"); try { @@ -1761,11 +1647,9 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath); destFile = new QemuImgFile(rbdDestFile, destFormat); - logger.debug( - "Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); + logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); qemu.convert(srcFile, destFile); - logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " - + rbdDestPath); + logger.debug("Successfully converted source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath); /* We have to stat the RBD image to see how big it became afterwards */ Rados r = new Rados(destPool.getAuthUserName()); @@ -1782,32 +1666,26 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt RbdImageInfo rbdInfo = image.stat(); newDisk.setSize(rbdInfo.size); newDisk.setVirtualSize(rbdInfo.size); - logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " - + toHumanReadableSize(rbdInfo.size) + " bytes long"); + logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + toHumanReadableSize(rbdInfo.size) + " bytes long"); rbd.close(image); r.ioCtxDestroy(io); } catch (QemuImgException | LibvirtException e) { String srcFilename = srcFile != null ? srcFile.getFileName() : null; String destFilename = destFile != null ? destFile.getFileName() : null; - logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, - destFilename, e.getMessage())); + logger.error(String.format("Failed to convert from %s to %s the error was: %s", srcFilename, destFilename, e.getMessage())); newDisk = null; } catch (RadosException e) { - logger.error( - "A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); + logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; } catch (RbdException e) { - logger.error( - "A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); + logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage()); newDisk = null; } } else { /** - * We let Qemu-Img do the work here. Although we could work with librbd and have - * that do the cloning - * it doesn't benefit us. It's better to keep the current code in place which - * works + We let Qemu-Img do the work here. Although we could work with librbd and have that do the cloning + it doesn't benefit us. It's better to keep the current code in place which works */ srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, sourcePath)); srcFile.setFormat(sourceFormat); @@ -1821,8 +1699,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt try { qemu.convert(srcFile, destFile); } catch (QemuImgException | LibvirtException e) { - logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() - + " the error was: " + e.getMessage()); + logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage()); newDisk = null; } } @@ -1836,7 +1713,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt @Override public boolean refresh(KVMStoragePool pool) { - LibvirtStoragePool libvirtPool = (LibvirtStoragePool) pool; + LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; StoragePool virtPool = libvirtPool.getPool(); try { refreshPool(virtPool); @@ -1863,4 +1740,4 @@ private void deleteVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtEx private void deleteDirVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException { Script.runSimpleBashScript("rm -r --interactive=never " + vol.getPath()); } -} \ No newline at end of file +} diff --git a/test/integration/plugins/ontap/OVERVIEW.html b/test/integration/plugins/ontap/OVERVIEW.html new file mode 100644 index 000000000000..c17f796bf252 --- /dev/null +++ b/test/integration/plugins/ontap/OVERVIEW.html @@ -0,0 +1,1969 @@ + + + + + + + + ONTAP Integration Tests — Team Overview + + + + + + +
+
Apache CloudStack · NetApp ONTAP Plugin
+

Integration Tests — Team Overview

+
+ A guide to the Marvin-based end-to-end test suite: how it works, how it's structured, and how to read the + code. + Written for engineers seeing these tests for the first time. +
+
+ + + + + +
+ + + + + +
+ + +
+ +
+
🏗
+

Big Picture

+
+ +

+ These are end-to-end integration tests for the NetApp ONTAP primary storage plugin in Apache + CloudStack. + Every test drives real CloudStack API calls and then cross-checks the result on the actual ONTAP + system. + Both sides must agree for a test to pass. +

+ +
+
+ 🧪 +
Test Code
+
Python
Marvin framework
your laptop
+
+ +
+
+
CS API :8096
+
+ +
+ ☁️ +
CloudStack
+
Management server
KVM agent
10.193.56.62
+
+ +
+
+
ONTAP REST :443
+
+ +
+ 🔷 +
NetApp ONTAP
+
ONTAP REST API
SVM: vs0
10.196.38.187
+
+
+ +
+
+
☁️
+

CloudStack side

+

Pool state, volume listing, VM state — verified via + listStoragePools, listVolumes, listVirtualMachines. +

+
+
+
🔷
+

ONTAP side

+

FlexVol state, LUNs, igroups, export policies, LUN-maps — verified via + direct ONTAP REST API calls from OntapRestClient.

+
+
+
🔗
+

Both must agree

+

A test only passes if the CS API and the ONTAP REST API both report + the expected state. Orphaned ONTAP objects cause test failures.

+
+
+
+ + +
+ +
+
📂
+

Directory Layout

+
+ +

All test files live under test/integration/plugins/ontap/. The tree mirrors the protocol + × concern matrix.

+ +
+ test/integration/plugins/ontap/ + ├── ontap.cfg ← environment + config: IPs, credentials, zone info + ├── ontap_test_base.py ← + shared base class + ONTAP REST client (imported by all test files) + ├── TEST_CASES.html ← test + case reference (this repo) + ├── OVERVIEW.html ← this + file + │ + ├── nfs3/ ← NFS3 protocol + tests + │ ├── pool/ + │ │ ├── test_pool_lifecycle.py 8 tests — create/disable/enable/maintenance/delete + │ │ ├── test_pool_with_volumes.py 7 tests — same lifecycle with a live CS volume + │ │ └── test_zone_scoped_pool.py 4 tests — zone scope (attachZone) + │ ├── volume/ + │ │ └── test_volume_lifecycle.py 5 tests — CS volume create/delete semantics + │ └── instance/ + │ └── test_vm_volume_attach.py 8 tests — pool + VM + hot attach/detach + │ + └── iscsi/ ← iSCSI protocol + tests (mirrors nfs3/) + ├── pool/ + │ ├── test_pool_lifecycle.py 8 tests — iSCSI pool + igroup assertions + │ ├── test_pool_with_volumes.py 7 tests — pool with LUN-backed volume + │ └── test_zone_scoped_pool.py 4 tests — zone scope + ├── volume/ + │ └── test_volume_lifecycle.py 5 tests — LUN create/delete per CS volume + └── instance/ + └── test_vm_volume_attach.py 8 tests — pool + VM + LUN-map lifecycle +
+ +
+
💡
+
+ Why is ontap_test_base.py in the parent folder? + All 10 test files share the same base class. Keeping it at the top level means one import + (from ontap_test_base import …) works from any subdirectory — as long as you set + PYTHONPATH=test/integration/plugins/ontap when running the tests. +
+
+
+ + +
+ +
+
🔬
+

The Marvin Framework

+
+ +

Marvin is CloudStack's own Python-based integration test framework. It ships inside the + CloudStack repo at tools/marvin/.

+ +
+
+

What Marvin gives you

+
    +
  • cloudstackTestCase — base class for all test classes
    • +
  • getClsTestClient() — reads ontap.cfg, connects to CloudStack + API and MySQL
    • +
  • getApiClient() — pre-authenticated CloudStack API client
    • +
  • getParsedTestDataConfig() — parsed ontap.cfg as a Python dict +
    • +
  • Auto-discovery of all test_NN_* methods and runs them sorted
    • +
  • @attr(tags=[…]) — tag-based test filtering
    • +
+
+
+

What Marvin does NOT do

+
    +
  • Marvin does not talk to ONTAP — that's our OntapRestClient
    • +
  • Marvin does not spin up CloudStack — you need a running management server
    • +
  • Marvin does not clean up after failed tests automatically — tearDownClass + handles it
    • +
  • Marvin is not pytest — it uses Python's unittest runner under the hood via + nosetests
    • +
+
+
+ +
+
⚠️
+
+ Running Marvin — always use python3 -m nose, not the installed + nosetests binary. On macOS the binary may have a stale shebang pointing to a + non-existent Python from the CLT toolchain. The -m nose form always uses the + correct interpreter. +
+
+
+ + +
+ +
+
🔍
+

Test File Anatomy

+
+ +

Every test file follows the same 5-part structure. The example below is from + nfs3/pool/test_pool_lifecycle.py.

+ +
+ 
+# ① Apache 2.0 license header (required on every source file)
+# Licensed to the Apache Software Foundation ...
+
+"""
+② Module docstring — workflow summary, prerequisites, run command
+Sequential workflow integration tests for NFS3 primary storage pool.
+Workflow: 01 Create  02 Disable  03 Enable  04 Maintenance ...
+"""
+
+# ③ Imports — Marvin + ONTAP base classes
+from marvin.cloudstackAPI import createStoragePool as createStoragePoolAPI
+from marvin.lib.base import StoragePool
+from ontap_test_base import OntapRestClient, OntapTestBase
+
+# ④ TestData — config values + createStoragePool parameters
+class TestData:
+    def __init__(self, storage_ip, svm_name, username, password, ...):
+        self.testdata = {
+            "primaryStorage": {
+                "managed": True, "capacitybytes": 3355443200,
+                "details": { "protocol": "NFS3", "storageIP": storage_ip, ... }
+            }, ...
+        }
+
+# ⑤ Test class — sequential numbered methods
+class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase):
+
+    pool      = None   # ← class-level shared state
+    volume    = None
+    pool_ep_name = None
+
+    @classmethod
+    def setUpClass(cls): ...   # connect, resolve zone/cluster/hosts
+
+    def _create_pool(self): ...  # helper — NOT a test
+
+    @attr(tags=["nfs3_workflow"], required_hardware=True)
+    def test_01_create_primary_storage_pool(self): ...
+    def test_02_disable_storage_pool(self): ...
+    def test_03_enable_storage_pool(self): ...
+
+
+
+ ① + Apache 2.0 license header — required by repo policy. Checked by + Apache RAT on every PR. +
+
+ ② + Module docstring — tells the reader what workflow the file covers and + how to run it standalone. +
+
+ ③ + Marvin API imports + our shared ontap_test_base. No + credentials in code. +
+
+ ④ + TestData holds all config values. It reads from + ontap.cfg via setUpClass — never hard-codes IPs or + passwords. +
+
+ ⑤ + The test class. Methods named test_NN_* are discovered + and run in sorted order by nosetests. +
+
+
+
+ + +
+ +
+
💡
+

Key Code Patterns

+
+

Four patterns appear in every test file. Understanding these is the key to reading any test.

+ + +
+
+
1
+
Class-level state — always self.__class__.attr
+ Most common mistake +
+
+
+
+
+ ❌ Wrong +
+ 
+def test_01_create_pool(self):
+    pool = self._create_pool()
+    self.pool = pool  # instance attr
+                      # ← GONE after test_01 ends!
+
+def test_02_disable_pool(self):
+    self.pool.id  # ← AttributeError
+
+
+
+ ✓ Correct +
+ 
+def test_01_create_pool(self):
+    pool = self._create_pool()
+    self.__class__.pool = pool
+    # ↑ class attr — survives all tests
+
+def test_02_disable_pool(self):
+    self.__class__.pool.id  # ✓ works
+
+
+
+
ℹ️
+
nosetests creates a new instance of the test class for every test + method. Instance attributes (self.pool) are thrown away between tests. + Class attributes (self.__class__.pool) persist for the lifetime of the + class — i.e., for the whole suite.
+
+
+
+ + +
+
+
2
+
Guard assertions — fail fast with a clear message
+
+
+
+
+ Python + any test file — first line of every test after + test_01 +
+ 
+@attr(tags=["nfs3_workflow"], required_hardware=True)
+def test_03_enable_storage_pool(self):
+    self.assertIsNotNone(
+        self.__class__.pool,
+        "Pool absent — test_01 must pass first"
+    )
+    # rest of test ...
+
+
+
✅
+
Without this guard, a missing pool causes an AttributeError + deep in the test body — confusing to read. With the guard, the failure message + immediately tells you which earlier test to fix.
+
+
+
+ + +
+
+
3
+
Creating a storage pool — indexed details[N].key syntax +
+ Critical +
+
+
+
+ Python + _create_pool() helper — same pattern in every file +
+ 
+def _create_pool(self):
+    ps = self.testdata["primaryStorage"]
+    cmd = createStoragePoolAPI.createStoragePoolCmd()
+    cmd.name       = "OntapNFS3_12345"
+    cmd.url        = "nfs://10.196.38.187/ontap"
+    cmd.zoneid     = self.zone.id
+    cmd.clusterid  = self.cluster.id
+    cmd.podid      = self.cluster.podid
+    cmd.scope      = "CLUSTER"
+    cmd.provider   = "NetApp ONTAP"
+    cmd.tags       = "ontap-nfs3"
+    cmd.managed    = True
+
+    count = 1
+    for key, value in ps["details"].items():
+        setattr(cmd, "details[{}].{}".format(count, key), value)
+        count += 1
+    # ↑ This produces: details[1].protocol="NFS3", details[2].storageIP=..., etc.
+    # NEVER use StoragePool.create() — it does not support this indexed syntax.
+
+    response = self.apiClient.createStoragePool(cmd)
+    return StoragePool(response.__dict__)
+
+
+
⚠️
+
The CloudStack API for createStoragePool passes plugin-specific details as + numbered index parameters (details[1].key, + details[1].value, …). The Marvin helper StoragePool.create() + does not generate this format. Always build the command manually as shown above.
+
+
+
+ + +
+
+
4
+
Polling for async state changes
+
+
+
+
+ Python + inherited from OntapTestBase._poll_pool_state() +
+ 
+# CloudStack operations are asynchronous — state changes take time.
+# Never read state directly after an API call:
+def test_04_enter_maintenance_mode(self):
+    cmd = enableStorageMaintenance.enableStorageMaintenanceCmd()
+    cmd.id = self.__class__.pool.id
+    self.apiClient.enableStorageMaintenance(cmd)
+
+    result = self._poll_pool_state(
+        self.__class__.pool.id,
+        "Maintenance",
+        timeout=120          # seconds to wait
+    )
+    self.assertEqual(result.state, "Maintenance")
+
+# _poll_pool_state() calls listStoragePools every 5s until
+# the state matches or timeout is exceeded.
+
+
+
+
+ + +
+ +
+
🏛
+

Shared Base Class — ontap_test_base.py

+
+ +

+ All 10 test classes extend OntapTestBase. It handles the boilerplate that would + otherwise appear in every file: connecting to CloudStack, resolving zone/cluster/hosts, creating a + test account, and cleaning up after the suite runs. +

+ +
+
+

OntapTestBase — what setUpClass does

+
+
+
+
1
+
+
+
+
Connect to CloudStack
+
Reads ontap.cfg, opens API client on port 8096, + opens MySQL connection.
+
+
+
+
+
2
+
+
+
+
Resolve zone, pod, cluster
+
Calls get_zone(), list_clusters() + to find the first available KVM cluster.
+
+
+
+
+
3
+
+
+
+
List cluster hosts
+
Calls listHosts to get all KVM hosts — needed + for export policy and igroup assertions.
+
+
+
+
+
4
+
+
+
+
Create test account + disk offering
+
Creates a temporary CloudStack account and a matching disk + offering used for volumes.
+
+
+
+
+
5
+
+
+
+
tearDownClass (cleanup)
+
Best-effort: force-deletes the pool, volume, disk offering, + account. Runs even if tests fail.
+
+
+
+
+ +
+

OntapTestBase — methods you call in tests

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
MethodPurpose
_poll_pool_state(id, state, timeout)Polls listStoragePools until target state or + timeout
_create_volume(pool_id)Creates a CloudStack data volume on the given pool
_delete_pool(pool_id, forced)Enters Maintenance then calls deleteStoragePool +
_parse_pool_details(pool)Extracts key→value dict from pool details attribute
+
+ +

Class attributes set by setUpClass

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AttributeWhat it holds
cls.zoneFirst available CloudStack zone
cls.clusterFirst KVM cluster in that zone
cls.cluster_hostsList of KVM hosts in the cluster
cls.accountTemporary test account object
cls.domainRoot domain
cls.ontapOntapRestClient instance (set by each subclass)
+
+
+
+ + + +

OntapRestClient — the ONTAP side verifier

+

OntapRestClient is a thin HTTPS wrapper around the ONTAP REST API. Every assertion that + starts with "ONTAP:" uses one of these methods.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodONTAP REST endpoint calledUsed to assert
get_volume(name)GET /api/storage/volumes?name=<n>FlexVol exists and state == "online"
get_export_policy(name)GET /api/protocols/nfs/export-policies?name=<n>NFS3 export policy exists with correct client IPs
get_data_lifs(svm_name)GET /api/network/ip/interfaces?svm.name=<n>At least one NFS/iSCSI data LIF is present on SVM
get_igroup(svm_name, name)GET /api/protocols/san/igroups?name=<n>iSCSI igroup exists; host IQN is in its initiator list
list_luns_in_volume(svm, vol_name)GET /api/storage/luns?location.volume.name=<n>LUN created/removed inside the pool's FlexVol
list_lun_maps_for_volume(svm, vol_name)GET /api/protocols/san/lun-maps?lun.location.volume.name=<n>LUN-map created on attach, removed on VM stop/detach
list_files_in_volume(svm, vol_name)GET /api/storage/volumes/<uuid>/filesData file for volume UUID present after NFS3 attach
+
+ +
+
+ Python — example assertion using OntapRestClient + from test_01 in nfs3/pool/test_pool_lifecycle.py +
+ 
+# After createStoragePool succeeds on CloudStack side,
+# verify the corresponding ONTAP FlexVol was created and is online:
+ontap_vol = self.ontap.get_volume(pool.name)
+
+self.assertIsNotNone(
+    ontap_vol,
+    "ONTAP FlexVol not found for pool '%s'" % pool.name
+)
+self.assertEqual(
+    ontap_vol.get("state"), "online",
+    "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state")
+)
+
+
+ + +
+ +
+
⚙️
+

Configuration — ontap.cfg

+
+ +

ontap.cfg is a JSON file that Marvin reads at startup to find CloudStack, MySQL, and + ONTAP. Never commit real credentials. The file is gitignored.

+ +
+
+
+
+ JSON + ontap.cfg — skeleton +
+ 
+{
+  "mgtSvr": [{
+    "mgtSvrIp": "<CS_IP>",
+    "port":     8096,
+    "user":     "admin",
+    "passwd":   "password"
+  }],
+  "dbSvr": {
+    "dbSvr":  "<CS_IP>",
+    "port":   3306,
+    "user":   "cloud",
+    "passwd": "cloud"
+  },
+  "ontap": {
+    "storageIP": "<ONTAP_IP>",
+    "svmName":   "vs0",
+    "username":  "admin",
+    "password":  "<pw>"
+  }
+}
+
+
+ +
+

Key fields explained

+
+
mgtSvr[0].mgtSvrIp
+
CloudStack management server IP — where the CS API runs
+
mgtSvr[0].port
+
8096 = integration API (no auth). Must be enabled in CS + config.
+
dbSvr.dbSvr
+
MySQL server IP — Marvin uses this for direct DB queries
+
ontap.storageIP
+
ONTAP cluster management IP — used by OntapRestClient for + REST calls
+
ontap.svmName
+
The SVM (Storage Virtual Machine) that hosts NFS and iSCSI services +
+
ontap.username/password
+
ONTAP admin credentials — used for REST API authentication only
+
+
+
+
+ + +
+ +
+
▶️
+

Running the Tests

+
+ +
+
📌
+
+ Always run from the repo root — PYTHONPATH must point at the + ontap/ folder so Python can find ontap_test_base.py when running files + in subdirectories. +
+
+ +
+
+
+
+
+   All ONTAP tests (~60–90 min) +
+
+

+ PYTHONPATH=test/integration/plugins/ontap \
+ python3 -m nose --with-marvin \
+     --marvin-config=test/integration/plugins/ontap/ontap.cfg \
+     test/integration/plugins/ontap/ -v +

+
+
+ +
+
+
+
+
+   Single suite +
+
+

+ PYTHONPATH=test/integration/plugins/ontap \
+ python3 -m nose --with-marvin \
+     --marvin-config=test/integration/plugins/ontap/ontap.cfg \
+     test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py + -v +

+
+
+ +
+
+
+
+
+   By tag — only iSCSI tests +
+
+

+ PYTHONPATH=test/integration/plugins/ontap \
+ python3 -m nose --with-marvin \
+     --marvin-config=test/integration/plugins/ontap/ontap.cfg \
+     -a tags=iscsi_workflow \
+     test/integration/plugins/ontap/ -v +

+
+
+ +

Where to find test results

+
+
+

/tmp/marvin_last_run.txt
stdout + stderr summary of the last + run

+
+
+

/tmp/MarvinLogs/<timestamp>/
results.txt — per-test + pass/fail · runinfo.txt — full API trace

+
+
+
+ + +
+ +
+
🔄
+

Test Execution Flow

+
+ +

Here is exactly what happens when you run a test suite, step by step.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PhaseWho runs itWhat happens
Startupnosetests / MarvinReads ontap.cfg, connects to CloudStack API port 8096, + opens MySQL connection
setUpClassTest classResolves zone → pod → cluster → hosts; creates test account + disk + offering; creates OntapRestClient
test_01Test methodCreates storage pool via CloudStack API; asserts CS state; asserts + ONTAP FlexVol state; stores pool in class attr
test_02 … test_NTest methodsEach reads state from the previous step via class attrs; performs one + CloudStack operation; asserts both CS and ONTAP outcomes
tearDownClassOntapTestBaseBest-effort cleanup: force-delete pool (enters Maintenance first), + delete volume, delete disk offering, delete account. Runs even if tests failed.
+
+ +
+
🔗
+
+ Sequential dependency — every test in a suite depends on the one before it. If + test_02 fails, tests 03–08 will hit the guard assertion and fail immediately with a clear + message. Fix earlier failures first. The test order is enforced by alphabetic sorting of method + names — that's why they're all named test_01_…, test_02_… etc. +
+
+ +
+

NFS3 vs iSCSI — what changes between protocols

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AspectNFS3iSCSI
Pool URLnfs://<ip>/ontapiscsi://<ip>/ontap
ONTAP object per poolFlexVol + NFS export policyFlexVol + one igroup per KVM host
ONTAP object per CS volumeNone — FlexVol is shared across volumesOne LUN inside the pool's FlexVol
Host connectivity verified viaget_export_policy() — checks client IP rulesget_igroup() — checks IQN initiator in igroup
VM start/stop ONTAP effectExport policy retained (NFS mount persists)LUN-map removed on stop; re-created on start
CS tagsontap-nfs3ontap-iscsi
+
+
+
+ +
+
+ +
+ Apache CloudStack · NetApp ONTAP Plugin · Integration Test Team Overview · 2026-07-10 +
+ + + + \ No newline at end of file diff --git a/test/integration/plugins/ontap/README.md b/test/integration/plugins/ontap/README.md index f6d2fe44e6d3..e8130c331fb2 100644 --- a/test/integration/plugins/ontap/README.md +++ b/test/integration/plugins/ontap/README.md @@ -16,13 +16,282 @@ specific language governing permissions and limitations under the License. --> -# ONTAP plugin — Marvin integration tests +# NetApp ONTAP Integration Tests — README -Add Marvin tests here (e.g. `test_ontap_smoke.py`). They can be included in Apache upstream PRs when ready. +This folder contains end-to-end integration tests for the NetApp ONTAP primary storage plugin in Apache CloudStack. The tests use the **Marvin** framework to drive real CloudStack API calls against a live management server and verify outcomes on a real ONTAP storage system. CI wiring: - - Bundles: `private-cicd/marvin/bundles.txt` - Zone config: `private-cicd/marvin/zones/` (downstream only) -Follow patterns in `test/integration/plugins/solidfire/` and `test/integration/plugins/linstor/`. +--- + +## Directory layout + +``` +test/integration/plugins/ontap/ +├── ontap.cfg # Environment config (IPs, credentials, zone info) +├── ontap_test_base.py # Shared base class and ONTAP REST client +├── TEST_CASES.md # Full test case reference table (62 tests) +├── README.md # This file +│ +├── nfs3/ +│ ├── pool/ +│ │ ├── test_pool_lifecycle.py # Pool create/disable/enable/maintenance/delete +│ │ ├── test_pool_with_volumes.py # Same lifecycle with a CS volume present +│ │ └── test_zone_scoped_pool.py # Zone-scoped pool (attachZone) +│ ├── volume/ +│ │ └── test_volume_lifecycle.py # Volume create/delete/negative-delete +│ └── instance/ +│ └── test_vm_volume_attach.py # Pool + volume + VM + attach/detach +│ +└── iscsi/ + ├── pool/ + │ ├── test_pool_lifecycle.py # iSCSI pool lifecycle + igroup assertions + │ ├── test_pool_with_volumes.py # Same lifecycle with a LUN-backed volume + │ └── test_zone_scoped_pool.py # Zone-scoped iSCSI pool + ├── volume/ + │ └── test_volume_lifecycle.py # LUN create/delete/negative-delete + └── instance/ + └── test_vm_volume_attach.py # Pool + LUN + VM + attach/LUN-map lifecycle +``` + +--- + +## What is being tested + +The ONTAP plugin (`plugins/storage/volume/ontap/`) integrates CloudStack's primary storage API with the NetApp ONTAP REST API. Every test suite verifies **both sides** of an operation: + +1. **CloudStack side** — the expected `listStoragePools` / `listVolumes` / `listVirtualMachines` state after each API call. +2. **ONTAP side** — the actual ONTAP object state (FlexVol, LUN, igroup, export policy, LUN-map) via direct REST API queries. + +### NFS3 vs iSCSI — key differences + +| Aspect | NFS3 | iSCSI | +|--------|------|-------| +| ONTAP object per pool | FlexVol + export policy | FlexVol + igroup per KVM host | +| ONTAP object per CS volume | None (FlexVol is shared) | One LUN inside the FlexVol | +| Host connectivity | NFS mount | iSCSI login (IQN-based) | +| Volume detach from running VM | Works via virtio hot-unplug | Requires KVM guest to support SCSI hot-unplug | + +--- + +## Prerequisites + +Before running any test: + +1. **CloudStack management server** running with the ONTAP plugin deployed (jar in `/usr/share/cloudstack-management/lib/`). +2. **Integration API port 8096 enabled** — run on the management server: + ```sql + UPDATE configuration SET value='8096' WHERE name='integration.api.port'; + ``` + Then restart: `systemctl restart cloudstack-management` +3. **MySQL accessible remotely** from your laptop (port 3306). If not: + ```bash + sudo sed -i 's/^bind-address.*/bind-address = 0.0.0.0/' /etc/mysql/mysql.conf.d/mysqld.cnf + sudo iptables -I INPUT -p tcp --dport 3306 -j ACCEPT + sudo systemctl restart mysql + ``` +4. **ONTAP SVM** with NFS3 service and/or iSCSI service enabled, and at least one data LIF per protocol. +5. **KVM cluster** registered in CloudStack. For iSCSI tests, every KVM host must have iSCSI configured (its `storageUrl` starts with `iqn.`). +6. **`ontap.cfg` populated** — see the [Configuration](#configuration) section. + +### Python / Marvin setup + +```bash +# Install Marvin from the repo's bundled tarball +python3 -m pip install --user \ + "$(ls tools/marvin/dist/Marvin-*.tar.gz | tail -1)" + +# Verify +python3 -c "import marvin; print('Marvin OK')" +``` + +--- + +## Configuration — `ontap.cfg` + +`ontap.cfg` is a JSON file that tells Marvin where CloudStack and ONTAP are. **Never commit real credentials.** + +Key sections: + +```json +{ + "mgtSvr": [{ "mgtSvrIp": "", "port": 8096, "user": "admin", "passwd": "password" }], + "dbSvr": { "dbSvr": "", "port": 3306, "user": "cloud", "passwd": "cloud" }, + "ontap": { "storageIP": "", "svmName": "", "username": "admin", "password": "" } +} +``` + +The test classes read `storageIP`, `svmName`, `username`, and `password` from the `ontap` section at runtime. **No credentials appear in test code.** + +--- + +## Running the tests + +**Always run from the repo root** so that `PYTHONPATH` picks up `ontap_test_base.py`: + +```bash +# All ONTAP tests (takes ~60–90 min) +PYTHONPATH=test/integration/plugins/ontap \ +python3 -m nose --with-marvin \ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \ + test/integration/plugins/ontap/ -v + +# Single suite (e.g. NFS3 pool lifecycle) +PYTHONPATH=test/integration/plugins/ontap \ +python3 -m nose --with-marvin \ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \ + test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py -v + +# By tag (e.g. all iSCSI workflow tests) +PYTHONPATH=test/integration/plugins/ontap \ +python3 -m nose --with-marvin \ + --marvin-config=test/integration/plugins/ontap/ontap.cfg \ + -a tags=iscsi_workflow \ + test/integration/plugins/ontap/ -v +``` + +> **Important:** `PYTHONPATH=test/integration/plugins/ontap` is always required. The test files in subdirectories import `ontap_test_base` from the parent directory; without this prefix, Python cannot find it. + +Test results are written to: +- `/tmp/marvin_last_run.txt` — stdout/stderr summary +- `/tmp/MarvinLogs//results.txt` — per-test pass/fail +- `/tmp/MarvinLogs//runinfo.txt` — full trace with API call details + +--- + +## Code structure — how a test file is organised + +Every test file follows the same layout: + +``` +1. Apache 2.0 license header +2. Module docstring ← workflow summary, prerequisites, run command +3. Imports +4. TestData class ← holds all config values read from ontap.cfg; builds the + createStoragePool command parameters +5. Test class (extends OntapTestBase) + ├── Class-level state attributes (pool, volume, vm, etc.) initialised to None + ├── setUpClass() ← connects to CloudStack; creates a test account and + │ disk offering; resolves zone/cluster/hosts + ├── tearDownClass() ← best-effort cleanup: deletes pool (forced=True), + │ volume, account, disk offering + ├── Helper methods ← _create_pool(), _create_volume(), _poll_pool_state(), + │ _lun_maps() (iSCSI only), etc. + └── test_01 … test_N ← sequential, numbered test methods +``` + +### Key patterns to know + +**Sequential state sharing — always use `self.__class__.`** + +Tests share state via class attributes, never instance attributes: +```python +# Correct +self.__class__.pool = pool +pool = self.__class__.pool + +# Wrong — state is lost between test method invocations +self.pool = pool +``` + +**Guard assertion at the start of every test (except test_01)** + +Every test after the first starts with an assertion that the previous step's resource exists. This produces a clear, readable failure message instead of a confusing `AttributeError`: +```python +def test_03_enable_storage_pool(self): + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") +``` + +**Creating a storage pool — always use indexed `details[N].key` syntax** + +The CloudStack API for `createStoragePool` requires plugin details to be passed as indexed parameters. **Never call `StoragePool.create()` directly** — it does not support this syntax: +```python +count = 1 +for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 +``` + +**Polling for async state changes** + +CloudStack operations are asynchronous. Use `_poll_pool_state()` rather than reading state immediately after an API call: +```python +result = self._poll_pool_state(pool.id, "Maintenance", timeout=120) +self.assertEqual(result.state, "Maintenance") +``` + +--- + +## Shared base — `ontap_test_base.py` + +`OntapTestBase` provides everything individual test classes inherit: + +| What | Purpose | +|------|---------| +| `_setup_cloudstack_resources()` | Creates a test account, domain, disk offering; resolves zone/cluster/hosts | +| `tearDownClass()` | Best-effort cleanup: deletes pool (forced=True), volume, disk offering, account | +| `_poll_pool_state(pool_id, state, timeout)` | Polls `listStoragePools` until pool reaches the target state | +| `_create_volume(pool_id)` | Creates a CloudStack data volume on the given pool | +| `_delete_pool(pool_id, forced)` | Enters Maintenance then calls `deleteStoragePool` | +| `_parse_pool_details(pool)` | Extracts key→value pairs from the pool's `details` list | +| `OntapRestClient` | Thin HTTPS client for ONTAP REST API calls | + +### `OntapRestClient` methods at a glance + +| Method | What it checks | Used in | +|--------|---------------|---------| +| `get_volume(name)` | FlexVol existence and state | All suites | +| `get_export_policy(name)` | NFS export policy existence | NFS3 suites | +| `get_data_lifs(svm_name)` | NFS data LIF count | NFS3 pool lifecycle | +| `get_igroup(svm_name, name)` | iSCSI igroup existence and initiator list | iSCSI suites | +| `list_luns_in_volume(svm_name, vol_name)` | LUNs present in a FlexVol | iSCSI volume/instance suites | +| `list_lun_maps_for_volume(svm_name, vol_name)` | Active LUN-maps for a volume | iSCSI instance suite | +| `list_files_in_volume(svm_name, vol_name)` | Files inside a FlexVol | NFS3 instance suite | + +--- + +## Test suite quick reference + +| Suite | File | Tests | What it covers | +|-------|------|-------|---------------| +| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete | +| NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 7 | Same + live volume present; negative delete guard | +| NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 4 | Zone scope — all hosts connected via `attachZone` | +| NFS3 Volume Lifecycle | `nfs3/volume/test_volume_lifecycle.py` | 5 | Volume is metadata-only; FlexVol unchanged on delete | +| NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle with hot-plug/detach | +| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete + igroups | +| iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 7 | Same + live LUN present; negative delete guard | +| iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 4 | Zone scope | +| iSCSI Volume Lifecycle | `iscsi/volume/test_volume_lifecycle.py` | 5 | LUN created per CS volume; LUN removed on delete | +| iSCSI VM + Volume Attach | `iscsi/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle; LUN-maps on VM start/stop/detach | + +For the goal, dependencies, and exact success criteria of every individual test, see [TEST_CASES.md](TEST_CASES.md). + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|-------------|-----| +| `ModuleNotFoundError: No module named 'ontap_test_base'` | Missing `PYTHONPATH` prefix | Prefix every run with `PYTHONPATH=test/integration/plugins/ontap` | +| `Marvin Init Failed` | CloudStack API unreachable | Check `mgtSvrIp:8096` is reachable; restart `cloudstack-management` | +| `Lost connection to MySQL` | MySQL not accepting remote connections | Enable remote MySQL access (see Prerequisites §3) | +| `sh: python: command not found` (repeated) | Marvin internal call — harmless on macOS | Ignore; Marvin Init still succeeds | +| Pool state never reaches `Maintenance` | KVM agent not responding | Check `cloudstack-agent` on KVM host; verify host is connected in CloudStack UI | +| iSCSI `test_07` error 530 | KVM guest does not ACK SCSI hot-unplug | Known environment limitation — see TEST_CASES.md Suite 10 note | +| ONTAP REST `401 Unauthorized` | Wrong credentials in `ontap.cfg` | Verify `username`/`password` under `ontap` section | +| `No ready KVM user template available` | Template still downloading | Wait for template `isready=true` in the CloudStack UI, then rerun | + +--- + +## Adding new test cases + +1. Pick the existing file closest to what you need and copy its structure. +2. Read `ontap_test_base.py` for the exact method signatures you can reuse. +3. Copy the `_create_pool()` helper from an existing file that matches your protocol — **never** call `StoragePool.create()`. +4. Number your methods `test_01`, `test_02`, … and add `@attr(tags=[""], required_hardware=True)` to each. +5. Use `self.__class__.` for all state shared between test methods. +6. Syntax-check before the first full run: `python3 -m py_compile .py` +7. Add your test cases to [TEST_CASES.md](TEST_CASES.md). diff --git a/test/integration/plugins/ontap/TEST_CASES.html b/test/integration/plugins/ontap/TEST_CASES.html new file mode 100644 index 000000000000..1bd11a96fe10 --- /dev/null +++ b/test/integration/plugins/ontap/TEST_CASES.html @@ -0,0 +1,2046 @@ + + + + + + + + ONTAP Integration Test Cases + + + + + + +
+
Apache CloudStack · NetApp ONTAP Plugin
+

Integration Test Case Reference

+
+ Marvin end-to-end tests — all 62 test cases across 10 suites — NFS3 & iSCSI protocols +
+
+ + +
+
+
10
+
Suites
+
+
+
62
+
Test Cases
+
+
+
61
+
Passing
+
+
+
1
+
Deferred
+
+
+
52
+
Positive
+
+
+
4
+
Negative
+
+
+
6
+
Cleanup
+
+
+ + +
+ Legend + ✓ positive + ✗ negative + ↩ cleanup + ⚠ deferred +   + 🗄 NFS3 + 💾 iSCSI + 🖥 VM attach + 🌐 Zone scope +
+ + +
+ + +
+
+
🎯
+
+
Goal
+
CloudStack workflow step being exercised
+
+
+
+
🔗
+
+
Depends on
+
Earlier tests that must pass — class-level state they produce
+
+
+
+
☁️
+
+
CloudStack criteria
+
What the CS API must return for the assertion to pass
+
+
+
+
🔷
+
+
ONTAP criteria
+
What the ONTAP REST API must show — FlexVol, LUN, igroup, export policy
+
+
+
+ + + + +
+ + +
+
+
+
Suite 01
+
NFS3 Pool Lifecycle
+
+ 🗄 NFS3 + Cluster scope + nfs3_workflow + nfs3/pool/test_pool_lifecycle.py +
+
+
8tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_primary_storage_poolCreate cluster-scoped NFS3 poolsetUpClass +
    +
  • pool.state == "Up"
    • +
  • pool.type == "NetworkFilesystem"
    • +
  • nfsmountopts contains vers=3
    • +
+ 		+
    +
  • FlexVol exists, state == "online"
    • +
  • Export policy exists with each cluster host IP
    • +
  • ≥1 NFS data LIF on SVM
    • +
+ 		✓ positive
02test_02_disable_storage_poolDisable the pooltest_01 (pool)pool.state == "Disabled" +
    +
  • FlexVol still online
    • +
  • Export policy still present
    • +
+ 		✓ positive
03test_03_enable_storage_poolRe-enable the pooltest_02pool.state == "Up" +
    +
  • FlexVol still online
    • +
  • Export policy still present
    • +
+ 		✓ positive
04test_04_enter_maintenance_modePut pool into maintenancetest_03pool.state == "Maintenance" +
    +
  • FlexVol still online
    • +
  • Export policy unchanged (CS-only state change)
    • +
+ 		✓ positive
05test_05_cancel_maintenance_modeCancel maintenance, return to servicetest_04pool.state == "Up" +
    +
  • FlexVol still online
    • +
  • Export policy still present
    • +
+ 		✓ positive
06test_06_delete_pool_from_maintenanceEnter maintenance then permanently delete pooltest_05Pool not found in listStoragePools +
    +
  • FlexVol deleted
    • +
  • Export policy deleted
    • +
+ 		✓ positive
07test_07_create_volume_on_poolCreate fresh pool + allocate a data volumetest_06 (new pool) +
    +
  • pool.state == "Up"
    • +
  • Volume object non-None
    • +
+ 		+
    +
  • FlexVol online
    • +
  • Export policy present
    • +
+ 		✓ positive
08test_08_delete_volume_and_poolDelete volume then force-delete pooltest_07 (pool, volume) +
    +
  • Volume not listed
    • +
  • Pool not listed
    • +
+ 		+
    +
  • FlexVol deleted
    • +
  • Export policy deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 02
+
NFS3 Pool with Volumes
+
+ 🗄 NFS3 + Cluster scope + nfs3_workflow + nfs3/pool/test_pool_with_volumes.py +
+
+
7tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_pool_and_volumeCreate NFS3 pool + allocate a data volumesetUpClass +
    +
  • pool.state == "Up"
    • +
  • Volume non-None
    • +
+ 		+
    +
  • FlexVol online
    • +
  • Export policy present
    • +
+ 		✓ positive
02test_02_disable_pool_volume_survivesDisable pool — volume must survivetest_01 +
    +
  • pool.state == "Disabled"
    • +
  • Volume still in listVolumes
    • +
+ 		FlexVol still online✓ positive
03test_03_enable_pool_volume_intactRe-enable pool with volume presenttest_02 +
    +
  • pool.state == "Up"
    • +
  • Volume still listed
    • +
+ 		FlexVol still online✓ positive
04test_04_enter_maintenance_volume_presentEnter maintenance with volume presenttest_03 +
    +
  • pool.state == "Maintenance"
    • +
  • Volume still listed
    • +
+ 		FlexVol still online✓ positive
05test_05_cancel_maintenance_with_volumeCancel maintenance with volume — verifies NFS3 cancel-maintenance fixtest_04 +
    +
  • pool.state == "Up"
    • +
  • Volume still listed
    • +
+ 		FlexVol still online✓ positive
06test_06_forced_false_delete_rejectedDelete with forced=False while volume present — must be rejectedtest_05 +
    +
  • CloudstackAPIException raised
    • +
  • Pool still in Maintenance
    • +
+ 		No ONTAP objects removed✗ negative
07test_07_force_delete_pool_and_cleanupCancel maintenance, delete volume, force-delete pooltest_06 +
    +
  • Pool not listed
    • +
  • Volume not listed
    • +
+ 		+
    +
  • FlexVol deleted
    • +
  • Export policy deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 03
+
NFS3 Zone-Scoped Pool
+
+ 🗄 NFS3 + 🌐 Zone scope + zone_pool + nfs3/pool/test_zone_scoped_pool.py +
+
+
4tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_zone_scoped_poolCreate zone-scoped NFS3 pool — CS calls attachZone()setUpClasspool.state == "Up" +
    +
  • FlexVol online
    • +
  • Export policy has every host IP in zone
    • +
  • ≥1 NFS data LIF
    • +
+ 		✓ positive
02test_02_disable_zone_scoped_poolDisable zone-scoped pooltest_01pool.state == "Disabled"FlexVol unchanged; export policy unchanged✓ positive
03test_03_enable_zone_scoped_poolRe-enable zone-scoped pooltest_02pool.state == "Up"FlexVol unchanged; export policy unchanged✓ positive
04test_04_delete_zone_scoped_poolEnter maintenance then delete pooltest_03Pool not listed +
    +
  • FlexVol deleted
    • +
  • Export policy deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 04
+
NFS3 Volume Lifecycle
+
+ 🗄 NFS3 + Cluster scope + nfs3_volume + nfs3/volume/test_volume_lifecycle.py +
+
+
5tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_pool_and_volumeCreate NFS3 pool + data volumesetUpClass +
    +
  • pool.state == "Up"
    • +
  • Volume non-None
    • +
+ 		FlexVol online; export policy present — no new ONTAP object + per volume✓ positive
02test_02_delete_volumeDelete CS volume — only the CS record is removed for NFS3test_01Volume not in listVolumesFlexVol still online and unaffected✓ positive
03test_03_recreate_volume_for_delete_testsRe-create volume (setup for negative tests)test_02New volume object non-NoneFlexVol still online✓ positive
04test_04_forced_false_delete_with_volume_failsEnter maintenance; deleteStoragePool(forced=False) must be rejected + test_03 +
    +
  • CloudstackAPIException raised
    • +
  • Pool still in Maintenance
    • +
+ 		No ONTAP objects removed✗ negative
05test_05_delete_volume_and_force_delete_poolDelete volume then force-delete pool from Maintenancetest_04 +
    +
  • Volume not listed
    • +
  • Pool not listed
    • +
+ 		+
    +
  • FlexVol deleted
    • +
  • Export policy deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 05
+
NFS3 VM + Volume Attach
+
+ 🗄 NFS3 + 🖥 VM attach + vm_volume_workflow + nfs3/instance/test_vm_volume_attach.py +
+
+
8tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_nfs3_poolCreate NFS3 ONTAP primary storage poolsetUpClasspool.state == "Up"FlexVol online; export policy present✓ positive
02test_02_create_ontap_data_volumeAllocate a CloudStack data volume on the ONTAP pooltest_01 (pool)Volume non-None and in listVolumesFlexVol still online✓ positive
03test_03_deploy_vmDeploy VM using first available ready KVM templatetest_02vm.state == "Running"n/a✓ positive
04test_04_attach_volume_to_vmHot-attach ONTAP data volume to running VMtest_03 (vm, volume) +
    +
  • volume.virtualmachineid == vm.id
    • +
  • Attach job succeeds
    • +
+ 		+
    +
  • FlexVol online
    • +
  • Data file for volume UUID present (list_files_in_volume)
    • +
+ 		✓ positive
05test_05_stop_vm_export_retainedStop VM with volume attached — export policy must be retainedtest_04vm.state == "Stopped" +
    +
  • FlexVol still online
    • +
  • Export policy still present
    • +
+ 		✓ positive
06test_06_start_vm_volume_accessibleStart stopped VMtest_05vm.state == "Running"FlexVol still online✓ positive
07test_07_detach_volume_from_vmHot-detach ONTAP volume from running VMtest_06 (vm, volume) +
    +
  • volume.virtualmachineid cleared
    • +
  • volume.state == "Ready"
    • +
+ 		+
    +
  • FlexVol still online
    • +
  • Data file still present (NFS3: file persists until deleteVolume)
    • +
+ 		✓ positive
08test_08_destroy_vm_and_cleanupDestroy VM (expunge), delete volume, delete pooltest_07 +
    +
  • VM not listed
    • +
  • Volume not listed
    • +
  • Pool not listed
    • +
+ 		+
    +
  • FlexVol deleted
    • +
  • Export policy deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 06
+
iSCSI Pool Lifecycle
+
+ 💾 iSCSI + Cluster scope + iscsi_workflow + iscsi/pool/test_pool_lifecycle.py +
+
+
8tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_primary_storage_poolCreate cluster-scoped iSCSI poolsetUpClass +
    +
  • pool.state == "Up"
    • +
  • pool.type == "Iscsi"
    • +
+ 		+
    +
  • FlexVol online
    • +
  • igroup per host with host IQN as initiator
    • +
+ 		✓ positive
02test_02_disable_storage_poolDisable pooltest_01pool.state == "Disabled"FlexVol still online✓ positive
03test_03_enable_storage_poolRe-enable pooltest_02pool.state == "Up"FlexVol still online✓ positive
04test_04_enter_maintenance_modeEnter maintenancetest_03pool.state == "Maintenance" +
    +
  • FlexVol still online
    • +
  • igroups unchanged
    • +
+ 		✓ positive
05test_05_cancel_maintenance_modeCancel maintenancetest_04pool.state == "Up"FlexVol still online✓ positive
06test_06_enter_maintenance_and_delete_poolEnter maintenance then force-delete pooltest_05Pool not listed +
    +
  • FlexVol deleted
    • +
  • All igroups for cluster hosts deleted
    • +
+ 		✓ positive
07test_07_create_volume_on_poolCreate fresh pool + allocate a data volume (creates a LUN)test_06 (new pool) +
    +
  • pool.state == "Up"
    • +
  • Volume non-None
    • +
+ 		+
    +
  • FlexVol online
    • +
  • ≥1 LUN in FlexVol
    • +
+ 		✓ positive
08test_08_delete_volume_and_poolDelete volume (removes LUN), enter maintenance, force-delete pooltest_07 +
    +
  • Volume not listed
    • +
  • Pool not listed
    • +
+ 		+
    +
  • LUN removed
    • +
  • FlexVol deleted
    • +
  • igroups deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 07
+
iSCSI Pool with Volumes
+
+ 💾 iSCSI + Cluster scope + iscsi_workflow + iscsi/pool/test_pool_with_volumes.py +
+
+
7tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_pool_and_volumeCreate iSCSI pool + data volume (creates LUN)setUpClass +
    +
  • pool.state == "Up"
    • +
  • Volume non-None
    • +
+ 		+
    +
  • FlexVol online
    • +
  • ≥1 LUN in FlexVol
    • +
+ 		✓ positive
02test_02_disable_pool_volume_survivesDisable pool with LUN-backed volume presenttest_01 +
    +
  • pool.state == "Disabled"
    • +
  • Volume still listed
    • +
+ 		+
    +
  • FlexVol still online
    • +
  • LUN still present
    • +
+ 		✓ positive
03test_03_enable_pool_volume_intactRe-enable pool with LUN presenttest_02 +
    +
  • pool.state == "Up"
    • +
  • Volume still listed
    • +
+ 		+
    +
  • FlexVol still online
    • +
  • LUN still present
    • +
+ 		✓ positive
04test_04_enter_maintenance_volume_presentEnter maintenance with LUN presenttest_03 +
    +
  • pool.state == "Maintenance"
    • +
  • Volume still listed
    • +
+ 		+
    +
  • FlexVol still online
    • +
  • LUN still present
    • +
+ 		✓ positive
05test_05_cancel_maintenance_volume_presentCancel maintenance with LUN presenttest_04 +
    +
  • pool.state == "Up"
    • +
  • Volume still listed
    • +
+ 		+
    +
  • FlexVol still online
    • +
  • LUN still present
    • +
+ 		✓ positive
06test_06_forced_false_delete_rejecteddeleteStoragePool(forced=False) with LUN present — must be rejected + test_05 +
    +
  • CloudstackAPIException raised
    • +
  • Pool still in Maintenance
    • +
+ 		No ONTAP objects removed✗ negative
07test_07_delete_volume_and_force_delete_poolDelete volume (LUN removed) then force-delete pooltest_06 +
    +
  • Volume not listed
    • +
  • Pool not listed
    • +
+ 		+
    +
  • LUN removed
    • +
  • FlexVol deleted
    • +
  • igroups deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 08
+
iSCSI Zone-Scoped Pool
+
+ 💾 iSCSI + 🌐 Zone scope + iscsi_zone_pool + iscsi/pool/test_zone_scoped_pool.py +
+
+
4tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_zone_scoped_poolCreate zone-scoped iSCSI pool — CS calls attachZone()setUpClasspool.state == "Up" +
    +
  • FlexVol online
    • +
  • igroup per cluster host with host IQN
    • +
+ 		✓ positive
02test_02_disable_zone_scoped_poolDisable zone-scoped pooltest_01pool.state == "Disabled"FlexVol unchanged; igroups unchanged✓ positive
03test_03_enable_zone_scoped_poolRe-enable zone-scoped pooltest_02pool.state == "Up"FlexVol unchanged; igroups unchanged✓ positive
04test_04_delete_zone_scoped_poolEnter maintenance then delete pooltest_03Pool not listed +
    +
  • FlexVol deleted
    • +
  • All igroups deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 09
+
iSCSI Volume Lifecycle
+
+ 💾 iSCSI + Cluster scope + iscsi_volume + iscsi/volume/test_volume_lifecycle.py +
+
+
5tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_pool_and_volumeCreate iSCSI pool + data volume — LUN is created inside FlexVolsetUpClass +
    +
  • pool.state == "Up"
    • +
  • Volume non-None
    • +
+ 		+
    +
  • FlexVol online
    • +
  • ≥1 LUN in FlexVol
    • +
+ 		✓ positive
02test_02_delete_volumeDelete volume — LUN is removed from FlexVoltest_01Volume not in listVolumes +
    +
  • LUN not in FlexVol
    • +
  • FlexVol still online
    • +
+ 		✓ positive
03test_03_recreate_volume_for_delete_testsRe-create volume — LUN is re-created (setup for negative test)test_02New volume non-NoneLUN present in FlexVol again✓ positive
04test_04_forced_false_delete_with_volume_failsEnter maintenance; deleteStoragePool(forced=False) must be rejected + with LUN presenttest_03 +
    +
  • CloudstackAPIException raised
    • +
  • Pool still in Maintenance
    • +
+ 		No ONTAP objects removed✗ negative
05test_05_delete_volume_and_force_delete_poolDelete volume (LUN removed) then force-delete pooltest_04 +
    +
  • Volume not listed
    • +
  • Pool not listed
    • +
+ 		+
    +
  • LUN removed
    • +
  • FlexVol deleted
    • +
  • igroups deleted
    • +
+ 		↩ cleanup
+
+
+ + +
+
+
+
Suite 10
+
iSCSI VM + Volume Attach
+
+ 💾 iSCSI + 🖥 VM attach + iscsi_vm_workflow + iscsi/instance/test_vm_volume_attach.py +
+
+
8tests
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Test methodGoalDepends onCloudStack criteriaONTAP criteriaType
01test_01_create_iscsi_poolCreate iSCSI primary storage poolsetUpClass +
    +
  • pool.state == "Up"
    • +
  • pool.type == "Iscsi"
    • +
+ 		+
    +
  • FlexVol online
    • +
  • igroup per host with host IQN
    • +
+ 		✓ positive
02test_02_create_ontap_data_volumeAllocate data volume — LUN created in FlexVoltest_01 (pool)Volume non-None≥1 LUN in FlexVol✓ positive
03test_03_deploy_vmDeploy VM; verify 0 LUN-maps before attachtest_02vm.state == "Running"0 LUN-maps (list_lun_maps_for_volume returns empty)✓ positive
04test_04_attach_volume_to_vmHot-attach iSCSI volume to running VM — LUN-map is createdtest_03 (vm, volume)volume.virtualmachineid == vm.id≥1 LUN-map linking LUN to host's igroup✓ positive
05test_05_stop_vm_lun_unmappedStop VM — LUN-maps must be removedtest_04vm.state == "Stopped" +
    +
  • 0 LUN-maps
    • +
  • LUN still present in FlexVol
    • +
+ 		✓ positive
06test_06_start_vm_lun_remappedStart VM — LUN-maps must be re-createdtest_05vm.state == "Running"≥1 LUN-map re-created✓ positive
07test_07_detach_volume_from_vmHot-detach iSCSI volume from running VMtest_06 (vm, volume) +
    +
  • volume.virtualmachineid cleared
    • +
  • 0 LUN-maps
    • +
+ 		LUN still in FlexVol⚠ deferred
08test_08_destroy_vm_and_cleanupDestroy VM (expunge), delete volume, delete pooltest_07 +
    +
  • VM not listed
    • +
  • Volume not listed
    • +
  • Pool not listed
    • +
+ 		+
    +
  • FlexVol deleted
    • +
  • All LUNs + igroups deleted
    • +
+ 		↩ cleanup
+
+
+
⚠️
+
+ test_07 — iSCSI hot-detach (deferred) + iSCSI hot-detach from a running VM relies on the KVM guest acknowledging SCSI device removal. + On this environment the guest does not acknowledge in time, causing CloudStack error 530. + This is a KVM-host-level or guest-template limitation, not a test code defect. + All other 61 tests pass. +
+
+
+ +
+ + +
+
Cross-suite summary
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#SuiteProtocolScopePositiveNegativeCleanupTotalStatus
01NFS3 Pool LifecycleNFS3Cluster7—18✓ All pass
02NFS3 Pool with VolumesNFS3Cluster5117✓ All pass
03NFS3 Zone-Scoped PoolNFS3Zone3—14✓ All pass
04NFS3 Volume LifecycleNFS3Cluster3115✓ All pass
05NFS3 VM + Volume AttachNFS3Cluster7—18✓ All pass
06iSCSI Pool LifecycleiSCSICluster7—18✓ All pass
07iSCSI Pool with VolumesiSCSICluster5117✓ All pass
08iSCSI Zone-Scoped PooliSCSIZone3—14✓ All pass
09iSCSI Volume LifecycleiSCSICluster3115✓ All pass
10iSCSI VM + Volume AttachiSCSICluster6—18⚠ 7 / 8
Total49496261 / 62
+
+
+ +
+ +
+ Apache CloudStack · NetApp ONTAP Plugin · Integration Test Case Reference · Generated 2026-07-10 +
+ + + + \ No newline at end of file diff --git a/test/integration/plugins/ontap/TEST_CASES.md b/test/integration/plugins/ontap/TEST_CASES.md new file mode 100644 index 000000000000..37fc07a0a33f --- /dev/null +++ b/test/integration/plugins/ontap/TEST_CASES.md @@ -0,0 +1,221 @@ +# ONTAP Integration Test Cases + +Complete reference for all 62 test cases across 10 test suites. +Each suite is sequential — tests must run in numbered order; each step builds on state created by the previous step. + +--- + +## How to read the tables + +| Column | Meaning | +|--------|---------| +| **Test method** | Exact Python method name | +| **Goal** | What CloudStack workflow step is being exercised | +| **Depends on** | Which earlier tests must have passed (class state they consume) | +| **CloudStack success criteria** | What the CS API must return for the test to pass | +| **ONTAP success criteria** | What the ONTAP REST API must show for the test to pass | +| **Type** | `positive` = happy path, `negative` = tests a rejection/error condition, `cleanup` = teardown step | + +--- + +## Suite 1 — NFS3 Pool Lifecycle + +**File:** `nfs3/pool/test_pool_lifecycle.py` +**Class:** `TestOntapNFS3PrimaryStorageWorkflow` +**Tag:** `nfs3_workflow` +**Total:** 8 tests | **Scope:** cluster-scoped NFS3 pool, no volumes for tests 01–06 + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped NFS3 primary storage pool | setUpClass (zone, cluster, account) | `pool.state == "Up"`, `pool.type == "NetworkFilesystem"`, `nfsmountopts` contains `vers=3` | FlexVol exists and `state == "online"`, export policy exists with each cluster host IP as a rule, at least one NFS data LIF present on SVM | positive | +| 02 | `test_02_disable_storage_pool` | Disable the pool (admin operation) | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | +| 03 | `test_03_enable_storage_pool` | Re-enable the pool | test_02 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 04 | `test_04_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_03 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | +| 05 | `test_05_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_04 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 06 | `test_06_delete_pool_from_maintenance` | Enter maintenance then permanently delete the pool | test_05 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol deleted (not found by `GET /api/storage/volumes?name=`); export policy deleted | positive | +| 07 | `test_07_create_volume_on_pool` | Create a second fresh pool and allocate a CloudStack data volume on it | test_06 (pool deleted; creates new pool) | New `pool.state == "Up"`; `createVolume` returns non-None volume object | FlexVol `online` after volume allocation; export policy present | positive | +| 08 | `test_08_delete_volume_and_pool` | Delete the volume then force-delete the pool | test_07 (`pool`, `volume`) | Volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | positive | + +--- + +## Suite 2 — NFS3 Pool with Volumes + +**File:** `nfs3/pool/test_pool_with_volumes.py` +**Class:** `TestOntapNFS3PoolWithVolumes` +**Tag:** `nfs3_workflow` +**Total:** 7 tests | **Scope:** cluster-scoped NFS3 pool with a live CloudStack volume throughout + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_pool_and_volume` | Create NFS3 pool and immediately allocate a data volume | setUpClass | `pool.state == "Up"`, volume object non-None | FlexVol `online`; export policy present | positive | +| 02 | `test_02_disable_pool_volume_survives` | Disable pool while a volume exists — volume must survive | test_01 (`pool`, `volume`) | `pool.state == "Disabled"`; volume still listed in `listVolumes` | FlexVol still `online` | positive | +| 03 | `test_03_enable_pool_volume_intact` | Re-enable pool with volume present | test_02 | `pool.state == "Up"`; volume still listed | FlexVol still `online` | positive | +| 04 | `test_04_enter_maintenance_volume_present` | Enter maintenance while volume present | test_03 | `pool.state == "Maintenance"`; volume still listed | FlexVol still `online` | positive | +| 05 | `test_05_cancel_maintenance_with_volume` | Cancel maintenance with volume — verifies the NFS3 cancel-maintenance fix | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online` | positive | +| 06 | `test_06_forced_false_delete_rejected` | Attempt to delete pool (forced=False) with volume present — must be rejected | test_05 | `deleteStoragePool(forced=False)` raises `CloudstackAPIException`; pool still listed in `Maintenance` state | FlexVol still `online`; no ONTAP objects removed | negative | +| 07 | `test_07_force_delete_pool_and_cleanup` | Cancel maintenance, delete volume, then force-delete pool | test_06 | Pool no longer listed; volume no longer listed | FlexVol deleted; export policy deleted | cleanup | + +--- + +## Suite 3 — NFS3 Zone-Scoped Pool + +**File:** `nfs3/pool/test_zone_scoped_pool.py` +**Class:** `TestOntapZoneScopedPool` +**Tag:** `zone_pool` +**Total:** 4 tests | **Scope:** zone-scoped NFS3 pool (scope=ZONE, all hosts in zone connected) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped NFS3 pool; CloudStack calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; export policy exists and contains **every** cluster host IP; at least one NFS data LIF present | positive | +| 02 | `test_02_disable_zone_scoped_pool` | Disable the zone-scoped pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol unchanged; export policy unchanged | positive | +| 03 | `test_03_enable_zone_scoped_pool` | Re-enable the zone-scoped pool | test_02 | `pool.state == "Up"` | FlexVol unchanged; export policy unchanged | positive | +| 04 | `test_04_delete_zone_scoped_pool` | Enter maintenance and force-delete the zone-scoped pool | test_03 | Pool no longer listed | FlexVol deleted; export policy deleted | positive | + +--- + +## Suite 4 — NFS3 Volume Lifecycle + +**File:** `nfs3/volume/test_volume_lifecycle.py` +**Class:** `TestOntapNFS3VolumeLifecycle` +**Tag:** `nfs3_volume` +**Total:** 5 tests | **Scope:** NFS3 CloudStack volume create/delete semantics (NFS3 volumes are metadata-only in CS) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_pool_and_volume` | Create NFS3 pool and allocate a CloudStack data volume | setUpClass | `pool.state == "Up"`; volume object non-None | FlexVol `online` after volume allocation; export policy present — **no new ONTAP object per volume** (FlexVol is shared) | positive | +| 02 | `test_02_delete_volume` | Delete the CS data volume — for NFS3 only the CS record is removed | test_01 (`pool`, `volume`) | Volume no longer listed in `listVolumes` | FlexVol still `online` and **unaffected**; export policy still present | positive | +| 03 | `test_03_recreate_volume_for_delete_tests` | Re-create a volume on the pool (setup for negative tests 04–05) | test_02 | New volume object non-None | FlexVol still `online` | positive | +| 04 | `test_04_forced_false_delete_with_volume_fails` | Enter maintenance then attempt `deleteStoragePool(forced=False)` while volume exists — must be rejected | test_03 (`pool`, `volume`) | `deleteStoragePool(forced=False)` raises `CloudstackAPIException`; pool still in `Maintenance` state | No ONTAP objects removed | negative | +| 05 | `test_05_delete_volume_and_force_delete_pool` | Delete volume from Maintenance, then force-delete pool | test_04 | Volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | positive | + +--- + +## Suite 5 — NFS3 VM + Volume Attach + +**File:** `nfs3/instance/test_vm_volume_attach.py` +**Class:** `TestOntapVMVolumeAttach` +**Tag:** `vm_volume_workflow` +**Total:** 8 tests | **Scope:** end-to-end — NFS3 pool, data volume, running VM, attach/detach lifecycle + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_nfs3_pool` | Create NFS3 ONTAP primary storage pool | setUpClass (zone, cluster, template) | `pool.state == "Up"` | FlexVol `online`; export policy present | positive | +| 02 | `test_02_create_ontap_data_volume` | Allocate a CloudStack data volume on the ONTAP pool | test_01 (`pool`) | Volume non-None and listed in `listVolumes` | FlexVol still `online` | positive | +| 03 | `test_03_deploy_vm` | Deploy a VM using the first available ready KVM template | test_02 (`pool`, `volume`) | `vm.state == "Running"`; template auto-selected from `listTemplates` | n/a | positive | +| 04 | `test_04_attach_volume_to_vm` | Attach the ONTAP data volume to the running VM (hot-plug) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id`; `attachVolume` job succeeds | FlexVol `online`; after attach, a data file matching volume UUID present in FlexVol (`list_files_in_volume`) | positive | +| 05 | `test_05_stop_vm_export_retained` | Stop the running VM with volume attached | test_04 | `vm.state == "Stopped"` | FlexVol still `online`; NFS export policy still present | positive | +| 06 | `test_06_start_vm_volume_accessible` | Start the stopped VM | test_05 | `vm.state == "Running"` | FlexVol still `online` | positive | +| 07 | `test_07_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM (TDS Detach NFS3) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | +| 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM no longer listed; volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | cleanup | + +--- + +## Suite 6 — iSCSI Pool Lifecycle + +**File:** `iscsi/pool/test_pool_lifecycle.py` +**Class:** `TestOntapISCSIPoolLifecycle` +**Tag:** `iscsi_workflow` +**Total:** 8 tests | **Scope:** cluster-scoped iSCSI pool, no volumes for tests 01–06 + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "Iscsi"` | FlexVol `online`; one igroup per cluster host (named `cs_{svmName}_{hostShortName}`) with host IQN as initiator | positive | +| 02 | `test_02_disable_storage_pool` | Disable the pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol still `online` | positive | +| 03 | `test_03_enable_storage_pool` | Re-enable the pool | test_02 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 04 | `test_04_enter_maintenance_mode` | Put pool into maintenance | test_03 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | +| 05 | `test_05_cancel_maintenance_mode` | Cancel maintenance | test_04 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 06 | `test_06_enter_maintenance_and_delete_pool` | Enter maintenance then force-delete the pool | test_05 | Pool no longer listed | FlexVol deleted; all igroups for cluster hosts deleted | positive | +| 07 | `test_07_create_volume_on_pool` | Create a second fresh pool and allocate a CloudStack data volume (creates a LUN) | test_06 (new pool) | New `pool.state == "Up"`; volume object non-None | FlexVol `online`; ≥1 LUN present inside FlexVol (`list_luns_in_volume`) | positive | +| 08 | `test_08_delete_volume_and_pool` | Delete the volume (removes LUN), enter maintenance, force-delete pool | test_07 (`pool`, `volume`) | Volume no longer listed; pool no longer listed | LUN no longer in FlexVol; FlexVol deleted; igroups deleted | positive | + +--- + +## Suite 7 — iSCSI Pool with Volumes + +**File:** `iscsi/pool/test_pool_with_volumes.py` +**Class:** `TestOntapISCSIPoolWithVolumes` +**Tag:** `iscsi_workflow` +**Total:** 7 tests | **Scope:** cluster-scoped iSCSI pool with a live CloudStack volume (LUN) throughout + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_pool_and_volume` | Create iSCSI pool and allocate a data volume (creates LUN) | setUpClass | `pool.state == "Up"`; volume non-None | FlexVol `online`; ≥1 LUN in FlexVol | positive | +| 02 | `test_02_disable_pool_volume_survives` | Disable pool with volume present | test_01 (`pool`, `volume`) | `pool.state == "Disabled"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 03 | `test_03_enable_pool_volume_intact` | Re-enable pool with volume | test_02 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 04 | `test_04_enter_maintenance_volume_present` | Enter maintenance with volume | test_03 | `pool.state == "Maintenance"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 05 | `test_05_cancel_maintenance_volume_present` | Cancel maintenance with volume (TDS iSCSI cancel maintenance) | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 06 | `test_06_forced_false_delete_rejected` | Attempt `deleteStoragePool(forced=False)` with LUN-backed volume present — must be rejected | test_05 | `CloudstackAPIException` raised; pool still in `Maintenance` | No ONTAP objects removed | negative | +| 07 | `test_07_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_06 (`pool`, `volume`) | Volume gone; pool gone | LUN removed; FlexVol deleted; igroups deleted | cleanup | + +--- + +## Suite 8 — iSCSI Zone-Scoped Pool + +**File:** `iscsi/pool/test_zone_scoped_pool.py` +**Class:** `TestOntapISCSIZoneScopedPool` +**Tag:** `iscsi_zone_pool` +**Total:** 4 tests | **Scope:** zone-scoped iSCSI pool (scope=ZONE) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped iSCSI pool; CS calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; igroup per cluster host, each with host IQN as initiator | positive | +| 02 | `test_02_disable_zone_scoped_pool` | Disable pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol unchanged; igroups unchanged | positive | +| 03 | `test_03_enable_zone_scoped_pool` | Re-enable pool | test_02 | `pool.state == "Up"` | FlexVol unchanged; igroups unchanged | positive | +| 04 | `test_04_delete_zone_scoped_pool` | Enter maintenance then delete pool | test_03 | Pool no longer listed | FlexVol deleted; all igroups deleted | positive | + +--- + +## Suite 9 — iSCSI Volume Lifecycle + +**File:** `iscsi/volume/test_volume_lifecycle.py` +**Class:** `TestOntapISCSIVolumeLifecycle` +**Tag:** `iscsi_volume` +**Total:** 5 tests | **Scope:** iSCSI CloudStack volume create/delete semantics (each CS volume maps to an ONTAP LUN) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_pool_and_volume` | Create iSCSI pool and allocate a data volume — a LUN is created inside the pool's FlexVol | setUpClass | `pool.state == "Up"`; volume non-None | FlexVol `online`; ≥1 LUN in FlexVol (`list_luns_in_volume`) | positive | +| 02 | `test_02_delete_volume` | Delete the volume — the LUN is removed from the FlexVol | test_01 (`pool`, `volume`) | Volume no longer listed | LUN no longer in FlexVol; FlexVol itself still `online` | positive | +| 03 | `test_03_recreate_volume_for_delete_tests` | Re-create a volume (LUN re-created) — setup for negative tests | test_02 | New volume non-None | LUN present in FlexVol again | positive | +| 04 | `test_04_forced_false_delete_with_volume_fails` | Enter maintenance then attempt `deleteStoragePool(forced=False)` with LUN present — must be rejected | test_03 (`pool`, `volume`) | `CloudstackAPIException` raised; pool still in `Maintenance` | No ONTAP objects removed | negative | +| 05 | `test_05_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_04 | Volume gone; pool gone | LUN removed; FlexVol deleted; igroups deleted | positive | + +--- + +## Suite 10 — iSCSI VM + Volume Attach + +**File:** `iscsi/instance/test_vm_volume_attach.py` +**Class:** `TestOntapVMVolumeAttachISCSI` +**Tag:** `iscsi_vm_workflow` +**Total:** 8 tests | **Scope:** end-to-end — iSCSI pool, data volume (LUN), running VM, attach/stop/start/detach lifecycle + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_create_iscsi_pool` | Create iSCSI ONTAP primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "Iscsi"` | FlexVol `online`; igroup per cluster host with host IQN | positive | +| 02 | `test_02_create_ontap_data_volume` | Allocate a CloudStack data volume (creates a LUN in the FlexVol) | test_01 (`pool`) | Volume non-None | ≥1 LUN in FlexVol | positive | +| 03 | `test_03_deploy_vm` | Deploy VM using first ready KVM template; verify 0 LUN-maps exist before attach | test_02 (`volume`) | `vm.state == "Running"`; 0 LUN-maps on ONTAP | 0 LUN-maps (`list_lun_maps_for_volume` returns empty) | positive | +| 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created (TDS SN 27) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | +| 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed (TDS VM Stop iSCSI) | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | +| 06 | `test_06_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created (TDS VM Start iSCSI) | test_05 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | +| 07 | `test_07_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM (TDS Detach iSCSI) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps; LUN still in FlexVol | positive ⚠️ | +| 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM gone; volume gone; pool gone | FlexVol deleted; all LUNs and igroups deleted | cleanup | + +> ⚠️ **test_07 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. All other 61 tests pass. + +--- + +## Cross-suite summary + +| Suite | Protocol | Scope | Tests | Status | +|-------|---------|-------|-------|--------| +| NFS3 Pool Lifecycle | NFS3 | Cluster | 8 | ✅ | +| NFS3 Pool with Volumes | NFS3 | Cluster | 7 | ✅ | +| NFS3 Zone-Scoped Pool | NFS3 | Zone | 4 | ✅ | +| NFS3 Volume Lifecycle | NFS3 | Cluster | 5 | ✅ | +| NFS3 VM + Volume Attach | NFS3 | Cluster | 8 | ✅ | +| iSCSI Pool Lifecycle | iSCSI | Cluster | 8 | ✅ | +| iSCSI Pool with Volumes | iSCSI | Cluster | 7 | ✅ | +| iSCSI Zone-Scoped Pool | iSCSI | Zone | 4 | ✅ | +| iSCSI Volume Lifecycle | iSCSI | Cluster | 5 | ✅ | +| iSCSI VM + Volume Attach | iSCSI | Cluster | 8 | ⚠️ 7/8 | +| **Total** | | | **62** | **61 passing** | diff --git a/test/integration/plugins/ontap/manual_cancel_maint_test.py b/test/integration/plugins/ontap/manual_cancel_maint_test.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test/integration/plugins/ontap/ontap.cfg b/test/integration/plugins/ontap/ontap.cfg index 4bf9d17dc499..a609742f48f0 100644 --- a/test/integration/plugins/ontap/ontap.cfg +++ b/test/integration/plugins/ontap/ontap.cfg @@ -1,47 +1,17 @@ { "zones": [ { - "name": "Zone-ONTAP", - "dns1": "8.8.8.8", - "internal_dns1": "8.8.8.8", - "guestcidraddress": "10.1.1.0/24", - "physical_networks": [ - { - "broadcastdomainrange": "Zone", - "name": "physical_network", - "traffictypes": [ - {"typ": "Guest"}, - {"typ": "Management"}, - {"typ": "Storage"} - ], - "providers": [ - { - "broadcastdomainrange": "ZONE", - "name": "VirtualRouter" - } - ] - } - ], "pods": [ { - "name": "Pod-ONTAP", - "gateway": "10.193.56.1", - "startip": "10.193.56.10", - "endip": "10.193.56.50", - "netmask": "255.255.255.128", "clusters": [ { - "clustername": "KVM-Cluster-ONTAP", - "hypervisor": "KVM", - "clustertype": "CloudManaged", "hosts": [ { "url": "http://10.193.56.65", "username": "root", "password": "netapp1!" } - ], - "primaryStorages": [] + ] } ] } @@ -49,7 +19,7 @@ } ], "dbSvr": { - "dbSvr": "10.193.56.65", + "dbSvr": "10.193.56.62", "passwd": "", "db": "cloud", "port": 3306, @@ -60,7 +30,7 @@ }, "mgtSvr": [ { - "mgtSvrIp": "10.193.56.65", + "mgtSvrIp": "10.193.56.62", "port": 8096, "user": "admin", "passwd": "password", @@ -73,7 +43,24 @@ "username": "admin", "password": "netapp1!" }, - "TestData": { - "Path": "test/integration/plugins/ontap/ontap.cfg" + "storagePool": { + "storagePoolScope": "CLUSTER", + "storagePoolProvider": "NetApp ONTAP", + "capacitybytes": null, + "protocols": { + "iscsi": { + "enabled": true, + "storagePoolTags": "ontap-iscsi" + }, + "nfs3": { + "enabled": true, + "storagePoolTags": "ontap-nfs3" + } + } + }, + "cloudstack": { + "zoneName": null, + "clusterName": null, + "domainName": "ROOT" } } diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index 74e02067aa45..43739acb113e 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -57,14 +57,6 @@ # --------------------------------------------------------------------------- def _parse_pool_details(pool): - """ - Convert a StoragePool object's ``details`` attribute to a plain Python dict, - regardless of how Marvin chose to represent it. - - Note: listStoragePools only returns a subset of detail keys - (volumeUUID, exportPolicyName, exportPolicyId). For the full set - use the pool object returned directly by createStoragePool. - """ details_raw = getattr(pool, "details", None) if not details_raw: return {} @@ -236,16 +228,6 @@ def list_files_in_volume(self, vol_name, path="/"): # --------------------------------------------------------------------------- class OntapTestBase(cloudstackTestCase): - """ - Shared base for sequential ONTAP primary-storage workflow tests. - - Subclasses must: - - Set ``_vol_name_prefix`` to distinguish volume names per protocol. - - Define ``setUpClass`` that builds ``cls.testdata``, creates - ``cls.ontap`` and ``cls.svm_name``, then calls - ``cls._setup_cloudstack_resources(config, account_testdata)``. - - Define ``_create_pool`` (protocol-specific URL scheme and name). - """ # ---- shared state (set/cleared by individual tests) ---------------- pool = None diff --git a/test/integration/plugins/ontap/probe_test.py b/test/integration/plugins/ontap/probe_test.py deleted file mode 100644 index 8ed1ca03da29..000000000000 --- a/test/integration/plugins/ontap/probe_test.py +++ /dev/null @@ -1,35 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -import json -from marvin.cloudstackAPI import listTemplates, listServiceOfferings, listNetworks -from marvin.cloudstackTestCase import cloudstackTestCase - -class ProbeResources(cloudstackTestCase): - @classmethod - def setUpClass(cls): - tc = super(ProbeResources, cls).getClsTestClient() - cls.api = tc.getApiClient() - - def test_01_probe(self): - out = {} - cmd = listTemplates.listTemplatesCmd() - cmd.templatefilter = "executable" - resp = self.api.listTemplates(cmd) - out["templates"] = [{"id": t.id, "name": t.name, "hypervisor": getattr(t,"hypervisor","?"), "status": getattr(t,"status","?")} for t in (resp or [])] - cmd2 = listServiceOfferings.listServiceOfferingsCmd() - resp2 = self.api.listServiceOfferings(cmd2) - out["offerings"] = [{"id": s.id, "name": s.name, "cpu": getattr(s,"cpunumber","?"), "mem": getattr(s,"memory","?")} for s in (resp2 or [])] - cmd3 = listNetworks.listNetworksCmd() - cmd3.listall = True - resp3 = self.api.listNetworks(cmd3) - out["networks"] = [{"id": n.id, "name": n.name, "type": getattr(n,"type","?"), "state": getattr(n,"state","?")} for n in (resp3 or [])] - with open("/tmp/cs_probe_out.json","w") as f: - json.dump(out, f, indent=2) - self.assertTrue(True) diff --git a/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py b/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py deleted file mode 100644 index acdd4fe33ee8..000000000000 --- a/test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py +++ /dev/null @@ -1,710 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage pool. - -Tests are numbered test_01 ... test_11 and must run in that order. Each step -builds on the shared state established by the previous step. - -Workflow: - 01 Create primary storage pool - 02 Disable storage pool - 03 Enable storage pool - 04 Enter maintenance mode - 05 Cancel maintenance mode - 06 Create a data volume on the pool - 07 Enter maintenance mode (pool has a volume) - 08 Cancel maintenance mode (pool has a volume) - 09 Delete the data volume - 10 Enter maintenance mode and delete the storage pool - 11 Create a second pool, attach a volume, enter maintenance, - then force-delete the pool (volume still present) - -Prerequisites: - - CloudStack management server with the NetApp ONTAP plugin deployed - - KVM cluster where every host has iSCSI configured (storageUrl starts with iqn.) - - ONTAP SVM with iSCSI service enabled and at least one iSCSI data LIF - - ontap.cfg populated with real values - -Running: - nosetests --with-marvin \\ - --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ - test/integration/plugins/ontap/test_ontap_create_primary_storage_iscsi.py -v -""" - -import base64 -import logging -import random - -from nose.plugins.attrib import attr - -from marvin.cloudstackAPI import ( - createStoragePool as createStoragePoolAPI, - deleteVolume as deleteVolumeAPI, - enableStorageMaintenance, - cancelStorageMaintenance, - updateStoragePool as updateStoragePoolAPI, -) -from marvin.lib.base import StoragePool -from marvin.lib.common import list_storage_pools - -from ontap_test_base import OntapRestClient, OntapTestBase - -logger = logging.getLogger("TestOntapISCSIWorkflow") - - -# --------------------------------------------------------------------------- -# Test data -# --------------------------------------------------------------------------- - -class TestData: - account = "account" - ontap = "ontap" - primaryStorage = "primaryStorage" - provider = "provider" - scope = "scope" - tags = "tags" - - DETAIL_USERNAME = "username" - DETAIL_PASSWORD = "password" - DETAIL_SVM_NAME = "svmName" - DETAIL_PROTOCOL = "protocol" - DETAIL_STORAGE_IP = "storageIP" - - ONTAP_MIN_VOLUME_SIZE = 1677721600 - - def __init__(self, storage_ip, svm_name, username, password, - scope="CLUSTER", provider="NetApp ONTAP", - tags="ontap-iscsi", capacitybytes=None): - if capacitybytes is None: - capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 - encoded_password = base64.b64encode(password.encode()).decode() - self.testdata = { - TestData.ontap: { - TestData.DETAIL_STORAGE_IP: storage_ip, - TestData.DETAIL_SVM_NAME: svm_name, - TestData.DETAIL_USERNAME: username, - TestData.DETAIL_PASSWORD: password, - }, - TestData.account: { - "email": "ontap-iscsi-wf@test.com", - "firstname": "ONTAP", - "lastname": "iSCSI-WF", - "username": "ontap_iscsi_wf_%d" % random.randint(0, 9999), - "password": "password", - }, - TestData.primaryStorage: { - "name": "OntapISCSI_%d" % random.randint(0, 9999), - TestData.scope: scope, - TestData.provider: provider, - TestData.tags: tags, - "capacitybytes": capacitybytes, - "managed": True, - "details": { - TestData.DETAIL_USERNAME: username, - TestData.DETAIL_PASSWORD: encoded_password, - TestData.DETAIL_SVM_NAME: svm_name, - TestData.DETAIL_PROTOCOL: "ISCSI", - TestData.DETAIL_STORAGE_IP: storage_ip, - }, - }, - } - - -# --------------------------------------------------------------------------- -# iSCSI path helpers -# --------------------------------------------------------------------------- - -def _igroup_name(svm_name, host_name): - """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" - short = host_name.split(".")[0] - import re - sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) - return "cs_%s_%s" % (svm_name, sanitized) - - -def _lun_path(vol_name, lun_name): - """Mirror OntapStorageUtils.getLunName: /vol/{volName}/{lunName}""" - return "/vol/%s/%s" % (vol_name, lun_name) - - -# --------------------------------------------------------------------------- -# Sequential workflow test class -# --------------------------------------------------------------------------- - -class TestOntapISCSIPrimaryStorageWorkflow(OntapTestBase): - - # ---- iSCSI-specific state (set/cleared by individual tests) -------- - _vol_name_prefix = "OntapISCSIVol" - lun_path = None # ONTAP LUN path of cls.volume - lun_path2 = None # ONTAP LUN path of cls.volume2 - - @classmethod - def setUpClass(cls): - testclient = super( - TestOntapISCSIPrimaryStorageWorkflow, cls - ).getClsTestClient() - - cls.apiClient = testclient.getApiClient() - cls.dbConnection = testclient.getDbConnection() - config = testclient.getParsedTestDataConfig() - - ontap_cfg = config.get("ontap", {}) - storage_ip = ontap_cfg.get("storageIP", "") - svm_name = ontap_cfg.get("svmName", "") - username = ontap_cfg.get("username", "") - password = ontap_cfg.get("password", "") - scope = ontap_cfg.get("storagePoolScope", "CLUSTER") - provider = ontap_cfg.get("storagePoolProvider", "NetApp ONTAP") - tags = ontap_cfg.get("storagePoolTags", "ontap-iscsi") - capacitybytes = ontap_cfg.get("capacitybytes", None) - - cls.testdata = TestData( - storage_ip, svm_name, username, password, - scope=scope, provider=provider, tags=tags, - capacitybytes=capacitybytes, - ).testdata - cls.ontap = OntapRestClient(storage_ip, username, password) - cls.svm_name = svm_name - - cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) - - # No per-test tearDown — state intentionally persists between steps. - - # ------------------------------------------------------------------ - # Helpers - # ------------------------------------------------------------------ - - def _create_pool(self): - ps = self.testdata[TestData.primaryStorage] - storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] - pool_name = "OntapISCSI_%d" % random.randint(0, 99999) - - cmd = createStoragePoolAPI.createStoragePoolCmd() - cmd.name = pool_name - cmd.url = "iscsi://%s/ontap" % storage_ip - cmd.zoneid = self.zone.id - cmd.clusterid = self.cluster.id - cmd.podid = self.cluster.podid - cmd.scope = ps[TestData.scope] - cmd.provider = ps[TestData.provider] - cmd.tags = ps[TestData.tags] - cmd.capacitybytes = ps["capacitybytes"] - cmd.hypervisor = "KVM" - cmd.managed = True - - count = 1 - for key, value in ps["details"].items(): - setattr(cmd, "details[{}].{}".format(count, key), value) - count += 1 - - response = self.apiClient.createStoragePool(cmd) - return StoragePool(response.__dict__) - - # ------------------------------------------------------------------ - # Step 01 - Create primary storage pool - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_01_create_primary_storage_pool(self): - """ - Create an iSCSI primary storage pool and verify: - - CloudStack state is Up, type is Iscsi - - ONTAP: FlexVol exists and is online - - ONTAP: one igroup per cluster host exists with the correct IQN initiator - """ - pool = self._create_pool() - self.__class__.pool = pool - - self.assertEqual( - pool.state, "Up", - "Pool state should be 'Up', got '%s'" % pool.state - ) - self.assertEqual( - pool.type, "Iscsi", - "Pool type should be 'Iscsi', got '%s'" % pool.type - ) - - # ONTAP: FlexVol must be online - ontap_vol = self.ontap.get_volume(pool.name) - self.assertIsNotNone( - ontap_vol, - "ONTAP FlexVol not found for pool '%s'" % pool.name - ) - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") - ) - - # ONTAP: igroup must exist for each cluster host that has an IQN - for host in self.cluster_hosts: - iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) - if not iqn or not iqn.startswith("iqn."): - continue # host not iSCSI-enabled; skip igroup check for it - igroup_name = _igroup_name(self.svm_name, host.name) - igroup = self.ontap.get_igroup(self.svm_name, igroup_name) - self.assertIsNotNone( - igroup, - "ONTAP igroup '%s' not found for host '%s'" % (igroup_name, host.name) - ) - initiator_names = [ - i.get("name", "") for i in igroup.get("initiators", []) - ] - self.assertIn( - iqn, initiator_names, - "Host IQN '%s' not in igroup '%s' initiators: %s" - % (iqn, igroup_name, initiator_names) - ) - - # ------------------------------------------------------------------ - # Step 02 - Disable storage pool - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_02_disable_storage_pool(self): - """ - Disable the pool and verify: - - CloudStack reports Disabled - - ONTAP: FlexVol is still online (disable is a CS-only state change) - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") - - cmd = updateStoragePoolAPI.updateStoragePoolCmd() - cmd.id = self.__class__.pool.id - cmd.enabled = False - self.apiClient.updateStoragePool(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) - self.assertEqual(result.state, "Disabled") - - # ONTAP: disable must not touch the FlexVol - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should still be 'online' after disable, got '%s'" - % ontap_vol.get("state") - ) - - # ------------------------------------------------------------------ - # Step 03 - Enable storage pool - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_03_enable_storage_pool(self): - """ - Re-enable the pool and verify: - - CloudStack reports Up - - ONTAP: FlexVol is still online (enable is a CS-only state change) - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") - - cmd = updateStoragePoolAPI.updateStoragePoolCmd() - cmd.id = self.__class__.pool.id - cmd.enabled = True - self.apiClient.updateStoragePool(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) - self.assertEqual(result.state, "Up") - - # ONTAP: enable must not touch the FlexVol - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online' after enable, got '%s'" - % ontap_vol.get("state") - ) - - # ------------------------------------------------------------------ - # Step 04 - Enter maintenance mode - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_04_enter_maintenance_mode(self): - """ - Put the pool into maintenance mode and verify: - - CloudStack reports Maintenance - - ONTAP: FlexVol is still online (maintenance is a CS-only state change) - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") - - cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() - cmd.id = self.__class__.pool.id - self.apiClient.enableStorageMaintenance(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) - self.assertEqual(result.state, "Maintenance") - - # ONTAP: maintenance must not touch the FlexVol - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" - % ontap_vol.get("state") - ) - - # ------------------------------------------------------------------ - # Step 05 - Cancel maintenance mode - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_05_cancel_maintenance_mode(self): - """ - Cancel maintenance and verify: - - CloudStack reports Up - - ONTAP: FlexVol is still online - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") - - cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() - cmd.id = self.__class__.pool.id - self.apiClient.cancelStorageMaintenance(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) - self.assertEqual(result.state, "Up") - - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" - % ontap_vol.get("state") - ) - - # ------------------------------------------------------------------ - # Step 06 - Create a data volume on the pool - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_06_create_volume(self): - """ - Allocate a data volume on the iSCSI pool and verify: - - CloudStack returns a volume id - - ONTAP: a LUN is created inside the FlexVol at /vol/{poolName}/{volName} - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") - if not self.disk_offering_id: - self.skipTest("No disk offering available - skipping volume steps") - - try: - vol = self._create_volume(self.__class__.pool.id) - except Exception as e: - self.skipTest("createVolume failed (iSCSI may require an attached VM): %s" % e) - - self.__class__.volume = vol - vol_id = getattr(vol, "id", None) - self.assertIsNotNone(vol_id, "Volume creation returned no id") - - # ONTAP: a LUN must exist inside the FlexVol - luns = self.ontap.list_luns_in_volume(self.svm_name, self.__class__.pool.name) - self.assertTrue( - len(luns) > 0, - "No LUNs found in ONTAP FlexVol '%s' after volume creation" - % self.__class__.pool.name - ) - self.__class__.lun_path = luns[0].get("name") # cache for later steps - - # ------------------------------------------------------------------ - # Step 07 - Enter maintenance mode (pool has a volume) - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_07_enter_maintenance_mode_with_volume(self): - """ - Enter maintenance mode while the pool holds a data volume and verify: - - CloudStack reports Maintenance - - ONTAP: FlexVol still online and LUN still present (maintenance - does not affect ONTAP data plane) - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") - - cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() - cmd.id = self.__class__.pool.id - self.apiClient.enableStorageMaintenance(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) - self.assertEqual(result.state, "Maintenance") - - # ONTAP: FlexVol and LUN must be untouched - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared during maintenance (with volume)") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online' during maintenance, got '%s'" - % ontap_vol.get("state") - ) - if getattr(self.__class__, "lun_path", None): - lun = self.ontap.get_lun(self.svm_name, self.__class__.lun_path) - self.assertIsNotNone( - lun, - "ONTAP LUN '%s' disappeared during maintenance" % self.__class__.lun_path - ) - - # ------------------------------------------------------------------ - # Step 08 - Cancel maintenance mode (pool has a volume) - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_08_cancel_maintenance_mode_with_volume(self): - """ - Cancel maintenance mode while the pool still holds the volume and verify: - - CloudStack reports Up - - ONTAP: FlexVol online and LUN still present - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") - - cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() - cmd.id = self.__class__.pool.id - self.apiClient.cancelStorageMaintenance(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=120) - self.assertEqual(result.state, "Up") - - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after cancel maintenance (with volume)") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online' after cancel maintenance, got '%s'" - % ontap_vol.get("state") - ) - if getattr(self.__class__, "lun_path", None): - lun = self.ontap.get_lun(self.svm_name, self.__class__.lun_path) - self.assertIsNotNone( - lun, - "ONTAP LUN '%s' disappeared after cancel maintenance" % self.__class__.lun_path - ) - - # ------------------------------------------------------------------ - # Step 09 - Delete the volume - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_09_delete_volume(self): - """ - Delete the data volume and verify: - - ONTAP: the LUN is removed from the FlexVol - - ONTAP: the FlexVol itself is still online (only the LUN is gone) - """ - if self.__class__.volume is None: - self.skipTest("No volume from test_06 - skipping") - - vol_id = self.__class__.volume.id - lun_path = getattr(self.__class__, "lun_path", None) - cmd = deleteVolumeAPI.deleteVolumeCmd() - cmd.id = vol_id - self.apiClient.deleteVolume(cmd) - self.__class__.volume = None - self.__class__.lun_path = None - - logger.info("Volume %s deleted" % vol_id) - - # ONTAP: LUN must be gone - if lun_path: - lun = self.ontap.get_lun(self.svm_name, lun_path) - self.assertIsNone( - lun, - "ONTAP LUN '%s' still exists after volume deletion" % lun_path - ) - - # ONTAP: FlexVol must still be online - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after volume deletion") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should still be 'online' after volume deletion, got '%s'" - % ontap_vol.get("state") - ) - - # ------------------------------------------------------------------ - # Step 10 - Enter maintenance mode and delete the storage pool - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_10_enter_maintenance_and_delete_pool(self): - """ - Enter maintenance mode then delete the pool. - Verifies the pool is removed from CloudStack and the backing ONTAP - FlexVol is deleted. - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") - pool = self.__class__.pool - pool_name = pool.name - - maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() - maint_cmd.id = pool.id - self.apiClient.enableStorageMaintenance(maint_cmd) - self._poll_pool_state(pool.id, "Maintenance", timeout=120) - - self._delete_pool(pool.id) - self.__class__.pool = None - - # CloudStack: pool must be gone - try: - remaining = list_storage_pools(self.apiClient, id=pool.id) - except Exception: - remaining = None - self.assertFalse(remaining, "Pool still listed in CloudStack after deletion") - - # ONTAP: FlexVol must be deleted - ontap_vol = self.ontap.get_volume(pool_name) - self.assertIsNone( - ontap_vol, - "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name - ) - - # ONTAP: igroups for each cluster host must be deleted - for host in self.cluster_hosts: - iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) - if not iqn or not iqn.startswith("iqn."): - continue - igroup_name = _igroup_name(self.svm_name, host.name) - igroup = self.ontap.get_igroup(self.svm_name, igroup_name) - self.assertIsNone( - igroup, - "ONTAP igroup '%s' still exists after pool deletion" % igroup_name - ) - - # ------------------------------------------------------------------ - # Step 11 - Create pool + volume, enter maintenance, force-delete - # ------------------------------------------------------------------ - - @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_11_create_pool_volume_maintenance_force_delete(self): - """ - Validates the forced=True behaviour of deleteStoragePool. - - CloudStack distinguishes two volume categories on a pool: - - non-destroyed (Allocated/Ready): active volumes - - destroyed (Destroy state) : soft-deleted, awaiting GC expunge - - forced=False → fails if ANY volume record exists on the pool (any state) - forced=True → fails only if non-destroyed volumes exist; - if only destroyed volumes remain CloudStack force-expunges - them and removes the pool. - - This test covers the two reliable halves of that contract: - - Step 1 Create pool + allocate a data volume (non-destroyed). - Step 2 Enter maintenance mode. - Step 3 forced=False delete MUST FAIL — non-destroyed volume present. - Step 4 Soft-delete the volume (deleteVolume API). - Step 5 forced=True delete MUST SUCCEED — handles any remaining state - (immediately-expunged or still Destroyed — both pass). - Step 6 Assert pool is gone from CloudStack and ONTAP. - - Note: The Destroyed-only scenario (forced=True succeeds where forced=False - would still fail) requires a VM lifecycle to produce Destroyed volumes and - is covered by higher-level system tests rather than this FT suite. - """ - if not self.disk_offering_id: - self.skipTest( - "No disk offering available; force-delete test requires a volume " - "to be present on the pool." - ) - - pool2 = self._create_pool() - self.__class__.pool2 = pool2 - self.assertEqual( - pool2.state, "Up", - "Pool2 state should be 'Up', got '%s'" % pool2.state - ) - - # Step 1: allocate a data volume — must succeed for this test to be valid - try: - vol2 = self._create_volume(pool2.id) - self.__class__.volume2 = vol2 - except Exception as e: - self.skipTest( - "createVolume failed (iSCSI may require an attached VM): %s" % e - ) - self.assertIsNotNone(getattr(vol2, "id", None), - "Volume2 creation returned no id") - - # ONTAP: LUN must be created in pool2's FlexVol - luns2 = self.ontap.list_luns_in_volume(self.svm_name, pool2.name) - self.assertTrue( - len(luns2) > 0, - "No LUNs found in ONTAP FlexVol '%s' after volume2 creation" % pool2.name - ) - self.__class__.lun_path2 = luns2[0].get("name") - - # Step 2: enter maintenance - maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() - maint_cmd.id = pool2.id - self.apiClient.enableStorageMaintenance(maint_cmd) - self._poll_pool_state(pool2.id, "Maintenance", timeout=120) - - # ONTAP: LUN still present during maintenance - if self.__class__.lun_path2: - lun2 = self.ontap.get_lun(self.svm_name, self.__class__.lun_path2) - self.assertIsNotNone( - lun2, - "ONTAP LUN '%s' disappeared during maintenance (pool2)" % self.__class__.lun_path2 - ) - - # Step 3: forced=False must FAIL — active (non-destroyed) volume present - from marvin.cloudstackException import CloudstackAPIException - with self.assertRaises(CloudstackAPIException, - msg="deleteStoragePool (forced=False) should fail " - "when a non-destroyed volume is on the pool"): - self._delete_pool(pool2.id, forced=False) - - # Step 4: soft-delete the volume via the deleteVolume API - del_vol_cmd = deleteVolumeAPI.deleteVolumeCmd() - del_vol_cmd.id = self.__class__.volume2.id - self.apiClient.deleteVolume(del_vol_cmd) - self.__class__.volume2 = None - - # ONTAP: LUN must be gone after deleteVolume - if self.__class__.lun_path2: - lun2 = self.ontap.get_lun(self.svm_name, self.__class__.lun_path2) - self.assertIsNone( - lun2, - "ONTAP LUN '%s' still exists after deleteVolume (pool2)" % self.__class__.lun_path2 - ) - self.__class__.lun_path2 = None - - # Step 5: forced=True must SUCCEED — handles any remaining volume state - self._delete_pool(pool2.id, forced=True) - self.__class__.pool2 = None - - # Step 6: assert CloudStack and ONTAP cleaned up - try: - remaining = list_storage_pools(self.apiClient, id=pool2.id) - except Exception: - remaining = None - self.assertFalse(remaining, "Pool2 still listed in CloudStack after force-deletion") - - # ONTAP: FlexVol and igroups must be deleted - ontap_vol = self.ontap.get_volume(pool2.name) - self.assertIsNone( - ontap_vol, - "ONTAP FlexVol '%s' still exists after force-deletion" % pool2.name - ) - for host in self.cluster_hosts: - iqn = getattr(host, "storageurl", None) or getattr(host, "StorageUrl", None) - if not iqn or not iqn.startswith("iqn."): - continue - igroup_name = _igroup_name(self.svm_name, host.name) - igroup = self.ontap.get_igroup(self.svm_name, igroup_name) - self.assertIsNone( - igroup, - "ONTAP igroup '%s' still exists after pool2 force-deletion" % igroup_name - ) diff --git a/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py b/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py deleted file mode 100644 index adbc61579292..000000000000 --- a/test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py +++ /dev/null @@ -1,404 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. - -Tests are numbered test_01 ... test_04 and must run in that order. Each step -builds on the shared state established by the previous step. - -Workflow: - 01 Create primary storage pool - 02 Disable storage pool - 03 Enable storage pool - 04 Enter maintenance mode - -Prerequisites: - - CloudStack management server with the NetApp ONTAP plugin deployed - - KVM cluster registered in CloudStack - - ONTAP SVM with NFS3 service enabled and at least one NFS data LIF - - ontap.cfg populated with real values - -Running: - nosetests --with-marvin \\ - --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ - test/integration/plugins/ontap/test_ontap_create_primary_storage_nfs3.py -v - -Note: Tests 01-04 share class-level state (sequential). Running a single test -with -m "test_NN" will invoke setUpClass but the guard assertion will fail -immediately if earlier steps have not yet run. Always run the full suite. -""" - -import base64 -import logging -import random - -from nose.plugins.attrib import attr - -from marvin.cloudstackAPI import ( - createStoragePool as createStoragePoolAPI, - enableStorageMaintenance, - updateStoragePool as updateStoragePoolAPI, -) -from marvin.lib.base import StoragePool -from marvin.lib.common import list_storage_pools - -from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details - -logger = logging.getLogger("TestOntapNFS3Workflow") - - -# --------------------------------------------------------------------------- -# Test data -# --------------------------------------------------------------------------- - -class TestData: - account = "account" - ontap = "ontap" - primaryStorage = "primaryStorage" - provider = "provider" - scope = "scope" - tags = "tags" - - DETAIL_USERNAME = "username" - DETAIL_PASSWORD = "password" - DETAIL_SVM_NAME = "svmName" - DETAIL_PROTOCOL = "protocol" - DETAIL_STORAGE_IP = "storageIP" - DETAIL_VOLUME_UUID = "volumeUUID" - DETAIL_VOLUME_NAME = "volumeName" - DETAIL_DATA_LIF = "dataLIF" - DETAIL_NFS_MOUNT_OPTS = "nfsmountopts" - - ONTAP_MIN_VOLUME_SIZE = 1677721600 - - def __init__(self, storage_ip, svm_name, username, password, - protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", - tags="ontap-nfs3", capacitybytes=None): - if capacitybytes is None: - capacitybytes = TestData.ONTAP_MIN_VOLUME_SIZE * 2 - encoded_password = base64.b64encode(password.encode()).decode() - self.testdata = { - TestData.ontap: { - TestData.DETAIL_STORAGE_IP: storage_ip, - TestData.DETAIL_SVM_NAME: svm_name, - TestData.DETAIL_USERNAME: username, - TestData.DETAIL_PASSWORD: password, - }, - TestData.account: { - "email": "ontap-nfs3-wf@test.com", - "firstname": "ONTAP", - "lastname": "NFS3-WF", - "username": "ontap_nfs3_wf_%d" % random.randint(0, 9999), - "password": "password", - }, - TestData.primaryStorage: { - "name": "OntapNFS3_%d" % random.randint(0, 9999), - TestData.scope: scope, - TestData.provider: provider, - TestData.tags: tags, - "capacitybytes": capacitybytes, - "managed": True, - "details": { - TestData.DETAIL_USERNAME: username, - TestData.DETAIL_PASSWORD: encoded_password, - TestData.DETAIL_SVM_NAME: svm_name, - TestData.DETAIL_PROTOCOL: protocol, - TestData.DETAIL_STORAGE_IP: storage_ip, - }, - }, - } - - -# --------------------------------------------------------------------------- -# Sequential workflow test class -# --------------------------------------------------------------------------- - -class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): - - # ---- NFS3-specific shared state ------------------------------------ - pool_ep_name = None # NFS export policy name for pool - cluster_host_ips = None - - _vol_name_prefix = "OntapNFS3Vol" - - @classmethod - def setUpClass(cls): - testclient = super( - TestOntapNFS3PrimaryStorageWorkflow, cls - ).getClsTestClient() - - cls.apiClient = testclient.getApiClient() - cls.dbConnection = testclient.getDbConnection() - config = testclient.getParsedTestDataConfig() - - ontap_cfg = config.get("ontap", {}) - storage_ip = ontap_cfg.get("storageIP", "") - svm_name = ontap_cfg.get("svmName", "") - username = ontap_cfg.get("username", "") - password = ontap_cfg.get("password", "") - protocol = ontap_cfg.get("protocol", "NFS3") - scope = ontap_cfg.get("storagePoolScope", "CLUSTER") - provider = ontap_cfg.get("storagePoolProvider", "NetApp ONTAP") - tags = ontap_cfg.get("storagePoolTags", "ontap-nfs3") - capacitybytes = ontap_cfg.get("capacitybytes", None) - - cls.testdata = TestData( - storage_ip, svm_name, username, password, - protocol=protocol, scope=scope, provider=provider, - tags=tags, capacitybytes=capacitybytes, - ).testdata - cls.ontap = OntapRestClient(storage_ip, username, password) - cls.svm_name = svm_name - - cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) - - # Resolve cluster host IPs for export policy rule assertions - cls.cluster_host_ips = [ - h.ipaddress for h in cls.cluster_hosts - if getattr(h, "ipaddress", None) - ] - - # No per-test tearDown — state intentionally persists between steps. - - # ------------------------------------------------------------------ - # Helpers - # ------------------------------------------------------------------ - - def _create_pool(self): - ps = self.testdata[TestData.primaryStorage] - storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] - pool_name = "OntapNFS3_%d" % random.randint(0, 99999) - - cmd = createStoragePoolAPI.createStoragePoolCmd() - cmd.name = pool_name - cmd.url = "nfs://%s/ontap" % storage_ip - cmd.zoneid = self.zone.id - cmd.clusterid = self.cluster.id - cmd.podid = self.cluster.podid - cmd.scope = ps[TestData.scope] - cmd.provider = ps[TestData.provider] - cmd.tags = ps[TestData.tags] - cmd.capacitybytes = ps["capacitybytes"] - cmd.hypervisor = "KVM" - cmd.managed = True - - count = 1 - for key, value in ps["details"].items(): - setattr(cmd, "details[{}].{}".format(count, key), value) - count += 1 - - response = self.apiClient.createStoragePool(cmd) - return StoragePool(response.__dict__) - - def _get_export_policy_name(self, pool): - """Extract the export policy name from pool creation response details.""" - details = _parse_pool_details(pool) - ep_name = details.get("exportPolicyName") - if not ep_name: - # Fallback: plugin typically uses cs-{svmName}-{poolName} - ep_name = "cs-%s-%s" % (self.svm_name, pool.name) - return ep_name - - def _assert_export_policy_has_host_ips(self, ep_name): - """Assert that the export policy exists and its rules include each cluster host IP.""" - policy = self.ontap.get_export_policy(ep_name) - self.assertIsNotNone( - policy, - "Export policy '%s' not found on ONTAP" % ep_name - ) - if not self.cluster_host_ips: - return # no host IPs registered; skip rule-level check - all_clients = [] - for rule in policy.get("rules", []): - for client in rule.get("clients", []): - all_clients.append(client.get("match", "")) - for ip in self.cluster_host_ips: - self.assertTrue( - any(ip in c for c in all_clients), - "Host IP '%s' not found in export policy '%s' rules: %s" - % (ip, ep_name, all_clients) - ) - - # ------------------------------------------------------------------ - # Step 01 — Create primary storage pool - # ------------------------------------------------------------------ - - @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_01_create_primary_storage_pool(self): - """ - Create an NFS3 primary storage pool and verify: - - CloudStack state is Up, type is NetworkFilesystem - - nfsmountopts contains 'vers=3' - - ONTAP: FlexVol exists and is online - - ONTAP: NFS export policy exists with cluster host IP rules - - ONTAP: at least one NFS data LIF is present on the SVM - """ - pool = self._create_pool() - self.__class__.pool = pool - - self.assertEqual( - pool.state, "Up", - "Pool state should be 'Up', got '%s'" % pool.state - ) - self.assertEqual( - pool.type, "NetworkFilesystem", - "Pool type should be 'NetworkFilesystem', got '%s'" % pool.type - ) - - # Verify nfsmountopts via listStoragePools - listed = list_storage_pools(self.apiClient, id=pool.id) - self.assertIsNotNone(listed, "listStoragePools returned None for pool %s" % pool.id) - nfs_opts = getattr(listed[0], "nfsmountopts", "") - self.assertIn( - "vers=3", nfs_opts, - "nfsmountopts should contain 'vers=3', got '%s'" % nfs_opts - ) - - # ONTAP: FlexVol must be online - ontap_vol = self.ontap.get_volume(pool.name) - self.assertIsNotNone( - ontap_vol, - "ONTAP FlexVol not found for pool '%s'" % pool.name - ) - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") - ) - - # ONTAP: export policy must exist with host IP rules - ep_name = self._get_export_policy_name(pool) - self.__class__.pool_ep_name = ep_name - self._assert_export_policy_has_host_ips(ep_name) - - # ONTAP: at least one NFS data LIF must be present - lifs = self.ontap.get_data_lifs(self.svm_name) - self.assertTrue( - len(lifs) > 0, - "No NFS data LIFs found on SVM '%s'" % self.svm_name - ) - - # ------------------------------------------------------------------ - # Step 02 — Disable storage pool - # ------------------------------------------------------------------ - - @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_02_disable_storage_pool(self): - """ - Disable the pool and verify: - - CloudStack reports Disabled - - ONTAP: FlexVol is still online and export policy unchanged - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") - - cmd = updateStoragePoolAPI.updateStoragePoolCmd() - cmd.id = self.__class__.pool.id - cmd.enabled = False - self.apiClient.updateStoragePool(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Disabled", timeout=60) - self.assertEqual(result.state, "Disabled") - - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after disable") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should still be 'online' after disable, got '%s'" - % ontap_vol.get("state") - ) - if self.__class__.pool_ep_name: - policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) - self.assertIsNotNone( - policy, - "Export policy '%s' should still exist after disable" - % self.__class__.pool_ep_name - ) - - # ------------------------------------------------------------------ - # Step 03 — Enable storage pool - # ------------------------------------------------------------------ - - @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_03_enable_storage_pool(self): - """ - Re-enable the pool and verify: - - CloudStack reports Up - - ONTAP: FlexVol is still online and export policy unchanged - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") - - cmd = updateStoragePoolAPI.updateStoragePoolCmd() - cmd.id = self.__class__.pool.id - cmd.enabled = True - self.apiClient.updateStoragePool(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Up", timeout=60) - self.assertEqual(result.state, "Up") - - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after enable") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should be 'online' after enable, got '%s'" - % ontap_vol.get("state") - ) - if self.__class__.pool_ep_name: - policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) - self.assertIsNotNone( - policy, - "Export policy '%s' should still exist after enable" - % self.__class__.pool_ep_name - ) - - # ------------------------------------------------------------------ - # Step 04 — Enter maintenance mode - # ------------------------------------------------------------------ - - @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_04_enter_maintenance_mode(self): - """ - Put the pool into maintenance mode and verify: - - CloudStack reports Maintenance - - ONTAP: FlexVol is still online and export policy unchanged - (maintenance is a CS-only state change) - """ - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") - - cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() - cmd.id = self.__class__.pool.id - self.apiClient.enableStorageMaintenance(cmd) - - result = self._poll_pool_state(self.__class__.pool.id, "Maintenance", timeout=120) - self.assertEqual(result.state, "Maintenance") - - ontap_vol = self.ontap.get_volume(self.__class__.pool.name) - self.assertIsNotNone(ontap_vol, "ONTAP FlexVol disappeared after entering maintenance") - self.assertEqual( - ontap_vol.get("state"), "online", - "ONTAP FlexVol should still be 'online' in maintenance, got '%s'" - % ontap_vol.get("state") - ) - if self.__class__.pool_ep_name: - policy = self.ontap.get_export_policy(self.__class__.pool_ep_name) - self.assertIsNotNone( - policy, - "Export policy '%s' should still exist during maintenance" - % self.__class__.pool_ep_name - ) - - -