diff -Nru nova-25.1.1/debian/changelog nova-25.1.1/debian/changelog --- nova-25.1.1/debian/changelog 2023-05-15 20:11:40.000000000 +0000 +++ nova-25.1.1/debian/changelog 2023-05-31 20:43:41.000000000 +0000 @@ -1,3 +1,14 @@ +nova (3:25.1.1-0ubuntu1.1) jammy-security; urgency=medium + + * SECURITY UPDATE: Unauthorized File Access (LP: #2021980) + - debian/patches/CVE-2023-2088-1.patch: Use force=True for os-brick + disconnect during delete. + - debian/patches/CVE-2023-2088-2.patch: Enable use of service user + token with admin context. + - CVE-2023-2088 + + -- Corey Bryant Wed, 31 May 2023 16:43:41 -0400 + nova (3:25.1.1-0ubuntu1) jammy; urgency=medium * New stable point release for OpenStack Yoga (LP: #2019759). diff -Nru nova-25.1.1/debian/patches/CVE-2023-2088-1.patch nova-25.1.1/debian/patches/CVE-2023-2088-1.patch --- nova-25.1.1/debian/patches/CVE-2023-2088-1.patch 1970-01-01 00:00:00.000000000 +0000 +++ nova-25.1.1/debian/patches/CVE-2023-2088-1.patch 2023-05-31 20:43:41.000000000 +0000 @@ -0,0 +1,1138 @@ +From 4d8efa2d196f72fdde33136a0b50c4ee8da3c941 Mon Sep 17 00:00:00 2001 +From: melanie witt +Date: Wed, 15 Feb 2023 22:37:40 +0000 +Subject: [PATCH] Use force=True for os-brick disconnect during delete + +The 'force' parameter of os-brick's disconnect_volume() method allows +callers to ignore flushing errors and ensure that devices are being +removed from the host. + +We should use force=True when we are going to delete an instance to +avoid leaving leftover devices connected to the compute host which +could then potentially be reused to map to volumes to an instance that +should not have access to those volumes. + +We can use force=True even when disconnecting a volume that will not be +deleted on termination because os-brick will always attempt to flush +and disconnect gracefully before forcefully removing devices. + +Closes-Bug: #2004555 + +Change-Id: I3629b84d3255a8fe9d8a7cea8c6131d7c40899e8 +(cherry picked from commit db455548a12beac1153ce04eca5e728d7b773901) +(cherry picked from commit efb01985db88d6333897018174649b425feaa1b4) +(cherry picked from commit 8b4b99149a35663fc11d7d163082747b1b210b4d) +--- + .../admin/configuration/cross-cell-resize.rst | 2 +- + doc/source/admin/configuration/index.rst | 1 + + .../configuration/service-user-token.rst | 59 +++++++++++++++++ + doc/source/admin/live-migration-usage.rst | 2 +- + .../admin/migrate-instance-with-snapshot.rst | 2 +- + doc/source/admin/support-compute.rst | 64 ------------------- + doc/source/install/compute-install-obs.rst | 20 ++++++ + doc/source/install/compute-install-rdo.rst | 20 ++++++ + doc/source/install/compute-install-ubuntu.rst | 20 ++++++ + doc/source/install/controller-install-obs.rst | 20 ++++++ + doc/source/install/controller-install-rdo.rst | 20 ++++++ + .../install/controller-install-ubuntu.rst | 20 ++++++ + nova/cmd/status.py | 11 ++++ + nova/tests/unit/cmd/test_status.py | 16 +++++ + nova/tests/unit/virt/hyperv/test_vmops.py | 2 +- + nova/tests/unit/virt/hyperv/test_volumeops.py | 26 ++++++-- + nova/tests/unit/virt/libvirt/test_driver.py | 61 ++++++++++++++++-- + .../virt/libvirt/volume/test_fibrechannel.py | 20 ++++++ + .../unit/virt/libvirt/volume/test_iscsi.py | 9 +++ + .../unit/virt/libvirt/volume/test_lightos.py | 8 ++- + .../unit/virt/libvirt/volume/test_nvme.py | 8 ++- + .../unit/virt/libvirt/volume/test_scaleio.py | 8 ++- + .../unit/virt/libvirt/volume/test_storpool.py | 16 ++++- + .../virt/libvirt/volume/test_vzstorage.py | 8 ++- + nova/virt/hyperv/vmops.py | 2 +- + nova/virt/hyperv/volumeops.py | 12 ++-- + nova/virt/libvirt/driver.py | 7 +- + nova/virt/libvirt/volume/fibrechannel.py | 7 +- + nova/virt/libvirt/volume/fs.py | 2 +- + nova/virt/libvirt/volume/iscsi.py | 7 +- + nova/virt/libvirt/volume/lightos.py | 7 +- + nova/virt/libvirt/volume/nvme.py | 6 +- + nova/virt/libvirt/volume/quobyte.py | 2 +- + nova/virt/libvirt/volume/scaleio.py | 7 +- + nova/virt/libvirt/volume/smbfs.py | 2 +- + nova/virt/libvirt/volume/storpool.py | 5 +- + nova/virt/libvirt/volume/volume.py | 2 +- + nova/virt/libvirt/volume/vzstorage.py | 5 +- + .../service-user-token-421d067c16257782.yaml | 11 ++++ + 39 files changed, 413 insertions(+), 114 deletions(-) + create mode 100644 doc/source/admin/configuration/service-user-token.rst + create mode 100644 releasenotes/notes/service-user-token-421d067c16257782.yaml + +diff --git a/doc/source/admin/configuration/cross-cell-resize.rst b/doc/source/admin/configuration/cross-cell-resize.rst +index e51e425774..0c34fd13f5 100644 +--- a/doc/source/admin/configuration/cross-cell-resize.rst ++++ b/doc/source/admin/configuration/cross-cell-resize.rst +@@ -284,7 +284,7 @@ Troubleshooting + Timeouts + ~~~~~~~~ + +-Configure a :ref:`service user ` in case the user token ++Configure a :ref:`service user ` in case the user token + times out, e.g. during the snapshot and download of a large server image. + + If RPC calls are timing out with a ``MessagingTimeout`` error in the logs, +diff --git a/doc/source/admin/configuration/index.rst b/doc/source/admin/configuration/index.rst +index 233597b1fe..f5b6fde9da 100644 +--- a/doc/source/admin/configuration/index.rst ++++ b/doc/source/admin/configuration/index.rst +@@ -19,6 +19,7 @@ A list of config options based on different topics can be found below: + .. toctree:: + :maxdepth: 1 + ++ /admin/configuration/service-user-token + /admin/configuration/api + /admin/configuration/resize + /admin/configuration/cross-cell-resize +diff --git a/doc/source/admin/configuration/service-user-token.rst b/doc/source/admin/configuration/service-user-token.rst +new file mode 100644 +index 0000000000..740730af1d +--- /dev/null ++++ b/doc/source/admin/configuration/service-user-token.rst +@@ -0,0 +1,59 @@ ++.. _service_user_token: ++ ++=================== ++Service User Tokens ++=================== ++ ++.. note:: ++ ++ Configuration of service user tokens is **required** for every Nova service ++ for security reasons. See https://bugs.launchpad.net/nova/+bug/2004555 for ++ details. ++ ++Configure Nova to send service user tokens alongside regular user tokens when ++making REST API calls to other services. The identity service (Keystone) will ++authenticate a request using the service user token if the regular user token ++has expired. ++ ++This is important when long-running operations such as live migration or ++snapshot take long enough to exceed the expiry of the user token. Without the ++service token, if a long-running operation exceeds the expiry of the user ++token, post operations such as cleanup after a live migration could fail when ++Nova calls other service APIs like block-storage (Cinder) or networking ++(Neutron). ++ ++The service token is also used by services to validate whether the API caller ++is a service. Some service APIs are restricted to service users only. ++ ++To set up service tokens, create a ``nova`` service user and ``service`` role ++in the identity service (Keystone) and assign the ``service`` role to the ++``nova`` service user. ++ ++Then, configure the :oslo.config:group:`service_user` section of the Nova ++configuration file, for example: ++ ++.. code-block:: ini ++ ++ [service_user] ++ send_service_user_token = true ++ auth_url = https://104.130.216.102/identity ++ auth_strategy = keystone ++ auth_type = password ++ project_domain_name = Default ++ project_name = service ++ user_domain_name = Default ++ username = nova ++ password = secretservice ++ ... ++ ++And configure the other identity options as necessary for the service user, ++much like you would configure nova to work with the image service (Glance) or ++networking service (Neutron). ++ ++.. note:: ++ ++ Please note that the role assigned to the :oslo.config:group:`service_user` ++ needs to be in the configured ++ :oslo.config:option:`keystone_authtoken.service_token_roles` of other ++ services such as block-storage (Cinder), image (Glance), and networking ++ (Neutron). +diff --git a/doc/source/admin/live-migration-usage.rst b/doc/source/admin/live-migration-usage.rst +index 783ab5e27c..a1e7f18756 100644 +--- a/doc/source/admin/live-migration-usage.rst ++++ b/doc/source/admin/live-migration-usage.rst +@@ -320,4 +320,4 @@ To make live-migration succeed, you have several options: + + If live migrations routinely timeout or fail during cleanup operations due + to the user token timing out, consider configuring nova to use +-:ref:`service user tokens `. ++:ref:`service user tokens `. +diff --git a/doc/source/admin/migrate-instance-with-snapshot.rst b/doc/source/admin/migrate-instance-with-snapshot.rst +index 65059679ab..230431091e 100644 +--- a/doc/source/admin/migrate-instance-with-snapshot.rst ++++ b/doc/source/admin/migrate-instance-with-snapshot.rst +@@ -67,7 +67,7 @@ Create a snapshot of the instance + + If snapshot operations routinely fail because the user token times out + while uploading a large disk image, consider configuring nova to use +- :ref:`service user tokens `. ++ :ref:`service user tokens `. + + #. Use the :command:`openstack image list` command to check the status + until the status is ``ACTIVE``: +diff --git a/doc/source/admin/support-compute.rst b/doc/source/admin/support-compute.rst +index 8522e51d79..31e32fd1dd 100644 +--- a/doc/source/admin/support-compute.rst ++++ b/doc/source/admin/support-compute.rst +@@ -478,67 +478,3 @@ Ensure the ``compute`` endpoint in the identity service catalog is pointing + at ``/v2.1`` instead of ``/v2``. The former route supports microversions, + while the latter route is considered the legacy v2.0 compatibility-mode + route which renders all requests as if they were made on the legacy v2.0 API. +- +- +-.. _user_token_timeout: +- +-User token times out during long-running operations +---------------------------------------------------- +- +-Problem +-~~~~~~~ +- +-Long-running operations such as live migration or snapshot can sometimes +-overrun the expiry of the user token. In such cases, post operations such +-as cleaning up after a live migration can fail when the nova-compute service +-needs to cleanup resources in other services, such as in the block-storage +-(cinder) or networking (neutron) services. +- +-For example: +- +-.. code-block:: console +- +- 2018-12-17 13:47:29.591 16987 WARNING nova.virt.libvirt.migration [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] [instance: ead8ecc3-f473-4672-a67b-c44534c6042d] Live migration not completed after 2400 sec +- 2018-12-17 13:47:30.097 16987 WARNING nova.virt.libvirt.driver [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] [instance: ead8ecc3-f473-4672-a67b-c44534c6042d] Migration operation was cancelled +- 2018-12-17 13:47:30.299 16987 ERROR nova.virt.libvirt.driver [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] [instance: ead8ecc3-f473-4672-a67b-c44534c6042d] Live Migration failure: operation aborted: migration job: canceled by client: libvirtError: operation aborted: migration job: canceled by client +- 2018-12-17 13:47:30.685 16987 INFO nova.compute.manager [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] [instance: ead8ecc3-f473-4672-a67b-c44534c6042d] Swapping old allocation on 3e32d595-bd1f-4136-a7f4-c6703d2fbe18 held by migration 17bec61d-544d-47e0-a1c1-37f9d7385286 for instance +- 2018-12-17 13:47:32.450 16987 ERROR nova.volume.cinder [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] Delete attachment failed for attachment 58997d5b-24f0-4073-819e-97916fb1ee19. Error: The request you have made requires authentication. (HTTP 401) Code: 401: Unauthorized: The request you have made requires authentication. (HTTP 401) +- +-Solution +-~~~~~~~~ +- +-Configure nova to use service user tokens to supplement the regular user token +-used to initiate the operation. The identity service (keystone) will then +-authenticate a request using the service user token if the user token has +-already expired. +- +-To use, create a service user in the identity service similar as you would when +-creating the ``nova`` service user. +- +-Then configure the :oslo.config:group:`service_user` section of the nova +-configuration file, for example: +- +-.. code-block:: ini +- +- [service_user] +- send_service_user_token = True +- auth_type = password +- project_domain_name = Default +- project_name = service +- user_domain_name = Default +- password = secretservice +- username = nova +- auth_url = https://104.130.216.102/identity +- ... +- +-And configure the other identity options as necessary for the service user, +-much like you would configure nova to work with the image service (glance) +-or networking service. +- +-.. note:: +- +- Please note that the role of the :oslo.config:group:`service_user` you +- configure needs to be a superset of +- :oslo.config:option:`keystone_authtoken.service_token_roles` (The option +- :oslo.config:option:`keystone_authtoken.service_token_roles` is configured +- in cinder, glance and neutron). +diff --git a/doc/source/install/compute-install-obs.rst b/doc/source/install/compute-install-obs.rst +index c5c1d29fb3..c227b6eba4 100644 +--- a/doc/source/install/compute-install-obs.rst ++++ b/doc/source/install/compute-install-obs.rst +@@ -92,6 +92,26 @@ Install and configure components + Comment out or remove any other options in the ``[keystone_authtoken]`` + section. + ++ * In the ``[service_user]`` section, configure :ref:`service user ++ tokens `: ++ ++ .. path /etc/nova/nova.conf ++ .. code-block:: ini ++ ++ [service_user] ++ send_service_user_token = true ++ auth_url = https://controller/identity ++ auth_strategy = keystone ++ auth_type = password ++ project_domain_name = Default ++ project_name = service ++ user_domain_name = Default ++ username = nova ++ password = NOVA_PASS ++ ++ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in ++ the Identity service. ++ + * In the ``[DEFAULT]`` section, configure the ``my_ip`` option: + + .. path /etc/nova/nova.conf +diff --git a/doc/source/install/compute-install-rdo.rst b/doc/source/install/compute-install-rdo.rst +index 0a5ad685a6..0c6203a667 100644 +--- a/doc/source/install/compute-install-rdo.rst ++++ b/doc/source/install/compute-install-rdo.rst +@@ -84,6 +84,26 @@ Install and configure components + Comment out or remove any other options in the ``[keystone_authtoken]`` + section. + ++ * In the ``[service_user]`` section, configure :ref:`service user ++ tokens `: ++ ++ .. path /etc/nova/nova.conf ++ .. code-block:: ini ++ ++ [service_user] ++ send_service_user_token = true ++ auth_url = https://controller/identity ++ auth_strategy = keystone ++ auth_type = password ++ project_domain_name = Default ++ project_name = service ++ user_domain_name = Default ++ username = nova ++ password = NOVA_PASS ++ ++ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in ++ the Identity service. ++ + * In the ``[DEFAULT]`` section, configure the ``my_ip`` option: + + .. path /etc/nova/nova.conf +diff --git a/doc/source/install/compute-install-ubuntu.rst b/doc/source/install/compute-install-ubuntu.rst +index 8605c73316..baf0585e52 100644 +--- a/doc/source/install/compute-install-ubuntu.rst ++++ b/doc/source/install/compute-install-ubuntu.rst +@@ -74,6 +74,26 @@ Install and configure components + Comment out or remove any other options in the + ``[keystone_authtoken]`` section. + ++ * In the ``[service_user]`` section, configure :ref:`service user ++ tokens `: ++ ++ .. path /etc/nova/nova.conf ++ .. code-block:: ini ++ ++ [service_user] ++ send_service_user_token = true ++ auth_url = https://controller/identity ++ auth_strategy = keystone ++ auth_type = password ++ project_domain_name = Default ++ project_name = service ++ user_domain_name = Default ++ username = nova ++ password = NOVA_PASS ++ ++ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in ++ the Identity service. ++ + * In the ``[DEFAULT]`` section, configure the ``my_ip`` option: + + .. path /etc/nova/nova.conf +diff --git a/doc/source/install/controller-install-obs.rst b/doc/source/install/controller-install-obs.rst +index 18499612c3..01b7bb0f5a 100644 +--- a/doc/source/install/controller-install-obs.rst ++++ b/doc/source/install/controller-install-obs.rst +@@ -260,6 +260,26 @@ Install and configure components + Comment out or remove any other options in the ``[keystone_authtoken]`` + section. + ++ * In the ``[service_user]`` section, configure :ref:`service user ++ tokens `: ++ ++ .. path /etc/nova/nova.conf ++ .. code-block:: ini ++ ++ [service_user] ++ send_service_user_token = true ++ auth_url = https://controller/identity ++ auth_strategy = keystone ++ auth_type = password ++ project_domain_name = Default ++ project_name = service ++ user_domain_name = Default ++ username = nova ++ password = NOVA_PASS ++ ++ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in ++ the Identity service. ++ + * In the ``[DEFAULT]`` section, configure the ``my_ip`` option to use the + management interface IP address of the controller node: + +diff --git a/doc/source/install/controller-install-rdo.rst b/doc/source/install/controller-install-rdo.rst +index fd2419631e..b6098f1776 100644 +--- a/doc/source/install/controller-install-rdo.rst ++++ b/doc/source/install/controller-install-rdo.rst +@@ -247,6 +247,26 @@ Install and configure components + Comment out or remove any other options in the ``[keystone_authtoken]`` + section. + ++ * In the ``[service_user]`` section, configure :ref:`service user ++ tokens `: ++ ++ .. path /etc/nova/nova.conf ++ .. code-block:: ini ++ ++ [service_user] ++ send_service_user_token = true ++ auth_url = https://controller/identity ++ auth_strategy = keystone ++ auth_type = password ++ project_domain_name = Default ++ project_name = service ++ user_domain_name = Default ++ username = nova ++ password = NOVA_PASS ++ ++ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in ++ the Identity service. ++ + * In the ``[DEFAULT]`` section, configure the ``my_ip`` option to use the + management interface IP address of the controller node: + +diff --git a/doc/source/install/controller-install-ubuntu.rst b/doc/source/install/controller-install-ubuntu.rst +index 7282b0b2e2..1363a98ba8 100644 +--- a/doc/source/install/controller-install-ubuntu.rst ++++ b/doc/source/install/controller-install-ubuntu.rst +@@ -237,6 +237,26 @@ Install and configure components + Comment out or remove any other options in the ``[keystone_authtoken]`` + section. + ++ * In the ``[service_user]`` section, configure :ref:`service user ++ tokens `: ++ ++ .. path /etc/nova/nova.conf ++ .. code-block:: ini ++ ++ [service_user] ++ send_service_user_token = true ++ auth_url = https://controller/identity ++ auth_strategy = keystone ++ auth_type = password ++ project_domain_name = Default ++ project_name = service ++ user_domain_name = Default ++ username = nova ++ password = NOVA_PASS ++ ++ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in ++ the Identity service. ++ + * In the ``[DEFAULT]`` section, configure the ``my_ip`` option to use the + management interface IP address of the controller node: + +diff --git a/nova/cmd/status.py b/nova/cmd/status.py +index 8a7041b062..2f310f0871 100644 +--- a/nova/cmd/status.py ++++ b/nova/cmd/status.py +@@ -336,6 +336,15 @@ https://docs.openstack.org/latest/nova/admin/hw_machine_type.html""")) + + return upgradecheck.Result(upgradecheck.Code.SUCCESS) + ++ def _check_service_user_token(self): ++ if not CONF.service_user.send_service_user_token: ++ msg = (_(""" ++Service user token configuration is required for all Nova services. ++For more details see the following: ++https://docs.openstack.org/latest/nova/admin/configuration/service-user-token.html""")) # noqa ++ return upgradecheck.Result(upgradecheck.Code.FAILURE, msg) ++ return upgradecheck.Result(upgradecheck.Code.SUCCESS) ++ + # The format of the check functions is to return an upgradecheck.Result + # object with the appropriate upgradecheck.Code and details set. If the + # check hits warnings or failures then those should be stored in the +@@ -361,6 +370,8 @@ https://docs.openstack.org/latest/nova/admin/hw_machine_type.html""")) + (_('Older than N-1 computes'), _check_old_computes), + # Added in Wallaby + (_('hw_machine_type unset'), _check_machine_type_set), ++ # Added in Bobcat ++ (_('Service User Token Configuration'), _check_service_user_token), + ) + + +diff --git a/nova/tests/unit/cmd/test_status.py b/nova/tests/unit/cmd/test_status.py +index ba85590697..2d33c890b7 100644 +--- a/nova/tests/unit/cmd/test_status.py ++++ b/nova/tests/unit/cmd/test_status.py +@@ -502,3 +502,19 @@ class TestCheckMachineTypeUnset(test.NoDBTestCase): + upgradecheck.Code.SUCCESS, + result.code + ) ++ ++ ++class TestUpgradeCheckServiceUserToken(test.NoDBTestCase): ++ ++ def setUp(self): ++ super().setUp() ++ self.cmd = status.UpgradeCommands() ++ ++ def test_service_user_token_not_configured(self): ++ result = self.cmd._check_service_user_token() ++ self.assertEqual(upgradecheck.Code.FAILURE, result.code) ++ ++ def test_service_user_token_configured(self): ++ self.flags(send_service_user_token=True, group='service_user') ++ result = self.cmd._check_service_user_token() ++ self.assertEqual(upgradecheck.Code.SUCCESS, result.code) +diff --git a/nova/tests/unit/virt/hyperv/test_vmops.py b/nova/tests/unit/virt/hyperv/test_vmops.py +index 1a71045ea2..0110b595c7 100644 +--- a/nova/tests/unit/virt/hyperv/test_vmops.py ++++ b/nova/tests/unit/virt/hyperv/test_vmops.py +@@ -1129,7 +1129,7 @@ class VMOpsTestCase(test_base.HyperVBaseTestCase): + mock_unplug_vifs.assert_called_once_with( + mock_instance, mock.sentinel.fake_network_info) + mock_disconnect_volumes.assert_called_once_with( +- mock.sentinel.FAKE_BD_INFO) ++ mock.sentinel.FAKE_BD_INFO, force=True) + mock_delete_disk_files.assert_called_once_with( + mock_instance.name) + +diff --git a/nova/tests/unit/virt/hyperv/test_volumeops.py b/nova/tests/unit/virt/hyperv/test_volumeops.py +index da7262085d..4a088b6030 100644 +--- a/nova/tests/unit/virt/hyperv/test_volumeops.py ++++ b/nova/tests/unit/virt/hyperv/test_volumeops.py +@@ -140,7 +140,13 @@ class VolumeOpsTestCase(test_base.HyperVBaseTestCase): + + self._volumeops.disconnect_volumes(block_device_info) + fake_volume_driver.disconnect_volume.assert_called_once_with( +- block_device_mapping[0]['connection_info']) ++ block_device_mapping[0]['connection_info'], force=False) ++ ++ # Verify force=True ++ fake_volume_driver.disconnect_volume.reset_mock() ++ self._volumeops.disconnect_volumes(block_device_info, force=True) ++ fake_volume_driver.disconnect_volume.assert_called_once_with( ++ block_device_mapping[0]['connection_info'], force=True) + + @mock.patch('time.sleep') + @mock.patch.object(volumeops.VolumeOps, '_get_volume_driver') +@@ -180,7 +186,7 @@ class VolumeOpsTestCase(test_base.HyperVBaseTestCase): + + if attach_failed: + fake_volume_driver.disconnect_volume.assert_called_once_with( +- fake_conn_info) ++ fake_conn_info, force=False) + mock_sleep.assert_has_calls( + [mock.call(CONF.hyperv.volume_attach_retry_interval)] * + CONF.hyperv.volume_attach_retry_count) +@@ -202,7 +208,13 @@ class VolumeOpsTestCase(test_base.HyperVBaseTestCase): + mock_get_volume_driver.assert_called_once_with( + mock.sentinel.conn_info) + fake_volume_driver.disconnect_volume.assert_called_once_with( +- mock.sentinel.conn_info) ++ mock.sentinel.conn_info, force=False) ++ ++ # Verify force=True ++ fake_volume_driver.disconnect_volume.reset_mock() ++ self._volumeops.disconnect_volume(mock.sentinel.conn_info, force=True) ++ fake_volume_driver.disconnect_volume.assert_called_once_with( ++ mock.sentinel.conn_info, force=True) + + @mock.patch.object(volumeops.VolumeOps, '_get_volume_driver') + def test_detach_volume(self, mock_get_volume_driver): +@@ -346,7 +358,13 @@ class BaseVolumeDriverTestCase(test_base.HyperVBaseTestCase): + self._base_vol_driver.disconnect_volume(conn_info) + + self._conn.disconnect_volume.assert_called_once_with( +- conn_info['data']) ++ conn_info['data'], force=False) ++ ++ # Verify force=True ++ self._conn.disconnect_volume.reset_mock() ++ self._base_vol_driver.disconnect_volume(conn_info, force=True) ++ self._conn.disconnect_volume.assert_called_once_with( ++ conn_info['data'], force=True) + + @mock.patch.object(volumeops.BaseVolumeDriver, '_get_disk_res_path') + def _test_get_disk_resource_path_by_conn_info(self, +diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py +index 33d16851b4..0eada9ee14 100644 +--- a/nova/tests/unit/virt/libvirt/test_driver.py ++++ b/nova/tests/unit/virt/libvirt/test_driver.py +@@ -9222,7 +9222,7 @@ class LibvirtConnTestCase(test.NoDBTestCase, + drvr._disconnect_volume( + self.context, fake_connection_info, fake_instance_1) + mock_volume_driver.disconnect_volume.assert_called_once_with( +- fake_connection_info, fake_instance_1) ++ fake_connection_info, fake_instance_1, force=False) + + @mock.patch.object(libvirt_driver.LibvirtDriver, '_detach_encryptor') + @mock.patch('nova.objects.InstanceList.get_uuids_by_host') +@@ -9596,7 +9596,12 @@ class LibvirtConnTestCase(test.NoDBTestCase, + device_name='vdc', + ), + mock.call.detach_encryptor(**encryption), +- mock.call.disconnect_volume(connection_info, instance)]) ++ mock.call.disconnect_volume( ++ connection_info, ++ instance, ++ force=False, ++ ) ++ ]) + get_device_conf_func = mock_detach_with_retry.mock_calls[0][1][2] + self.assertEqual(mock_guest.get_disk, get_device_conf_func.func) + self.assertEqual(('vdc',), get_device_conf_func.args) +@@ -19811,16 +19816,64 @@ class LibvirtConnTestCase(test.NoDBTestCase, + self.context, + mock.sentinel.connection_info, + instance, +- destroy_secrets=False ++ destroy_secrets=False, ++ force=True + ), + mock.call( + self.context, + mock.sentinel.connection_info, + instance, +- destroy_secrets=True ++ destroy_secrets=True, ++ force=True + ) + ]) + ++ @mock.patch('nova.virt.libvirt.driver.LibvirtDriver._get_volume_driver') ++ @mock.patch( ++ 'nova.virt.libvirt.driver.LibvirtDriver._should_disconnect_target', ++ new=mock.Mock(return_value=True)) ++ @mock.patch('nova.virt.libvirt.driver.LibvirtDriver._detach_encryptor', ++ new=mock.Mock()) ++ @mock.patch('nova.virt.libvirt.driver.LibvirtDriver._undefine_domain', ++ new=mock.Mock()) ++ @mock.patch('nova.virt.libvirt.driver.LibvirtDriver._get_vpmems', ++ new=mock.Mock(return_value=None)) ++ def test_cleanup_disconnect_volume(self, mock_vol_driver): ++ """Verify that we call disconnect_volume() with force=True ++ ++ cleanup() is called by destroy() when an instance is being deleted and ++ force=True should be passed down to os-brick's disconnect_volume() ++ call, which will ensure removal of devices regardless of errors. ++ ++ We need to ensure that devices are removed when an instance is being ++ deleted to avoid leaving leftover devices that could later be ++ erroneously connected by external entities (example: multipathd) to ++ instances that should not have access to the volumes. ++ ++ See https://bugs.launchpad.net/nova/+bug/2004555 for details. ++ """ ++ connection_info = mock.MagicMock() ++ block_device_info = { ++ 'block_device_mapping': [ ++ { ++ 'connection_info': connection_info ++ } ++ ] ++ } ++ instance = objects.Instance(self.context, **self.test_instance) ++ drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI()) ++ ++ drvr.cleanup( ++ self.context, ++ instance, ++ network_info={}, ++ block_device_info=block_device_info, ++ destroy_vifs=False, ++ destroy_disks=False, ++ ) ++ mock_vol_driver.return_value.disconnect_volume.assert_called_once_with( ++ connection_info, instance, force=True) ++ + @mock.patch.object(libvirt_driver.LibvirtDriver, '_get_volume_encryption') + @mock.patch.object(libvirt_driver.LibvirtDriver, '_allow_native_luksv1') + def test_swap_volume_native_luks_blocked(self, mock_allow_native_luksv1, +diff --git a/nova/tests/unit/virt/libvirt/volume/test_fibrechannel.py b/nova/tests/unit/virt/libvirt/volume/test_fibrechannel.py +index 89a59f2f1a..f0d403e300 100644 +--- a/nova/tests/unit/virt/libvirt/volume/test_fibrechannel.py ++++ b/nova/tests/unit/virt/libvirt/volume/test_fibrechannel.py +@@ -81,3 +81,23 @@ class LibvirtFibreChannelVolumeDriverTestCase( + self.assertEqual(requested_size, new_size) + libvirt_driver.connector.extend_volume.assert_called_once_with( + connection_info['data']) ++ ++ def test_disconnect_volume(self): ++ device_path = '/dev/fake-dev' ++ connection_info = {'data': {'device_path': device_path}} ++ ++ libvirt_driver = fibrechannel.LibvirtFibreChannelVolumeDriver( ++ self.fake_host) ++ libvirt_driver.connector.disconnect_volume = mock.MagicMock() ++ libvirt_driver.disconnect_volume( ++ connection_info, mock.sentinel.instance) ++ ++ libvirt_driver.connector.disconnect_volume.assert_called_once_with( ++ connection_info['data'], connection_info['data'], force=False) ++ ++ # Verify force=True ++ libvirt_driver.connector.disconnect_volume.reset_mock() ++ libvirt_driver.disconnect_volume( ++ connection_info, mock.sentinel.instance, force=True) ++ libvirt_driver.connector.disconnect_volume.assert_called_once_with( ++ connection_info['data'], connection_info['data'], force=True) +diff --git a/nova/tests/unit/virt/libvirt/volume/test_iscsi.py b/nova/tests/unit/virt/libvirt/volume/test_iscsi.py +index f8a64abea5..540c9c822d 100644 +--- a/nova/tests/unit/virt/libvirt/volume/test_iscsi.py ++++ b/nova/tests/unit/virt/libvirt/volume/test_iscsi.py +@@ -57,10 +57,19 @@ class LibvirtISCSIVolumeDriverTestCase( + device=device_path)) + libvirt_driver.disconnect_volume(connection_info, + mock.sentinel.instance) ++ libvirt_driver.connector.disconnect_volume.assert_called_once_with( ++ connection_info['data'], None, force=False) + + msg = mock_LOG_warning.call_args_list[0] + self.assertIn('Ignoring VolumeDeviceNotFound', msg[0][0]) + ++ # Verify force=True ++ libvirt_driver.connector.disconnect_volume.reset_mock() ++ libvirt_driver.disconnect_volume( ++ connection_info, mock.sentinel.instance, force=True) ++ libvirt_driver.connector.disconnect_volume.assert_called_once_with( ++ connection_info['data'], None, force=True) ++ + def test_extend_volume(self): + device_path = '/dev/fake-dev' + connection_info = {'data': {'device_path': device_path}} +diff --git a/nova/tests/unit/virt/libvirt/volume/test_lightos.py b/nova/tests/unit/virt/libvirt/volume/test_lightos.py +index 67fead13df..1eb9583d4c 100644 +--- a/nova/tests/unit/virt/libvirt/volume/test_lightos.py ++++ b/nova/tests/unit/virt/libvirt/volume/test_lightos.py +@@ -62,7 +62,13 @@ class LibvirtLightVolumeDriverTestCase(test_volume.LibvirtVolumeBaseTestCase): + connection_info = {'data': disk_info} + lightos_driver.disconnect_volume(connection_info, None) + lightos_driver.connector.disconnect_volume.assert_called_once_with( +- disk_info, None) ++ disk_info, None, force=False) ++ ++ # Verify force=True ++ lightos_driver.connector.disconnect_volume.reset_mock() ++ lightos_driver.disconnect_volume(connection_info, None, force=True) ++ lightos_driver.connector.disconnect_volume.assert_called_once_with( ++ disk_info, None, force=True) + + @mock.patch('os_brick.initiator.connector.InitiatorConnector.factory', + new=mock.Mock(return_value=mock.Mock())) +diff --git a/nova/tests/unit/virt/libvirt/volume/test_nvme.py b/nova/tests/unit/virt/libvirt/volume/test_nvme.py +index 5159f3aaf6..2803903e9f 100644 +--- a/nova/tests/unit/virt/libvirt/volume/test_nvme.py ++++ b/nova/tests/unit/virt/libvirt/volume/test_nvme.py +@@ -77,7 +77,13 @@ class LibvirtNVMEVolumeDriverTestCase(test_volume.LibvirtVolumeBaseTestCase): + connection_info = {'data': disk_info} + nvme_driver.disconnect_volume(connection_info, None) + nvme_driver.connector.disconnect_volume.assert_called_once_with( +- disk_info, None) ++ disk_info, None, force=False) ++ ++ # Verify force=True ++ nvme_driver.connector.disconnect_volume.reset_mock() ++ nvme_driver.disconnect_volume(connection_info, None, force=True) ++ nvme_driver.connector.disconnect_volume.assert_called_once_with( ++ disk_info, None, force=True) + + @mock.patch('os_brick.initiator.connector.InitiatorConnector.factory', + new=mock.Mock(return_value=mock.Mock())) +diff --git a/nova/tests/unit/virt/libvirt/volume/test_scaleio.py b/nova/tests/unit/virt/libvirt/volume/test_scaleio.py +index 6d9247cd2d..ed5ab08a6e 100644 +--- a/nova/tests/unit/virt/libvirt/volume/test_scaleio.py ++++ b/nova/tests/unit/virt/libvirt/volume/test_scaleio.py +@@ -49,7 +49,13 @@ class LibvirtScaleIOVolumeDriverTestCase( + conn = {'data': mock.sentinel.conn_data} + sio.disconnect_volume(conn, mock.sentinel.instance) + sio.connector.disconnect_volume.assert_called_once_with( +- mock.sentinel.conn_data, None) ++ mock.sentinel.conn_data, None, force=False) ++ ++ # Verify force=True ++ sio.connector.disconnect_volume.reset_mock() ++ sio.disconnect_volume(conn, mock.sentinel.instance, force=True) ++ sio.connector.disconnect_volume.assert_called_once_with( ++ mock.sentinel.conn_data, None, force=True) + + @mock.patch('os_brick.initiator.connector.InitiatorConnector.factory', + new=mock.Mock(return_value=mock.Mock())) +diff --git a/nova/tests/unit/virt/libvirt/volume/test_storpool.py b/nova/tests/unit/virt/libvirt/volume/test_storpool.py +index e14954f148..9ceac07260 100644 +--- a/nova/tests/unit/virt/libvirt/volume/test_storpool.py ++++ b/nova/tests/unit/virt/libvirt/volume/test_storpool.py +@@ -53,9 +53,11 @@ class MockStorPoolConnector(object): + } + return {'type': 'block', 'path': test_attached[v]['path']} + +- def disconnect_volume(self, connection_info, device_info): ++ def disconnect_volume(self, connection_info, device_info, **kwargs): + self.inst.assertIn('client_id', connection_info) + self.inst.assertIn('volume', connection_info) ++ self.inst.assertIn('force', kwargs) ++ self.inst.assertEqual(self.inst.force, kwargs.get('force')) + + v = connection_info['volume'] + if v not in test_attached: +@@ -86,6 +88,11 @@ class MockStorPoolInitiator(object): + class LibvirtStorPoolVolumeDriverTestCase( + test_volume.LibvirtVolumeBaseTestCase): + ++ def setUp(self): ++ super().setUp() ++ # This is for testing the force flag of disconnect_volume() ++ self.force = False ++ + def mock_storpool(f): + def _config_inner_inner1(inst, *args, **kwargs): + @mock.patch( +@@ -175,3 +182,10 @@ class LibvirtStorPoolVolumeDriverTestCase( + + libvirt_driver.disconnect_volume(ci_2, mock.sentinel.instance) + self.assertDictEqual({}, test_attached) ++ ++ # Connect the volume again so we can detach it again ++ libvirt_driver.connect_volume(ci_2, mock.sentinel.instance) ++ # Verify force=True ++ self.force = True ++ libvirt_driver.disconnect_volume( ++ ci_2, mock.sentinel.instance, force=True) +diff --git a/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py b/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py +index 883cebb55a..032ceb4fe5 100644 +--- a/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py ++++ b/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py +@@ -95,7 +95,13 @@ class LibvirtVZStorageTestCase(test_volume.LibvirtVolumeBaseTestCase): + conn = {'data': mock.sentinel.conn_data} + drv.disconnect_volume(conn, mock.sentinel.instance) + drv.connector.disconnect_volume.assert_called_once_with( +- mock.sentinel.conn_data, None) ++ mock.sentinel.conn_data, None, force=False) ++ ++ # Verify force=True ++ drv.connector.disconnect_volume.reset_mock() ++ drv.disconnect_volume(conn, mock.sentinel.instance, force=True) ++ drv.connector.disconnect_volume.assert_called_once_with( ++ mock.sentinel.conn_data, None, force=True) + + def test_libvirt_vzstorage_driver_get_config(self): + libvirt_driver = vzstorage.LibvirtVZStorageVolumeDriver(self.fake_host) +diff --git a/nova/virt/hyperv/vmops.py b/nova/virt/hyperv/vmops.py +index 3ec7e90c30..08adeada76 100644 +--- a/nova/virt/hyperv/vmops.py ++++ b/nova/virt/hyperv/vmops.py +@@ -747,7 +747,7 @@ class VMOps(object): + # should be disconnected even if the VM doesn't exist anymore, + # so they are not leaked. + self.unplug_vifs(instance, network_info) +- self._volumeops.disconnect_volumes(block_device_info) ++ self._volumeops.disconnect_volumes(block_device_info, force=True) + + if destroy_disks: + self._delete_disk_files(instance_name) +diff --git a/nova/virt/hyperv/volumeops.py b/nova/virt/hyperv/volumeops.py +index da5b40f375..d2bfed2441 100644 +--- a/nova/virt/hyperv/volumeops.py ++++ b/nova/virt/hyperv/volumeops.py +@@ -59,10 +59,10 @@ class VolumeOps(object): + for vol in volumes: + self.attach_volume(vol['connection_info'], instance_name) + +- def disconnect_volumes(self, block_device_info): ++ def disconnect_volumes(self, block_device_info, force=False): + mapping = driver.block_device_info_get_mapping(block_device_info) + for vol in mapping: +- self.disconnect_volume(vol['connection_info']) ++ self.disconnect_volume(vol['connection_info'], force=force) + + def attach_volume(self, connection_info, instance_name, + disk_bus=constants.CTRL_TYPE_SCSI): +@@ -116,9 +116,9 @@ class VolumeOps(object): + volume_driver.set_disk_qos_specs(connection_info, + qos_specs) + +- def disconnect_volume(self, connection_info): ++ def disconnect_volume(self, connection_info, force=False): + volume_driver = self._get_volume_driver(connection_info) +- volume_driver.disconnect_volume(connection_info) ++ volume_driver.disconnect_volume(connection_info, force=force) + + def detach_volume(self, connection_info, instance_name): + LOG.debug("Detaching volume: %(connection_info)s " +@@ -231,8 +231,8 @@ class BaseVolumeDriver(object): + def connect_volume(self, connection_info): + return self._connector.connect_volume(connection_info['data']) + +- def disconnect_volume(self, connection_info): +- self._connector.disconnect_volume(connection_info['data']) ++ def disconnect_volume(self, connection_info, force=False): ++ self._connector.disconnect_volume(connection_info['data'], force=force) + + def get_disk_resource_path(self, connection_info): + disk_paths = self._connector.get_volume_paths(connection_info['data']) +diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py +index 542943282f..615a009e06 100644 +--- a/nova/virt/libvirt/driver.py ++++ b/nova/virt/libvirt/driver.py +@@ -1639,7 +1639,7 @@ class LibvirtDriver(driver.ComputeDriver): + try: + self._disconnect_volume( + context, connection_info, instance, +- destroy_secrets=destroy_secrets) ++ destroy_secrets=destroy_secrets, force=True) + except Exception as exc: + with excutils.save_and_reraise_exception() as ctxt: + if cleanup_instance_disks: +@@ -1956,7 +1956,7 @@ class LibvirtDriver(driver.ComputeDriver): + return (False if connection_count > 1 else True) + + def _disconnect_volume(self, context, connection_info, instance, +- encryption=None, destroy_secrets=True): ++ encryption=None, destroy_secrets=True, force=False): + self._detach_encryptor( + context, + connection_info, +@@ -1968,7 +1968,8 @@ class LibvirtDriver(driver.ComputeDriver): + multiattach = connection_info.get('multiattach', False) + if self._should_disconnect_target( + context, instance, multiattach, vol_driver, volume_id): +- vol_driver.disconnect_volume(connection_info, instance) ++ vol_driver.disconnect_volume( ++ connection_info, instance, force=force) + else: + LOG.info('Detected multiple connections on this host for ' + 'volume: %(volume)s, skipping target disconnect.', +diff --git a/nova/virt/libvirt/volume/fibrechannel.py b/nova/virt/libvirt/volume/fibrechannel.py +index b50db3aa1c..1f890c95c1 100644 +--- a/nova/virt/libvirt/volume/fibrechannel.py ++++ b/nova/virt/libvirt/volume/fibrechannel.py +@@ -59,7 +59,7 @@ class LibvirtFibreChannelVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): + connection_info['data']['multipath_id'] = \ + device_info['multipath_id'] + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Detach the volume from instance_name.""" + + LOG.debug("calling os-brick to detach FC Volume", instance=instance) +@@ -69,11 +69,12 @@ class LibvirtFibreChannelVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): + # the 2nd param of disconnect_volume and be consistent + # with the rest of the connectors. + self.connector.disconnect_volume(connection_info['data'], +- connection_info['data']) ++ connection_info['data'], ++ force=force) + LOG.debug("Disconnected FC Volume", instance=instance) + + super(LibvirtFibreChannelVolumeDriver, +- self).disconnect_volume(connection_info, instance) ++ self).disconnect_volume(connection_info, instance, force=force) + + def extend_volume(self, connection_info, instance, requested_size): + """Extend the volume.""" +diff --git a/nova/virt/libvirt/volume/fs.py b/nova/virt/libvirt/volume/fs.py +index 5fb9af4a52..992ef45016 100644 +--- a/nova/virt/libvirt/volume/fs.py ++++ b/nova/virt/libvirt/volume/fs.py +@@ -116,7 +116,7 @@ class LibvirtMountedFileSystemVolumeDriver(LibvirtBaseFileSystemVolumeDriver, + connection_info['data']['device_path'] = \ + self._get_device_path(connection_info) + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Disconnect the volume.""" + vol_name = connection_info['data']['name'] + mountpoint = self._get_mount_path(connection_info) +diff --git a/nova/virt/libvirt/volume/iscsi.py b/nova/virt/libvirt/volume/iscsi.py +index 564bac14cc..2b25972a49 100644 +--- a/nova/virt/libvirt/volume/iscsi.py ++++ b/nova/virt/libvirt/volume/iscsi.py +@@ -66,19 +66,20 @@ class LibvirtISCSIVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): + + connection_info['data']['device_path'] = device_info['path'] + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Detach the volume from instance_name.""" + + LOG.debug("calling os-brick to detach iSCSI Volume", instance=instance) + try: +- self.connector.disconnect_volume(connection_info['data'], None) ++ self.connector.disconnect_volume( ++ connection_info['data'], None, force=force) + except os_brick_exception.VolumeDeviceNotFound as exc: + LOG.warning('Ignoring VolumeDeviceNotFound: %s', exc) + return + LOG.debug("Disconnected iSCSI Volume", instance=instance) + + super(LibvirtISCSIVolumeDriver, +- self).disconnect_volume(connection_info, instance) ++ self).disconnect_volume(connection_info, instance, force=force) + + def extend_volume(self, connection_info, instance, requested_size): + """Extend the volume.""" +diff --git a/nova/virt/libvirt/volume/lightos.py b/nova/virt/libvirt/volume/lightos.py +index d6d393994e..6a22bf6dc6 100644 +--- a/nova/virt/libvirt/volume/lightos.py ++++ b/nova/virt/libvirt/volume/lightos.py +@@ -42,14 +42,15 @@ class LibvirtLightOSVolumeDriver(libvirt_volume.LibvirtVolumeDriver): + LOG.debug("Connecting NVMe volume with device_info %s", device_info) + connection_info['data']['device_path'] = device_info['path'] + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Detach the volume from the instance.""" + LOG.debug("Disconnecting NVMe disk. instance:%s, volume_id:%s", + connection_info.get("instance", ""), + connection_info.get("volume_id", "")) +- self.connector.disconnect_volume(connection_info['data'], None) ++ self.connector.disconnect_volume( ++ connection_info['data'], None, force=force) + super(LibvirtLightOSVolumeDriver, self).disconnect_volume( +- connection_info, instance) ++ connection_info, instance, force=force) + + def extend_volume(self, connection_info, instance, requested_size=None): + """Extend the volume.""" +diff --git a/nova/virt/libvirt/volume/nvme.py b/nova/virt/libvirt/volume/nvme.py +index 7436552812..e2977c3572 100644 +--- a/nova/virt/libvirt/volume/nvme.py ++++ b/nova/virt/libvirt/volume/nvme.py +@@ -45,13 +45,13 @@ class LibvirtNVMEVolumeDriver(libvirt_volume.LibvirtVolumeDriver): + + connection_info['data']['device_path'] = device_info['path'] + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Detach the volume from the instance.""" + LOG.debug("Disconnecting NVMe disk", instance=instance) + self.connector.disconnect_volume( +- connection_info['data'], None) ++ connection_info['data'], None, force=force) + super(LibvirtNVMEVolumeDriver, +- self).disconnect_volume(connection_info, instance) ++ self).disconnect_volume(connection_info, instance, force=force) + + def extend_volume(self, connection_info, instance, requested_size): + """Extend the volume.""" +diff --git a/nova/virt/libvirt/volume/quobyte.py b/nova/virt/libvirt/volume/quobyte.py +index bb7a770e57..2eb4bcfb42 100644 +--- a/nova/virt/libvirt/volume/quobyte.py ++++ b/nova/virt/libvirt/volume/quobyte.py +@@ -189,7 +189,7 @@ class LibvirtQuobyteVolumeDriver(fs.LibvirtBaseFileSystemVolumeDriver): + instance=instance) + + @utils.synchronized('connect_qb_volume') +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Disconnect the volume.""" + + mount_path = self._get_mount_path(connection_info) +diff --git a/nova/virt/libvirt/volume/scaleio.py b/nova/virt/libvirt/volume/scaleio.py +index 7c414c2870..04a9423e8e 100644 +--- a/nova/virt/libvirt/volume/scaleio.py ++++ b/nova/virt/libvirt/volume/scaleio.py +@@ -57,12 +57,13 @@ class LibvirtScaleIOVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): + instance=instance) + connection_info['data']['device_path'] = device_info['path'] + +- def disconnect_volume(self, connection_info, instance): +- self.connector.disconnect_volume(connection_info['data'], None) ++ def disconnect_volume(self, connection_info, instance, force=False): ++ self.connector.disconnect_volume( ++ connection_info['data'], None, force=force) + LOG.debug("Disconnected volume", instance=instance) + + super(LibvirtScaleIOVolumeDriver, self).disconnect_volume( +- connection_info, instance) ++ connection_info, instance, force=force) + + def extend_volume(self, connection_info, instance, requested_size): + LOG.debug("calling os-brick to extend ScaleIO Volume", +diff --git a/nova/virt/libvirt/volume/smbfs.py b/nova/virt/libvirt/volume/smbfs.py +index d112af750c..9de1ce23cd 100644 +--- a/nova/virt/libvirt/volume/smbfs.py ++++ b/nova/virt/libvirt/volume/smbfs.py +@@ -52,7 +52,7 @@ class LibvirtSMBFSVolumeDriver(fs.LibvirtBaseFileSystemVolumeDriver): + device_path = self._get_device_path(connection_info) + connection_info['data']['device_path'] = device_path + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Disconnect the volume.""" + smbfs_share = connection_info['data']['export'] + mount_path = self._get_mount_path(connection_info) +diff --git a/nova/virt/libvirt/volume/storpool.py b/nova/virt/libvirt/volume/storpool.py +index 0e71221f5b..e6dffca39a 100644 +--- a/nova/virt/libvirt/volume/storpool.py ++++ b/nova/virt/libvirt/volume/storpool.py +@@ -47,10 +47,11 @@ class LibvirtStorPoolVolumeDriver(libvirt_volume.LibvirtVolumeDriver): + device_info, instance=instance) + connection_info['data']['device_path'] = device_info['path'] + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + LOG.debug("Detaching StorPool volume %s", + connection_info['data']['volume'], instance=instance) +- self.connector.disconnect_volume(connection_info['data'], None) ++ self.connector.disconnect_volume( ++ connection_info['data'], None, force=force) + LOG.debug("Detached StorPool volume", instance=instance) + + def extend_volume(self, connection_info, instance, requested_size): +diff --git a/nova/virt/libvirt/volume/volume.py b/nova/virt/libvirt/volume/volume.py +index 6d650c80e6..f76c3618b2 100644 +--- a/nova/virt/libvirt/volume/volume.py ++++ b/nova/virt/libvirt/volume/volume.py +@@ -135,7 +135,7 @@ class LibvirtBaseVolumeDriver(object): + """Connect the volume.""" + pass + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Disconnect the volume.""" + pass + +diff --git a/nova/virt/libvirt/volume/vzstorage.py b/nova/virt/libvirt/volume/vzstorage.py +index 85ffb45076..babfdef55c 100644 +--- a/nova/virt/libvirt/volume/vzstorage.py ++++ b/nova/virt/libvirt/volume/vzstorage.py +@@ -126,9 +126,10 @@ class LibvirtVZStorageVolumeDriver(fs.LibvirtBaseFileSystemVolumeDriver): + + return _connect_volume(connection_info, instance) + +- def disconnect_volume(self, connection_info, instance): ++ def disconnect_volume(self, connection_info, instance, force=False): + """Detach the volume from instance_name.""" + LOG.debug("calling os-brick to detach Vzstorage Volume", + instance=instance) +- self.connector.disconnect_volume(connection_info['data'], None) ++ self.connector.disconnect_volume( ++ connection_info['data'], None, force=force) + LOG.debug("Disconnected Vzstorage Volume", instance=instance) +diff --git a/releasenotes/notes/service-user-token-421d067c16257782.yaml b/releasenotes/notes/service-user-token-421d067c16257782.yaml +new file mode 100644 +index 0000000000..d3af14fbb8 +--- /dev/null ++++ b/releasenotes/notes/service-user-token-421d067c16257782.yaml +@@ -0,0 +1,11 @@ ++upgrade: ++ - | ++ Configuration of service user tokens is now **required** for all Nova services ++ to ensure security of block-storage volume data. ++ ++ All Nova configuration files must configure the ``[service_user]`` section as ++ described in the `documentation`__. ++ ++ See https://bugs.launchpad.net/nova/+bug/2004555 for more details. ++ ++ __ https://docs.openstack.org/nova/latest/admin/configuration/service-user-token.html +-- +2.39.2 + diff -Nru nova-25.1.1/debian/patches/CVE-2023-2088-2.patch nova-25.1.1/debian/patches/CVE-2023-2088-2.patch --- nova-25.1.1/debian/patches/CVE-2023-2088-2.patch 1970-01-01 00:00:00.000000000 +0000 +++ nova-25.1.1/debian/patches/CVE-2023-2088-2.patch 2023-05-31 20:43:41.000000000 +0000 @@ -0,0 +1,170 @@ +From 98c3e3707c08a07f7ca5996086b165512f604ad6 Mon Sep 17 00:00:00 2001 +From: melanie witt +Date: Tue, 9 May 2023 03:11:25 +0000 +Subject: [PATCH] Enable use of service user token with admin context + +When the [service_user] section is configured in nova.conf, nova will +have the ability to send a service user token alongside the user's +token. The service user token is sent when nova calls other services' +REST APIs to authenticate as a service, and service calls can sometimes +have elevated privileges. + +Currently, nova does not however have the ability to send a service user +token with an admin context. This means that when nova makes REST API +calls to other services with an anonymous admin RequestContext (such as +in nova-manage or periodic tasks), it will not be authenticated as a +service. + +This adds a keyword argument to service_auth.get_auth_plugin() to +enable callers to provide a user_auth object instead of attempting to +extract the user_auth from the RequestContext. + +The cinder and neutron client modules are also adjusted to make use of +the new user_auth keyword argument so that nova calls made with +anonymous admin request contexts can authenticate as a service when +configured. + +Related-Bug: #2004555 + +Change-Id: I14df2d55f4b2f0be58f1a6ad3f19e48f7a6bfcb4 +(cherry picked from commit 41c64b94b0af333845e998f6cc195e72ca5ab6bc) +(cherry picked from commit 1f781423ee4224c0871ab4aafec191bb2f7ef0e4) +(cherry picked from commit 0d6dd6c67f56c9d4ed36246d14f119da6bca0a5a) +--- + nova/network/neutron.py | 8 +++++--- + nova/service_auth.py | 6 ++++-- + nova/tests/unit/network/test_neutron.py | 16 ++++++++++++++++ + nova/tests/unit/test_service_auth.py | 10 ++++++++++ + nova/tests/unit/volume/test_cinder.py | 11 +++++++++++ + nova/volume/cinder.py | 8 +++++--- + 6 files changed, 51 insertions(+), 8 deletions(-) + +diff --git a/nova/network/neutron.py b/nova/network/neutron.py +index 1e703658f8..faf455d9b8 100644 +--- a/nova/network/neutron.py ++++ b/nova/network/neutron.py +@@ -223,13 +223,15 @@ def _get_auth_plugin(context, admin=False): + # support some services (metadata API) where an admin context is used + # without an auth token. + global _ADMIN_AUTH ++ user_auth = None + if admin or (context.is_admin and not context.auth_token): + if not _ADMIN_AUTH: + _ADMIN_AUTH = _load_auth_plugin(CONF) +- return _ADMIN_AUTH ++ user_auth = _ADMIN_AUTH + +- if context.auth_token: +- return service_auth.get_auth_plugin(context) ++ if context.auth_token or user_auth: ++ # When user_auth = None, user_auth will be extracted from the context. ++ return service_auth.get_auth_plugin(context, user_auth=user_auth) + + # We did not get a user token and we should not be using + # an admin token so log an error +diff --git a/nova/service_auth.py b/nova/service_auth.py +index f5ae0646d8..aa8fd8fa12 100644 +--- a/nova/service_auth.py ++++ b/nova/service_auth.py +@@ -30,8 +30,10 @@ def reset_globals(): + _SERVICE_AUTH = None + + +-def get_auth_plugin(context): +- user_auth = context.get_auth_plugin() ++def get_auth_plugin(context, user_auth=None): ++ # user_auth may be passed in when the RequestContext is anonymous, such as ++ # when get_admin_context() is used for API calls by nova-manage. ++ user_auth = user_auth or context.get_auth_plugin() + + if CONF.service_user.send_service_user_token: + global _SERVICE_AUTH +diff --git a/nova/tests/unit/network/test_neutron.py b/nova/tests/unit/network/test_neutron.py +index 5cde8d482d..458777c3a3 100644 +--- a/nova/tests/unit/network/test_neutron.py ++++ b/nova/tests/unit/network/test_neutron.py +@@ -143,6 +143,22 @@ class TestNeutronClient(test.NoDBTestCase): + self.assertIsInstance(cl.httpclient.auth, + service_token.ServiceTokenAuthWrapper) + ++ @mock.patch('nova.service_auth._SERVICE_AUTH') ++ @mock.patch('nova.network.neutron._ADMIN_AUTH') ++ @mock.patch.object(ks_loading, 'load_auth_from_conf_options') ++ def test_admin_with_service_token( ++ self, mock_load, mock_admin_auth, mock_service_auth ++ ): ++ self.flags(send_service_user_token=True, group='service_user') ++ ++ admin_context = context.get_admin_context() ++ ++ cl = neutronapi.get_client(admin_context) ++ self.assertIsInstance(cl.httpclient.auth, ++ service_token.ServiceTokenAuthWrapper) ++ self.assertEqual(mock_admin_auth, cl.httpclient.auth.user_auth) ++ self.assertEqual(mock_service_auth, cl.httpclient.auth.service_auth) ++ + @mock.patch.object(client.Client, "list_networks", + side_effect=exceptions.Unauthorized()) + def test_Unauthorized_user(self, mock_list_networks): +diff --git a/nova/tests/unit/test_service_auth.py b/nova/tests/unit/test_service_auth.py +index db2a2e2899..ceb2a93b02 100644 +--- a/nova/tests/unit/test_service_auth.py ++++ b/nova/tests/unit/test_service_auth.py +@@ -55,3 +55,13 @@ class ServiceAuthTestCase(test.NoDBTestCase): + result = service_auth.get_auth_plugin(self.ctx) + self.assertEqual(1, mock_load.call_count) + self.assertNotIsInstance(result, service_token.ServiceTokenAuthWrapper) ++ ++ @mock.patch.object(ks_loading, 'load_auth_from_conf_options', ++ new=mock.Mock()) ++ def test_get_auth_plugin_user_auth(self): ++ self.flags(send_service_user_token=True, group='service_user') ++ user_auth = mock.Mock() ++ ++ result = service_auth.get_auth_plugin(self.ctx, user_auth=user_auth) ++ ++ self.assertEqual(user_auth, result.user_auth) +diff --git a/nova/tests/unit/volume/test_cinder.py b/nova/tests/unit/volume/test_cinder.py +index f4ee7383d4..ffa46ce2aa 100644 +--- a/nova/tests/unit/volume/test_cinder.py ++++ b/nova/tests/unit/volume/test_cinder.py +@@ -1275,3 +1275,14 @@ class CinderClientTestCase(test.NoDBTestCase): + admin_ctx = context.get_admin_context() + params = cinder._get_cinderclient_parameters(admin_ctx) + self.assertEqual(params[0], mock_admin_auth) ++ ++ @mock.patch('nova.service_auth._SERVICE_AUTH') ++ @mock.patch('nova.volume.cinder._ADMIN_AUTH') ++ def test_admin_context_without_user_token_but_with_service_token( ++ self, mock_admin_auth, mock_service_auth ++ ): ++ self.flags(send_service_user_token=True, group='service_user') ++ admin_ctx = context.get_admin_context() ++ params = cinder._get_cinderclient_parameters(admin_ctx) ++ self.assertEqual(mock_admin_auth, params[0].user_auth) ++ self.assertEqual(mock_service_auth, params[0].service_auth) +diff --git a/nova/volume/cinder.py b/nova/volume/cinder.py +index 01efcfec19..f5328148d2 100644 +--- a/nova/volume/cinder.py ++++ b/nova/volume/cinder.py +@@ -91,12 +91,14 @@ def _get_auth(context): + # from them generated from 'context.get_admin_context' + # which only set is_admin=True but is without token. + # So add load_auth_plugin when this condition appear. ++ user_auth = None + if context.is_admin and not context.auth_token: + if not _ADMIN_AUTH: + _ADMIN_AUTH = _load_auth_plugin(CONF) +- return _ADMIN_AUTH +- else: +- return service_auth.get_auth_plugin(context) ++ user_auth = _ADMIN_AUTH ++ ++ # When user_auth = None, user_auth will be extracted from the context. ++ return service_auth.get_auth_plugin(context, user_auth=user_auth) + + + # NOTE(efried): Bug #1752152 +-- +2.39.2 + diff -Nru nova-25.1.1/debian/patches/CVE-2023-2088.patch nova-25.1.1/debian/patches/CVE-2023-2088.patch --- nova-25.1.1/debian/patches/CVE-2023-2088.patch 2023-05-15 20:11:40.000000000 +0000 +++ nova-25.1.1/debian/patches/CVE-2023-2088.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,1135 +0,0 @@ -From f0a3dcfb9c2ae3483f9f6a15da248742115fe278 Mon Sep 17 00:00:00 2001 -From: melanie witt -Date: Wed, 15 Feb 2023 22:37:40 +0000 -Subject: [PATCH] Use force=True for os-brick disconnect during delete - -The 'force' parameter of os-brick's disconnect_volume() method allows -callers to ignore flushing errors and ensure that devices are being -removed from the host. - -We should use force=True when we are going to delete an instance to -avoid leaving leftover devices connected to the compute host which -could then potentially be reused to map to volumes to an instance that -should not have access to those volumes. - -We can use force=True even when disconnecting a volume that will not be -deleted on termination because os-brick will always attempt to flush -and disconnect gracefully before forcefully removing devices. - -Closes-Bug: #2004555 - -Change-Id: I3629b84d3255a8fe9d8a7cea8c6131d7c40899e8 ---- - .../admin/configuration/cross-cell-resize.rst | 2 +- - doc/source/admin/configuration/index.rst | 1 + - .../configuration/service-user-token.rst | 59 +++++++++++++++++ - doc/source/admin/live-migration-usage.rst | 2 +- - .../admin/migrate-instance-with-snapshot.rst | 2 +- - doc/source/admin/support-compute.rst | 64 ------------------- - doc/source/install/compute-install-obs.rst | 20 ++++++ - doc/source/install/compute-install-rdo.rst | 20 ++++++ - doc/source/install/compute-install-ubuntu.rst | 20 ++++++ - doc/source/install/controller-install-obs.rst | 20 ++++++ - doc/source/install/controller-install-rdo.rst | 20 ++++++ - .../install/controller-install-ubuntu.rst | 20 ++++++ - nova/cmd/status.py | 11 ++++ - nova/tests/unit/cmd/test_status.py | 16 +++++ - nova/tests/unit/virt/hyperv/test_vmops.py | 2 +- - nova/tests/unit/virt/hyperv/test_volumeops.py | 26 ++++++-- - nova/tests/unit/virt/libvirt/test_driver.py | 61 ++++++++++++++++-- - .../virt/libvirt/volume/test_fibrechannel.py | 20 ++++++ - .../unit/virt/libvirt/volume/test_iscsi.py | 9 +++ - .../unit/virt/libvirt/volume/test_lightos.py | 8 ++- - .../unit/virt/libvirt/volume/test_nvme.py | 8 ++- - .../unit/virt/libvirt/volume/test_scaleio.py | 8 ++- - .../unit/virt/libvirt/volume/test_storpool.py | 16 ++++- - .../virt/libvirt/volume/test_vzstorage.py | 8 ++- - nova/virt/hyperv/vmops.py | 2 +- - nova/virt/hyperv/volumeops.py | 12 ++-- - nova/virt/libvirt/driver.py | 7 +- - nova/virt/libvirt/volume/fibrechannel.py | 7 +- - nova/virt/libvirt/volume/fs.py | 2 +- - nova/virt/libvirt/volume/iscsi.py | 7 +- - nova/virt/libvirt/volume/lightos.py | 7 +- - nova/virt/libvirt/volume/nvme.py | 6 +- - nova/virt/libvirt/volume/quobyte.py | 2 +- - nova/virt/libvirt/volume/scaleio.py | 7 +- - nova/virt/libvirt/volume/smbfs.py | 2 +- - nova/virt/libvirt/volume/storpool.py | 5 +- - nova/virt/libvirt/volume/volume.py | 2 +- - nova/virt/libvirt/volume/vzstorage.py | 5 +- - .../service-user-token-421d067c16257782.yaml | 11 ++++ - 39 files changed, 413 insertions(+), 114 deletions(-) - create mode 100644 doc/source/admin/configuration/service-user-token.rst - create mode 100644 releasenotes/notes/service-user-token-421d067c16257782.yaml - -diff --git a/doc/source/admin/configuration/cross-cell-resize.rst b/doc/source/admin/configuration/cross-cell-resize.rst -index e51e425774..0c34fd13f5 100644 ---- a/doc/source/admin/configuration/cross-cell-resize.rst -+++ b/doc/source/admin/configuration/cross-cell-resize.rst -@@ -284,7 +284,7 @@ Troubleshooting - Timeouts - ~~~~~~~~ - --Configure a :ref:`service user ` in case the user token -+Configure a :ref:`service user ` in case the user token - times out, e.g. during the snapshot and download of a large server image. - - If RPC calls are timing out with a ``MessagingTimeout`` error in the logs, -diff --git a/doc/source/admin/configuration/index.rst b/doc/source/admin/configuration/index.rst -index 233597b1fe..f5b6fde9da 100644 ---- a/doc/source/admin/configuration/index.rst -+++ b/doc/source/admin/configuration/index.rst -@@ -19,6 +19,7 @@ A list of config options based on different topics can be found below: - .. toctree:: - :maxdepth: 1 - -+ /admin/configuration/service-user-token - /admin/configuration/api - /admin/configuration/resize - /admin/configuration/cross-cell-resize -diff --git a/doc/source/admin/configuration/service-user-token.rst b/doc/source/admin/configuration/service-user-token.rst -new file mode 100644 -index 0000000000..740730af1d ---- /dev/null -+++ b/doc/source/admin/configuration/service-user-token.rst -@@ -0,0 +1,59 @@ -+.. _service_user_token: -+ -+=================== -+Service User Tokens -+=================== -+ -+.. note:: -+ -+ Configuration of service user tokens is **required** for every Nova service -+ for security reasons. See https://bugs.launchpad.net/nova/+bug/2004555 for -+ details. -+ -+Configure Nova to send service user tokens alongside regular user tokens when -+making REST API calls to other services. The identity service (Keystone) will -+authenticate a request using the service user token if the regular user token -+has expired. -+ -+This is important when long-running operations such as live migration or -+snapshot take long enough to exceed the expiry of the user token. Without the -+service token, if a long-running operation exceeds the expiry of the user -+token, post operations such as cleanup after a live migration could fail when -+Nova calls other service APIs like block-storage (Cinder) or networking -+(Neutron). -+ -+The service token is also used by services to validate whether the API caller -+is a service. Some service APIs are restricted to service users only. -+ -+To set up service tokens, create a ``nova`` service user and ``service`` role -+in the identity service (Keystone) and assign the ``service`` role to the -+``nova`` service user. -+ -+Then, configure the :oslo.config:group:`service_user` section of the Nova -+configuration file, for example: -+ -+.. code-block:: ini -+ -+ [service_user] -+ send_service_user_token = true -+ auth_url = https://104.130.216.102/identity -+ auth_strategy = keystone -+ auth_type = password -+ project_domain_name = Default -+ project_name = service -+ user_domain_name = Default -+ username = nova -+ password = secretservice -+ ... -+ -+And configure the other identity options as necessary for the service user, -+much like you would configure nova to work with the image service (Glance) or -+networking service (Neutron). -+ -+.. note:: -+ -+ Please note that the role assigned to the :oslo.config:group:`service_user` -+ needs to be in the configured -+ :oslo.config:option:`keystone_authtoken.service_token_roles` of other -+ services such as block-storage (Cinder), image (Glance), and networking -+ (Neutron). -diff --git a/doc/source/admin/live-migration-usage.rst b/doc/source/admin/live-migration-usage.rst -index 32c67c2b0a..dc27574f91 100644 ---- a/doc/source/admin/live-migration-usage.rst -+++ b/doc/source/admin/live-migration-usage.rst -@@ -320,4 +320,4 @@ To make live-migration succeed, you have several options: - - If live migrations routinely timeout or fail during cleanup operations due - to the user token timing out, consider configuring nova to use --:ref:`service user tokens `. -+:ref:`service user tokens `. -diff --git a/doc/source/admin/migrate-instance-with-snapshot.rst b/doc/source/admin/migrate-instance-with-snapshot.rst -index 65059679ab..230431091e 100644 ---- a/doc/source/admin/migrate-instance-with-snapshot.rst -+++ b/doc/source/admin/migrate-instance-with-snapshot.rst -@@ -67,7 +67,7 @@ Create a snapshot of the instance - - If snapshot operations routinely fail because the user token times out - while uploading a large disk image, consider configuring nova to use -- :ref:`service user tokens `. -+ :ref:`service user tokens `. - - #. Use the :command:`openstack image list` command to check the status - until the status is ``ACTIVE``: -diff --git a/doc/source/admin/support-compute.rst b/doc/source/admin/support-compute.rst -index 8522e51d79..31e32fd1dd 100644 ---- a/doc/source/admin/support-compute.rst -+++ b/doc/source/admin/support-compute.rst -@@ -478,67 +478,3 @@ Ensure the ``compute`` endpoint in the identity service catalog is pointing - at ``/v2.1`` instead of ``/v2``. The former route supports microversions, - while the latter route is considered the legacy v2.0 compatibility-mode - route which renders all requests as if they were made on the legacy v2.0 API. -- -- --.. _user_token_timeout: -- --User token times out during long-running operations ----------------------------------------------------- -- --Problem --~~~~~~~ -- --Long-running operations such as live migration or snapshot can sometimes --overrun the expiry of the user token. In such cases, post operations such --as cleaning up after a live migration can fail when the nova-compute service --needs to cleanup resources in other services, such as in the block-storage --(cinder) or networking (neutron) services. -- --For example: -- --.. code-block:: console -- -- 2018-12-17 13:47:29.591 16987 WARNING nova.virt.libvirt.migration [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] [instance: ead8ecc3-f473-4672-a67b-c44534c6042d] Live migration not completed after 2400 sec -- 2018-12-17 13:47:30.097 16987 WARNING nova.virt.libvirt.driver [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] [instance: ead8ecc3-f473-4672-a67b-c44534c6042d] Migration operation was cancelled -- 2018-12-17 13:47:30.299 16987 ERROR nova.virt.libvirt.driver [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] [instance: ead8ecc3-f473-4672-a67b-c44534c6042d] Live Migration failure: operation aborted: migration job: canceled by client: libvirtError: operation aborted: migration job: canceled by client -- 2018-12-17 13:47:30.685 16987 INFO nova.compute.manager [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] [instance: ead8ecc3-f473-4672-a67b-c44534c6042d] Swapping old allocation on 3e32d595-bd1f-4136-a7f4-c6703d2fbe18 held by migration 17bec61d-544d-47e0-a1c1-37f9d7385286 for instance -- 2018-12-17 13:47:32.450 16987 ERROR nova.volume.cinder [req-7bc758de-b2e4-461b-a971-f79be6cd4703 313d1247d7b845da9c731eec53e50a26 2f693c782fa748c2baece8db95b4ba5b - default default] Delete attachment failed for attachment 58997d5b-24f0-4073-819e-97916fb1ee19. Error: The request you have made requires authentication. (HTTP 401) Code: 401: Unauthorized: The request you have made requires authentication. (HTTP 401) -- --Solution --~~~~~~~~ -- --Configure nova to use service user tokens to supplement the regular user token --used to initiate the operation. The identity service (keystone) will then --authenticate a request using the service user token if the user token has --already expired. -- --To use, create a service user in the identity service similar as you would when --creating the ``nova`` service user. -- --Then configure the :oslo.config:group:`service_user` section of the nova --configuration file, for example: -- --.. code-block:: ini -- -- [service_user] -- send_service_user_token = True -- auth_type = password -- project_domain_name = Default -- project_name = service -- user_domain_name = Default -- password = secretservice -- username = nova -- auth_url = https://104.130.216.102/identity -- ... -- --And configure the other identity options as necessary for the service user, --much like you would configure nova to work with the image service (glance) --or networking service. -- --.. note:: -- -- Please note that the role of the :oslo.config:group:`service_user` you -- configure needs to be a superset of -- :oslo.config:option:`keystone_authtoken.service_token_roles` (The option -- :oslo.config:option:`keystone_authtoken.service_token_roles` is configured -- in cinder, glance and neutron). -diff --git a/doc/source/install/compute-install-obs.rst b/doc/source/install/compute-install-obs.rst -index c5c1d29fb3..c227b6eba4 100644 ---- a/doc/source/install/compute-install-obs.rst -+++ b/doc/source/install/compute-install-obs.rst -@@ -92,6 +92,26 @@ Install and configure components - Comment out or remove any other options in the ``[keystone_authtoken]`` - section. - -+ * In the ``[service_user]`` section, configure :ref:`service user -+ tokens `: -+ -+ .. path /etc/nova/nova.conf -+ .. code-block:: ini -+ -+ [service_user] -+ send_service_user_token = true -+ auth_url = https://controller/identity -+ auth_strategy = keystone -+ auth_type = password -+ project_domain_name = Default -+ project_name = service -+ user_domain_name = Default -+ username = nova -+ password = NOVA_PASS -+ -+ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in -+ the Identity service. -+ - * In the ``[DEFAULT]`` section, configure the ``my_ip`` option: - - .. path /etc/nova/nova.conf -diff --git a/doc/source/install/compute-install-rdo.rst b/doc/source/install/compute-install-rdo.rst -index 0a5ad685a6..0c6203a667 100644 ---- a/doc/source/install/compute-install-rdo.rst -+++ b/doc/source/install/compute-install-rdo.rst -@@ -84,6 +84,26 @@ Install and configure components - Comment out or remove any other options in the ``[keystone_authtoken]`` - section. - -+ * In the ``[service_user]`` section, configure :ref:`service user -+ tokens `: -+ -+ .. path /etc/nova/nova.conf -+ .. code-block:: ini -+ -+ [service_user] -+ send_service_user_token = true -+ auth_url = https://controller/identity -+ auth_strategy = keystone -+ auth_type = password -+ project_domain_name = Default -+ project_name = service -+ user_domain_name = Default -+ username = nova -+ password = NOVA_PASS -+ -+ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in -+ the Identity service. -+ - * In the ``[DEFAULT]`` section, configure the ``my_ip`` option: - - .. path /etc/nova/nova.conf -diff --git a/doc/source/install/compute-install-ubuntu.rst b/doc/source/install/compute-install-ubuntu.rst -index 8605c73316..baf0585e52 100644 ---- a/doc/source/install/compute-install-ubuntu.rst -+++ b/doc/source/install/compute-install-ubuntu.rst -@@ -74,6 +74,26 @@ Install and configure components - Comment out or remove any other options in the - ``[keystone_authtoken]`` section. - -+ * In the ``[service_user]`` section, configure :ref:`service user -+ tokens `: -+ -+ .. path /etc/nova/nova.conf -+ .. code-block:: ini -+ -+ [service_user] -+ send_service_user_token = true -+ auth_url = https://controller/identity -+ auth_strategy = keystone -+ auth_type = password -+ project_domain_name = Default -+ project_name = service -+ user_domain_name = Default -+ username = nova -+ password = NOVA_PASS -+ -+ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in -+ the Identity service. -+ - * In the ``[DEFAULT]`` section, configure the ``my_ip`` option: - - .. path /etc/nova/nova.conf -diff --git a/doc/source/install/controller-install-obs.rst b/doc/source/install/controller-install-obs.rst -index 18499612c3..01b7bb0f5a 100644 ---- a/doc/source/install/controller-install-obs.rst -+++ b/doc/source/install/controller-install-obs.rst -@@ -260,6 +260,26 @@ Install and configure components - Comment out or remove any other options in the ``[keystone_authtoken]`` - section. - -+ * In the ``[service_user]`` section, configure :ref:`service user -+ tokens `: -+ -+ .. path /etc/nova/nova.conf -+ .. code-block:: ini -+ -+ [service_user] -+ send_service_user_token = true -+ auth_url = https://controller/identity -+ auth_strategy = keystone -+ auth_type = password -+ project_domain_name = Default -+ project_name = service -+ user_domain_name = Default -+ username = nova -+ password = NOVA_PASS -+ -+ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in -+ the Identity service. -+ - * In the ``[DEFAULT]`` section, configure the ``my_ip`` option to use the - management interface IP address of the controller node: - -diff --git a/doc/source/install/controller-install-rdo.rst b/doc/source/install/controller-install-rdo.rst -index fd2419631e..b6098f1776 100644 ---- a/doc/source/install/controller-install-rdo.rst -+++ b/doc/source/install/controller-install-rdo.rst -@@ -247,6 +247,26 @@ Install and configure components - Comment out or remove any other options in the ``[keystone_authtoken]`` - section. - -+ * In the ``[service_user]`` section, configure :ref:`service user -+ tokens `: -+ -+ .. path /etc/nova/nova.conf -+ .. code-block:: ini -+ -+ [service_user] -+ send_service_user_token = true -+ auth_url = https://controller/identity -+ auth_strategy = keystone -+ auth_type = password -+ project_domain_name = Default -+ project_name = service -+ user_domain_name = Default -+ username = nova -+ password = NOVA_PASS -+ -+ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in -+ the Identity service. -+ - * In the ``[DEFAULT]`` section, configure the ``my_ip`` option to use the - management interface IP address of the controller node: - -diff --git a/doc/source/install/controller-install-ubuntu.rst b/doc/source/install/controller-install-ubuntu.rst -index 7282b0b2e2..1363a98ba8 100644 ---- a/doc/source/install/controller-install-ubuntu.rst -+++ b/doc/source/install/controller-install-ubuntu.rst -@@ -237,6 +237,26 @@ Install and configure components - Comment out or remove any other options in the ``[keystone_authtoken]`` - section. - -+ * In the ``[service_user]`` section, configure :ref:`service user -+ tokens `: -+ -+ .. path /etc/nova/nova.conf -+ .. code-block:: ini -+ -+ [service_user] -+ send_service_user_token = true -+ auth_url = https://controller/identity -+ auth_strategy = keystone -+ auth_type = password -+ project_domain_name = Default -+ project_name = service -+ user_domain_name = Default -+ username = nova -+ password = NOVA_PASS -+ -+ Replace ``NOVA_PASS`` with the password you chose for the ``nova`` user in -+ the Identity service. -+ - * In the ``[DEFAULT]`` section, configure the ``my_ip`` option to use the - management interface IP address of the controller node: - -diff --git a/nova/cmd/status.py b/nova/cmd/status.py -index 29e4a5d01e..4a4e28d7e8 100644 ---- a/nova/cmd/status.py -+++ b/nova/cmd/status.py -@@ -271,6 +271,15 @@ https://docs.openstack.org/latest/nova/admin/hw_machine_type.html""")) - - return upgradecheck.Result(upgradecheck.Code.SUCCESS) - -+ def _check_service_user_token(self): -+ if not CONF.service_user.send_service_user_token: -+ msg = (_(""" -+Service user token configuration is required for all Nova services. -+For more details see the following: -+https://docs.openstack.org/latest/nova/admin/configuration/service-user-token.html""")) # noqa -+ return upgradecheck.Result(upgradecheck.Code.FAILURE, msg) -+ return upgradecheck.Result(upgradecheck.Code.SUCCESS) -+ - # The format of the check functions is to return an upgradecheck.Result - # object with the appropriate upgradecheck.Code and details set. If the - # check hits warnings or failures then those should be stored in the -@@ -294,6 +303,8 @@ https://docs.openstack.org/latest/nova/admin/hw_machine_type.html""")) - (_('Older than N-1 computes'), _check_old_computes), - # Added in Wallaby - (_('hw_machine_type unset'), _check_machine_type_set), -+ # Added in Bobcat -+ (_('Service User Token Configuration'), _check_service_user_token), - ) - - -diff --git a/nova/tests/unit/cmd/test_status.py b/nova/tests/unit/cmd/test_status.py -index f5fcc168ee..c6a0ab2d52 100644 ---- a/nova/tests/unit/cmd/test_status.py -+++ b/nova/tests/unit/cmd/test_status.py -@@ -446,3 +446,19 @@ class TestCheckMachineTypeUnset(test.NoDBTestCase): - upgradecheck.Code.SUCCESS, - result.code - ) -+ -+ -+class TestUpgradeCheckServiceUserToken(test.NoDBTestCase): -+ -+ def setUp(self): -+ super().setUp() -+ self.cmd = status.UpgradeCommands() -+ -+ def test_service_user_token_not_configured(self): -+ result = self.cmd._check_service_user_token() -+ self.assertEqual(upgradecheck.Code.FAILURE, result.code) -+ -+ def test_service_user_token_configured(self): -+ self.flags(send_service_user_token=True, group='service_user') -+ result = self.cmd._check_service_user_token() -+ self.assertEqual(upgradecheck.Code.SUCCESS, result.code) -diff --git a/nova/tests/unit/virt/hyperv/test_vmops.py b/nova/tests/unit/virt/hyperv/test_vmops.py -index 07e1774f9a..1e3e50f92b 100644 ---- a/nova/tests/unit/virt/hyperv/test_vmops.py -+++ b/nova/tests/unit/virt/hyperv/test_vmops.py -@@ -1129,7 +1129,7 @@ class VMOpsTestCase(test_base.HyperVBaseTestCase): - mock_unplug_vifs.assert_called_once_with( - mock_instance, mock.sentinel.fake_network_info) - mock_disconnect_volumes.assert_called_once_with( -- mock.sentinel.FAKE_BD_INFO) -+ mock.sentinel.FAKE_BD_INFO, force=True) - mock_delete_disk_files.assert_called_once_with( - mock_instance.name) - -diff --git a/nova/tests/unit/virt/hyperv/test_volumeops.py b/nova/tests/unit/virt/hyperv/test_volumeops.py -index 66d2c2527f..f289d03632 100644 ---- a/nova/tests/unit/virt/hyperv/test_volumeops.py -+++ b/nova/tests/unit/virt/hyperv/test_volumeops.py -@@ -141,7 +141,13 @@ class VolumeOpsTestCase(test_base.HyperVBaseTestCase): - - self._volumeops.disconnect_volumes(block_device_info) - fake_volume_driver.disconnect_volume.assert_called_once_with( -- block_device_mapping[0]['connection_info']) -+ block_device_mapping[0]['connection_info'], force=False) -+ -+ # Verify force=True -+ fake_volume_driver.disconnect_volume.reset_mock() -+ self._volumeops.disconnect_volumes(block_device_info, force=True) -+ fake_volume_driver.disconnect_volume.assert_called_once_with( -+ block_device_mapping[0]['connection_info'], force=True) - - @mock.patch('time.sleep') - @mock.patch.object(volumeops.VolumeOps, '_get_volume_driver') -@@ -181,7 +187,7 @@ class VolumeOpsTestCase(test_base.HyperVBaseTestCase): - - if attach_failed: - fake_volume_driver.disconnect_volume.assert_called_once_with( -- fake_conn_info) -+ fake_conn_info, force=False) - mock_sleep.assert_has_calls( - [mock.call(CONF.hyperv.volume_attach_retry_interval)] * - CONF.hyperv.volume_attach_retry_count) -@@ -203,7 +209,13 @@ class VolumeOpsTestCase(test_base.HyperVBaseTestCase): - mock_get_volume_driver.assert_called_once_with( - mock.sentinel.conn_info) - fake_volume_driver.disconnect_volume.assert_called_once_with( -- mock.sentinel.conn_info) -+ mock.sentinel.conn_info, force=False) -+ -+ # Verify force=True -+ fake_volume_driver.disconnect_volume.reset_mock() -+ self._volumeops.disconnect_volume(mock.sentinel.conn_info, force=True) -+ fake_volume_driver.disconnect_volume.assert_called_once_with( -+ mock.sentinel.conn_info, force=True) - - @mock.patch.object(volumeops.VolumeOps, '_get_volume_driver') - def test_detach_volume(self, mock_get_volume_driver): -@@ -347,7 +359,13 @@ class BaseVolumeDriverTestCase(test_base.HyperVBaseTestCase): - self._base_vol_driver.disconnect_volume(conn_info) - - self._conn.disconnect_volume.assert_called_once_with( -- conn_info['data']) -+ conn_info['data'], force=False) -+ -+ # Verify force=True -+ self._conn.disconnect_volume.reset_mock() -+ self._base_vol_driver.disconnect_volume(conn_info, force=True) -+ self._conn.disconnect_volume.assert_called_once_with( -+ conn_info['data'], force=True) - - @mock.patch.object(volumeops.BaseVolumeDriver, '_get_disk_res_path') - def _test_get_disk_resource_path_by_conn_info(self, -diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py -index 2b58c7df8b..1eefe038cc 100644 ---- a/nova/tests/unit/virt/libvirt/test_driver.py -+++ b/nova/tests/unit/virt/libvirt/test_driver.py -@@ -9743,7 +9743,7 @@ class LibvirtConnTestCase(test.NoDBTestCase, - drvr._disconnect_volume( - self.context, fake_connection_info, fake_instance_1) - mock_volume_driver.disconnect_volume.assert_called_once_with( -- fake_connection_info, fake_instance_1) -+ fake_connection_info, fake_instance_1, force=False) - - @mock.patch.object(libvirt_driver.LibvirtDriver, '_detach_encryptor') - @mock.patch('nova.objects.InstanceList.get_uuids_by_host') -@@ -10117,7 +10117,12 @@ class LibvirtConnTestCase(test.NoDBTestCase, - device_name='vdc', - ), - mock.call.detach_encryptor(**encryption), -- mock.call.disconnect_volume(connection_info, instance)]) -+ mock.call.disconnect_volume( -+ connection_info, -+ instance, -+ force=False, -+ ) -+ ]) - get_device_conf_func = mock_detach_with_retry.mock_calls[0][1][2] - self.assertEqual(mock_guest.get_disk, get_device_conf_func.func) - self.assertEqual(('vdc',), get_device_conf_func.args) -@@ -20495,16 +20500,64 @@ class LibvirtConnTestCase(test.NoDBTestCase, - self.context, - mock.sentinel.connection_info, - instance, -- destroy_secrets=False -+ destroy_secrets=False, -+ force=True - ), - mock.call( - self.context, - mock.sentinel.connection_info, - instance, -- destroy_secrets=True -+ destroy_secrets=True, -+ force=True - ) - ]) - -+ @mock.patch('nova.virt.libvirt.driver.LibvirtDriver._get_volume_driver') -+ @mock.patch( -+ 'nova.virt.libvirt.driver.LibvirtDriver._should_disconnect_target', -+ new=mock.Mock(return_value=True)) -+ @mock.patch('nova.virt.libvirt.driver.LibvirtDriver._detach_encryptor', -+ new=mock.Mock()) -+ @mock.patch('nova.virt.libvirt.driver.LibvirtDriver._undefine_domain', -+ new=mock.Mock()) -+ @mock.patch('nova.virt.libvirt.driver.LibvirtDriver._get_vpmems', -+ new=mock.Mock(return_value=None)) -+ def test_cleanup_disconnect_volume(self, mock_vol_driver): -+ """Verify that we call disconnect_volume() with force=True -+ -+ cleanup() is called by destroy() when an instance is being deleted and -+ force=True should be passed down to os-brick's disconnect_volume() -+ call, which will ensure removal of devices regardless of errors. -+ -+ We need to ensure that devices are removed when an instance is being -+ deleted to avoid leaving leftover devices that could later be -+ erroneously connected by external entities (example: multipathd) to -+ instances that should not have access to the volumes. -+ -+ See https://bugs.launchpad.net/nova/+bug/2004555 for details. -+ """ -+ connection_info = mock.MagicMock() -+ block_device_info = { -+ 'block_device_mapping': [ -+ { -+ 'connection_info': connection_info -+ } -+ ] -+ } -+ instance = objects.Instance(self.context, **self.test_instance) -+ drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI()) -+ -+ drvr.cleanup( -+ self.context, -+ instance, -+ network_info={}, -+ block_device_info=block_device_info, -+ destroy_vifs=False, -+ destroy_disks=False, -+ ) -+ mock_vol_driver.return_value.disconnect_volume.assert_called_once_with( -+ connection_info, instance, force=True) -+ - @mock.patch.object(libvirt_driver.LibvirtDriver, '_get_volume_encryption') - @mock.patch.object(libvirt_driver.LibvirtDriver, '_allow_native_luksv1') - def test_swap_volume_native_luks_blocked(self, mock_allow_native_luksv1, -diff --git a/nova/tests/unit/virt/libvirt/volume/test_fibrechannel.py b/nova/tests/unit/virt/libvirt/volume/test_fibrechannel.py -index 06065322f6..55054652c3 100644 ---- a/nova/tests/unit/virt/libvirt/volume/test_fibrechannel.py -+++ b/nova/tests/unit/virt/libvirt/volume/test_fibrechannel.py -@@ -81,3 +81,23 @@ class LibvirtFibreChannelVolumeDriverTestCase( - self.assertEqual(requested_size, new_size) - libvirt_driver.connector.extend_volume.assert_called_once_with( - connection_info['data']) -+ -+ def test_disconnect_volume(self): -+ device_path = '/dev/fake-dev' -+ connection_info = {'data': {'device_path': device_path}} -+ -+ libvirt_driver = fibrechannel.LibvirtFibreChannelVolumeDriver( -+ self.fake_host) -+ libvirt_driver.connector.disconnect_volume = mock.MagicMock() -+ libvirt_driver.disconnect_volume( -+ connection_info, mock.sentinel.instance) -+ -+ libvirt_driver.connector.disconnect_volume.assert_called_once_with( -+ connection_info['data'], connection_info['data'], force=False) -+ -+ # Verify force=True -+ libvirt_driver.connector.disconnect_volume.reset_mock() -+ libvirt_driver.disconnect_volume( -+ connection_info, mock.sentinel.instance, force=True) -+ libvirt_driver.connector.disconnect_volume.assert_called_once_with( -+ connection_info['data'], connection_info['data'], force=True) -diff --git a/nova/tests/unit/virt/libvirt/volume/test_iscsi.py b/nova/tests/unit/virt/libvirt/volume/test_iscsi.py -index bd516b1dd6..a1111e0d12 100644 ---- a/nova/tests/unit/virt/libvirt/volume/test_iscsi.py -+++ b/nova/tests/unit/virt/libvirt/volume/test_iscsi.py -@@ -57,10 +57,19 @@ class LibvirtISCSIVolumeDriverTestCase( - device=device_path)) - libvirt_driver.disconnect_volume(connection_info, - mock.sentinel.instance) -+ libvirt_driver.connector.disconnect_volume.assert_called_once_with( -+ connection_info['data'], None, force=False) - - msg = mock_LOG_warning.call_args_list[0] - self.assertIn('Ignoring VolumeDeviceNotFound', msg[0][0]) - -+ # Verify force=True -+ libvirt_driver.connector.disconnect_volume.reset_mock() -+ libvirt_driver.disconnect_volume( -+ connection_info, mock.sentinel.instance, force=True) -+ libvirt_driver.connector.disconnect_volume.assert_called_once_with( -+ connection_info['data'], None, force=True) -+ - def test_extend_volume(self): - device_path = '/dev/fake-dev' - connection_info = {'data': {'device_path': device_path}} -diff --git a/nova/tests/unit/virt/libvirt/volume/test_lightos.py b/nova/tests/unit/virt/libvirt/volume/test_lightos.py -index 8a85d73059..f97a696a53 100644 ---- a/nova/tests/unit/virt/libvirt/volume/test_lightos.py -+++ b/nova/tests/unit/virt/libvirt/volume/test_lightos.py -@@ -62,7 +62,13 @@ class LibvirtLightVolumeDriverTestCase(test_volume.LibvirtVolumeBaseTestCase): - connection_info = {'data': disk_info} - lightos_driver.disconnect_volume(connection_info, None) - lightos_driver.connector.disconnect_volume.assert_called_once_with( -- disk_info, None) -+ disk_info, None, force=False) -+ -+ # Verify force=True -+ lightos_driver.connector.disconnect_volume.reset_mock() -+ lightos_driver.disconnect_volume(connection_info, None, force=True) -+ lightos_driver.connector.disconnect_volume.assert_called_once_with( -+ disk_info, None, force=True) - - @mock.patch('os_brick.initiator.connector.InitiatorConnector.factory', - new=mock.Mock(return_value=mock.Mock())) -diff --git a/nova/tests/unit/virt/libvirt/volume/test_nvme.py b/nova/tests/unit/virt/libvirt/volume/test_nvme.py -index 3f593841fa..42ef0adc8d 100644 ---- a/nova/tests/unit/virt/libvirt/volume/test_nvme.py -+++ b/nova/tests/unit/virt/libvirt/volume/test_nvme.py -@@ -77,7 +77,13 @@ class LibvirtNVMEVolumeDriverTestCase(test_volume.LibvirtVolumeBaseTestCase): - connection_info = {'data': disk_info} - nvme_driver.disconnect_volume(connection_info, None) - nvme_driver.connector.disconnect_volume.assert_called_once_with( -- disk_info, None) -+ disk_info, None, force=False) -+ -+ # Verify force=True -+ nvme_driver.connector.disconnect_volume.reset_mock() -+ nvme_driver.disconnect_volume(connection_info, None, force=True) -+ nvme_driver.connector.disconnect_volume.assert_called_once_with( -+ disk_info, None, force=True) - - @mock.patch('os_brick.initiator.connector.InitiatorConnector.factory', - new=mock.Mock(return_value=mock.Mock())) -diff --git a/nova/tests/unit/virt/libvirt/volume/test_scaleio.py b/nova/tests/unit/virt/libvirt/volume/test_scaleio.py -index f0fcba1deb..7d93691d9d 100644 ---- a/nova/tests/unit/virt/libvirt/volume/test_scaleio.py -+++ b/nova/tests/unit/virt/libvirt/volume/test_scaleio.py -@@ -49,7 +49,13 @@ class LibvirtScaleIOVolumeDriverTestCase( - conn = {'data': mock.sentinel.conn_data} - sio.disconnect_volume(conn, mock.sentinel.instance) - sio.connector.disconnect_volume.assert_called_once_with( -- mock.sentinel.conn_data, None) -+ mock.sentinel.conn_data, None, force=False) -+ -+ # Verify force=True -+ sio.connector.disconnect_volume.reset_mock() -+ sio.disconnect_volume(conn, mock.sentinel.instance, force=True) -+ sio.connector.disconnect_volume.assert_called_once_with( -+ mock.sentinel.conn_data, None, force=True) - - @mock.patch('os_brick.initiator.connector.InitiatorConnector.factory', - new=mock.Mock(return_value=mock.Mock())) -diff --git a/nova/tests/unit/virt/libvirt/volume/test_storpool.py b/nova/tests/unit/virt/libvirt/volume/test_storpool.py -index 678d4f8eb4..a3252b8525 100644 ---- a/nova/tests/unit/virt/libvirt/volume/test_storpool.py -+++ b/nova/tests/unit/virt/libvirt/volume/test_storpool.py -@@ -53,9 +53,11 @@ class MockStorPoolConnector(object): - } - return {'type': 'block', 'path': test_attached[v]['path']} - -- def disconnect_volume(self, connection_info, device_info): -+ def disconnect_volume(self, connection_info, device_info, **kwargs): - self.inst.assertIn('client_id', connection_info) - self.inst.assertIn('volume', connection_info) -+ self.inst.assertIn('force', kwargs) -+ self.inst.assertEqual(self.inst.force, kwargs.get('force')) - - v = connection_info['volume'] - if v not in test_attached: -@@ -86,6 +88,11 @@ class MockStorPoolInitiator(object): - class LibvirtStorPoolVolumeDriverTestCase( - test_volume.LibvirtVolumeBaseTestCase): - -+ def setUp(self): -+ super().setUp() -+ # This is for testing the force flag of disconnect_volume() -+ self.force = False -+ - def mock_storpool(f): - def _config_inner_inner1(inst, *args, **kwargs): - @mock.patch( -@@ -175,3 +182,10 @@ class LibvirtStorPoolVolumeDriverTestCase( - - libvirt_driver.disconnect_volume(ci_2, mock.sentinel.instance) - self.assertDictEqual({}, test_attached) -+ -+ # Connect the volume again so we can detach it again -+ libvirt_driver.connect_volume(ci_2, mock.sentinel.instance) -+ # Verify force=True -+ self.force = True -+ libvirt_driver.disconnect_volume( -+ ci_2, mock.sentinel.instance, force=True) -diff --git a/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py b/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py -index 168efee944..c9e455b193 100644 ---- a/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py -+++ b/nova/tests/unit/virt/libvirt/volume/test_vzstorage.py -@@ -95,7 +95,13 @@ class LibvirtVZStorageTestCase(test_volume.LibvirtVolumeBaseTestCase): - conn = {'data': mock.sentinel.conn_data} - drv.disconnect_volume(conn, mock.sentinel.instance) - drv.connector.disconnect_volume.assert_called_once_with( -- mock.sentinel.conn_data, None) -+ mock.sentinel.conn_data, None, force=False) -+ -+ # Verify force=True -+ drv.connector.disconnect_volume.reset_mock() -+ drv.disconnect_volume(conn, mock.sentinel.instance, force=True) -+ drv.connector.disconnect_volume.assert_called_once_with( -+ mock.sentinel.conn_data, None, force=True) - - def test_libvirt_vzstorage_driver_get_config(self): - libvirt_driver = vzstorage.LibvirtVZStorageVolumeDriver(self.fake_host) -diff --git a/nova/virt/hyperv/vmops.py b/nova/virt/hyperv/vmops.py -index 3ec7e90c30..08adeada76 100644 ---- a/nova/virt/hyperv/vmops.py -+++ b/nova/virt/hyperv/vmops.py -@@ -747,7 +747,7 @@ class VMOps(object): - # should be disconnected even if the VM doesn't exist anymore, - # so they are not leaked. - self.unplug_vifs(instance, network_info) -- self._volumeops.disconnect_volumes(block_device_info) -+ self._volumeops.disconnect_volumes(block_device_info, force=True) - - if destroy_disks: - self._delete_disk_files(instance_name) -diff --git a/nova/virt/hyperv/volumeops.py b/nova/virt/hyperv/volumeops.py -index da5b40f375..d2bfed2441 100644 ---- a/nova/virt/hyperv/volumeops.py -+++ b/nova/virt/hyperv/volumeops.py -@@ -59,10 +59,10 @@ class VolumeOps(object): - for vol in volumes: - self.attach_volume(vol['connection_info'], instance_name) - -- def disconnect_volumes(self, block_device_info): -+ def disconnect_volumes(self, block_device_info, force=False): - mapping = driver.block_device_info_get_mapping(block_device_info) - for vol in mapping: -- self.disconnect_volume(vol['connection_info']) -+ self.disconnect_volume(vol['connection_info'], force=force) - - def attach_volume(self, connection_info, instance_name, - disk_bus=constants.CTRL_TYPE_SCSI): -@@ -116,9 +116,9 @@ class VolumeOps(object): - volume_driver.set_disk_qos_specs(connection_info, - qos_specs) - -- def disconnect_volume(self, connection_info): -+ def disconnect_volume(self, connection_info, force=False): - volume_driver = self._get_volume_driver(connection_info) -- volume_driver.disconnect_volume(connection_info) -+ volume_driver.disconnect_volume(connection_info, force=force) - - def detach_volume(self, connection_info, instance_name): - LOG.debug("Detaching volume: %(connection_info)s " -@@ -231,8 +231,8 @@ class BaseVolumeDriver(object): - def connect_volume(self, connection_info): - return self._connector.connect_volume(connection_info['data']) - -- def disconnect_volume(self, connection_info): -- self._connector.disconnect_volume(connection_info['data']) -+ def disconnect_volume(self, connection_info, force=False): -+ self._connector.disconnect_volume(connection_info['data'], force=force) - - def get_disk_resource_path(self, connection_info): - disk_paths = self._connector.get_volume_paths(connection_info['data']) -diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py -index fe48960296..ff70c9b744 100644 ---- a/nova/virt/libvirt/driver.py -+++ b/nova/virt/libvirt/driver.py -@@ -1657,7 +1657,7 @@ class LibvirtDriver(driver.ComputeDriver): - try: - self._disconnect_volume( - context, connection_info, instance, -- destroy_secrets=destroy_secrets) -+ destroy_secrets=destroy_secrets, force=True) - except Exception as exc: - with excutils.save_and_reraise_exception() as ctxt: - if cleanup_instance_disks: -@@ -1974,7 +1974,7 @@ class LibvirtDriver(driver.ComputeDriver): - return (False if connection_count > 1 else True) - - def _disconnect_volume(self, context, connection_info, instance, -- encryption=None, destroy_secrets=True): -+ encryption=None, destroy_secrets=True, force=False): - self._detach_encryptor( - context, - connection_info, -@@ -1986,7 +1986,8 @@ class LibvirtDriver(driver.ComputeDriver): - multiattach = connection_info.get('multiattach', False) - if self._should_disconnect_target( - context, instance, multiattach, vol_driver, volume_id): -- vol_driver.disconnect_volume(connection_info, instance) -+ vol_driver.disconnect_volume( -+ connection_info, instance, force=force) - else: - LOG.info('Detected multiple connections on this host for ' - 'volume: %(volume)s, skipping target disconnect.', -diff --git a/nova/virt/libvirt/volume/fibrechannel.py b/nova/virt/libvirt/volume/fibrechannel.py -index 22c65e99c0..1752f6d0cc 100644 ---- a/nova/virt/libvirt/volume/fibrechannel.py -+++ b/nova/virt/libvirt/volume/fibrechannel.py -@@ -59,7 +59,7 @@ class LibvirtFibreChannelVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): - connection_info['data']['multipath_id'] = \ - device_info['multipath_id'] - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Detach the volume from instance_name.""" - - LOG.debug("calling os-brick to detach FC Volume", instance=instance) -@@ -69,11 +69,12 @@ class LibvirtFibreChannelVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): - # the 2nd param of disconnect_volume and be consistent - # with the rest of the connectors. - self.connector.disconnect_volume(connection_info['data'], -- connection_info['data']) -+ connection_info['data'], -+ force=force) - LOG.debug("Disconnected FC Volume", instance=instance) - - super(LibvirtFibreChannelVolumeDriver, -- self).disconnect_volume(connection_info, instance) -+ self).disconnect_volume(connection_info, instance, force=force) - - def extend_volume(self, connection_info, instance, requested_size): - """Extend the volume.""" -diff --git a/nova/virt/libvirt/volume/fs.py b/nova/virt/libvirt/volume/fs.py -index 5fb9af4a52..992ef45016 100644 ---- a/nova/virt/libvirt/volume/fs.py -+++ b/nova/virt/libvirt/volume/fs.py -@@ -116,7 +116,7 @@ class LibvirtMountedFileSystemVolumeDriver(LibvirtBaseFileSystemVolumeDriver, - connection_info['data']['device_path'] = \ - self._get_device_path(connection_info) - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Disconnect the volume.""" - vol_name = connection_info['data']['name'] - mountpoint = self._get_mount_path(connection_info) -diff --git a/nova/virt/libvirt/volume/iscsi.py b/nova/virt/libvirt/volume/iscsi.py -index 564bac14cc..2b25972a49 100644 ---- a/nova/virt/libvirt/volume/iscsi.py -+++ b/nova/virt/libvirt/volume/iscsi.py -@@ -66,19 +66,20 @@ class LibvirtISCSIVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): - - connection_info['data']['device_path'] = device_info['path'] - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Detach the volume from instance_name.""" - - LOG.debug("calling os-brick to detach iSCSI Volume", instance=instance) - try: -- self.connector.disconnect_volume(connection_info['data'], None) -+ self.connector.disconnect_volume( -+ connection_info['data'], None, force=force) - except os_brick_exception.VolumeDeviceNotFound as exc: - LOG.warning('Ignoring VolumeDeviceNotFound: %s', exc) - return - LOG.debug("Disconnected iSCSI Volume", instance=instance) - - super(LibvirtISCSIVolumeDriver, -- self).disconnect_volume(connection_info, instance) -+ self).disconnect_volume(connection_info, instance, force=force) - - def extend_volume(self, connection_info, instance, requested_size): - """Extend the volume.""" -diff --git a/nova/virt/libvirt/volume/lightos.py b/nova/virt/libvirt/volume/lightos.py -index d6d393994e..6a22bf6dc6 100644 ---- a/nova/virt/libvirt/volume/lightos.py -+++ b/nova/virt/libvirt/volume/lightos.py -@@ -42,14 +42,15 @@ class LibvirtLightOSVolumeDriver(libvirt_volume.LibvirtVolumeDriver): - LOG.debug("Connecting NVMe volume with device_info %s", device_info) - connection_info['data']['device_path'] = device_info['path'] - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Detach the volume from the instance.""" - LOG.debug("Disconnecting NVMe disk. instance:%s, volume_id:%s", - connection_info.get("instance", ""), - connection_info.get("volume_id", "")) -- self.connector.disconnect_volume(connection_info['data'], None) -+ self.connector.disconnect_volume( -+ connection_info['data'], None, force=force) - super(LibvirtLightOSVolumeDriver, self).disconnect_volume( -- connection_info, instance) -+ connection_info, instance, force=force) - - def extend_volume(self, connection_info, instance, requested_size=None): - """Extend the volume.""" -diff --git a/nova/virt/libvirt/volume/nvme.py b/nova/virt/libvirt/volume/nvme.py -index 7436552812..e2977c3572 100644 ---- a/nova/virt/libvirt/volume/nvme.py -+++ b/nova/virt/libvirt/volume/nvme.py -@@ -45,13 +45,13 @@ class LibvirtNVMEVolumeDriver(libvirt_volume.LibvirtVolumeDriver): - - connection_info['data']['device_path'] = device_info['path'] - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Detach the volume from the instance.""" - LOG.debug("Disconnecting NVMe disk", instance=instance) - self.connector.disconnect_volume( -- connection_info['data'], None) -+ connection_info['data'], None, force=force) - super(LibvirtNVMEVolumeDriver, -- self).disconnect_volume(connection_info, instance) -+ self).disconnect_volume(connection_info, instance, force=force) - - def extend_volume(self, connection_info, instance, requested_size): - """Extend the volume.""" -diff --git a/nova/virt/libvirt/volume/quobyte.py b/nova/virt/libvirt/volume/quobyte.py -index bb7a770e57..2eb4bcfb42 100644 ---- a/nova/virt/libvirt/volume/quobyte.py -+++ b/nova/virt/libvirt/volume/quobyte.py -@@ -189,7 +189,7 @@ class LibvirtQuobyteVolumeDriver(fs.LibvirtBaseFileSystemVolumeDriver): - instance=instance) - - @utils.synchronized('connect_qb_volume') -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Disconnect the volume.""" - - mount_path = self._get_mount_path(connection_info) -diff --git a/nova/virt/libvirt/volume/scaleio.py b/nova/virt/libvirt/volume/scaleio.py -index 7c414c2870..04a9423e8e 100644 ---- a/nova/virt/libvirt/volume/scaleio.py -+++ b/nova/virt/libvirt/volume/scaleio.py -@@ -57,12 +57,13 @@ class LibvirtScaleIOVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): - instance=instance) - connection_info['data']['device_path'] = device_info['path'] - -- def disconnect_volume(self, connection_info, instance): -- self.connector.disconnect_volume(connection_info['data'], None) -+ def disconnect_volume(self, connection_info, instance, force=False): -+ self.connector.disconnect_volume( -+ connection_info['data'], None, force=force) - LOG.debug("Disconnected volume", instance=instance) - - super(LibvirtScaleIOVolumeDriver, self).disconnect_volume( -- connection_info, instance) -+ connection_info, instance, force=force) - - def extend_volume(self, connection_info, instance, requested_size): - LOG.debug("calling os-brick to extend ScaleIO Volume", -diff --git a/nova/virt/libvirt/volume/smbfs.py b/nova/virt/libvirt/volume/smbfs.py -index d112af750c..9de1ce23cd 100644 ---- a/nova/virt/libvirt/volume/smbfs.py -+++ b/nova/virt/libvirt/volume/smbfs.py -@@ -52,7 +52,7 @@ class LibvirtSMBFSVolumeDriver(fs.LibvirtBaseFileSystemVolumeDriver): - device_path = self._get_device_path(connection_info) - connection_info['data']['device_path'] = device_path - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Disconnect the volume.""" - smbfs_share = connection_info['data']['export'] - mount_path = self._get_mount_path(connection_info) -diff --git a/nova/virt/libvirt/volume/storpool.py b/nova/virt/libvirt/volume/storpool.py -index 0e71221f5b..e6dffca39a 100644 ---- a/nova/virt/libvirt/volume/storpool.py -+++ b/nova/virt/libvirt/volume/storpool.py -@@ -47,10 +47,11 @@ class LibvirtStorPoolVolumeDriver(libvirt_volume.LibvirtVolumeDriver): - device_info, instance=instance) - connection_info['data']['device_path'] = device_info['path'] - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - LOG.debug("Detaching StorPool volume %s", - connection_info['data']['volume'], instance=instance) -- self.connector.disconnect_volume(connection_info['data'], None) -+ self.connector.disconnect_volume( -+ connection_info['data'], None, force=force) - LOG.debug("Detached StorPool volume", instance=instance) - - def extend_volume(self, connection_info, instance, requested_size): -diff --git a/nova/virt/libvirt/volume/volume.py b/nova/virt/libvirt/volume/volume.py -index 6d650c80e6..f76c3618b2 100644 ---- a/nova/virt/libvirt/volume/volume.py -+++ b/nova/virt/libvirt/volume/volume.py -@@ -135,7 +135,7 @@ class LibvirtBaseVolumeDriver(object): - """Connect the volume.""" - pass - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Disconnect the volume.""" - pass - -diff --git a/nova/virt/libvirt/volume/vzstorage.py b/nova/virt/libvirt/volume/vzstorage.py -index 85ffb45076..babfdef55c 100644 ---- a/nova/virt/libvirt/volume/vzstorage.py -+++ b/nova/virt/libvirt/volume/vzstorage.py -@@ -126,9 +126,10 @@ class LibvirtVZStorageVolumeDriver(fs.LibvirtBaseFileSystemVolumeDriver): - - return _connect_volume(connection_info, instance) - -- def disconnect_volume(self, connection_info, instance): -+ def disconnect_volume(self, connection_info, instance, force=False): - """Detach the volume from instance_name.""" - LOG.debug("calling os-brick to detach Vzstorage Volume", - instance=instance) -- self.connector.disconnect_volume(connection_info['data'], None) -+ self.connector.disconnect_volume( -+ connection_info['data'], None, force=force) - LOG.debug("Disconnected Vzstorage Volume", instance=instance) -diff --git a/releasenotes/notes/service-user-token-421d067c16257782.yaml b/releasenotes/notes/service-user-token-421d067c16257782.yaml -new file mode 100644 -index 0000000000..d3af14fbb8 ---- /dev/null -+++ b/releasenotes/notes/service-user-token-421d067c16257782.yaml -@@ -0,0 +1,11 @@ -+upgrade: -+ - | -+ Configuration of service user tokens is now **required** for all Nova services -+ to ensure security of block-storage volume data. -+ -+ All Nova configuration files must configure the ``[service_user]`` section as -+ described in the `documentation`__. -+ -+ See https://bugs.launchpad.net/nova/+bug/2004555 for more details. -+ -+ __ https://docs.openstack.org/nova/latest/admin/configuration/service-user-token.html --- -2.34.1 - diff -Nru nova-25.1.1/debian/patches/series nova-25.1.1/debian/patches/series --- nova-25.1.1/debian/patches/series 2023-05-15 20:11:40.000000000 +0000 +++ nova-25.1.1/debian/patches/series 2023-05-31 20:43:41.000000000 +0000 @@ -2,4 +2,5 @@ drop-sphinxcontrib-rsvgconverter.patch drop-sphinx-feature-classification.patch arm-console-patch.patch -#CVE-2023-2088.patch +CVE-2023-2088-1.patch +CVE-2023-2088-2.patch