From 65257b28906408c857875852fe031855f4aa3c95 Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Mon, 6 Jun 2022 16:33:21 -0400 Subject: [PATCH] vfio-pci/zdev: add open/close device hooks During vfio-pci open_device, pass the KVM associated with the vfio group (if one exists). This is needed in order to pass a special indicator (GISA) to firmware to allow zPCI interpretation facilities to be used for only the specific KVM associated with the vfio-pci device. During vfio-pci close_device, unregister the notifier. Signed-off-by: Matthew Rosato Acked-by: Alex Williamson Reviewed-by: Pierre Morel Link: https://lore.kernel.org/r/20220606203325.110625-18-mjrosato@linux.ibm.com Signed-off-by: Christian Borntraeger (backported from commit 8061d1c31f1a018281bc9877ecce44bfc779e21d) [MR: the upstream version of this patch relied on 421cfe6596f6 which removed the VFIO_GROUP_NOTIFY_SET_KVM notifier. Backporting that patch would require a large number of vfio pre-reqs affecting all platforms. This backport instead implements a group notifier and leaves base vfio unchanged.] Signed-off-by: Matthew Rosato --- arch/s390/include/asm/pci.h | 1 + drivers/vfio/pci/vfio_pci_core.c | 11 ++++++ drivers/vfio/pci/vfio_pci_zdev.c | 60 ++++++++++++++++++++++++++++++++ include/linux/vfio_pci_core.h | 10 ++++++ 4 files changed, 82 insertions(+) diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index d49b5e94398e..cef3fef77471 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -195,6 +195,7 @@ struct zpci_dev { struct s390_domain *s390_domain; /* s390 IOMMU domain data */ struct kvm_zdev *kzdev; struct mutex kzdev_lock; + struct notifier_block kvm_group_nb; /* Jammy-specific */ }; static inline bool zdev_enabled(struct zpci_dev *zdev) diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index f3916e6b16b9..4f1aceadd4fb 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -299,8 +299,17 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) pci_write_config_word(pdev, PCI_COMMAND, cmd); } + ret = vfio_pci_zdev_open_device(vdev); + if (ret) { + kfree(vdev->pci_saved_state); + vdev->pci_saved_state = NULL; + pci_disable_device(pdev); + return ret; + } + ret = vfio_config_init(vdev); if (ret) { + vfio_pci_zdev_close_device(vdev); kfree(vdev->pci_saved_state); vdev->pci_saved_state = NULL; pci_disable_device(pdev); @@ -395,6 +404,8 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) vdev->needs_reset = true; + vfio_pci_zdev_close_device(vdev); + /* * If we have saved state, restore it. If we can reset the device, * even better. Resetting with current state seems better than diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c index ea4c0d2b0663..e37d0574d6c7 100644 --- a/drivers/vfio/pci/vfio_pci_zdev.c +++ b/drivers/vfio/pci/vfio_pci_zdev.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -136,3 +137,62 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, return ret; } + +static int vfio_pci_zdev_group_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct zpci_dev *zdev = container_of(nb, struct zpci_dev, + kvm_group_nb); + + if (action == VFIO_GROUP_NOTIFY_SET_KVM) { + if (data) { + if (kvm_s390_pci_register_kvm(zdev, data)) + return NOTIFY_BAD; + } + } + + return NOTIFY_OK; +} + +int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev) +{ + struct zpci_dev *zdev = to_zpci(vdev->pdev); + unsigned long events = VFIO_GROUP_NOTIFY_SET_KVM; + int ret; + + if (!zdev) + return -ENODEV; + + /* + * Jammy-specific: backporting upstream commit 421cfe6596f6cb would + * require a large number of pre-req series to also be applied, + * affecting vfio across all platforms. + * Instead implement a vfio KVM group notifier here which only + * affects vfio-pci-zdev. Rather than relying on the kvm pointer + * being provided in the vdev, get it from the notifier. Sice we know + * the kvm will already be set, we can then immediately unregister the + * notifier. + */ + zdev->kvm_group_nb.notifier_call = vfio_pci_zdev_group_notifier; + ret = vfio_register_notifier(&vdev->pdev->dev, VFIO_GROUP_NOTIFY, + &events, &zdev->kvm_group_nb); + if (ret != 0) { + pr_warn("vfio_register_notifier for group failed: %d\n", ret); + return ret; + } + + vfio_unregister_notifier(&vdev->pdev->dev, VFIO_GROUP_NOTIFY, + &zdev->kvm_group_nb); + + return 0; +} + +void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev) +{ + struct zpci_dev *zdev = to_zpci(vdev->pdev); + + if (!zdev || !zdev->kzdev) + return; + + kvm_s390_pci_unregister_kvm(zdev); +} diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index db6a60298ffa..f22d5a382c15 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -209,12 +209,22 @@ static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) #ifdef CONFIG_VFIO_PCI_ZDEV_KVM extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, struct vfio_info_cap *caps); +int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev); +void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev); #else static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, struct vfio_info_cap *caps) { return -ENODEV; } + +static inline int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev) +{ + return 0; +} + +static inline void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev) +{} #endif /* Will be exported for vfio pci drivers usage */ -- 2.40.1