Compiled from various sources:
- https://forum.level1techs.com/t/vfio-pass-through-working-on-9070xt/227194/44
- https://forum.level1techs.com/t/solved-vfio-vm-some-games-do-not-run/237677
Disable ROM BAR on the GPU hostdev definition with <rom bar="off"/>. Do for both, VGA and Audio, For example:
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</source>
<rom bar="off"/>
<address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x03" slot="0x00" function="0x1"/>
</source>
<rom bar="off"/>
<address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
</hostdev>Do not let vfio-pci driver to takeover the GPU when the VM is not running, so do not use the vfio-pci-override.sh script mentioned in this gist https://gist.github.com/gdesatrigraha/aabfe6aee7a6d5e641fb4cd27e3dae86
Save and run the following shell script:
#!/bin/bash
# Our virtio vm name
VM_NAME='<vm-name>'
# populate from `lspci -nnk`
# for example for my system, the output is:
# 03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Navi 48 [Radeon RX 9070/9070 XT/9070 GRE] [1002:7550] (rev c0)
# 03:00.1 Audio device [0403]: Advanced Micro Devices, Inc. [AMD/ATI] Navi 48 HDMI/DP Audio Controller [1002:ab40]
VENDOR_ID='1002'
VGA_DEVICE_ID='7550'
AUDIO_DEVICE_ID='ab40'
VGA_PCI_ID='0000:03:00.0'
AUDIO_PCI_ID='0000:03:00.1'
cat << EOF | sudo tee "/etc/libvirt/hooks/qemu" > /dev/null
#!/bin/bash
#
# Libvirt QEMU hook script dispatcher.
#
# This script is executed by libvirt when a QEMU domain is started,
# stopped, or other events occur. It dispatches the hook to the
# appropriate script in /etc/libvirt/hooks/qemu.d/
#
# Arguments:
# $1: Domain name
# $2: Hook name (prepare, start, started, stopped, release)
# $3: State name (begin, end)
# $4: Type (qemu, lxc)
set -e
GUEST_NAME="$1"
HOOK_NAME="$2"
STATE_NAME="$3"
TYPE="$4"
HOOK_DIR="/etc/libvirt/hooks/qemu.d"
HOOK_PATH="$HOOK_DIR/$GUEST_NAME/$HOOK_NAME/$STATE_NAME"
# Pass all arguments to the hook script.
if [ -d "$HOOK_PATH" ]; then
for script in "$HOOK_PATH"/*; do
if [ -x "$script" ]; then
"$script" "$@"
fi
done
fi
exit 0
EOF
cat << EOF | sudo tee "/etc/libvirt/hooks/qemu.d/${VM_NAME}/prepare/begin/gpu.sh" > /dev/null
#!/bin/bash
#
# Unbind GPU from amdgpu and bind to vfio-pci
#
# Unbind from amdgpu
echo '${VGA_PCI_ID}' > '/sys/bus/pci/devices/${VGA_PCI_ID}/driver/unbind'
echo '${AUDIO_PCI_ID}' > '/sys/bus/pci/devices/${AUDIO_PCI_ID}/driver/unbind'
# Bind to vfio-pci
modprobe vfio-pci
echo '${VENDOR_ID} ${VGA_DEVICE_ID}' > '/sys/bus/pci/drivers/vfio-pci/new_id'
echo '${VENDOR_ID} ${AUDIO_DEVICE_ID}' > '/sys/bus/pci/drivers/vfio-pci/new_id'
sleep 1s
EOF
cat << EOF | sudo tee "/etc/libvirt/hooks/qemu.d/${VM_NAME}/release/end/gpu.sh" > /dev/null
!/bin/bash
#
# Unbind GPU from vfio-pci and bind back to amdgpu
#
# Unbind from vfio-pci
echo '${VGA_PCI_ID}' > '/sys/bus/pci/devices/0000:03:00.0/driver/unbind'
echo '${AUDIO_PCI_ID}' > '/sys/bus/pci/devices/0000:03:00.1/driver/unbind'
sleep 1s
# Bind back to amdgpu and snd_hda_intel
echo '${VGA_PCI_ID}' > '/sys/bus/pci/drivers/amdgpu/bind'
echo '${AUDIO_PCI_ID}' > '/sys/bus/pci/drivers/snd_hda_intel/bind'
sleep 1s
EOF