virt-install支持VF

原始CentOS自带QEMU是不支持RDMA的，需要重新编译，可以参考我的上一篇文档：

QEMU源码编译CentOS

物理机BIOS已开启SR-IOV，就开始使用SR-IOV

SR-IOV创建出的虚拟网卡称为VF（Virtual Function），如下命令可以查看网卡物理端口ens4f0/1（称PF）最大支持创建的VF均为8个

# cat /sys/class/net/ens4f0/device/sriov_totalvfs
8
# cat /sys/class/net/ens4f1/device/sriov_totalvfs
8
ens4f0单个网口虚拟出6个VF

# echo 6 > /sys/class/net/ens4f0/device/sriov_numvfs
 
# lspci|grep Mellanox
b1:00.0 Ethernet controller: Mellanox Technologies MT2894 Family [ConnectX-6 Lx]
b1:00.1 Ethernet controller: Mellanox Technologies MT2894 Family [ConnectX-6 Lx]
b1:00.2 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.3 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.4 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.5 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.6 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.7 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
 
# ip link |grep ens4
261: ens4f0v0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
262: ens4f0v1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
263: ens4f0v2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
264: ens4f0v3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
265: ens4f0v4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
266: ens4f0v5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
18: ens4f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
19: ens4f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
 
# ip link show ens4f0v0
261: ens4f0v0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 56:ba:79:b5:fb:3a brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v1
262: ens4f0v1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 42:f9:c8:62:be:fd brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v2
263: ens4f0v2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 2e:2b:21:22:a7:da brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v3
264: ens4f0v3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 22:cd:f8:8e:8b:39 brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v4
265: ens4f0v4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether b6:b1:22:d5:28:46 brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v5
266: ens4f0v5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether be:64:4f:36:e0:f7 brd ff:ff:ff:ff:ff:ff

虚拟网卡已经准备就绪了，开始创建KVM了，并且制定mac=${VF MAC}

rm -rf /var/lib/libvirt/images/mytest.raw
qemu-img create -f raw /var/lib/libvirt/images/mytest.raw 120G
chown qemu:qemu /var/lib/libvirt/images/mytest.raw

virt-install \
        --name mytest \
        --vcpus 2 \
        --cpu host-passthrough \
        --memory 8348 \
        --location ./CentOS8.4_x86_64.iso,kernel=images/pxeboot/vmlinuz,initrd=images/pxeboot/initrd.img \
        --extra-args "inst.stage2=hd:LABEL=XXXXXX inst.ks=file:/ks_preload.cfg mgmt_net=192.128.134.112/24@192.128.134.1@preload-mytest@ sys_drive=auto" \
        --disk path=/var/lib/libvirt/images/mytest.img,format=qcow2,size=120,bus=scsi,cache=writeback \
        "--qemu-commandline= -drive file=/var/lib/libvirt/images/mytest.raw,format=raw,if=none,id=D1 -device nvme,drive=D1,serial=1" \
        --os-variant centos7.0 \
        --network network=host-bridge,model=virtio \
        --network network=cluster_E,model=virtio,mac=56:ba:79:b5:fb:3a \
        --graphics vnc \
        --noautoconsole