SR-IOV Configuration

SR-IOV (Single Root I/O Virtualization) allows virtual machines to directly access physical network adapter Virtual Functions (VFs), bypassing the hypervisor virtual switch for near-native I/O throughput and ultra-low latency.

Prerequisites

Hardware Requirements

CPU with VT-d (Intel) or AMD-Vi (AMD) support
SR-IOV capable NIC (Intel 82599, X710, Mellanox ConnectX, etc.)
Motherboard with IOMMU support

Enable IOMMU

# Intel VT-d
# Edit /etc/default/grub
GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt"

# AMD-Vi
GRUB_CMDLINE_LINUX="amd_iommu=on iommu=pt"

# Update GRUB
update-grub
reboot

# Verify IOMMU
dmesg | grep -i iommu

Enable SR-IOV

Check SR-IOV Support

# Check if NIC supports SR-IOV
lspci -vvv -s <NIC_PCI_ADDRESS> | grep -i "sr-iov"

# Example output:
# Capabilities: [150] Single Root I/O Virtualization (SR-IOV)

Enable VFs

# Enable 8 VFs on eth2
echo 8 > /sys/class/net/eth2/device/sriov_numvfs

# Verify VFs created
lspci | grep -i "Virtual Function"

# Make persistent
cat > /etc/systemd/system/sriov-eth2.service <<EOF
[Unit]
Description=Enable SR-IOV on eth2
After=network.target

[Service]
Type=oneshot
ExecStart=/bin/sh -c 'echo 8 > /sys/class/net/eth2/device/sriov_numvfs'
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOF

systemctl enable sriov-eth2.service

Configure VF

Bind VF to VFIO

# Get VF PCI address
lspci | grep "Virtual Function" | head -1
# Example: 03:10.0 Ethernet controller: Intel Corporation 82599 VF

# Get vendor and device ID
lspci -n -s 03:10.0
# Example: 03:10.0 0200: 8086:10ed (rev 01)

# Unbind from current driver
echo 0000:03:10.0 > /sys/bus/pci/drivers/ixgbevf/unbind

# Bind to vfio-pci
modprobe vfio-pci
echo "8086 10ed" > /sys/bus/pci/drivers/vfio-pci/new_id
echo 0000:03:10.0 > /sys/bus/pci/drivers/vfio-pci/bind

Configure VF Network

# Set MAC address
ip link set eth2 vf 0 mac aa:bb:cc:dd:ee:01

# Set VLAN
ip link set eth2 vf 0 vlan 100

# Set rate limit (in Mbps)
ip link set eth2 vf 0 rate 1000

PiVirt Configuration

Network Configuration

network:
  sriov:
    - name: "sriov-net"
      pf: "eth2"
      vfs: 8
      vf_driver: "vfio-pci"

VM Configuration

spec:
  nics:
    - name: eth0
      network: sriov-net
      model: vfio-pci
      pci_address: "03:10.0"
      mac: "aa:bb:cc:dd:ee:01"

Performance Tuning

CPU Pinning

spec:
  cpu:
    cores: 4
    pinning:
      - vcpu: 0
        host_cpu: 0
      - vcpu: 1
        host_cpu: 1
      - vcpu: 2
        host_cpu: 2
      - vcpu: 3
        host_cpu: 3

NUMA Awareness

spec:
  numa:
    nodes:
      - id: 0
        cpus: "0-3"
        memory: "8Gi"

Hugepages

spec:
  memory:
    size: 8Gi
    hugepages:
      enabled: true
      size: "2Mi"

Monitoring

Check VF Status

# List VFs
ip link show eth2

# Check VF stats
ethtool -S eth2 | grep -i vf

Performance Metrics

# Check network stats
ethtool -S eth2

# Check VF throughput
iperf3 -c 10.10.1.11 -t 60

Troubleshooting

VF Not Created

# Check SR-IOV support
lspci -vvv -s <NIC_PCI_ADDRESS> | grep -i "sr-iov"

# Check IOMMU
dmesg | grep -i iommu

# Check kernel parameters
cat /proc/cmdline | grep iommu

VFIO Bind Failed

# Check if VFIO is loaded
lsmod | grep vfio

# Check IOMMU groups
ls -la /sys/kernel/iommu_groups/

# Check VF PCI address
lspci -n -s <VF_PCI_ADDRESS>

VM Can’t Access VF

# Check VF binding
ls -la /sys/bus/pci/drivers/vfio-pci/

# Check VM config
pvcli vm get web-01 -o json | jq '.spec.nics'

# Check QEMU logs
journalctl -u pivirtd -f | grep -i vfio