[CentOS] Xen pci passthru problems with kernel -164.6.1

Tue Nov 10 12:01:32 UTC 2009
Miguel Di Ciurcio Filho <miguel at ic.unicamp.br>

Hi there,

After updating a server from CentOS 5.3 to 5.4 my Xen pci-passthru setup 
has some troubles.

The server has two NICs, one used by dom0 and the other assigned to a 
domU and one SCSI controler assigned to another domU. It has been 
working fine since CentOS 5.1, when I did the initial setup.

After upgrading to 5.4, I get this error when starting the domU with one 
of the NICs assigned:

Error: pci: improper device assignment specified: pci: 0000:07:00.0 must 
be co-assigned to the same guest with 0000:07:00.1, but it is not owned 
by pciback.

On the other hand, the domU with the SCSI controler boots, but the 
driver inside the domU prints a lot of awful errors and does not work.

I've booted the domU with the SCSI controller using the -128.7.1 kernel 
and everything works fine as before -164.6.1.

Any clues on why I can't assign just one NIC to the domU anymore?

# ls -l /sys/bus/pci/drivers/pciback/
total 0
lrwxrwxrwx 1 root root    0 Nov 10 09:55 0000:07:00.1 -> 
../../../../devices/pci0000:00/0000:00:02.0/0000:01:00.0/0000:02:02.0/0000:07:00.1
lrwxrwxrwx 1 root root    0 Nov 10 09:55 0000:08:03.0 -> 
../../../../devices/pci0000:00/0000:00:02.0/0000:01:00.3/0000:08:03.0
--w------- 1 root root 4096 Nov 10 09:55 bind
lrwxrwxrwx 1 root root    0 Nov 10 09:55 module -> 
../../../../module/pciback
--w------- 1 root root 4096 Nov 10 09:55 new_id
--w------- 1 root root 4096 Nov 10 09:55 new_slot
-rw------- 1 root root    0 Nov 10 08:35 permissive
-rw------- 1 root root    0 Nov 10 08:35 quirks
--w------- 1 root root 4096 Nov 10 09:55 remove_id
--w------- 1 root root 4096 Nov 10 09:55 remove_slot
-r-------- 1 root root 4096 Nov 10 09:55 slots
--w------- 1 root root 4096 Nov 10 09:55 unbind


# Script do create the initrds
KVER="2.6.18-164.6.1.el5xen"

mkinitrd -f --omit-scsi-modules --omit-raid-modules \
--with=e1000e --with=xennet --with=xenblk \
--preload=xenblk /boot/initrd-$KVER-domU.img $KVER

mkinitrd -f --preload=pciback /boot/initrd-$KVER.img $KVER


# cat /etc/modprobe.conf
alias eth0 e1000e
options netloop nloopbacks=0
options pciback hide=(0000:07:00.1)(0000:08:03.0)
alias scsi_hostadapter2 megaraid_sas # this is the RAID controller
alias scsi_hostadapter3 ata_piix

# lspci |egrep "(Ether|SCSI)"
07:00.0 Ethernet controller: Intel Corporation 80003ES2LAN Gigabit 
Ethernet Controller (Copper) (rev 01)
07:00.1 Ethernet controller: Intel Corporation 80003ES2LAN Gigabit 
Ethernet Controller (Copper) (rev 01)
08:03.0 SCSI storage controller: LSI Logic / Symbios Logic 53c1030 PCI-X 
Fusion-MPT Dual Ultra320 SCSI (rev 08)


# uname -a
Linux salvador.ic.unicamp.br 2.6.18-164.6.1.el5xen #1 SMP Tue Nov 3 
16:48:13 EST 2009 x86_64 x86_64 x86_64 GNU/Linux

# domU-with-NIC.cfg
kernel = "/boot/vmlinuz-2.6.18-164.6.1.el5xen"
ramdisk = "/boot/initrd-2.6.18-164.6.1.el5xen-domU.img"
memory = "4096"
name = "taquaral"
disk = ['phy:/dev/volumes/taquaral-root,xvda1,w',
         'phy:/dev/volumes/taquaral-home,xvda2,w',
         'phy:/dev/volumes/taquaral-home-staff,xvda3,w',
         'phy:/dev/volumes/taquaral-home-adm,xvda4,w',
         'phy:/dev/volumes/taquaral-home-admsis,xvda5,w',
         'phy:/dev/volumes/taquaral-home-phd,xvda6,w',
         'phy:/dev/volumes/taquaral-home-spec,xvda9,w',
         'phy:/dev/volumes/taquaral-swap,xvda7,w',
         'phy:/dev/volumes/taquaral-var,xvda8,w']
vif = ['mac=00:16:3e:57:13:a5, bridge=vlan0']
vcpus = 4
extra = "console=xvc0"
root = "/dev/xvda1"
pci = ["0000:07:00.1"]

# domU-with-SCSI.cfg
kernel = "/boot/vmlinuz-2.6.18-128.7.1.el5xen"
ramdisk = "/boot/initrd-2.6.18-128.7.1.el5xen-domU.img"
memory = "2048"
name = "flamboyant"
disk = ['phy:/dev/volumes/flamboyant-root,xvda1,w', 
'phy:/dev/volumes/flamboyant-var,xvda2,w', 
'phy:/dev/volumes/flamboyant-l,xvda3,w', 
'phy:/dev/volumes/flamboyant-swap,xvda4,w']
vif = ['mac=00:16:e3:55:55:55, bridge=br0', 'mac=00:16:3e:55:55:10, 
bridge=vlan0']
vcpus = 2
extra = "console=xvc0"
root = "/dev/xvda1"
pci = ["0000:08:03.0"]

# menu.lst
title CentOS (2.6.18-164.6.1.el5xen)
         root (hd0,0)
         kernel /xen.gz-2.6.18-164.6.1.el5 dom0_mem=1G
         module /vmlinuz-2.6.18-164.6.1.el5xen ro root=LABEL=/ enforcing=0
         module /initrd-2.6.18-164.6.1.el5xen.img