[CentOS] dm-multipath use

Thu Jun 26 08:01:58 UTC 2008
Plant, Dean <dean.plant at roke.co.uk>

Geoff Galitz wrote:
> Are folks in the Centos community succesfully using
> device-mapper-multipath? I am looking to deploy it for error handling
> on our iSCSI setup but there seems to be little traffic about this
> package on the Centos forums, as far as I can tell, and there seems
> to be a number of small issues based on my reading the dm-multipath
> developer lists and related resources. 
> 
> -geoff

Here's my notes from working with our SAN. Please check thoroughly as
these instructions worked for me but I had to learn this from scratch
and there maybe mistakes. One thing I never figured how to achieve was
to rescan an existing ISCSI device for changes if you resize a partition
on the SAN. I have always had to reboot to get the new partition size to
be seen.

###############################
ISCSI notes.

# yum -y install iscsi-initiator-utils lsscsi device-mapper-multipath

# service iscsi start

Add the ISCSI targets

iscsiadm -m discovery -t sendtargets -p 192.168.100.6
iscsiadm -m discovery -t sendtargets -p 192.168.100.2
iscsiadm -m discovery -t sendtargets -p 192.168.100.8
iscsiadm -m discovery -t sendtargets -p 192.168.100.4

# lsscsi
[0:0:0:0]    disk    VMware,  VMware Virtual S 1.0   /dev/sda
[1:0:0:0]    disk    COMPELNT Compellent Vol   0306  -
[2:0:0:0]    disk    COMPELNT Compellent Vol   0306  -
[3:0:0:0]    disk    COMPELNT Compellent Vol   0306  -
[4:0:0:0]    disk    COMPELNT Compellent Vol   0306  -

service multipathd start
chkconfig multipathd on



Configuring SAN volumes without reboot

Rescan for ISCSI devices

# iscsiadm -m session -R

Add your partitions For this example I created 2 partitions

Let the running kernel see the new partitions

# partprobe

fdisk /dev/mapper/mpath2

We need to tell the mapper about the new partitions

# ls -l /dev/mapper/mpath2*
brw-rw---- 1 root disk 253, 7 Jan 30 15:42 /dev/mapper/mpath2

# kpartx -l /dev/mapper/mpath2

# ls -l /dev/mapper/mpath2*
brw-rw---- 1 root disk 253, 7 Jan 30 15:42 /dev/mapper/mpath2
brw-rw---- 1 root disk 253, 8 Jan 30 15:43 /dev/mapper/mpath2p1
brw-rw---- 1 root disk 253, 9 Jan 30 15:43 /dev/mapper/mpath2p2

# mke2fs -j /dev/mapper/mpath2p1
# mke2fs -j /dev/mapper/mpath2p2


# iscsiadm -m node
192.168.100.6:3260,0 iqn.2002-03.com.compellent:5000d310000a630a
192.168.100.2:3260,0 iqn.2002-03.com.compellent:5000d310000a6302
192.168.100.4:3260,0 iqn.2002-03.com.compellent:5000d310000a6304
192.168.100.8:3260,0 iqn.2002-03.com.compellent:5000d310000a630c

[root at test2 ~]# iscsiadm -m  session
tcp: [12] 192.168.100.6:3260,0
iqn.2002-03.com.compellent:5000d310000a630a
tcp: [13] 192.168.100.2:3260,0
iqn.2002-03.com.compellent:5000d310000a6302
tcp: [14] 192.168.100.4:3260,0
iqn.2002-03.com.compellent:5000d310000a6304
tcp: [15] 192.168.100.8:3260,0
iqn.2002-03.com.compellent:5000d310000a630c

# iscsiadm -m discovery
192.168.100.2:3260 via sendtargets
192.168.100.6:3260 via sendtargets
192.168.100.4:3260 via sendtargets
192.168.100.8:3260 via sendtargets


To automatically mount a file system during startup
you must have the partition entry in /etc/fstab marked with the
"_netdev"
option. For example this would mount a iscsi disk sdb:

        /dev/sdb /mnt/iscsi ext3 _netdev 0 0


NOTES WHEN USING LVM WITH MULTIPATH

http://www.redhat.com/docs/manuals/enterprise/RHEL-5-manual/en-US/RHEL51
0/DM_Multipath/multipath_logical_volumes.html

When you create an LVM logical volume that uses active/passive multipath
arrays as the underlying physical devices, you should include filters in
the lvm.conf to exclude the disks that underlie the multipath devices.
This is because if the array automatically changes the active path to
the passive path when it receives I/O, multipath will failover and
failback whenever LVM scans the passive path if these devices are not
filtered. For active/passive arrays that require a command to make the
passive path active, LVM prints a warning message when this occurs.

To filter all SCSI devices in the multipath configuration file
(lvm.conf), include the following filter in the devices section of the
file. 


filter = [ "r/disk/", "r/sd.*/", "a/.*/" ]

A filter to allow sda but disallow all other sd* drives


filter = [ "a|/dev/sda|","r/disk/", "r/sd.*/", "a/.*/" ]


CREATE LVM on top of a SAN MULTI PATH

vi /etc/lvm/lvm.conf

#    preferred_names = [ ]
# DAP
     preferred_names = [ "^/dev/mpath/", "^/dev/[hs]d" ]

#   filter = [ "a/.*/" ]
# DAP
    filter = [ "a|/dev/sda|","r/disk/", "r/sd.*/", "a/.*/" ]


pvcreate /dev/mpath/mpath2p1

# pvdisplay /dev/mpath/mpath2p1
  --- Physical volume ---
  PV Name               /dev/mpath/mpath2p1
  VG Name               VGSAN00
  PV Size               1019.72 MB / not usable 3.72 MB
  Allocatable           yes (but full)
  PE Size (KByte)       4096
  Total PE              254
  Free PE               0
  Allocated PE          254
  PV UUID               ZjqvDp-mxMh-xbuV-CFql-QzaB-cC4l-Eo7RNl

vgcreate VGSAN00 /dev/mpath/mpath2p1

# vgdisplay VGSAN00
  --- Volume group ---
  VG Name               VGSAN00
  System ID
  Format                lvm2
  Metadata Areas        1
  Metadata Sequence No  3
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                0
  Open LV               0
  Max PV                0
  Cur PV                1
  Act PV                1
  VG Size               1016.00 MB
  PE Size               4.00 MB
  Total PE              254
  Alloc PE / Size       0 / 0
  Free  PE / Size       254 / 1016.00 MB
  VG UUID               bXv8IW-3eJT-RUXB-MLEo-gnfV-4VJv-j598PE

# lvcreate -l 254 -n data VGSAN00

# lvdisplay /dev/VGSAN00/data
  --- Logical volume ---
  LV Name                /dev/VGSAN00/data
  VG Name                VGSAN00
  LV UUID                UorNsT-0NuN-kj86-2YGq-gVvG-aJYu-m04bhj
  LV Write Access        read/write
  LV Status              available
  # open                 0
  LV Size                1016.00 MB
  Current LE             254
  Segments               1
  Allocation             inherit
  Read ahead sectors     0
  Block device           253:9


------------------------------------------------
vi /etc/multipath.conf

defaults {
        user_friendly_names yes
}
defaults {
        udev_dir                /dev
        polling_interval        10
        selector                "round-robin 0"
        path_grouping_policy    multibus
        getuid_callout          "/sbin/scsi_id -g -u -s /block/%n"
        prio_callout            /bin/true
        path_checker            readsector0
        rr_min_io               100
        rr_weight               priorities
        failback                immediate
        no_path_retry           fail
        user_friendly_name      yes
}

devices {
        device {
                vendor "COMPELNT"
                product "Compellent Vol"
                path_grouping_policy multibus
                getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
                path_checker readsector0
                path_selector "round-robin 0"
                hardware_handler "0"
                failback 15
                rr_weight priorities
                no_path_retry queue
        }
}
------------------------------------------------


# iscsiadm -m session -i
iscsiadm version 2.0-742
************************************
Session (sid 12) using module tcp:
************************************
TargetName: iqn.2002-03.com.compellent:5000d310000a630a
Portal Group Tag: 0
Network Portal: 192.168.100.6:3260
iSCSI Connection State: LOGGED IN
Internal iscsid Session State: NO CHANGE

************************
Negotiated iSCSI params:
************************
HeaderDigest: None
DataDigest: None
MaxRecvDataSegmentLength: 65536
MaxXmitDataSegmentLength: 65536
FirstBurstLength: 131072
MaxBurstLength: 262144
ImmediateData: Yes
InitialR2T: Yes
MaxOutstandingR2T: 1

************************
Attached SCSI devices:
************************
Host Number: 12 State: running

scsi12 Channel 00 Id 0 Lun: 0
Attached scsi disk sda          State: running

************************************
Session (sid 13) using module tcp:
************************************
TargetName: iqn.2002-03.com.compellent:5000d310000a6302
Portal Group Tag: 0
Network Portal: 192.168.100.2:3260
iSCSI Connection State: LOGGED IN
Internal iscsid Session State: NO CHANGE

************************
Negotiated iSCSI params:
************************
HeaderDigest: None
DataDigest: None
MaxRecvDataSegmentLength: 65536
MaxXmitDataSegmentLength: 65536
FirstBurstLength: 131072
MaxBurstLength: 262144
ImmediateData: Yes
InitialR2T: Yes
MaxOutstandingR2T: 1

************************
Attached SCSI devices:
************************
Host Number: 13 State: running

scsi13 Channel 00 Id 0 Lun: 0

************************************
Session (sid 14) using module tcp:
************************************
TargetName: iqn.2002-03.com.compellent:5000d310000a6304
Portal Group Tag: 0
Network Portal: 192.168.100.4:3260
iSCSI Connection State: LOGGED IN
Internal iscsid Session State: NO CHANGE

************************
Negotiated iSCSI params:
************************
HeaderDigest: None
DataDigest: None
MaxRecvDataSegmentLength: 65536
MaxXmitDataSegmentLength: 65536
FirstBurstLength: 131072
MaxBurstLength: 262144
ImmediateData: Yes
InitialR2T: Yes
MaxOutstandingR2T: 1

************************
Attached SCSI devices:
************************
Host Number: 14 State: running

scsi14 Channel 00 Id 0 Lun: 0

************************************
Session (sid 15) using module tcp:
************************************
TargetName: iqn.2002-03.com.compellent:5000d310000a630c
Portal Group Tag: 0
Network Portal: 192.168.100.8:3260
iSCSI Connection State: LOGGED IN
Internal iscsid Session State: NO CHANGE

************************
Negotiated iSCSI params:
************************
HeaderDigest: None
DataDigest: None
MaxRecvDataSegmentLength: 65536
MaxXmitDataSegmentLength: 65536
FirstBurstLength: 131072
MaxBurstLength: 262144
ImmediateData: Yes
InitialR2T: Yes
MaxOutstandingR2T: 1

************************
Attached SCSI devices:
************************
Host Number: 15 State: running

scsi15 Channel 00 Id 0 Lun: 0
Attached scsi disk sdb          State: running


Quick LVM

pvcreate /dev/mapper/mpath3p1
pvcreate /dev/mapper/mpath4p1
vgcreate vg1 /dev/mapper/mpath3p1
lvcreate -l 2559  -n data vg1
mke2fs -j /dev/mapper/vg1-data
mount /dev/mapper/vg1-data /test
vgextend vg1 /dev/mapper/mpath4p1
lvextend -l +100%FREE /dev/vg1/data
resize2fs /dev/vg1/data


LVm cheatsheat

# pvs
  PV                  VG         Fmt  Attr PSize  PFree
  /dev/mpath/mpath3p1 vg1        lvm2 a-   10.00G    0
  /dev/mpath/mpath4p1 vg1        lvm2 a-    4.99G    0
  /dev/sda2           VolGroup00 lvm2 a-    3.88G    0
# vgs
  VG         #PV #LV #SN Attr   VSize  VFree
  VolGroup00   1   2   0 wz--n-  3.88G    0
  vg1          2   1   0 wz--n- 14.99G    0
# lvs
  LV       VG         Attr   LSize   Origin Snap%  Move Log Copy%
  LogVol00 VolGroup00 -wi-ao   3.38G
  LogVol01 VolGroup00 -wi-ao 512.00M
  data     vg1        -wi-ao  14.99G
# pvscan
  PV /dev/mpath/mpath3p1   VG vg1          lvm2 [10.00 GB / 0    free]
  PV /dev/mpath/mpath4p1   VG vg1          lvm2 [4.99 GB / 0    free]
  PV /dev/sda2             VG VolGroup00   lvm2 [3.88 GB / 0    free]
  Total: 3 [18.86 GB] / in use: 3 [18.86 GB] / in no VG: 0 [0   ]
# vgscan
  Reading all physical volumes.  This may take a while...
  Found volume group "vg1" using metadata type lvm2
  Found volume group "VolGroup00" using metadata type lvm2
# lvscan
  ACTIVE            '/dev/vg1/data' [14.99 GB] inherit
  ACTIVE            '/dev/VolGroup00/LogVol00' [3.38 GB] inherit
  ACTIVE            '/dev/VolGroup00/LogVol01' [512.00 MB] inherit