前言
配置存储时一定会遇到 multipath 多路径的问题，不同的厂商比如 EMC PowerPath，Veritas VxDMP 等都有独立的多路径软件，而多路径软件的功能也很清晰主要用于IO流量负载均衡和故障切换恢复等。在 Linux 环境中 device-mapper-multipath 是一个免费的通用型多路径管理软件，其配置文件也非常简单，主要通过修改 /etc/multipath.conf 来调整。文章提供了《HPE 3PAR Red Hat Enterprise Linux和 Oracle Linux 实施指南》，也分享了自己配置 multipath 的实践过程，希望对大家有参考价值。
记录 3PAR8400 存储 Multipath 多路径配置
更新历史
2017年05月04日 - 初稿
阅读原文 - https://wsgzao.github.io/post/multipath/
扩展阅读
multipath - https://access.redhat.com/labsinfo/multipathhelper
[原]红旗上使用multipath复合多条路径 - http://www.linuxfly.org/post/513/
官方最佳实践手册
HPE 3PAR Red Hat Enterprise Linux和 Oracle Linux 实施指南
<iframe src="https://www.slideshare.net/slideshow/embed_code/key/MKQJhstxnv4JFy" width="479" height="511" frameborder="0" marginwidth="0" marginheight="0" scrolling="no" style="border:1px solid #CCC; border-width:1px; margin-bottom:5px; max-width: 100%;" allowfullscreen> </iframe> <div style="margin-bottom:5px"> </div>
什么是multipath
普通的电脑主机都是一个硬盘挂接到一个总线上，这里是一对一的关系。而到了有光纤组成的SAN环境，由于主机和存储通过了光纤交换机连接，这样的话，就构成了多对多的关系。也就是说，主机到存储可以有多条路径可以选择。主机到存储之间的IO由多条路径可以选择。
既然，每个主机到所对应的存储可以经过几条不同的路径，如果是同时使用的话，I/O流量如何分配？其中一条路径坏掉了，如何处理？还有在操作系统的角度来看，每条路径，操作系统会认为是一个实际存在的物理盘，但实际上只是通向同一个物理盘的不同路径而已，这样是在使用的时候，就给用户带来了困惑。多路径软件就是为了解决上面的问题应运而生的，多路径的主要功能就是和存储设备一起配合实现如下功能：
- 故障的切换和恢复
- IO流量的负载均衡
- 磁盘的虚拟化
为什么使用multipath
由于多路径软件是需要和存储在一起配合使用的，不同的厂商基于不同的操作系统，都提供了不同的版本。并且有的厂商，软件和硬件也不是一起卖的，如果要使用多路径软件的话，可能还需要向厂商购买license才行。比如EMC公司基于Linux下的多路径软件，就需要单独的购买license。其中，EMC提供的就是PowerPath，HDS提供的就是HDLM，Veritas提供的就是VxDMP。当然，使用系统自带的免费多路径软件包，同时也是一个比较通用的包，可以支持大多数存储厂商的设备，即使是一些不是出名的厂商，通过对配置文件进行稍作修改，也是可以支持并运行的很好。
比较重要的一点还是听从原厂工程师的建议根据实际的业务和存储策略使用合适的多路径软件。
配置multipath
原理看了一堆，实际配置还是比较简单的，配置文件只有一个：/etc/multipath.conf
#查看主机或者存储交换机上的WWN号，在存储上将LUN映射给需要的主机
cat /sys/class/fc_host/host*/port_name
0x2002d0431efb7f5d
0x2001d0431efb7f5d
#在系统内执行扫盘命令，没有命令先安装sg3_utils
yum install sg3_utils
rescan-scsi-bus.sh
#查看是否映射到对应的
fdisk -l
lsblk
#查看是否安装了multipath
yum install device-mapper-multipath
rpm -qa | grep device-mapper
device-mapper-libs-1.02.107-5.el7_2.2.x86_64
device-mapper-persistent-data-0.5.5-1.el7.x86_64
device-mapper-multipath-0.4.9-85.el7_2.4.x86_64
device-mapper-1.02.107-5.el7_2.2.x86_64
device-mapper-event-libs-1.02.107-5.el7_2.2.x86_64
device-mapper-event-1.02.107-5.el7_2.2.x86_64
device-mapper-multipath-libs-0.4.9-85.el7_2.4.x86_64
#拷贝默认的multipath.conf到/etc目录下，也可以使用mpathconf命令创建默认模板
cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/multipath.conf
mpathconf --enable --with_multipathd y
#查看本地存储wwid
/lib/udev/scsi_id --whitelisted --device=/dev/sda
36141877030bcca001d9c4d52106b9d90
#查看存储参数
cat /sys/block/sdb/device/vendor
3PARdata
cat /sys/block/sdb/device/model
VV
#添加本地盘到blacklist内,不同的存储和系统参考官方的最佳实践
vim /etc/multipath.conf
defaults {
polling_interval 10
user_friendly_names no
find_multipaths yes
}
blacklist {
wwid "<wwid_of_the_local_disk>"
}
devices {
device {
vendor "3PARdata"
product "VV"
path_grouping_policy group_by_prio
path_selector "round-robin 0"
path_checker tur
features "0"
hardware_handler "1 alua"
prio alua
failback immediate
rr_weight uniform
no_path_retry 18
rr_min_io_rq 1
detect_prio yes
# fast_io_fail_tmo 10
# dev_loss_tmo 14
}
}
#固定设备的别名
cat /etc/multipath/bindings
# Multipath bindings, Version : 1.0
# NOTE: this file is automatically maintained by the multipath program.
# You should not need to edit this file in normal circumstances.
#
# Format:
# alias wwid
#
mpatha 360002ac000000000000000260001a945
mpathb 360002ac000000000000000270001a945
mpathc 360002ac000000000000000280001a945
mpathd 360002ac000000000000000290001a945
mpathe 360002ac0000000000000002a0001a945
#编辑multipath.conf，增加以下字段
vim /etc/multipath.conf
multipaths {
multipath {
wwid 360002ac000000000000000260001a945
alias 3PAR8400_DEV_LUN26
}
multipath {
wwid 360002ac000000000000000270001a945
alias 3PAR8400_DEV_LUN27
}
multipath {
wwid 360002ac000000000000000280001a945
alias 3PAR8400_DEV_LUN28
}
multipath {
wwid 360002ac000000000000000290001a945
alias 3PAR8400_DEV_LUN29
}
multipath {
wwid 360002ac0000000000000002a0001a945
alias 3PAR8400_DEV_LUN30
}
}
#清空已有的multipath记录
multipath -F
#打印诊断信息
multipath -v3
#启用多路径守护程序以在引导时启动
mpathconf --enable
#启动多路径服务
service multipathd start
#如果在启动 multipath 守护程序后更改多路径配置文件，请运行以下命令以使更改生效。
service multipathd reload
#重启系统测试
init 6
#查看多路径当前状态
multipath -ll
#Enable extended logging on the qla2xxx driver
chmod u+x /sys/module/qla2xxx/parameters/ql2xextended_error_logging
echo "1" > /sys/module/qla2xxx/parameters/ql2xextended_error_logging
cat /sys/module/qla2xxx/parameters/ql2xextended_error_logging
#重启lvm服务是识别共享存储vg信息
systemctl restart lvm2-lvmetad.service
pvs
#屏蔽WARNING: duplicate提示信息
vim /etc/lvm/lvm.conf
# Configuration option devices/filter.
# Limit the block devices that are used by LVM commands.
# This is a list of regular expressions used to accept or reject block
# device path names. Each regex is delimited by a vertical bar '|'
# (or any character) and is preceded by 'a' to accept the path, or
# by 'r' to reject the path. The first regex in the list to match the
# path is used, producing the 'a' or 'r' result for the device.
# When multiple path names exist for a block device, if any path name
# matches an 'a' pattern before an 'r' pattern, then the device is
# accepted. If all the path names match an 'r' pattern first, then the
# device is rejected. Unmatching path names do not affect the accept
# or reject decision. If no path names for a device match a pattern,
# then the device is accepted. Be careful mixing 'a' and 'r' patterns,
# as the combination might produce unexpected results (test changes.)
# Run vgscan after changing the filter to regenerate the cache.
# See the use_lvmetad comment for a special case regarding filters.
#
# Example
# Accept every block device:
# filter = [ "a|.*/|" ]
# Reject the cdrom drive:
# filter = [ "r|/dev/cdrom|" ]
# Work with just loopback devices, e.g. for testing:
# filter = [ "a|loop|", "r|.*|" ]
# Accept all loop devices and ide drives except hdc:
# filter = [ "a|loop|", "r|/dev/hdc|", "a|/dev/ide|", "r|.*|" ]
# Use anchors to be very specific:
# filter = [ "a|^/dev/hda8$|", "r|.*/|" ]
#
# This configuration option has an automatic default value.
# filter = [ "a|.*/|" ]
filter = ["a/sda/","a/mapper/3PAR8400_DEV_LUN26p1/","r/.*/"]
#filter = ["a/sda/","r/.*/"]
# Configuration option devices/global_filter.
# Limit the block devices that are used by LVM system components.
# Because devices/filter may be overridden from the command line, it is
# not suitable for system-wide device filtering, e.g. udev and lvmetad.
# Use global_filter to hide devices from these LVM system components.
# The syntax is the same as devices/filter. Devices rejected by
# global_filter are not opened by LVM.
# This configuration option has an automatic default value.
# global_filter = [ "a|.*/|" ]
global_filter = ["a/sda/","a/mapper/3PAR8400_DEV_LUN26p1/","r/.*/"]
#一个简单的multipath.conf配置文件
[root@controller01 ~]# cat /etc/multipath.conf
# This is a basic configuration file with some examples, for device mapper
# multipath.
#
# For a complete list of the default configuration values, run either
# multipath -t
# or
# multipathd show config
#
# For a list of configuration options with descriptions, see the multipath.conf
# man page
## By default, devices with vendor = "IBM" and product = "S/390.*" are
## blacklisted. To enable mulitpathing on these devies, uncomment the
## following lines.
#blacklist_exceptions {
# device {
# vendor "IBM"
# product "S/390.*"
# }
#}
## Use user friendly names, instead of using WWIDs as names.
defaults {
user_friendly_names yes
find_multipaths yes
}
##
## Here is an example of how to configure some standard options.
##
#
#defaults {
# polling_interval 10
# path_selector "round-robin 0"
# path_grouping_policy multibus
# uid_attribute ID_SERIAL
# prio alua
# path_checker readsector0
# rr_min_io 100
# max_fds 8192
# rr_weight priorities
# failback immediate
# no_path_retry fail
# user_friendly_names yes
#}
##
## The wwid line in the following blacklist section is shown as an example
## of how to blacklist devices by wwid. The 2 devnode lines are the
## compiled in default blacklist. If you want to blacklist entire types
## of devices, such as all scsi devices, you should use a devnode line.
## However, if you want to blacklist specific devices, you should use
## a wwid line. Since there is no guarantee that a specific device will
## not change names on reboot (from /dev/sda to /dev/sdb for example)
## devnode lines are not recommended for blacklisting specific devices.
##
#blacklist {
# wwid 26353900f02796769
# devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
# devnode "^hd[a-z]"
#}
#multipaths {
# multipath {
# wwid 3600508b4000156d700012000000b0000
# alias yellow
# path_grouping_policy multibus
# path_selector "round-robin 0"
# failback manual
# rr_weight priorities
# no_path_retry 5
# }
# multipath {
# wwid 1DEC_____321816758474
# alias red
# }
#}
#devices {
# device {
# vendor "COMPAQ "
# product "HSV110 (C)COMPAQ"
# path_grouping_policy multibus
# path_checker readsector0
# path_selector "round-robin 0"
# hardware_handler "0"
# failback 15
# rr_weight priorities
# no_path_retry queue
# }
# device {
# vendor "COMPAQ "
# product "MSA1000 "
# path_grouping_policy multibus
# }
#}
multipaths {
multipath {
wwid 360002ac000000000000000260001a945
alias 3PAR8400_DEV_LUN26
}
multipath {
wwid 360002ac000000000000000270001a945
alias 3PAR8400_DEV_LUN27
}
multipath {
wwid 360002ac000000000000000280001a945
alias 3PAR8400_DEV_LUN28
}
multipath {
wwid 360002ac000000000000000290001a945
alias 3PAR8400_DEV_LUN29
}
multipath {
wwid 360002ac0000000000000002a0001a945
alias 3PAR8400_DEV_LUN30
}
multipath {
wwid 360002ac0000000000000002b0001a945
alias 3PAR8400_DEV_LUN31
}
multipath {
wwid 360002ac0000000000000002c0001a945
alias 3PAR8400_DEV_LUN32
}
multipath {
wwid 360002ac0000000000000002d0001a945
alias 3PAR8400_DEV_LUN33
}
multipath {
wwid 360002ac0000000000000002e0001a945
alias 3PAR8400_DEV_LUN34
}
multipath {
wwid 360002ac0000000000000002f0001a945
alias 3PAR8400_DEV_LUN35
}
multipath {
wwid 360002ac000000000000000320001a945
alias 3PAR8400_DEV_LUN36
}
multipath {
wwid 360002ac000000000000000330001a945
alias 3PAR8400_DEV_LUN37
}
multipath {
wwid 360002ac000000000000000340001a945
alias 3PAR8400_DEV_LUN38
}
multipath {
wwid 360002ac000000000000000350001a945
alias 3PAR8400_DEV_LUN39
}
multipath {
wwid 360002ac000000000000000360001a945
alias 3PAR8400_DEV_LUN40
}
multipath {
wwid 360002ac000000000000000370001a945
alias 3PAR8400_DEV_LUN41
}
multipath {
wwid 360002ac000000000000000380001a945
alias 3PAR8400_DEV_LUN42
}
multipath {
wwid 360002ac000000000000000390001a945
alias 3PAR8400_DEV_LUN43
}
multipath {
wwid 360002ac0000000000000003a0001a945
alias 3PAR8400_DEV_LUN44
}
multipath {
wwid 360002ac0000000000000003b0001a945
alias 3PAR8400_DEV_LUN45
}
multipath {
wwid 360002ac0000000000000003d0001a945
alias 3PAR8400_DEV_LUN46
}
multipath {
wwid 360002ac0000000000000003e0001a945
alias 3PAR8400_DEV_LUN47
}
multipath {
wwid 360002ac0000000000000003f0001a945
alias 3PAR8400_DEV_LUN48
}
multipath {
wwid 360002ac000000000000000400001a945
alias 3PAR8400_DEV_LUN49
}
multipath {
wwid 360002ac000000000000000410001a945
alias 3PAR8400_DEV_LUN50
}
}
