LINUX 高可用群集之 ~Corosync~

# vim /etc/hosts
172.16.27.1 node1.tanxw.com
172.16.27.2 node2.tanxw.com
# ssh-keygen -t rsa -P ‘’
# ssh-copy-id -i id_rsa.pub node2.tanxw.com

# yum -y install python-jinja2   解决依赖关系
# yum -y install ansible
安装好之后再去/etc/ansible下配置两个节点的hostname：
# vim /etc/ansible/hosts  把里面的内容全都注释掉，加下面你的节点hostname
[corosync]
node1.tanxw.com
node2.tanxw.com

# crontab -e
*/5 * * * * /usr/sbin/ntpdte 172.16.27.0 &> /dev/null   表示每隔5分钟同步一次

# mv /dev/random /dev/m
# ln /dev/urandom /dev/random  如果这把这个随机数墒池改了可以会产生随机数不够用，这个就要敲击键盘给这个墒池一些随机数；生成完这个key后把链接删除，再把墒池改回来；不过这样改可以会有点为安全，不过做测试的应该不要紧；
# corosync-keygen
# rm -rf /dev/random
# mv /dev/m /dev/random
对于corosync而言，我们各节点之间通信时必须要能够实现安全认证的，要用到一个密钥文件：
# corosync-keygen    # 生成密钥文件，用于双机通信互信，会生成一authkey的文件
# scp authkey corosync.conf node2.tanxw.com:/etc/corosync/   在配置好的节点上把这两个文件复制给另一个节点上的corosync的配置文件中去

crmsh-1.2.6-4.el6.x86_64.rpm
pssh-2.3.1-2.el6.x86_64.rpm
# yum -y install crmsh-1.2.6-4.el6.x86_64.rpm pssh-2.3.1-2.el6.x86_64.rpm

[root@node1 corosync]# crm configure    #进入crm命令行模式配置资源等
crm(live)configure# property    #切换到property目录下，可以用两次tab键进行补全和查看
usage: property [$id=<set_id>] <option>=<value>    # property的用法和格式
crm(live)configure# property stonith-enabled=false        #禁用stonith-enabled
crm(live)configure# verify        #检查设置的属性是否正确
crm(live)configure# commit    #检查没问题就可以提交了
crm(live)configure# show        #查看当前集群的所有配置信息
node node1.tanxw.com
node node2.tanxw.com        #两个节点
property $id="cib-bootstrap-options" \  
    dc-version="1.1.10-14.el6-368c726" \    #DC的版本号
    cluster-infrastructure="classic openais (with plugin)" \    #集群的基础架构，使用的是OpenAIS，插件式的
    expected-quorum-votes="2" \        #期望节点的票数
    stonith-enabled="false"        #禁用stonith功能
crm(live)configure#

crm(live)# configure
crm(live)configure# primitive webip ocf:heartbeat:IPaddr params ip=172.16.27.88
crm(live)configure# verify
crm(live)configure# commit
crm(live)configure# show
node node1.tanxw.com
node node2.tanxw.com
primitive webip ocf:heartbeat:IPaddr     params ip="172.16.27.88"
property $id="cib-bootstrap-options"     dc-version="1.1.10-14.el6-368c726"     cluster-infrastructure="classic openais (with plugin)"     expected-quorum-votes="2"     stonith-enabled="false"
crm(live)configure#

[root@node2 drbd.d]# fdisk /dev/sda
WARNING: DOS-compatible mode is deprecated. It‘s strongly recommended to
         switch off the mode (command ‘c‘) and change display units to
         sectors (command ‘u‘).
Command (m for help): p
Disk /dev/sda: 85.9 GB, 85899345920 bytes
255 heads, 63 sectors/track, 10443 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x000e89e2
   Device Boot      Start         End      Blocks   Id  System
/dev/sda1   *           1          26      204800   83  Linux
Partition 1 does not end on cylinder boundary.
/dev/sda2              26        7859    62914560   8e  Linux LVM
/dev/sda3            7859        9164    10489446   8e  Linux LVM
Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
e
Selected partition 4
First cylinder (9165-10443, default 9165):
Using default value 9165
Last cylinder, +cylinders or +size{K,M,G} (9165-10443, default 10443):
Using default value 10443
Command (m for help): n
First cylinder (9165-10443, default 9165):
Using default value 9165
Last cylinder, +cylinders or +size{K,M,G} (9165-10443, default 10443): +3G
Command (m for help): p
Disk /dev/sda: 85.9 GB, 85899345920 bytes
255 heads, 63 sectors/track, 10443 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x000e89e2
   Device Boot      Start         End      Blocks   Id  System
/dev/sda1   *           1          26      204800   83  Linux
Partition 1 does not end on cylinder boundary.
/dev/sda2              26        7859    62914560   8e  Linux LVM
/dev/sda3            7859        9164    10489446   8e  Linux LVM
/dev/sda4            9165       10443    10273567+   5  Extended
/dev/sda5            9165        9557     3156741   83  Linux
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
WARNING: Re-reading the partition table failed with error 16: Device or resource busy.
The kernel still uses the old table. The new table will be used at
the next reboot or after you run partprobe(8) or kpartx(8)
Syncing disks.
[root@node2 drbd.d]# kpartx -af /dev/sda
device-mapper: reload ioctl on sda1 failed: Invalid argument
create/reload failed on sda1
device-mapper: reload ioctl on sda2 failed: Invalid argument
create/reload failed on sda2
device-mapper: reload ioctl on sda3 failed: Invalid argument
create/reload failed on sda3
device-mapper: reload ioctl on sda4 failed: Invalid argument
create/reload failed on sda4
device-mapper: reload ioctl on sda5 failed: Invalid argument
create/reload failed on sda5
[root@node2 drbd.d]# partx -a /dev/sda
BLKPG: Device or resource busy
error adding partition 1
BLKPG: Device or resource busy
error adding partition 2
BLKPG: Device or resource busy
error adding partition 3
[root@node2 drbd.d]# partx -a /dev/sda
BLKPG: Device or resource busy
error adding partition 1
BLKPG: Device or resource busy
error adding partition 2
BLKPG: Device or resource busy
error adding partition 3
BLKPG: Device or resource busy
error adding partition 4
BLKPG: Device or resource busy
error adding partition 5
[root@node2 drbd.d]#

# cd /etc/drbd.d/
# vim mystore.res
resource mystore {                    #定义一个资源，用关键字resource；
  on node1.tanxw.com {                #on说明在哪个节点上，跟uname -n保持一致,有多少个节点就定义多少个；
    device    /dev/drbd0;            #在磁盘上表现的drbd叫什么名；
    disk      /dev/sda5;            #所使用的磁盘设备是哪个；
    address   172.16.27.1:7789;        #在node1这个节点上监听的套接字,默认监听在7789端口上；
    meta-disk internal;                #保存drbd元数据信息的，表示就放在自己的磁盘区分上，也可以放在外部的磁盘上；
  }
  on node2.tanxw.com {
    device    /dev/drbd0;
    disk      /dev/sda5;
    address   172.16.27.2:7789;
    meta-disk internal;
  }
}

# scp global_common.conf  mystore.res node2.tanxw.com:/etc/drbd.d/
# drdbadm create-md mystore  在各自的节点上初始化资源
# server drbd start   启动drbd
# watch -n 1 ‘cat /proc/drbd‘   实现查看磁盘信息
# drbdadm primary --force mystore   在节点上把其中一个提升为主的
# watch -n 1 ‘cat /proc/drbd‘   提升为主的之后再实现查看磁盘信息

# scp global_common.conf  mystore.res node2.tanxw.com:/etc/drbd.d/
# drdbadm create-md mystore  在各自的节点上初始化资源
# server drbd start   启动drbd
# watch -n 1 ‘cat /proc/drbd‘   实现查看磁盘信息
# drbdadm primary --force mystore   在节点上把其中一个提升为主的
# watch -n 1 ‘cat /proc/drbd‘   提升为主的之后再实现查看磁盘信息

# cd
# umount /dev/drbd0
# drbdadm secondary mystore        哪个是主的就在哪相节点上降级
# service drbd stop        两个节点都需要停止服务
# chkconfig drbd off

# yum -y install corosync pacemaker
# yum -y install crmsh pssh
# cp /etc/corosync.conf.example /etc/corosync/corosync.conf
# vim /etc/corosync/corosync.conf
compatibility: whitetank
totem {
        version: 2
        secauth: on
        threads: 0
        interface {
                ringnumber: 0
                bindnetaddr: 172.16.0.0
                mcastaddr: 226.98.188.188
                mcastport: 5405
                ttl: 1
        }
}
logging {
        fileline: off
        to_stderr: no
        to_logfile: yes
        to_syslog: no
        logfile: /var/log/cluster/corosync.log
        debug: off
        timestamp: on
        logger_subsys {
                subsys: AMF
                debug: off
        }
}
amf {
        mode: disabled
}
service {
        name:   pacemaker
        ver:    0
}
aisexce {
        user:   root
        group:  root
}
# mv /dev/random /dev/m
# ln /dev/urandom /dev/random  如果这把这个随机数墒池改了可以会产生随机数不够用，这个就要敲击键盘给这个墒池一些随机数；生成完这个key后把链接删除，再把墒池改回来；不过这样改可以会有点为安全，不过做测试的应该不要紧；
# corosync-keygen
# rm -rf /dev/random
# mv /dev/m /dev/random
再把改好的配置复制一份到别一个节点上去，也保留一份给另一个节点：
# scp -p authkey corosync.conf node2.tanxw.com:/etc/corosync/

# crm
# configure
# property stonith-enabled=false
# property no-quorum-policy=ignore
# rsc_defaults resource-stickiness=100
# verify
# commit
# show
# meta ocf:linbit:drbd   查看drbd的详细信息
# 定义资源,切换到configure中，mysqlstore定义资源名，drbd_resource=mystore这个是drbd名，后面定义的都是一些监控
# primitive mysqlstore ocf:linbit:drbd params drbd_resource=mystore op monitor role=Master interval=30s timeout=20s op monitor role=Slave interval=60s timeout=20s op start timeout=240s op stop timeout=100s
# verify    检查语法
# 定义主资源
# master ms_mysqlstore mysqlstore meta master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=”True”
# verify
#commit
定义共享文件的资源,并且让服务器切换时可以自动挂载，device=/dev/drbd0挂载的设备，directory=/drbd挂载点，两个挂载点的文件名要一致：
# primitive mysqlfs ocf:heartbeat:Filesystem params device=/dev/drbd0 directory=/drbd fstype=ext4 op monitor interval=30s timeout=40s on-fail-restart op start timeout=60s op stop timeout=60s
# verify
定义排列约束：
# collocation mysqlfs_with_ms_mysqlstore_master inf: mysqlfs ms_mysqlstore:Master
# verify
定义顺序约束：
# order mysqlfs_after_ms_mysqlstore_master mandatory: ms_mysqlstore:promote mysqlfs:start
# show
# commit

crm(live)# node standby node2.tanxw.com   把node2提升为备用的
crm(live)# node noline node2.tanxw.com    让node2上线
crm(live)# node standby node1.tanxw.com   把node1提升为备用的
crm(live)# node noline node1.tanxw.com    让node1上线，都可以随意切换一下再检测一下挂载的效果的；