概述

在 redis 高可用主从(Master-Replica)架构下, 通过 sentinel 实现 master 故障的自动切换, redis client 如何将写操作正确分发到 master 节点? 如何将读操作正确分发到多个 replica 节点以实现读负载均衡? 现提供如下基于 haproxy 自定义健康检查(tcp-check)的代理实现方案:

• haproxy 将写命令发送到 master 节点，将读命令根据权重发送到 master 或 replica 节点实现读负载均衡
• haproxy 自动下线异常的只读节点,待节点恢复后再重新启用
• haproxy 端口 16379 负责写, 端口 26379 负责读负载均衡

环境规划

主机名 Ip 地址 节点类型 c2 192.168.31.12 haproxy c7 192.168.31.17 redis-server 6381 (Master), redis-sentinel(26379) c8 192.168.31.18 redis-server 6381 (Replica), redis-sentinel(26379) c9 192.168.31.19 redis-server 6381 (Replica), redis-sentinel(26379)

本方案主要涉及的技术细节:

• Redis 主从环境
• Redis Sentinel 哨兵集群
• Haproxy 自定义健康检查(tcp-check)

编译安装

1、下载解压、编译安装 redis

wget http://download.redis.io/releases/redis-5.0.9.tar.gz
tar zxvf redis-5.0.9.tar.gz
# 默认 redis 相关二进制执行文件安装到 /usr/bin 目录内
cd redis-5.0.9.tar.gz && make && make install
# 创建 redis 数据及配置文件目录
mkdir -p /opt/redis/{6379,6381} && /etc/redis/{6379,6381}
cp redis.conf /etc/redis/{6379,6381}
# 创建 redis 运行用户及设置权限
useradd -r redis  && chown redis.redis -R /opt/redis/6381

2、Redis Server Systemd 启动脚本

cat > /usr/lib/systemd/system/redis.service << EOF
[Unit]
Description=Redis persistent key-value database
After=network.target

[Service]
LimitNOFILE=infinity
LimitNOFILE=102400
LimitNOFILE=102400
WorkingDirectory=/opt/redis/6381
ExecStart=/usr/bin/redis-server /etc/redis/6381/redis.conf --supervised systemd
ExecStop=/usr/bin/redis-cli shutdown
Type=notify
User=redis
Group=redis
RuntimeDirectory=redis
RuntimeDirectoryMode=0755

[Install]
WantedBy=multi-user.target
EOF

Redis 主从

1、Master 节点 Redis Server 配置

# cat /etc/redis/6381/redis.conf
# general config
daemonize yes
protected-mode no
port 6381
bind 0.0.0.0
databases 16
loglevel notice
pidfile "/opt/redis/6381/redis-server.pid"
logfile "/opt/redis/6381/redis-server.log"
timeout 0
maxmemory 128mb
tcp-keepalive 0
tcp-backlog 511
lua-time-limit 5000
slowlog-log-slower-than 1000
slowlog-max-len 128
requirepass "pass123"

# rdb 持久化 config
dir "/opt/redis/6381"
dbfilename "dump.rdb"
save 900 1
save 300 10
save 60 1000
rdbcompression yes
rdbchecksum yes
stop-writes-on-bgsave-error yes

# aof 持久化 config
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
aof-use-rdb-preamble yes
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-rewrite-incremental-fsync yes

# replication config
replica-priority 100
masterauth "pass123"
# replicaof 192.168.31.18 6381
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync yes
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
repl-timeout 60
repl-backlog-ttl 600
repl-backlog-size 10mb

2、Replica(Slave) 节点 Redis Server 配置, 基于 Master 配置添加或修改如下配置项

replica-priority 80
# 指定副本连接 master 时的验证密码
masterauth "pass123"
# 指定该副本的 master ip 及 port
replicaof 192.168.31.17 6381

3、确认 Redis 主从环境运行正常

redis-cli -p 6381 -h 192.168.31.17
192.168.31.17:6381> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=192.168.31.18,port=6381,state=online,offset=39771,lag=1
slave1:ip=192.168.31.19,port=6381,state=online,offset=39771,lag=1
master_replid:b090f53b06d56ba3145709f6dde72b7e820c3935
master_replid2:f39d8c822c424308c27e2a47599ed2cb685004c4
master_repl_offset:39905
second_repl_offset:26123
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1230
repl_backlog_histlen:38676

Redis 哨兵

1、Redis Sentinel 所有节点配置

# cat /etc/redis/6379/redis.conf
port 26379
protected-mode no
dir /opt/redis/26379
logfile sentinel_26379.log
# 是否为守护进程
daemonize yes
# 至少 2 个 sentinel 节点检测到 master 失效, 才确认 master 客观下线; m1 标识一个主从复制组
# 192.168.31.17 6381 为 redis master 节点 Ip 及 Port
sentinel monitor m1 192.168.31.17 6381 2
# 指定故障切换允许的毫秒数，超过这个时间，就认为故障切换失败
sentinel failover-timeout m1 180000
# 若 redis master 在 15s 内没有回应 PONG 或者是回复了一个错误消息, 则当前 sentinel 任务该 master 挂了(SDOWN)
sentinel down-after-milliseconds m1 15000
# 指定最多可以有多少个 replica 同时对新的 master 进行同步
sentinel parallel-syncs m1 1
# sentinel 连接 redis 时的密码验证
sentinel auth-pass m1 pass123

# 发生切换之后执行的一个自定义脚本
# sentinel notification-script <master-name> <script-path>
# sentinel client-reconfig-script <master-name> <script-path>

2、设置哨兵启动脚本

# cat /usr/lib/systemd/system/redis-sentinel.service
[Unit]
Description=Redis persistent key-value database
After=network.target

[Service]
LimitNOFILE=infinity
LimitNOFILE=102400
LimitNOFILE=102400
WorkingDirectory=/opt/redis/6379
ExecStart=/usr/bin/redis-sentinel /etc/redis/6379/redis.conf --supervised systemd
ExecStop=/usr/bin/redis-cli -p 26379 shutdown
Type=notify
User=redis
Group=redis
RuntimeDirectory=redis
RuntimeDirectoryMode=0755

[Install]
WantedBy=multi-user.target
EOF

3、确定 Redis Sentinel 运行是否正常

# linux shell 启动 sentinel 节点
systemctl start redis-sentinel
redis-cli -p 26379
127.0.0.1:26379> info sentinel
# 输出如下信息, 标识 master 地址 192.168.31.17, 有 2 个副本, 共有三个 sentinel 节点
sentinel_masters:1
sentinel_tilt:0
sentinel_running_scripts:0
sentinel_scripts_queue_length:0
sentinel_simulate_failure_flags:0
master0:name=m1,status=ok,address=192.168.31.17:6381,slaves=2,sentinels=3

Redis 读写分离

1、haproxy 配置文件 haproxy_redis.cfg

global
    daemon
    maxconn 30000

defaults
    mode tcp
    log global
    option tcplog
    option dontlognull
    maxconn 30000
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms

# HAProxy stats page
listen stats
    bind *:8888
    mode http
    stats enable
    # stats uri /status   # 默认路径 /haproxy?stats
    stats auth admin:admin

# redis 读负载均衡
listen redis-read
    bind *:26379
    mode tcp
    balance roundrobin
    option tcp-check
    # redis 密码认证
    tcp-check send AUTH\ pass123\r\n
    tcp-check expect string +OK
    # 读操作仅分发到 slave
    tcp-check send info\ replication\r\n
    tcp-check expect string role:slave
    # 读操作分发到 slave 和 master
    #tcp-check send PING\r\n
    #tcp-check expect string +PONG
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server redis-1 192.168.31.17:6381 maxconn 5000 check inter 2s
    server redis-2 192.168.31.18:6381 maxconn 5000 check inter 2s
    server redis-3 192.168.31.19:6381 maxconn 5000 check inter 2s

# redis 写流量到 redis master
frontend redis-write
    bind *:16379
    mode tcp
    option tcplog
    # 通过自己(redis-server)和 2 个 sentinel 确认本身是否为 master 节点
    use_backend redis-master-1 if { srv_is_up(redis-master-1/redis) } { nbsrv(mastercheck-redis-1) ge 2 }
    use_backend redis-master-2 if { srv_is_up(redis-master-2/redis) } { nbsrv(mastercheck-redis-2) ge 2 }
    use_backend redis-master-3 if { srv_is_up(redis-master-3/redis) } { nbsrv(mastercheck-redis-3) ge 2 }
    # 如果未找到有效的 master 时的默认后端选择
    default_backend redis-legacy

# 询问 redis-1(192.168.31.17:6381) 是否认为自己是 Master
backend redis-master-1
    mode tcp
    balance first
    option tcp-check
    #comment following 2 lines if your redis server doesn't requirepass
    tcp-check send AUTH\ pass123\r\n
    tcp-check expect string +OK
    tcp-check send info\ replication\r\n
    tcp-check expect string role:master
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server redis 192.168.31.17:6381 maxconn 5000 check inter 2s

backend redis-master-2
    mode tcp
    balance first
    option tcp-check
    tcp-check send AUTH\ pass123\r\n
    tcp-check expect string +OK
    tcp-check send info\ replication\r\n
    tcp-check expect string role:master
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server redis 192.168.31.18:6381 maxconn 5000 check inter 2s

backend redis-master-3
    mode tcp
    balance first
    option tcp-check
    tcp-check send AUTH\ pass123\r\n
    tcp-check expect string +OK
    tcp-check send info\ replication\r\n
    tcp-check expect string role:master
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server redis 192.168.31.19:6381 maxconn 5000 check inter 2s

# 询问三个 Sentinel节点当前 redis-1是否为 Master
backend mastercheck-redis-1
    mode tcp
    option tcp-check
    tcp-check send SENTINEL\ master\ m1\r\n
    tcp-check expect string 192.168.31.17
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server sentinel-1 192.168.31.17:26379 check inter 2s
    server sentinel-2 192.168.31.18:26379 check inter 2s
    server sentinel-3 192.168.31.19:26379 check inter 2s

backend mastercheck-redis-2
    mode tcp
    option tcp-check
    tcp-check send SENTINEL\ master\ m1\r\n
    tcp-check expect string 192.168.31.18
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server sentinel-1 192.168.31.17:26379 check inter 2s
    server sentinel-2 192.168.31.18:26379 check inter 2s
    server sentinel-3 192.168.31.19:26379 check inter 2s

backend mastercheck-redis-3
    mode tcp
    option tcp-check
    tcp-check send SENTINEL\ master\ m1\r\n
    tcp-check expect string 192.168.31.19
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server sentinel-1 192.168.31.17:26379 check inter 2s
    server sentinel-2 192.168.31.18:26379 check inter 2s
    server sentinel-3 192.168.31.19:26379 check inter 2s

# 通过 redis server 本身确定自己是否为 master 节点; 脑裂情况下存在多个 master 节点 该方式会导致数据混乱!!
backend redis-legacy
    mode tcp
    balance first
    option tcp-check
    tcp-check send AUTH\ pass123\r\n
    tcp-check expect string +OK
    tcp-check send info\ replication\r\n
    tcp-check expect string role:master
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server redis-1 192.168.31.17:6381 maxconn 5000 check inter 2s
    server redis-2 192.168.31.18:6381 maxconn 5000 check inter 2s
    server redis-3 192.168.31.19:6381 maxconn 5000 check inter 2s

测试验证

# 1、验证读写分离,可通过 redis server 的进程 id 可判断连接到哪个 redis 后端
redis-cli -h 192.168.31.12 -p 26379 -a pass123 info | grep process_id

# 2、通过 haproxy 状态页查看后端 redis 状态
http://192.168.31.12:8888/haproxy?stats

# 3、读写分离
haproxy 端口 16379 负责写(对应后端 master 节点), 端口 26379 负责读负载均衡(对应后端 replica 节点)