Load balancing, lvs + haproxy + nginx

How can we help?

Навигация

R1:

[root@r1 ~]# vtysh

Hello, this is FRRouting (version 8.2.2).
Copyright 1996-2005 Kunihiro Ishiguro, et al.

r1# show run
Building configuration...

Current configuration:
!
frr version 8.2.2
frr defaults traditional
hostname r1
log syslog informational
no ip forwarding
no ipv6 forwarding
!
interface enp0s8
 no ip ospf passive
exit
!
router ospf
 network 111.111.111.111/32 area 0
 network 192.168.12.0/24 area 0
 neighbor 192.168.12.2
exit
!
end
r1# sho ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, A - Babel, F - PBR, f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

K>* 0.0.0.0/0 [0/100] via 192.168.1.1, enp0s3, 2d00h58m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h58m
O   192.168.12.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h55m
C>* 192.168.12.0/24 is directly connected, enp0s8, 2d00h58m
O>* 192.168.23.0/24 [110/200] via 192.168.12.2, enp0s8, weight 1, 2d00h55m
r1# conf t
r1(config)# router ospf
r1(config-router)# no network 111.111.111.111/32 area 0
r1(config-router)# no neighbor 192.168.12.2
r1(config-router)# end
r1# wr
Note: this version of vtysh never writes vtysh.conf
Building Configuration...
Integrated configuration saved to /etc/frr/frr.conf
[OK]
r1# show ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, A - Babel, F - PBR, f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

K>* 0.0.0.0/0 [0/100] via 192.168.1.1, enp0s3, 2d00h58m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h58m
O   192.168.12.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h56m
C>* 192.168.12.0/24 is directly connected, enp0s8, 2d00h58m
O>* 192.168.23.0/24 [110/200] via 192.168.12.2, enp0s8, weight 1, 2d00h56m
r1# show run
Building configuration...

Current configuration:
!
frr version 8.2.2
frr defaults traditional
hostname r1
log syslog informational
no ip forwarding
no ipv6 forwarding
!
interface enp0s8
 no ip ospf passive
exit
!
router ospf
 network 192.168.12.0/24 area 0
exit
!
end
r1#

[root@r1 ~]# curl http://10.1.1.1
curl: (7) Failed connect to 10.1.1.1:80; Время ожидания соединения истекло
[root@r1 ~]# curl http://10.1.1.1
^[[A^C
[root@r1 ~]# curl http://10.1.1.1
^C
[root@r1 ~]# curl http://10.1.1.1
^C
[root@r1 ~]# curl http://10.1.1.1
^C
[root@r1 ~]# curl http://10.1.1.1
^[[A^C
[root@r1 ~]# curl http://10.1.1.1 # после отключения rp.filter
Hello from R6!
[root@r1 ~]# curl http://10.1.1.1
Hello from R7!

R2:

[root@r2 ~]# vtysh

Hello, this is FRRouting (version 8.2.2).
Copyright 1996-2005 Kunihiro Ishiguro, et al.

r2# show ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, A - Babel, F - PBR, f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

K>* 0.0.0.0/0 [0/100] via 192.168.1.1, enp0s3, 2d00h57m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h57m
O   192.168.12.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h57m
C>* 192.168.12.0/24 is directly connected, enp0s8, 2d00h57m
O   192.168.23.0/24 [110/100] is directly connected, enp0s9, weight 1, 2d00h57m
C>* 192.168.23.0/24 is directly connected, enp0s9, 2d00h57m
r2# sho run
Building configuration...

Current configuration:
!
frr version 8.2.2
frr defaults traditional
hostname r2
log syslog informational
no ipv6 forwarding
!
interface enp0s8
 no ip ospf passive
exit
!
router ospf
 network 192.168.12.0/24 area 0
 network 192.168.23.0/24 area 0
exit
!
end
[root@r2 ~]# rpm -qa | grep ipvsadm
[root@r2 ~]# yum install ipvsadm -y
...
Установлено:
  ipvsadm.x86_64 0:1.27-8.el7
...
[root@r2 ~]# ip link add dummy0 type dummy
RTNETLINK answers: File exists
[root@r2 ~]# ip addr add 10.1.1.1/32 dev dummy0
[root@r2 ~]# ip link set up dummy0
[root@r2 ~]# ip a
...
5: dummy0: <BROADCAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 4e:92:4f:27:a5:de brd ff:ff:ff:ff:ff:ff
    inet 10.1.1.1/32 scope global dummy0
       valid_lft forever preferred_lft forever
    inet6 fe80::4c92:4fff:fe27:a5de/64 scope link
       valid_lft forever preferred_lft forever
[root@r2 ~]# vtysh
r2# conf t
r2(config)# router ospf
r2(config-router)# network 10.1.1.1/32 area 0
r2(config-router)# end
r2# wr
[root@r2 ~]# ipvsadm -A -t 10.1.1.1:80 -s rr
[root@r2 ~]# ipvsadm -a -t 10.1.1.1:80 -r 192.168.35.5:80 -i
[root@r2 ~]# ipvsadm -a -t 10.1.1.1:80 -r 192.168.34.4:80 -i
[root@r2 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.1.1.1:80 rr
  -> 192.168.34.4:80              Tunnel  1      0          0
  -> 192.168.35.5:80              Tunnel  1      0          0
# после curl на r1 смотрим как балансирутся запросы
[root@r2 ~]# ipvsadm -Ln -c
IPVS connection entries
pro expire state       source             virtual            destination
TCP 00:41  SYN_RECV    192.168.12.1:49172 10.1.1.1:80        192.168.35.5:80
TCP 00:10  SYN_RECV    10.1.1.1:41390     10.1.1.1:80        192.168.34.4:80

[root@r2 ~]# sysctl -w net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.rp_filter = 0
[root@r2 ~]# sysctl -w net.ipv4.conf.dummy0.rp_filter=0
net.ipv4.conf.dummy0.rp_filter = 0
[root@r2 ~]# sysctl -w net.ipv4.conf.enp0s8.rp_filter=0
net.ipv4.conf.enp0s8.rp_filter = 0
[root@r2 ~]# sysctl -w net.ipv4.conf.enp0s9.rp_filter=0
net.ipv4.conf.enp0s9.rp_filter = 0
[root@r2 ~]# sysctl -w net.ipv4.conf.enp0s10.rp_filter=0
sysctl: cannot stat /proc/sys/net/ipv4/conf/enp0s10/rp_filter: Нет такого файла или каталога
[root@r2 ~]# sysctl -w net.ipv4.conf.tunl0.rp_filter=0
sysctl: cannot stat /proc/sys/net/ipv4/conf/tunl0/rp_filter: Нет такого файла или каталога

R3:

[root@r3 ~]# vtysh

Hello, this is FRRouting (version 8.2.2).
Copyright 1996-2005 Kunihiro Ishiguro, et al.

r3# show run
Building configuration...

Current configuration:
!
frr version 8.2.2
frr defaults traditional
hostname r3
log syslog informational
no ipv6 forwarding
!
router ospf
 network 192.168.23.0/24 area 0
exit
!
end
r3# sho ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, A - Babel, F - PBR, f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

K>* 0.0.0.0/0 [0/101] via 192.168.1.1, enp0s3, 2d00h53m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h53m
O>* 192.168.12.0/24 [110/200] via 192.168.23.2, enp0s8, weight 1, 2d00h53m
O   192.168.23.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h53m
C>* 192.168.23.0/24 is directly connected, enp0s8, 2d00h53m
C>* 192.168.34.0/24 is directly connected, enp0s9, 2d00h53m
C>* 192.168.35.0/24 is directly connected, enp0s10, 2d00h53m
r3# conf t
r3(config)# router ospf
r3(config-router)# network 192.168.34.0/24 area 0
r3(config-router)# network 192.168.35.0/24 area 0
r3(config-router)# end
r3# wr
Note: this version of vtysh never writes vtysh.conf
Building Configuration...
Integrated configuration saved to /etc/frr/frr.conf
[OK]
r3# sho ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, A - Babel, F - PBR, f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

K>* 0.0.0.0/0 [0/101] via 192.168.1.1, enp0s3, 2d00h58m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h58m
O>* 192.168.12.0/24 [110/200] via 192.168.23.2, enp0s8, weight 1, 2d00h58m
O   192.168.23.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h58m
C>* 192.168.23.0/24 is directly connected, enp0s8, 2d00h58m
O   192.168.34.0/24 [110/100] is directly connected, enp0s9, weight 1, 00:04:38
C>* 192.168.34.0/24 is directly connected, enp0s9, 2d00h58m
O   192.168.35.0/24 [110/100] is directly connected, enp0s10, weight 1, 00:04:23
C>* 192.168.35.0/24 is directly connected, enp0s10, 2d00h58m
r3# sho ip osf ne
% Unknown command: sho ip osf ne
r3# sho ip ospf neighbor

Neighbor ID     Pri State           Up Time         Dead Time Address         Interface                        RXmtL RqstL DBsmL
192.168.12.2      1 Full/DR         4233600d00h00m    37.136s 192.168.23.2    enp0s8:192.168.23.3                  0     0     0
192.168.34.4      1 Full/Backup     3m44s             35.295s 192.168.34.4    enp0s9:192.168.34.3                  0     0     0
192.168.35.5      1 Full/Backup     1m30s             31.493s 192.168.35.5    enp0s10:192.168.35.3                 0     0     0

[root@r3 ~]# sysctl -w net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.rp_filter = 0
[root@r3 ~]# sysctl -w net.ipv4.conf.dummy0.rp_filter=0
sysctl: cannot stat /proc/sys/net/ipv4/conf/dummy0/rp_filter: Нет такого файла или каталога
[root@r3 ~]# sysctl -w net.ipv4.conf.enp0s8.rp_filter=0
net.ipv4.conf.enp0s8.rp_filter = 0
[root@r3 ~]# sysctl -w net.ipv4.conf.enp0s9.rp_filter=0
net.ipv4.conf.enp0s9.rp_filter = 0
[root@r3 ~]# sysctl -w net.ipv4.conf.enp0s10.rp_filter=0
net.ipv4.conf.enp0s10.rp_filter = 0
[root@r3 ~]# sysctl -w net.ipv4.conf.tunl0.rp_filter=0
sysctl: cannot stat /proc/sys/net/ipv4/conf/tunl0/rp_filter: Нет такого файла или каталога
[root@r3 ~]#

R4:

[root@r4 ~]# yum install haproxy -y
...
Установлено:
  haproxy.x86_64 0:1.5.18-9.el7_9.1
[root@r4 ~]# ip link add dummy0 type dummy
RTNETLINK answers: File exists
[root@r4 ~]# ip addr add 10.1.1.1/32 dev dummy0
[root@r4 ~]# ip link set up dummy0
[root@r4 ~]# ip a
...
5: dummy0: <BROADCAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 52:51:78:89:e9:51 brd ff:ff:ff:ff:ff:ff
    inet 10.1.1.1/32 scope global dummy0
       valid_lft forever preferred_lft forever
    inet6 fe80::5051:78ff:fe89:e951/64 scope link
       valid_lft forever preferred_lft forever
[root@r4 ~]# modprobe ipip
[root@r4 ~]# ip link set up tunl0
[root@r5 ~]# ip a
...
6: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
[root@r4 ~]# vi /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend  main 10.1.1.1:80
    acl url_static       path_beg       -i /static /images /javascript /stylesheets
    acl url_static       path_end       -i .jpg .gif .png .css .js

    use_backend static          if url_static
    default_backend             app

#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
backend static
    balance     roundrobin
    server      static 127.0.0.1:4331 check

#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend app
    balance     roundrobin
    server  app1 192.168.50.6:80 check
    server  app2 192.168.50.7:80 check
[root@r4 ~]# sysctl -w net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.rp_filter = 0
[root@r4 ~]# sysctl -w net.ipv4.conf.dummy0.rp_filter=0
net.ipv4.conf.dummy0.rp_filter = 0
[root@r4 ~]# sysctl -w net.ipv4.conf.enp0s8.rp_filter=0
net.ipv4.conf.enp0s8.rp_filter = 0
[root@r4 ~]# sysctl -w net.ipv4.conf.tunl0.rp_filter=0
net.ipv4.conf.tunl0.rp_filter = 0

R5:

[root@r5 ~]# rpm -qa | grep haproxy
[root@r5 ~]# yum install haproxy -y
...                                                                                                                            
Установлено:
  haproxy.x86_64 0:1.5.18-9.el7_9.1
[root@r5 ~]# ip addr add 10.1.1.1/32 dev dummy0
[root@r5 ~]# ip link set up dummy0
[root@r5 ~]# ip a
...
5: dummy0: <BROADCAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether d6:72:ba:f5:a4:d2 brd ff:ff:ff:ff:ff:ff
    inet 10.1.1.1/32 scope global dummy0
       valid_lft forever preferred_lft forever
    inet6 fe80::d472:baff:fef5:a4d2/64 scope link
       valid_lft forever preferred_lft forever
...

[root@r5 ~]# vi /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend  main 10.1.1.1:80
    acl url_static       path_beg       -i /static /images /javascript /stylesheets
    acl url_static       path_end       -i .jpg .gif .png .css .js

    use_backend static          if url_static
    default_backend             app

#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
backend static
    balance     roundrobin
    server      static 127.0.0.1:4331 check

#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend app
    balance     roundrobin
    server  app1 192.168.50.6:80 check
    server  app2 192.168.50.7:80 check

[root@r5 ~]# vi /etc/haproxy/haproxy.cfg
[root@r5 ~]# sysctl -w net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.rp_filter = 0
[root@r5 ~]# sysctl -w net.ipv4.conf.dummy0.rp_filter=0
net.ipv4.conf.dummy0.rp_filter = 0
[root@r5 ~]# sysctl -w net.ipv4.conf.enp0s8.rp_filter=0
net.ipv4.conf.enp0s8.rp_filter = 0
[root@r5 ~]# sysctl -w net.ipv4.conf.tunl0.rp_filter=0
net.ipv4.conf.tunl0.rp_filter = 0

R6:

[root@r6 ~]# yum install epel-release -y
[root@r6 ~]# yum install nginx -y
[root@r6 ~]# vi /etc/nginx/nginx.conf
[root@r6 ~]# cd /usr/share/nginx/html/
[root@r6 html]# echo 'Hello from R6!' > index.html
[root@r6 html]# systemctl restart nginx
[root@r6 html]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
   Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
   Active: active (running) since Ср 2022-08-03 08:15:26 MSK; 11s ago
 ...
[root@r6 html]# curl http://localhost
Hello from R6!

R7:

[root@r7 ~]# yum install epel-release -y
[root@r7 ~]# yum install nginx -y
[root@r7 ~]# vi /etc/nginx/nginx.conf
[root@r7 ~]# echo 'Hello from R7!' > /usr/share/nginx/html/index.html
[root@r7 ~]# systemctl restart nginx
[root@r7 ~]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
   Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
   Active: active (running) since Ср 2022-08-03 08:14:40 MSK; 9s ago
 ...
[root@r7 ~]# curl http://localhost
Hello from R7!