How can we help?
Load balancing, lvs + haproxy + nginx
R1:
[root@r1 ~]# vtysh
Hello, this is FRRouting (version 8.2.2).
Copyright 1996-2005 Kunihiro Ishiguro, et al.
r1# show run
Building configuration...
Current configuration:
!
frr version 8.2.2
frr defaults traditional
hostname r1
log syslog informational
no ip forwarding
no ipv6 forwarding
!
interface enp0s8
no ip ospf passive
exit
!
router ospf
network 111.111.111.111/32 area 0
network 192.168.12.0/24 area 0
neighbor 192.168.12.2
exit
!
end
r1# sho ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
T - Table, A - Babel, F - PBR, f - OpenFabric,
> - selected route, * - FIB route, q - queued, r - rejected, b - backup
t - trapped, o - offload failure
K>* 0.0.0.0/0 [0/100] via 192.168.1.1, enp0s3, 2d00h58m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h58m
O 192.168.12.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h55m
C>* 192.168.12.0/24 is directly connected, enp0s8, 2d00h58m
O>* 192.168.23.0/24 [110/200] via 192.168.12.2, enp0s8, weight 1, 2d00h55m
r1# conf t
r1(config)# router ospf
r1(config-router)# no network 111.111.111.111/32 area 0
r1(config-router)# no neighbor 192.168.12.2
r1(config-router)# end
r1# wr
Note: this version of vtysh never writes vtysh.conf
Building Configuration...
Integrated configuration saved to /etc/frr/frr.conf
[OK]
r1# show ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
T - Table, A - Babel, F - PBR, f - OpenFabric,
> - selected route, * - FIB route, q - queued, r - rejected, b - backup
t - trapped, o - offload failure
K>* 0.0.0.0/0 [0/100] via 192.168.1.1, enp0s3, 2d00h58m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h58m
O 192.168.12.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h56m
C>* 192.168.12.0/24 is directly connected, enp0s8, 2d00h58m
O>* 192.168.23.0/24 [110/200] via 192.168.12.2, enp0s8, weight 1, 2d00h56m
r1# show run
Building configuration...
Current configuration:
!
frr version 8.2.2
frr defaults traditional
hostname r1
log syslog informational
no ip forwarding
no ipv6 forwarding
!
interface enp0s8
no ip ospf passive
exit
!
router ospf
network 192.168.12.0/24 area 0
exit
!
end
r1#
[root@r1 ~]# curl http://10.1.1.1
curl: (7) Failed connect to 10.1.1.1:80; Время ожидания соединения истекло
[root@r1 ~]# curl http://10.1.1.1
^[[A^C
[root@r1 ~]# curl http://10.1.1.1
^C
[root@r1 ~]# curl http://10.1.1.1
^C
[root@r1 ~]# curl http://10.1.1.1
^C
[root@r1 ~]# curl http://10.1.1.1
^[[A^C
[root@r1 ~]# curl http://10.1.1.1 # после отключения rp.filter
Hello from R6!
[root@r1 ~]# curl http://10.1.1.1
Hello from R7!
R2:
[root@r2 ~]# vtysh
Hello, this is FRRouting (version 8.2.2).
Copyright 1996-2005 Kunihiro Ishiguro, et al.
r2# show ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
T - Table, A - Babel, F - PBR, f - OpenFabric,
> - selected route, * - FIB route, q - queued, r - rejected, b - backup
t - trapped, o - offload failure
K>* 0.0.0.0/0 [0/100] via 192.168.1.1, enp0s3, 2d00h57m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h57m
O 192.168.12.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h57m
C>* 192.168.12.0/24 is directly connected, enp0s8, 2d00h57m
O 192.168.23.0/24 [110/100] is directly connected, enp0s9, weight 1, 2d00h57m
C>* 192.168.23.0/24 is directly connected, enp0s9, 2d00h57m
r2# sho run
Building configuration...
Current configuration:
!
frr version 8.2.2
frr defaults traditional
hostname r2
log syslog informational
no ipv6 forwarding
!
interface enp0s8
no ip ospf passive
exit
!
router ospf
network 192.168.12.0/24 area 0
network 192.168.23.0/24 area 0
exit
!
end
[root@r2 ~]# rpm -qa | grep ipvsadm
[root@r2 ~]# yum install ipvsadm -y
...
Установлено:
ipvsadm.x86_64 0:1.27-8.el7
...
[root@r2 ~]# ip link add dummy0 type dummy
RTNETLINK answers: File exists
[root@r2 ~]# ip addr add 10.1.1.1/32 dev dummy0
[root@r2 ~]# ip link set up dummy0
[root@r2 ~]# ip a
...
5: dummy0: <BROADCAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 4e:92:4f:27:a5:de brd ff:ff:ff:ff:ff:ff
inet 10.1.1.1/32 scope global dummy0
valid_lft forever preferred_lft forever
inet6 fe80::4c92:4fff:fe27:a5de/64 scope link
valid_lft forever preferred_lft forever
[root@r2 ~]# vtysh
r2# conf t
r2(config)# router ospf
r2(config-router)# network 10.1.1.1/32 area 0
r2(config-router)# end
r2# wr
[root@r2 ~]# ipvsadm -A -t 10.1.1.1:80 -s rr
[root@r2 ~]# ipvsadm -a -t 10.1.1.1:80 -r 192.168.35.5:80 -i
[root@r2 ~]# ipvsadm -a -t 10.1.1.1:80 -r 192.168.34.4:80 -i
[root@r2 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.1.1.1:80 rr
-> 192.168.34.4:80 Tunnel 1 0 0
-> 192.168.35.5:80 Tunnel 1 0 0
# после curl на r1 смотрим как балансирутся запросы
[root@r2 ~]# ipvsadm -Ln -c
IPVS connection entries
pro expire state source virtual destination
TCP 00:41 SYN_RECV 192.168.12.1:49172 10.1.1.1:80 192.168.35.5:80
TCP 00:10 SYN_RECV 10.1.1.1:41390 10.1.1.1:80 192.168.34.4:80
[root@r2 ~]# sysctl -w net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.rp_filter = 0
[root@r2 ~]# sysctl -w net.ipv4.conf.dummy0.rp_filter=0
net.ipv4.conf.dummy0.rp_filter = 0
[root@r2 ~]# sysctl -w net.ipv4.conf.enp0s8.rp_filter=0
net.ipv4.conf.enp0s8.rp_filter = 0
[root@r2 ~]# sysctl -w net.ipv4.conf.enp0s9.rp_filter=0
net.ipv4.conf.enp0s9.rp_filter = 0
[root@r2 ~]# sysctl -w net.ipv4.conf.enp0s10.rp_filter=0
sysctl: cannot stat /proc/sys/net/ipv4/conf/enp0s10/rp_filter: Нет такого файла или каталога
[root@r2 ~]# sysctl -w net.ipv4.conf.tunl0.rp_filter=0
sysctl: cannot stat /proc/sys/net/ipv4/conf/tunl0/rp_filter: Нет такого файла или каталога
R3:
[root@r3 ~]# vtysh
Hello, this is FRRouting (version 8.2.2).
Copyright 1996-2005 Kunihiro Ishiguro, et al.
r3# show run
Building configuration...
Current configuration:
!
frr version 8.2.2
frr defaults traditional
hostname r3
log syslog informational
no ipv6 forwarding
!
router ospf
network 192.168.23.0/24 area 0
exit
!
end
r3# sho ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
T - Table, A - Babel, F - PBR, f - OpenFabric,
> - selected route, * - FIB route, q - queued, r - rejected, b - backup
t - trapped, o - offload failure
K>* 0.0.0.0/0 [0/101] via 192.168.1.1, enp0s3, 2d00h53m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h53m
O>* 192.168.12.0/24 [110/200] via 192.168.23.2, enp0s8, weight 1, 2d00h53m
O 192.168.23.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h53m
C>* 192.168.23.0/24 is directly connected, enp0s8, 2d00h53m
C>* 192.168.34.0/24 is directly connected, enp0s9, 2d00h53m
C>* 192.168.35.0/24 is directly connected, enp0s10, 2d00h53m
r3# conf t
r3(config)# router ospf
r3(config-router)# network 192.168.34.0/24 area 0
r3(config-router)# network 192.168.35.0/24 area 0
r3(config-router)# end
r3# wr
Note: this version of vtysh never writes vtysh.conf
Building Configuration...
Integrated configuration saved to /etc/frr/frr.conf
[OK]
r3# sho ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
T - Table, A - Babel, F - PBR, f - OpenFabric,
> - selected route, * - FIB route, q - queued, r - rejected, b - backup
t - trapped, o - offload failure
K>* 0.0.0.0/0 [0/101] via 192.168.1.1, enp0s3, 2d00h58m
C>* 192.168.1.0/24 is directly connected, enp0s3, 2d00h58m
O>* 192.168.12.0/24 [110/200] via 192.168.23.2, enp0s8, weight 1, 2d00h58m
O 192.168.23.0/24 [110/100] is directly connected, enp0s8, weight 1, 2d00h58m
C>* 192.168.23.0/24 is directly connected, enp0s8, 2d00h58m
O 192.168.34.0/24 [110/100] is directly connected, enp0s9, weight 1, 00:04:38
C>* 192.168.34.0/24 is directly connected, enp0s9, 2d00h58m
O 192.168.35.0/24 [110/100] is directly connected, enp0s10, weight 1, 00:04:23
C>* 192.168.35.0/24 is directly connected, enp0s10, 2d00h58m
r3# sho ip osf ne
% Unknown command: sho ip osf ne
r3# sho ip ospf neighbor
Neighbor ID Pri State Up Time Dead Time Address Interface RXmtL RqstL DBsmL
192.168.12.2 1 Full/DR 4233600d00h00m 37.136s 192.168.23.2 enp0s8:192.168.23.3 0 0 0
192.168.34.4 1 Full/Backup 3m44s 35.295s 192.168.34.4 enp0s9:192.168.34.3 0 0 0
192.168.35.5 1 Full/Backup 1m30s 31.493s 192.168.35.5 enp0s10:192.168.35.3 0 0 0
[root@r3 ~]# sysctl -w net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.rp_filter = 0
[root@r3 ~]# sysctl -w net.ipv4.conf.dummy0.rp_filter=0
sysctl: cannot stat /proc/sys/net/ipv4/conf/dummy0/rp_filter: Нет такого файла или каталога
[root@r3 ~]# sysctl -w net.ipv4.conf.enp0s8.rp_filter=0
net.ipv4.conf.enp0s8.rp_filter = 0
[root@r3 ~]# sysctl -w net.ipv4.conf.enp0s9.rp_filter=0
net.ipv4.conf.enp0s9.rp_filter = 0
[root@r3 ~]# sysctl -w net.ipv4.conf.enp0s10.rp_filter=0
net.ipv4.conf.enp0s10.rp_filter = 0
[root@r3 ~]# sysctl -w net.ipv4.conf.tunl0.rp_filter=0
sysctl: cannot stat /proc/sys/net/ipv4/conf/tunl0/rp_filter: Нет такого файла или каталога
[root@r3 ~]#
R4:
[root@r4 ~]# yum install haproxy -y
...
Установлено:
haproxy.x86_64 0:1.5.18-9.el7_9.1
[root@r4 ~]# ip link add dummy0 type dummy
RTNETLINK answers: File exists
[root@r4 ~]# ip addr add 10.1.1.1/32 dev dummy0
[root@r4 ~]# ip link set up dummy0
[root@r4 ~]# ip a
...
5: dummy0: <BROADCAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 52:51:78:89:e9:51 brd ff:ff:ff:ff:ff:ff
inet 10.1.1.1/32 scope global dummy0
valid_lft forever preferred_lft forever
inet6 fe80::5051:78ff:fe89:e951/64 scope link
valid_lft forever preferred_lft forever
[root@r4 ~]# modprobe ipip
[root@r4 ~]# ip link set up tunl0
[root@r5 ~]# ip a
...
6: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
[root@r4 ~]# vi /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend main 10.1.1.1:80
acl url_static path_beg -i /static /images /javascript /stylesheets
acl url_static path_end -i .jpg .gif .png .css .js
use_backend static if url_static
default_backend app
#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
backend static
balance roundrobin
server static 127.0.0.1:4331 check
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend app
balance roundrobin
server app1 192.168.50.6:80 check
server app2 192.168.50.7:80 check
[root@r4 ~]# sysctl -w net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.rp_filter = 0
[root@r4 ~]# sysctl -w net.ipv4.conf.dummy0.rp_filter=0
net.ipv4.conf.dummy0.rp_filter = 0
[root@r4 ~]# sysctl -w net.ipv4.conf.enp0s8.rp_filter=0
net.ipv4.conf.enp0s8.rp_filter = 0
[root@r4 ~]# sysctl -w net.ipv4.conf.tunl0.rp_filter=0
net.ipv4.conf.tunl0.rp_filter = 0
R5:
[root@r5 ~]# rpm -qa | grep haproxy
[root@r5 ~]# yum install haproxy -y
...
Установлено:
haproxy.x86_64 0:1.5.18-9.el7_9.1
[root@r5 ~]# ip addr add 10.1.1.1/32 dev dummy0
[root@r5 ~]# ip link set up dummy0
[root@r5 ~]# ip a
...
5: dummy0: <BROADCAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether d6:72:ba:f5:a4:d2 brd ff:ff:ff:ff:ff:ff
inet 10.1.1.1/32 scope global dummy0
valid_lft forever preferred_lft forever
inet6 fe80::d472:baff:fef5:a4d2/64 scope link
valid_lft forever preferred_lft forever
...
[root@r5 ~]# vi /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend main 10.1.1.1:80
acl url_static path_beg -i /static /images /javascript /stylesheets
acl url_static path_end -i .jpg .gif .png .css .js
use_backend static if url_static
default_backend app
#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
backend static
balance roundrobin
server static 127.0.0.1:4331 check
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend app
balance roundrobin
server app1 192.168.50.6:80 check
server app2 192.168.50.7:80 check
[root@r5 ~]# vi /etc/haproxy/haproxy.cfg
[root@r5 ~]# sysctl -w net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.all.rp_filter = 0
[root@r5 ~]# sysctl -w net.ipv4.conf.dummy0.rp_filter=0
net.ipv4.conf.dummy0.rp_filter = 0
[root@r5 ~]# sysctl -w net.ipv4.conf.enp0s8.rp_filter=0
net.ipv4.conf.enp0s8.rp_filter = 0
[root@r5 ~]# sysctl -w net.ipv4.conf.tunl0.rp_filter=0
net.ipv4.conf.tunl0.rp_filter = 0
R6:
[root@r6 ~]# yum install epel-release -y
[root@r6 ~]# yum install nginx -y
[root@r6 ~]# vi /etc/nginx/nginx.conf
[root@r6 ~]# cd /usr/share/nginx/html/
[root@r6 html]# echo 'Hello from R6!' > index.html
[root@r6 html]# systemctl restart nginx
[root@r6 html]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
Active: active (running) since Ср 2022-08-03 08:15:26 MSK; 11s ago
...
[root@r6 html]# curl http://localhost
Hello from R6!
R7:
[root@r7 ~]# yum install epel-release -y
[root@r7 ~]# yum install nginx -y
[root@r7 ~]# vi /etc/nginx/nginx.conf
[root@r7 ~]# echo 'Hello from R7!' > /usr/share/nginx/html/index.html
[root@r7 ~]# systemctl restart nginx
[root@r7 ~]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
Active: active (running) since Ср 2022-08-03 08:14:40 MSK; 9s ago
...
[root@r7 ~]# curl http://localhost
Hello from R7!