由 SuKai June 25, 2022
前面文章介绍了Kube-OVN基于OVS/OVN将网络虚拟化带入云原生领域。下面我们通过下面的实验场景来看一下容器如何使用虚拟化网络的，从而了解Kube-OVN的基本工作原理。
实验环境
实验使用的是两台虚拟机：ovn-1为控制节点和计算节点，IP地址为：192.168.0.115。ovn-2为计算节点，IP地址为：192.168.0.114。
实验场景
1，两个主机上进入网络命名空间，配置不同网段进行通信
2，Docker运行容器，通过OVS internal port进行通信
3，Docker运行容器，通过veth pair进行通信
安装部署
#控制节点和计算节点
sudo apt install net-tools
sudo apt-get install python-six openssl -y
sudo apt-get install openvswitch-switch openvswitch-common -y
sudo systemctl disable apparmor
#控制节点
sudo apt-get install ovn-central ovn-common ovn-host -y
#计算节点
sudo apt-get install ovn-host ovn-common -y
配置ovn
设置ovn数据库允许tcp进行连接，默认只允许socket本地访问。
#控制节点
sudo ovn-nbctl set-connection ptcp:6641:0.0.0.0 -- set connection . inactivity_probe=60000
sudo ovn-sbctl set-connection ptcp:6642:0.0.0.0 -- set connection . inactivity_probe=60000
sudo ovs-appctl -t ovsdb-server ovsdb-server/add-remote ptcp:6640:192.168.0.115
配置ovs
设置ovs连接远程控制节点的数据库，配置节点间大二层网络封装协议类型。
#控制和计算节点
sudo ovs-vsctl set open . external-ids:ovn-remote=tcp:192.168.0.115:6642
sudo ovs-vsctl set open . external-ids:ovn-encap-type=geneve
sudo ovs-vsctl set open . external-ids:ovn-encap-ip=192.168.0.115 #本机IP
查看安装后状态
看到监听6641/6642端口，Chassis可以看到两个节点。
sukai@ovn-1:~$ ss -tlnp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 10 0.0.0.0:6641 0.0.0.0:*
LISTEN 0 10 0.0.0.0:6642 0.0.0.0:*
LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 127.0.0.1:6010 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 [::1]:6010 [::]:*
sukai@ovn-1:~$ sudo ovn-nbctl show
sukai@ovn-1:~$ sudo ovn-sbctl show
Chassis "6fa2da75-1393-4efe-984d-0c0310b95ad3"
hostname: ovn-1
Encap geneve
ip: "192.168.0.115"
options: {csum="true"}
Chassis "28a87e04-b738-4db0-a767-380091e1a752"
hostname: ovn-2
Encap geneve
ip: "192.168.0.114"
options: {csum="true"}
sukai@ovn-1:~$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:d4:b7:e1 brd ff:ff:ff:ff:ff:ff
inet 192.168.0.115/24 brd 192.168.0.255 scope global dynamic ens33
valid_lft 85932sec preferred_lft 85932sec
inet6 fe80::20c:29ff:fed4:b7e1/64 scope link
valid_lft forever preferred_lft forever
3: ens34: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:d4:b7:eb brd ff:ff:ff:ff:ff:ff
inet 192.168.100.10/24 brd 192.168.100.255 scope global ens34
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fed4:b7eb/64 scope link
valid_lft forever preferred_lft forever
4: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether b2:3d:a2:f9:17:b4 brd ff:ff:ff:ff:ff:ff
5: br-int: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether b6:32:20:a4:db:46 brd ff:ff:ff:ff:ff:ff
6: genev_sys_6081: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65000 qdisc noqueue master ovs-system state UNKNOWN group default qlen 1000
link/ether 5a:b1:c8:22:ac:7a brd ff:ff:ff:ff:ff:ff
inet6 fe80::58b1:c8ff:fe22:ac7a/64 scope link
valid_lft forever preferred_lft forever
场景一
创建路由器和两台交换机
chassis指定路由所在的主机为ovn-1
sudo ovn-nbctl create Logical_Router name=router options:chassis=6fa2da75-1393-4efe-984d-0c0310b95ad3
sudo ovn-nbctl ls-add lswitch1
sudo ovn-nbctl ls-add lswitch2
路由器添加两个端口
sudo ovn-nbctl lrp-add router lr-ls1 52:54:00:c1:68:50 10.0.0.1/24
sudo ovn-nbctl lrp-add router lr-ls2 52:54:00:c1:68:60 10.1.0.1/24
交换机1上添加连接路由器接口
sudo ovn-nbctl lsp-add lswitch1 ls1-lr1
sudo ovn-nbctl lsp-set-type ls1-lr1 router
sudo ovn-nbctl lsp-set-addresses ls1-lr1 52:54:00:c1:68:50
sudo ovn-nbctl lsp-set-options ls1-lr1 router-port=lr-ls1
交换机1上配置DHCP
dhcp_sw1=`sudo ovn-nbctl create DHCP_Options cidr=10.0.0.0/24 options="\"server_id\"=\"10.0.0.1\" \"server_mac\"=\"52:54:00:c1:68:50\" \"lease_time\"=\"3600\" \"router\"=\"10.0.0.1\""`
sudo ovn-nbctl lsp-set-dhcpv4-options ls1-vm1 $dhcp_sw1
交换机2上配置DHCP
dhcp_sw2=`sudo ovn-nbctl create DHCP_Options cidr=10.1.0.0/24 options="\"server_id\"=\"10.1.0.1\" \"server_mac\"=\"52:54:00:c1:68:60\" \"lease_time\"=\"3600\" \"router\"=\"10.1.0.1\""`
sudo ovn-nbctl lsp-set-dhcpv4-options ls2-vm2 $dhcp_sw2
交换机1上添加vm1接口
sudo ovn-nbctl lsp-add lswitch1 ls1-vm1
sudo ovn-nbctl lsp-set-addresses ls1-vm1 "02:ac:10:ff:01:30 10.0.0.10"
sudo ovn-nbctl lsp-set-port-security ls1-vm1 "02:ac:10:ff:01:30 10.0.0.10"
在ovn-1虚拟机上创建vm1网络命名空间，配置网卡
sudo ifconfig br-int up
sudo ip netns add vm1
sudo ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
sudo ip link set vm1 address 02:ac:10:ff:01:30
sudo ip link set vm1 netns vm1
sudo ovs-vsctl set Interface vm1 external_ids:iface-id=ls1-vm1
sudo pkill dhclient
sudo ip netns exec vm1 dhclient vm1
在ovn-2虚拟机上创建vm2网络命名空间，配置网卡
sudo ifconfig br-int up
sudo ip netns add vm2
sudo ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
sudo ip link set vm2 address 02:ac:10:ff:01:31
sudo ip link set vm2 netns vm2
sudo ovs-vsctl set Interface vm2 external_ids:iface-id=ls2-vm2
sudo pkill dhclient
sudo ip netns exec vm2 dhclient vm2
验证网络
sukai@ovn-1:~$ sudo ip netns exec vm1 /bin/bash
root@ovn-1:/home/sukai# ip a
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
7: vm1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 02:ac:10:ff:01:30 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.10/24 brd 10.0.0.255 scope global dynamic vm1
valid_lft 3541sec preferred_lft 3541sec
inet6 fe80::ac:10ff:feff:130/64 scope link
valid_lft forever preferred_lft forever
root@ovn-1:/home/sukai# ping 10.0.0.1
PING 10.0.0.1 (10.0.0.1) 56(84) bytes of data.
64 bytes from 10.0.0.1: icmp_seq=1 ttl=254 time=0.967 ms
64 bytes from 10.0.0.1: icmp_seq=2 ttl=254 time=0.858 ms
^C
--- 10.0.0.1 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1002ms
rtt min/avg/max/mdev = 0.858/0.912/0.967/0.054 ms
root@ovn-1:/home/sukai# ping 10.10.0.1
PING 10.10.0.1 (10.10.0.1) 56(84) bytes of data.
^C
--- 10.10.0.1 ping statistics ---
2 packets transmitted, 0 received, 100% packet loss, time 1018ms
root@ovn-1:/home/sukai# ping 10.1.0.20
PING 10.1.0.20 (10.1.0.20) 56(84) bytes of data.
64 bytes from 10.1.0.20: icmp_seq=1 ttl=63 time=2.41 ms
64 bytes from 10.1.0.20: icmp_seq=2 ttl=63 time=1.41 ms
^C
--- 10.1.0.20 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1003ms
rtt min/avg/max/mdev = 1.410/1.909/2.409/0.499 ms
查看ovn信息
sukai@ovn-1:~$ sudo ovn-nbctl show
switch 75a787a4-9e55-4542-acaa-e969bebfaf16 (lswitch1)
port ls1-vm1
addresses: ["02:ac:10:ff:01:30 10.0.0.10"]
port ls1-lr1
type: router
addresses: ["52:54:00:c1:68:50"]
router-port: lr-ls1
switch d85756bc-e7d3-42fb-ace1-15168e017736 (lswitch2)
port ls2-lr1
type: router
addresses: ["52:54:00:c1:68:60"]
router-port: lr-ls2
port ls2-vm2
addresses: ["02:ac:10:ff:01:31 10.1.0.20"]
router 3bccdebe-38de-450c-8e02-208eaff33398 (router)
port lr-ls2
mac: "52:54:00:c1:68:60"
networks: ["10.1.0.1/24"]
port lr-ls1
mac: "52:54:00:c1:68:50"
networks: ["10.0.0.1/24"]
sukai@ovn-1:~$ sudo ovn-sbctl show
Chassis "6fa2da75-1393-4efe-984d-0c0310b95ad3"
hostname: ovn-1
Encap geneve
ip: "192.168.0.115"
options: {csum="true"}
Port_Binding lr-ls2
Port_Binding ls1-vm1
Port_Binding ls2-lr1
Port_Binding ls1-lr1
Port_Binding lr-ls1
Chassis "28a87e04-b738-4db0-a767-380091e1a752"
hostname: ovn-2
Encap geneve
ip: "192.168.0.114"
options: {csum="true"}
Port_Binding ls2-vm2
场景二
在交换机1上添加接口
sudo ovn-nbctl lsp-add lswitch1 ls1-ctn1
sudo ovn-nbctl lsp-set-addresses ls1-ctn1 "02:ac:10:ff:01:33 10.0.0.30"
创建容器
创建一个无网络容器
sudo docker run -itd --net=none --name ctn-1 library/alpine /bin/sh
sukai@ovn-1:~$ sudo docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
ffb0b4da4866 alpine "sleep 3600d" 3 minutes ago Up 3 minutes cont-1
34670a048857 alpine "/bin/sh" 7 minutes ago Up 7 minutes ctn-1
sukai@ovn-1:~$ echo $NETNS
8608
sukai@ovn-1:~$ sudo docker exec -it ctn-1 /bin/sh
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
/ #
配置容器使用ovs internal port
NETNS=`sudo docker inspect -f '{{.State.Pid}}' ctn-1`
sudo ln -s /proc/$NETNS/ns/net /var/run/netns/$NETNS
sudo ovs-vsctl add-port br-int ctn1 -- set Interface ctn1 type=internal
sudo ip link set ctn1 netns $NETNS
sudo ovs-vsctl set Interface ctn1 external_ids:iface-id=ls1-ctn1
sudo ip netns exec $NETNS ip link set ctn1 address 02:ac:10:ff:01:33
sudo ip netns exec $NETNS ip link set dev ctn1 up
sudo ip netns exec $NETNS ip addr add 10.0.0.30/24 dev ctn1
sudo ip netns exec $NETNS ip route add default via 10.0.0.1
查看ovs信息
sukai@ovn-1:~$ sudo ovs-vsctl show
93fd4842-e0e9-4413-a99a-2c11e4180a0a
Bridge br-int
fail_mode: secure
Port ctn1
Interface ctn1
type: internal
Port vm1
Interface vm1
type: internal
Port br-int
Interface br-int
type: internal
Port ovn-28a87e-0
Interface ovn-28a87e-0
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.0.114"}
ovs_version: "2.13.5"
sukai@ovn-1:~$
验证网络
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
11: ctn1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 02:ac:10:ff:01:33 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.30/24 scope global ctn1
valid_lft forever preferred_lft forever
# ip route
default via 10.0.0.1 dev ctn1
10.0.0.0/24 dev ctn1 proto kernel scope link src 10.0.0.30
# ping -c 1 10.0.0.10
PING 10.0.0.10 (10.0.0.10) 56(84) bytes of data.
64 bytes from 10.0.0.10: icmp_seq=1 ttl=64 time=1.79 ms
--- 10.0.0.10 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.787/1.787/1.787/0.000 ms
# ping -c 1 10.0.0.1
PING 10.0.0.1 (10.0.0.1) 56(84) bytes of data.
64 bytes from 10.0.0.1: icmp_seq=1 ttl=254 time=1.47 ms
--- 10.0.0.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.472/1.472/1.472/0.000 ms
# ping -c 1 10.1.0.20
PING 10.1.0.20 (10.1.0.20) 56(84) bytes of data.
64 bytes from 10.1.0.20: icmp_seq=1 ttl=63 time=12.7 ms
--- 10.1.0.20 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 12.714/12.714/12.714/0.000 ms
#
场景三
在交换机2上添加容器接口
sudo ovn-nbctl lsp-add lswitch2 ls2-ctn2
sudo ovn-nbctl lsp-set-addresses ls2-ctn2 "02:ac:10:ff:01:34 10.1.0.40"
创建容器，通过veth pair方式使用ovs
创建一个veth对，一端接入br-int，一端接入容器veth2
sudo docker run -itd --net=none --name ctn-2 library/alpine /bin/sh
NETNS=`sudo docker inspect -f '{{.State.Pid}}' ctn-2`
sudo ln -s /proc/$NETNS/ns/net /var/run/netns/$NETNS
sudo ip link add veth8d5915b type veth peer name veth2
sudo ifconfig veth8d5915b up
sudo ovs-vsctl add-port br-int veth8d5915b
sudo ip link set veth2 netns $NETNS
sudo ovs-vsctl set Interface veth8d5915b external_ids:iface-id=ls2-ctn2
sudo ip netns exec $NETNS ip link set veth2 address 02:ac:10:ff:01:34
sudo ip netns exec $NETNS ip link set dev veth2 up
sudo ip netns exec $NETNS ip address add 10.1.0.40/24 dev veth2
sudo ip netns exec $NETNS ip route add default via 10.1.0.1
查看ovs/ovn信息
sukai@ovn-2:~$ sudo ovs-vsctl show
1ab53615-2604-4edf-ac6a-7a244803a25e
Bridge br-int
fail_mode: secure
Port ovn-6fa2da-0
Interface ovn-6fa2da-0
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.0.115"}
Port vm2
Interface vm2
type: internal
Port veth8d5915b
Interface veth8d5915b
Port br-int
Interface br-int
type: internal
ovs_version: "2.13.5"
sukai@ovn-1:~$ sudo ovn-nbctl show
switch 75a787a4-9e55-4542-acaa-e969bebfaf16 (lswitch1)
port ls1-vm1
addresses: ["02:ac:10:ff:01:30 10.0.0.10"]
port ls1-ctn1
addresses: ["02:ac:10:ff:01:33 10.0.0.30"]
port ls1-lr1
type: router
addresses: ["52:54:00:c1:68:50"]
router-port: lr-ls1
switch d85756bc-e7d3-42fb-ace1-15168e017736 (lswitch2)
port ls2-ctn2
addresses: ["02:ac:10:ff:01:34 10.1.0.40"]
port ls2-lr1
type: router
addresses: ["52:54:00:c1:68:60"]
router-port: lr-ls2
port ls2-vm2
addresses: ["02:ac:10:ff:01:31 10.1.0.20"]
router 3bccdebe-38de-450c-8e02-208eaff33398 (router)
port lr-ls2
mac: "52:54:00:c1:68:60"
networks: ["10.1.0.1/24"]
port lr-ls1
mac: "52:54:00:c1:68:50"
networks: ["10.0.0.1/24"]
sukai@ovn-1:~$ sudo ovn-sbctl show
Chassis "6fa2da75-1393-4efe-984d-0c0310b95ad3"
hostname: ovn-1
Encap geneve
ip: "192.168.0.115"
options: {csum="true"}
Port_Binding ls1-vm1
Port_Binding ls1-ctn1
Port_Binding lr-ls2
Port_Binding lr-ls1
Port_Binding ls1-lr1
Port_Binding ls2-lr1
Chassis "28a87e04-b738-4db0-a767-380091e1a752"
hostname: ovn-2
Encap geneve
ip: "192.168.0.114"
options: {csum="true"}
Port_Binding ls2-ctn2
Port_Binding ls2-vm2
验证网络
sukai@ovn-2:~$ sudo ifconfig veth8d5915b up
sukai@ovn-2:~$ sudo ip netns exec $NETNS /bin/sh
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
9: veth2@if10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 02:ac:10:ff:01:34 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.1.0.40/24 scope global veth2
valid_lft forever preferred_lft forever
# ip route
default via 10.1.0.1 dev veth2
10.1.0.0/24 dev veth2 proto kernel scope link src 10.1.0.40
# ping -c 1 10.1.0.1
PING 10.1.0.1 (10.1.0.1) 56(84) bytes of data.
64 bytes from 10.1.0.1: icmp_seq=1 ttl=254 time=4.97 ms
--- 10.1.0.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 4.965/4.965/4.965/0.000 ms
# ping -c 1 10.0.0.30
PING 10.0.0.30 (10.0.0.30) 56(84) bytes of data.
64 bytes from 10.0.0.30: icmp_seq=1 ttl=63 time=8.49 ms
--- 10.0.0.30 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 8.488/8.488/8.488/0.000 ms
总结
OVN让容器结合了虚拟化网络，容器运行平台具备了SDN能力。容器可以通过OVS Internal Port和Veth对两种方式接入网络。