由 SuKai June 22, 2022
下面从Kube-OVN代码看一下如何实现Pod和主机网络连通的,再在实验环境中按照Kube-OVN方式进行实验。
| Kube-OVN初始化ovn0
Kube-OVN在主机上添加一个OVS的internal port:ovn0,实现主机和OVS网络的通信。
1,将ovn0连接到OVS集成网桥br-int,类型为internal port,与交换机端口关联。
2,配置ovn0的MAC, IP, MTU并启动接口。
// InitNodeGateway init ovn0
func InitNodeGateway(config *Configuration) error {
...
return configureNodeNic(portName, ipAddr, gw, mac, config.MTU)
}
func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error {
ipStr := util.GetIpWithoutMask(ip)
raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--",
"set", "interface", util.NodeNic, "type=internal", "--",
"set", "interface", util.NodeNic, fmt.Sprintf("external_ids:iface-id=%s", portName),
fmt.Sprintf("external_ids:ip=%s", ipStr))
if err != nil {
klog.Errorf("failed to configure node nic %s: %v, %q", portName, err, raw)
return fmt.Errorf(raw)
}
if err = configureNic(util.NodeNic, ip, macAddr, mtu); err != nil {
return err
}
...
return nil
}
func configureNic(link, ip string, macAddr net.HardwareAddr, mtu int) error {
...
for _, addr := range ipAddMap {
if err = netlink.AddrAdd(nodeLink, &addr); err != nil {
return fmt.Errorf("can not add address %v to nic %s: %v", addr, link, err)
}
}
if err = netlink.LinkSetHardwareAddr(nodeLink, macAddr); err != nil {
return fmt.Errorf("can not set mac address to nic %s: %v", link, err)
}
if mtu > 0 {
if err = netlink.LinkSetMTU(nodeLink, mtu); err != nil {
return fmt.Errorf("can not set nic %s mtu: %v", link, err)
}
}
if nodeLink.Attrs().OperState != netlink.OperUp {
if err = netlink.LinkSetUp(nodeLink); err != nil {
return fmt.Errorf("can not set node nic %s up: %v", link, err)
}
}
return nil
}
查看Kube-OVN信息
sukai@ubuntuserver:~$ kubectl ko nbctl show
switch e0b180e4-1e4c-49e0-9bea-d5aa384196b0 (ovn-default)
port cert-manager-8568b6f9cb-wf74c.cert-manager
addresses: ["00:00:00:E9:A4:E5 10.244.0.15"]
port cert-manager-cainjector-c9c77b797-xwlgh.cert-manager
addresses: ["00:00:00:39:46:D0 10.244.0.14"]
port virt-api-748cfcdb48-qnz2r.kubevirt
addresses: ["00:00:00:9A:54:68 10.244.0.38"]
port hostpath-provisioner-csi-5mcbc.hostpath-provisioner
addresses: ["00:00:00:D0:28:68 10.244.0.18"]
port virt-operator-5fcd4ff76f-zkfdf.kubevirt
addresses: ["00:00:00:92:AD:47 10.244.1.0"]
port nginx.default
addresses: ["00:00:00:F3:E4:1C 10.244.0.5"]
port cdi-deployment-7d587548cd-w55fh.cdi
addresses: ["00:00:00:D3:7E:DF 10.244.0.34"]
port cdi-uploadproxy-6c64d5cd59-5lvkc.cdi
addresses: ["00:00:00:B2:C7:B7 10.244.0.35"]
port virt-controller-5756d69f5f-6467g.kubevirt
addresses: ["00:00:00:06:B7:19 10.244.0.39"]
port cdi-apiserver-c6cdc9489-g57k4.cdi
addresses: ["00:00:00:11:AF:5B 10.244.0.33"]
port coredns-6d4b75cb6d-dbfs4.kube-system
addresses: ["00:00:00:13:ED:58 10.244.0.37"]
port cert-manager-webhook-885b8ffcb-hrzjt.cert-manager
addresses: ["00:00:00:11:A1:8B 10.244.0.16"]
port vpc-nat-gw-default-natgw-674964fd4b-29qhn.kube-system
addresses: ["00:00:00:79:B9:3A 10.244.0.254"]
port coredns-6d4b75cb6d-568xm.kube-system
addresses: ["00:00:00:E5:7E:A5 10.244.0.36"]
port hostpath-provisioner-operator-7c8c499c77-46vk8.hostpath-provisioner
addresses: ["00:00:00:1D:58:B7 10.244.0.17"]
port kube-ovn-pinger-tb5t7.kube-system
addresses: ["00:00:00:A7:BB:47 10.244.0.4"]
port virt-handler-jr8m5.kubevirt
addresses: ["00:00:00:10:6F:DA 10.244.1.1"]
port virt-controller-5756d69f5f-nfjxn.kubevirt
addresses: ["00:00:00:70:72:C4 10.244.0.255"]
port cdi-operator-997bdf879-ht6kx.cdi
addresses: ["00:00:00:DD:1E:29 10.244.0.32"]
port virt-operator-5fcd4ff76f-7w25s.kubevirt
addresses: ["00:00:00:AA:B7:7C 10.244.0.40"]
port ovn-default-ovn-cluster
type: router
router-port: ovn-cluster-ovn-default
switch 0629dbe7-beab-45f1-8d42-0ce2f2a89a79 (mail263)
port localnet.mail263
type: localnet
addresses: ["unknown"]
port mailserver.mail263
addresses: ["00:00:00:68:56:C6 172.16.3.250"]
port ubuntu-a.mail263
addresses: ["00:00:00:d8:95:33 172.16.3.201"]
port webserver.mail263
addresses: ["00:00:00:68:56:C9 172.16.3.210"]
port ubuntu-c.mail263
addresses: ["00:00:00:1F:C5:8F 172.16.3.203"]
port ubuntu-b.mail263
addresses: ["00:00:00:f1:b6:40 172.16.3.202"]
port appserver.mail263
addresses: ["00:00:00:68:56:C4 172.16.3.211"]
switch c1908873-7721-425d-8c31-be6595119dcb (a-subnet1)
port tenant-a-nginx.tenant-a
addresses: ["00:00:00:DE:B6:8A 10.0.1.1"]
port vpc-nat-gw-a-natgw-84b4c776f8-fl6hl.kube-system
addresses: ["00:00:00:F7:24:82 10.0.1.254"]
port a-subnet1-tenant-a
type: router
router-port: tenant-a-a-subnet1
port vm-ubuntu.tenant-a
addresses: ["00:00:00:16:9B:42 10.0.1.6"]
port ubuntu.tenant-a
addresses: ["00:00:00:D8:03:11 10.0.1.2"]
port dns-test.tenant-a
addresses: ["00:00:00:03:F4:06 10.0.1.7"]
switch 73e8db33-6e71-4f1e-af8a-516afaf13019 (join)
port node-ubuntuserver
addresses: ["00:00:00:B0:35:E1 100.64.0.2"]
port join-ovn-cluster
type: router
router-port: ovn-cluster-join
router e2c95d29-f93b-4f70-95b7-0f1c5e3c982a (ovn-cluster)
port ovn-cluster-join
mac: "00:00:00:E2:21:46"
networks: ["100.64.0.1/16"]
port ovn-cluster-ovn-default
mac: "00:00:00:C9:39:03"
networks: ["10.244.0.1/16"]
port ovn-cluster-ovn-vpc-external-network
mac: "00:00:00:06:51:7F"
networks: ["172.16.4.2/24"]
router b43995f3-09a9-4a43-bac1-d31ebce6ee80 (tenant-a)
port tenant-a-a-subnet1
mac: "00:00:00:20:FE:62"
networks: ["10.0.1.254/24"]
| 实验
创建交换机和路由器端口
分配一个100.64.0.0/16子网用于每个主机节点接入OVS网络
# 创建交换机join
sudo ovn-nbctl ls-add join
# 路由器router上添加端口ovn-cluster-join
sudo ovn-nbctl lrp-add router ovn-cluster-join 00:00:00:E2:21:46 100.64.0.1/16
# 交换机join上添加端口join-ovn-cluster, 端口类型为router, 与路由器端口关联
sudo ovn-nbctl lsp-add join join-ovn-cluster
sudo ovn-nbctl lsp-set-type join-ovn-cluster router
sudo ovn-nbctl lsp-set-addresses join-ovn-cluster 00:00:00:E2:21:46
sudo ovn-nbctl lsp-set-options join-ovn-cluster router-port=ovn-cluster-join
主机节点添加OVS Internal Port接口
交换机上添加两台主机的端口,在两台主机上添加ovs internal port接口。
# 交换机添加主机ovn-1的端口node-ovn-1
sudo ovn-nbctl lsp-add join node-ovn-1
sudo ovn-nbctl lsp-set-addresses node-ovn-1 "00:00:00:B0:35:E2 100.64.0.2"
sudo ovn-nbctl lsp-add join node-ovn-2
sudo ovn-nbctl lsp-set-addresses node-ovn-2 "00:00:00:B0:35:E3 100.64.0.3"
# ovn-1节点
# OVS集成网桥br-int下添加ovn0接口,接口与交换机端口node-ovn-1关联
sudo ovs-vsctl add-port br-int ovn0 -- set interface ovn0 type=internal -- set interface ovn0 external_ids:iface-id=node-ovn-1
# 配置ovn0 IP地址, MAC地址, UP状态
sudo ip link set ovn0 address 00:00:00:B0:35:E2
sudo ip link set dev ovn0 up
sudo ip addr add 100.64.0.2/16 dev ovn0
# ovn-2节点
# OVS集成网桥br-int下添加ovn0接口,接口与交换机端口node-ovn-2关联
sudo ovs-vsctl add-port br-int ovn0 -- set interface ovn0 type=internal -- set interface ovn0 external_ids:iface-id=node-ovn-2
# 配置ovn0 IP地址, MAC地址, UP状态
sudo ip link set ovn0 address 00:00:00:B0:35:E3
sudo ip link set dev ovn0 up
sudo ip addr add 100.64.0.3/16 dev ovn0
路由器设置默认路由
sudo ovn-nbctl lr-route-add router "0.0.0.0/0" 100.64.0.1
主机上设置OVS网络路由
sudo ip route add 10.0.0.0/24 via 100.64.0.1
sudo ip route add 10.1.0.0/24 via 100.64.0.1
验证网络
查看OVN信息
sukai@ovn-1:~$ sudo ovn-nbctl show
switch d85756bc-e7d3-42fb-ace1-15168e017736 (lswitch2)
port ls2-ctn2
addresses: ["02:ac:10:ff:01:34 10.1.0.40"]
port ls2-lr1
type: router
addresses: ["52:54:00:c1:68:60"]
router-port: lr-ls2
port ls2-vm2
addresses: ["02:ac:10:ff:01:31 10.1.0.20"]
switch 75a787a4-9e55-4542-acaa-e969bebfaf16 (lswitch1)
port ls1-vm1
addresses: ["02:ac:10:ff:01:30 10.0.0.10"]
port ls1-ctn1
addresses: ["02:ac:10:ff:01:33 10.0.0.30"]
port ls1-lr1
type: router
addresses: ["52:54:00:c1:68:50"]
router-port: lr-ls1
switch 5e9e0b6d-332a-412e-b268-dd5dd7623975 (join)
port node-ovn-2
addresses: ["00:00:00:B0:35:E3 100.64.0.3"]
port node-ovn-1
addresses: ["00:00:00:B0:35:E2 100.64.0.2"]
port join-ovn-cluster
type: router
addresses: ["00:00:00:E2:21:46"]
router-port: ovn-cluster-join
router 3bccdebe-38de-450c-8e02-208eaff33398 (router)
port lr-ls2
mac: "52:54:00:c1:68:60"
networks: ["10.1.0.1/24"]
port ovn-cluster-join
mac: "00:00:00:E2:21:46"
networks: ["100.64.0.1/16"]
port lr-ls1
mac: "52:54:00:c1:68:50"
networks: ["10.0.0.1/24"]
sukai@ovn-1:~$
sukai@ovn-1:~$ sudo ovn-nbctl lr-route-list router
IPv4 Routes
0.0.0.0/0 100.64.0.1 dst-ip
sukai@ovn-1:~$
查看主机网络
sukai@ovn-1:~$ ip route
default via 192.168.0.1 dev ens33 proto dhcp src 192.168.0.115 metric 100
10.0.0.0/24 via 100.64.0.1 dev ovn0
10.1.0.0/24 via 100.64.0.1 dev ovn0
100.64.0.0/16 dev ovn0 proto kernel scope link src 100.64.0.2
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
192.168.0.0/24 dev ens33 proto kernel scope link src 192.168.0.115
192.168.0.1 dev ens33 proto dhcp scope link src 192.168.0.115 metric 100
192.168.100.0/24 dev ens34 proto kernel scope link src 192.168.100.10
sukai@ovn-1:~$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:d4:b7:e1 brd ff:ff:ff:ff:ff:ff
inet 192.168.0.115/24 brd 192.168.0.255 scope global dynamic ens33
valid_lft 52065sec preferred_lft 52065sec
inet6 fe80::20c:29ff:fed4:b7e1/64 scope link
valid_lft forever preferred_lft forever
3: ens34: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:d4:b7:eb brd ff:ff:ff:ff:ff:ff
inet 192.168.100.10/24 brd 192.168.100.255 scope global ens34
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fed4:b7eb/64 scope link
valid_lft forever preferred_lft forever
4: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 32:90:ed:d9:bf:1d brd ff:ff:ff:ff:ff:ff
6: br-int: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 00:00:00:b0:35:e2 brd ff:ff:ff:ff:ff:ff
inet6 fe80::b432:20ff:fea4:db46/64 scope link
valid_lft forever preferred_lft forever
8: genev_sys_6081: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65000 qdisc noqueue master ovs-system state UNKNOWN group default qlen 1000
link/ether 66:e9:3a:63:e0:74 brd ff:ff:ff:ff:ff:ff
inet6 fe80::64e9:3aff:fe63:e074/64 scope link
valid_lft forever preferred_lft forever
9: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
link/ether 02:42:0b:ab:c8:7b brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
10: ovn0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 00:00:00:b0:35:e2 brd ff:ff:ff:ff:ff:ff
inet 100.64.0.2/16 scope global ovn0
valid_lft forever preferred_lft forever
inet6 fe80::200:ff:feb0:35e2/64 scope link
valid_lft forever preferred_lft forever
主机和容器互ping
主机ping另一台主机上的容器
sukai@ovn-1:~$ ping 10.1.0.40 -c 1
PING 10.1.0.40 (10.1.0.40) 56(84) bytes of data.
64 bytes from 10.1.0.40: icmp_seq=1 ttl=63 time=2.03 ms
--- 10.1.0.40 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.025/2.025/2.025/0.000 ms
sukai@ovn-1:~$
容器ping另一台主机
sukai@ovn-1:~$ sudo docker exec -it ctn-1 /bin/sh
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
5: ctn1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
link/ether 02:ac:10:ff:01:33 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.30/24 scope global ctn1
valid_lft forever preferred_lft forever
/ # ping 192.168.0.114 -c 1
PING 192.168.0.114 (192.168.0.114): 56 data bytes
64 bytes from 192.168.0.114: seq=0 ttl=63 time=5.360 ms
--- 192.168.0.114 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 5.360/5.360/5.360 ms
/ # traceroute 192.168.0.114
traceroute to 192.168.0.114 (192.168.0.114), 30 hops max, 46 byte packets
1 10.0.0.1 (10.0.0.1) 1.241 ms 2.083 ms 4.653 ms
2 192.168.0.114 (192.168.0.114) 4.792 ms 2.884 ms 1.988 ms
/ #
通过每台主机上创建一个ovn0接口,经过路由,将主机网络和ovs实现网络互通。