Kube-OVN如何实现Pod和主机网络连通

SuKai June 22, 2022

下面从Kube-OVN代码看一下如何实现Pod和主机网络连通的,再在实验环境中按照Kube-OVN方式进行实验。

| Kube-OVN初始化ovn0

Kube-OVN在主机上添加一个OVS的internal port:ovn0,实现主机和OVS网络的通信。

1,将ovn0连接到OVS集成网桥br-int,类型为internal port,与交换机端口关联。

2,配置ovn0的MAC, IP, MTU并启动接口。

// InitNodeGateway init ovn0
func InitNodeGateway(config *Configuration) error {
...
	return configureNodeNic(portName, ipAddr, gw, mac, config.MTU)
}

func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error {
	ipStr := util.GetIpWithoutMask(ip)
	raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--",
		"set", "interface", util.NodeNic, "type=internal", "--",
		"set", "interface", util.NodeNic, fmt.Sprintf("external_ids:iface-id=%s", portName),
		fmt.Sprintf("external_ids:ip=%s", ipStr))
	if err != nil {
		klog.Errorf("failed to configure node nic %s: %v, %q", portName, err, raw)
		return fmt.Errorf(raw)
	}

	if err = configureNic(util.NodeNic, ip, macAddr, mtu); err != nil {
		return err
	}
...
	return nil
}

func configureNic(link, ip string, macAddr net.HardwareAddr, mtu int) error {
...
	for _, addr := range ipAddMap {
		if err = netlink.AddrAdd(nodeLink, &addr); err != nil {
			return fmt.Errorf("can not add address %v to nic %s: %v", addr, link, err)
		}
	}

	if err = netlink.LinkSetHardwareAddr(nodeLink, macAddr); err != nil {
		return fmt.Errorf("can not set mac address to nic %s: %v", link, err)
	}

	if mtu > 0 {
		if err = netlink.LinkSetMTU(nodeLink, mtu); err != nil {
			return fmt.Errorf("can not set nic %s mtu: %v", link, err)
		}
	}

	if nodeLink.Attrs().OperState != netlink.OperUp {
		if err = netlink.LinkSetUp(nodeLink); err != nil {
			return fmt.Errorf("can not set node nic %s up: %v", link, err)
		}
	}
	return nil
}

查看Kube-OVN信息

sukai@ubuntuserver:~$ kubectl ko nbctl show
switch e0b180e4-1e4c-49e0-9bea-d5aa384196b0 (ovn-default)
    port cert-manager-8568b6f9cb-wf74c.cert-manager
        addresses: ["00:00:00:E9:A4:E5 10.244.0.15"]
    port cert-manager-cainjector-c9c77b797-xwlgh.cert-manager
        addresses: ["00:00:00:39:46:D0 10.244.0.14"]
    port virt-api-748cfcdb48-qnz2r.kubevirt
        addresses: ["00:00:00:9A:54:68 10.244.0.38"]
    port hostpath-provisioner-csi-5mcbc.hostpath-provisioner
        addresses: ["00:00:00:D0:28:68 10.244.0.18"]
    port virt-operator-5fcd4ff76f-zkfdf.kubevirt
        addresses: ["00:00:00:92:AD:47 10.244.1.0"]
    port nginx.default
        addresses: ["00:00:00:F3:E4:1C 10.244.0.5"]
    port cdi-deployment-7d587548cd-w55fh.cdi
        addresses: ["00:00:00:D3:7E:DF 10.244.0.34"]
    port cdi-uploadproxy-6c64d5cd59-5lvkc.cdi
        addresses: ["00:00:00:B2:C7:B7 10.244.0.35"]
    port virt-controller-5756d69f5f-6467g.kubevirt
        addresses: ["00:00:00:06:B7:19 10.244.0.39"]
    port cdi-apiserver-c6cdc9489-g57k4.cdi
        addresses: ["00:00:00:11:AF:5B 10.244.0.33"]
    port coredns-6d4b75cb6d-dbfs4.kube-system
        addresses: ["00:00:00:13:ED:58 10.244.0.37"]
    port cert-manager-webhook-885b8ffcb-hrzjt.cert-manager
        addresses: ["00:00:00:11:A1:8B 10.244.0.16"]
    port vpc-nat-gw-default-natgw-674964fd4b-29qhn.kube-system
        addresses: ["00:00:00:79:B9:3A 10.244.0.254"]
    port coredns-6d4b75cb6d-568xm.kube-system
        addresses: ["00:00:00:E5:7E:A5 10.244.0.36"]
    port hostpath-provisioner-operator-7c8c499c77-46vk8.hostpath-provisioner
        addresses: ["00:00:00:1D:58:B7 10.244.0.17"]
    port kube-ovn-pinger-tb5t7.kube-system
        addresses: ["00:00:00:A7:BB:47 10.244.0.4"]
    port virt-handler-jr8m5.kubevirt
        addresses: ["00:00:00:10:6F:DA 10.244.1.1"]
    port virt-controller-5756d69f5f-nfjxn.kubevirt
        addresses: ["00:00:00:70:72:C4 10.244.0.255"]
    port cdi-operator-997bdf879-ht6kx.cdi
        addresses: ["00:00:00:DD:1E:29 10.244.0.32"]
    port virt-operator-5fcd4ff76f-7w25s.kubevirt
        addresses: ["00:00:00:AA:B7:7C 10.244.0.40"]
    port ovn-default-ovn-cluster
        type: router
        router-port: ovn-cluster-ovn-default
switch 0629dbe7-beab-45f1-8d42-0ce2f2a89a79 (mail263)
    port localnet.mail263
        type: localnet
        addresses: ["unknown"]
    port mailserver.mail263
        addresses: ["00:00:00:68:56:C6 172.16.3.250"]
    port ubuntu-a.mail263
        addresses: ["00:00:00:d8:95:33 172.16.3.201"]
    port webserver.mail263
        addresses: ["00:00:00:68:56:C9 172.16.3.210"]
    port ubuntu-c.mail263
        addresses: ["00:00:00:1F:C5:8F 172.16.3.203"]
    port ubuntu-b.mail263
        addresses: ["00:00:00:f1:b6:40 172.16.3.202"]
    port appserver.mail263
        addresses: ["00:00:00:68:56:C4 172.16.3.211"]
switch c1908873-7721-425d-8c31-be6595119dcb (a-subnet1)
    port tenant-a-nginx.tenant-a
        addresses: ["00:00:00:DE:B6:8A 10.0.1.1"]
    port vpc-nat-gw-a-natgw-84b4c776f8-fl6hl.kube-system
        addresses: ["00:00:00:F7:24:82 10.0.1.254"]
    port a-subnet1-tenant-a
        type: router
        router-port: tenant-a-a-subnet1
    port vm-ubuntu.tenant-a
        addresses: ["00:00:00:16:9B:42 10.0.1.6"]
    port ubuntu.tenant-a
        addresses: ["00:00:00:D8:03:11 10.0.1.2"]
    port dns-test.tenant-a
        addresses: ["00:00:00:03:F4:06 10.0.1.7"]
switch 73e8db33-6e71-4f1e-af8a-516afaf13019 (join)
    port node-ubuntuserver
        addresses: ["00:00:00:B0:35:E1 100.64.0.2"]
    port join-ovn-cluster
        type: router
        router-port: ovn-cluster-join
router e2c95d29-f93b-4f70-95b7-0f1c5e3c982a (ovn-cluster)
    port ovn-cluster-join
        mac: "00:00:00:E2:21:46"
        networks: ["100.64.0.1/16"]
    port ovn-cluster-ovn-default
        mac: "00:00:00:C9:39:03"
        networks: ["10.244.0.1/16"]
    port ovn-cluster-ovn-vpc-external-network
        mac: "00:00:00:06:51:7F"
        networks: ["172.16.4.2/24"]
router b43995f3-09a9-4a43-bac1-d31ebce6ee80 (tenant-a)
    port tenant-a-a-subnet1
        mac: "00:00:00:20:FE:62"
        networks: ["10.0.1.254/24"]

| 实验

创建交换机和路由器端口

分配一个100.64.0.0/16子网用于每个主机节点接入OVS网络

# 创建交换机join
sudo ovn-nbctl ls-add join

# 路由器router上添加端口ovn-cluster-join
sudo ovn-nbctl lrp-add router ovn-cluster-join 00:00:00:E2:21:46 100.64.0.1/16

# 交换机join上添加端口join-ovn-cluster, 端口类型为router, 与路由器端口关联
sudo ovn-nbctl lsp-add join join-ovn-cluster
sudo ovn-nbctl lsp-set-type join-ovn-cluster router
sudo ovn-nbctl lsp-set-addresses join-ovn-cluster 00:00:00:E2:21:46
sudo ovn-nbctl lsp-set-options join-ovn-cluster router-port=ovn-cluster-join

主机节点添加OVS Internal Port接口

交换机上添加两台主机的端口,在两台主机上添加ovs internal port接口。

# 交换机添加主机ovn-1的端口node-ovn-1
sudo ovn-nbctl lsp-add join node-ovn-1
sudo ovn-nbctl lsp-set-addresses node-ovn-1 "00:00:00:B0:35:E2 100.64.0.2"
sudo ovn-nbctl lsp-add join node-ovn-2
sudo ovn-nbctl lsp-set-addresses node-ovn-2 "00:00:00:B0:35:E3 100.64.0.3"

# ovn-1节点
# OVS集成网桥br-int下添加ovn0接口,接口与交换机端口node-ovn-1关联
sudo ovs-vsctl add-port br-int ovn0 -- set interface ovn0 type=internal -- set interface ovn0 external_ids:iface-id=node-ovn-1

# 配置ovn0 IP地址, MAC地址, UP状态
sudo ip link set ovn0 address 00:00:00:B0:35:E2
sudo ip link set dev ovn0 up
sudo ip addr add 100.64.0.2/16 dev ovn0


# ovn-2节点
# OVS集成网桥br-int下添加ovn0接口,接口与交换机端口node-ovn-2关联
sudo ovs-vsctl add-port br-int ovn0 -- set interface ovn0 type=internal -- set interface ovn0 external_ids:iface-id=node-ovn-2

# 配置ovn0 IP地址, MAC地址, UP状态
sudo ip link set ovn0 address 00:00:00:B0:35:E3
sudo ip link set dev ovn0 up
sudo ip addr add 100.64.0.3/16 dev ovn0

路由器设置默认路由

sudo ovn-nbctl lr-route-add router "0.0.0.0/0" 100.64.0.1

主机上设置OVS网络路由

sudo ip route add 10.0.0.0/24 via 100.64.0.1
sudo ip route add 10.1.0.0/24 via 100.64.0.1

验证网络

查看OVN信息

sukai@ovn-1:~$ sudo ovn-nbctl show
switch d85756bc-e7d3-42fb-ace1-15168e017736 (lswitch2)
    port ls2-ctn2
        addresses: ["02:ac:10:ff:01:34 10.1.0.40"]
    port ls2-lr1
        type: router
        addresses: ["52:54:00:c1:68:60"]
        router-port: lr-ls2
    port ls2-vm2
        addresses: ["02:ac:10:ff:01:31 10.1.0.20"]
switch 75a787a4-9e55-4542-acaa-e969bebfaf16 (lswitch1)
    port ls1-vm1
        addresses: ["02:ac:10:ff:01:30 10.0.0.10"]
    port ls1-ctn1
        addresses: ["02:ac:10:ff:01:33 10.0.0.30"]
    port ls1-lr1
        type: router
        addresses: ["52:54:00:c1:68:50"]
        router-port: lr-ls1
switch 5e9e0b6d-332a-412e-b268-dd5dd7623975 (join)
    port node-ovn-2
        addresses: ["00:00:00:B0:35:E3 100.64.0.3"]
    port node-ovn-1
        addresses: ["00:00:00:B0:35:E2 100.64.0.2"]
    port join-ovn-cluster
        type: router
        addresses: ["00:00:00:E2:21:46"]
        router-port: ovn-cluster-join
router 3bccdebe-38de-450c-8e02-208eaff33398 (router)
    port lr-ls2
        mac: "52:54:00:c1:68:60"
        networks: ["10.1.0.1/24"]
    port ovn-cluster-join
        mac: "00:00:00:E2:21:46"
        networks: ["100.64.0.1/16"]
    port lr-ls1
        mac: "52:54:00:c1:68:50"
        networks: ["10.0.0.1/24"]
sukai@ovn-1:~$

sukai@ovn-1:~$ sudo ovn-nbctl lr-route-list router
IPv4 Routes
                0.0.0.0/0                100.64.0.1 dst-ip
sukai@ovn-1:~$

查看主机网络

sukai@ovn-1:~$ ip route
default via 192.168.0.1 dev ens33 proto dhcp src 192.168.0.115 metric 100
10.0.0.0/24 via 100.64.0.1 dev ovn0
10.1.0.0/24 via 100.64.0.1 dev ovn0
100.64.0.0/16 dev ovn0 proto kernel scope link src 100.64.0.2
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
192.168.0.0/24 dev ens33 proto kernel scope link src 192.168.0.115
192.168.0.1 dev ens33 proto dhcp scope link src 192.168.0.115 metric 100
192.168.100.0/24 dev ens34 proto kernel scope link src 192.168.100.10
sukai@ovn-1:~$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:d4:b7:e1 brd ff:ff:ff:ff:ff:ff
    inet 192.168.0.115/24 brd 192.168.0.255 scope global dynamic ens33
       valid_lft 52065sec preferred_lft 52065sec
    inet6 fe80::20c:29ff:fed4:b7e1/64 scope link
       valid_lft forever preferred_lft forever
3: ens34: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:d4:b7:eb brd ff:ff:ff:ff:ff:ff
    inet 192.168.100.10/24 brd 192.168.100.255 scope global ens34
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fed4:b7eb/64 scope link
       valid_lft forever preferred_lft forever
4: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 32:90:ed:d9:bf:1d brd ff:ff:ff:ff:ff:ff
6: br-int: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 00:00:00:b0:35:e2 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::b432:20ff:fea4:db46/64 scope link
       valid_lft forever preferred_lft forever
8: genev_sys_6081: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65000 qdisc noqueue master ovs-system state UNKNOWN group default qlen 1000
    link/ether 66:e9:3a:63:e0:74 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::64e9:3aff:fe63:e074/64 scope link
       valid_lft forever preferred_lft forever
9: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
    link/ether 02:42:0b:ab:c8:7b brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever
10: ovn0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 00:00:00:b0:35:e2 brd ff:ff:ff:ff:ff:ff
    inet 100.64.0.2/16 scope global ovn0
       valid_lft forever preferred_lft forever
    inet6 fe80::200:ff:feb0:35e2/64 scope link
       valid_lft forever preferred_lft forever

主机和容器互ping

主机ping另一台主机上的容器

sukai@ovn-1:~$ ping 10.1.0.40 -c 1
PING 10.1.0.40 (10.1.0.40) 56(84) bytes of data.
64 bytes from 10.1.0.40: icmp_seq=1 ttl=63 time=2.03 ms

--- 10.1.0.40 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.025/2.025/2.025/0.000 ms
sukai@ovn-1:~$

容器ping另一台主机

sukai@ovn-1:~$ sudo docker exec -it ctn-1 /bin/sh
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
5: ctn1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN qlen 1000
    link/ether 02:ac:10:ff:01:33 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.30/24 scope global ctn1
       valid_lft forever preferred_lft forever
/ # ping 192.168.0.114 -c 1
PING 192.168.0.114 (192.168.0.114): 56 data bytes
64 bytes from 192.168.0.114: seq=0 ttl=63 time=5.360 ms

--- 192.168.0.114 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 5.360/5.360/5.360 ms
/ # traceroute 192.168.0.114
traceroute to 192.168.0.114 (192.168.0.114), 30 hops max, 46 byte packets
 1  10.0.0.1 (10.0.0.1)  1.241 ms  2.083 ms  4.653 ms
 2  192.168.0.114 (192.168.0.114)  4.792 ms  2.884 ms  1.988 ms
/ #

通过每台主机上创建一个ovn0接口,经过路由,将主机网络和ovs实现网络互通。