介绍几种 Pod 测试网络方法
在使用中Pod网络故障，一般都需要排查一下POD与节点直接通信问题
[root@kubernetes ~]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
busybox 1/1 Running 27 (22m ago) 4d23h 172.30.0.34 kubernetes <none> <none>
查看 Pod 网卡的虚拟地址对
方法一
# 通过进入 Pod 发现网卡是 `eth0@if8` 这个是虚拟网卡地址对，包含了一对信息
[root@kubernetes ~]# kubectl exec -it busybox sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
Defaulted container "busybox" out of: busybox, network-check (init)
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0@if8: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue
link/ether ee:3d:45:76:00:d7 brd ff:ff:ff:ff:ff:ff
inet 172.30.0.34/24 brd 172.30.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::ec3d:45ff:fe76:d7/64 scope link
valid_lft forever preferred_lft forever
/ #
# if8 意味着在Node 节点第 8 个网卡
[root@kubernetes ~]# ip a |grep 8
...
link/ether 1e:56:14:dc:f6:bd brd ff:ff:ff:ff:ff:ff link-netns cni-31146746-06b3-1147-4335-894a93765ef0
inet6 fe80::1c56:14ff:fedc:f6bd/64 scope link
8: vetha15924f4@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master cni0 state UP group default
inet6 fe80::f4d1:eaff:fe43:6cd4/64 scope link
# 再打开一个新的窗口，使用 tcpdump 抓 vetha15924f4 网卡包，并且在 busybox 容器 ping，可以证实这是一个地址对
# 尽管 ping 不通，但确实发包了
/ # ping 1.2.3.4
PING 1.2.3.4 (1.2.3.4): 56 data bytes
[root@kubernetes ~]# tcpdump -enp -i vetha15924f4 icmp and dst 1.2.3.4
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on vetha15924f4, link-type EN10MB (Ethernet), capture size 262144 bytes
14:50:08.828832 ee:3d:45:76:00:d7 > ba:6f:3e:c4:7d:ae, ethertype IPv4 (0x0800), length 98: 172.30.0.34 > 1.2.3.4: ICMP echo request, id 25, seq 26, length 64
14:50:09.835225 ee:3d:45:76:00:d7 > ba:6f:3e:c4:7d:ae, ethertype IPv4 (0x0800), length 98: 172.30.0.34 > 1.2.3.4: ICMP echo request, id 25, seq 27, length 64
14:50:10.843519 ee:3d:45:76:00:d7 > ba:6f:3e:c4:7d:ae, ethertype IPv4 (0x0800), length 98: 172.30.0.34 > 1.2.3.4: ICMP echo request, id 25, seq 28, length 64
方法二
# 通过进入 Pod 查看网卡设备的 iflink 查找到网卡的地址对端
[root@kubernetes ~]# kubectl exec -it busybox sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
Defaulted container "busybox" out of: busybox, network-check (init)
~ # cat /sys/class/net/eth0/iflink
8
# 进入 Node 节点查看网卡
[root@kubernetes ~]# ip link list
...
8: vetha15924f4@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master cni0 state UP mode DEFAULT group default
link/ether f6:d1:ea:43:6c:d4 brd ff:ff:ff:ff:ff:ff link-netns cni-796a359b-cf07-3d9e-012a-b3ade24e592a
# 同理查看 Node 节点的 iflink，也可知道网卡在 pod 中序号
[root@kubernetes ~]# cat /sys/class/net/vetha15924f4/iflink
2
/ # ip a
...
2: eth0@if8: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue
link/ether ee:3d:45:76:00:d7 brd ff:ff:ff:ff:ff:ff
inet 172.30.0.34/24 brd 172.30.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::ec3d:45ff:fe76:d7/64 scope link
valid_lft forever preferred_lft forever
查看网络命名空间
方法一
# 有时候 Pod 是没有 sh、bash 可以进入，但是有需要在 Pod 中发送请求，这个时候就需要进去它的网络命名空间执行命令
# Pod 本质上是在 Node 节点执行进程，只不过有 namespace 进行了资源隔离，cgroup 进行了资源限制
# 在 Node 节点查看 Pod 怎么执行的进程命令，可以通过 inspect 查看，这里是 containerd 运行时，所以通过 crictl 查看
[root@kubernetes ~]# crictl ps -a |grep busybox
6f0359ebdc325 27a71e19c9562 56 minutes ago Running busybox 27 02c4081825556 busybox
[root@kubernetes ~]# crictl inspect 6f0359ebdc325
{
...
"image": {
"image": "sha256:27a71e19c95622dce4d60d4a3760707495c9875f5c5322c5bd535214799593ce"
},
"command": [
"sleep",
"3600"
],
...
}
# 观察发现 command 是 sleep 3600，可以通过 ps -ef 查看到在 Node 节点进程号
[root@kubernetes ~]# ps -ef |grep sleep
root 3995 2838 0 Dec17 ? 00:00:00 sleep infinity
root 573426 6321 0 14:20 ? 00:00:00 sleep 3600
root 595414 581429 0 15:19 pts/0 00:00:00 grep --color=auto sleep
# 通过 nsenter 进入进程网络命名空间
[root@kubernetes ~]# nsenter -n -t 595647
[root@kubernetes ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0@if8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether ee:3d:45:76:00:d7 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 172.30.0.34/24 brd 172.30.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::ec3d:45ff:fe76:d7/64 scope link
valid_lft forever preferred_lft forever
[root@kubernetes ~]# ping 1.2.3.4
PING 1.2.3.4 (1.2.3.4) 56(84) bytes of data.
# 可以查看到进入网络命名空间执行 ping 也有效果
[root@kubernetes ~]# tcpdump -enp -i vetha15924f4 icmp and dst 1.2.3.4
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on vetha15924f4, link-type EN10MB (Ethernet), capture size 262144 bytes
15:21:15.890030 ee:3d:45:76:00:d7 > ba:6f:3e:c4:7d:ae, ethertype IPv4 (0x0800), length 98: 172.30.0.34 > 1.2.3.4: ICMP echo request, id 6175, seq 6, length 64
15:21:16.918592 ee:3d:45:76:00:d7 > ba:6f:3e:c4:7d:ae, ethertype IPv4 (0x0800), length 98: 172.30.0.34 > 1.2.3.4: ICMP echo request, id 6175, seq 7, length 64
方法二
# Node节点存在了所有网络命名空间，只需要知道 Pod IP 在哪个命名空间
[root@kubernetes ~]# ip netns list
cni-47c73196-c985-ebc7-df28-98c454db5602 (id: 18)
cni-144f32ab-8a3e-5524-eb62-0ab4332a1f0d (id: 24)
cni-7eaabfd4-1feb-ddbe-07ca-96b908d51b6c (id: 22)
cni-8ec55ffc-2be0-5bd6-a4ac-3660ab572d6d (id: 23)
cni-16d32417-feb0-ffa8-9920-5f8ce77cd7fa (id: 20)
cni-e3be713b-d61e-a60b-a6f8-55ced9137c32 (id: 11)
cni-eff5d250-08b8-928b-f35c-2738fef7e2b4 (id: 13)
cni-0ae2e9c5-5cbb-7d26-884f-b75571a80012 (id: 12)
cni-2c54fd12-4389-cac3-b3d9-dda8b8ac01db (id: 21)
cni-1eaf1985-b315-cc90-c578-66bf81e0b779 (id: 16)
cni-b840db0c-3022-a1b1-64f9-ec2c971ff921 (id: 14)
cni-6c368b6c-463e-e91f-2ce3-79ace6b887a3 (id: 15)
cni-fb424ad4-87dc-de40-d718-184776286e79 (id: 17)
cni-9922b196-a502-2c07-9930-ce0dfc299b7c (id: 19)
cni-6ff965cc-5c97-b094-20df-ce01ada8d72f (id: 8)
cni-29c5949b-1e3f-b4e7-7960-ce90cb754aab (id: 10)
cni-a0479552-2232-95b0-64f1-b6640f4149ee (id: 7)
cni-42b2a460-018e-13e7-1b48-8ba5ba560dfe (id: 9)
cni-6dedaa32-4625-9b7b-b485-6c6032d42545 (id: 6)
cni-f7eb1482-da7b-c086-2c58-1914c6b989c3 (id: 5)
cni-b2be2273-792e-7741-e6fe-d03b1784f4c8 (id: 4)
cni-7614fb13-c67c-ee61-39bf-1ba9bf51dcb7 (id: 3)
cni-796a359b-cf07-3d9e-012a-b3ade24e592a (id: 2)
cni-31146746-06b3-1147-4335-894a93765ef0 (id: 1)
cni-4527c9c8-16ba-c90e-f369-3ea130ed50b0 (id: 0)
# 使用了一个 for 循环，虽然每个命名空间都执行命令，但也正确找到了命名空间
[root@kubernetes ~]# for i in $(ip netns list | awk -F ' ' '{print$1}') ; do ip netns exec $i ip a |grep 172.30.0.34 && echo $i ;done
inet 172.30.0.34/24 brd 172.30.0.255 scope global eth0
cni-796a359b-cf07-3d9e-012a-b3ade24e592a
[root@kubernetes ~]# ip netns exec cni-796a359b-cf07-3d9e-012a-b3ade24e592a ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0@if8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
link/ether ee:3d:45:76:00:d7 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 172.30.0.34/24 brd 172.30.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::ec3d:45ff:fe76:d7/64 scope link
valid_lft forever preferred_lft forever
[root@kubernetes ~]# ip netns exec cni-796a359b-cf07-3d9e-012a-b3ade24e592a ping 1.2.3.4
PING 1.2.3.4 (1.2.3.4) 56(84) bytes of data.