What is a pause container? This acts as a parent container to other containers in the POD and is responsible for bringing up the network namespace.
[root@netlab6 etc]# kubectl create namespace demo
namespace/demo created
[root@netlab6 etc]# kubectl -n demo create deployment netops --image=baburaj/netops
deployment.apps/netops created
[root@netlab6 etc]# kubectl -n demo get po -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
netops-f49b8bdb7-fwmzb 1/1 Running 0 29s 192.168.191.65 netlab8
[root@netlab8 ~]# docker ps |grep netops
e5a08108d0d5 baburaj/netops "/usr/bin/nginx-star…" 5 minutes ago Up 5 minutes k8s_netops_netops-f49b8bdb7-fwmzb_demo_a65ea2e7-41c3-4be4-8cab-e40fd5955dca_0
35281cd4a860 k8s.gcr.io/pause:3.6 "/pause" 5 minutes ago Up 5 minutes k8s_POD_netops-f49b8bdb7-fwmzb_demo_a65ea2e7-41c3-4be4-8cab-e40fd5955dca_0
[root@netlab8 ~]# docker inspect k8s_netops_netops-f49b8bdb7-fwmzb_demo_a65ea2e7-41c3-4be4-8cab-e40fd5955dca_0 | jq '.[].State.Pid'
18484
[root@netlab8 ~]# nsenter -t 18484 -a
[root@netops-f49b8bdb7-fwmzb /]# ip addr show
1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: tunl0@NONE: mtu 1480 qdisc noop state DOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if7: mtu 1480 qdisc noqueue state UP group default
link/ether 5e:bf:1f:df:67:ce brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.191.65/32 scope global eth0
valid_lft forever preferred_lft forever
[root@netlab8 ~]# cd /var/run/
[root@netlab8 run]# ln -s /var/run/docker/netns netns
[root@netlab8 run]# ip netns list
c09b7f2fae29 (id: 0)
default
[root@netlab8 run]# ip link show | grep -B1 c09b7f2fae29
7: cali861b4e7784e@if4: mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns c09b7f2fae29
[root@netlab6 etc]# kubectl -n demo create deployment netops --image=baburaj/netops
deployment.apps/netops created
[root@netlab6 etc]# kubectl -n demo scale deployment.apps/netops --replicas=4
deployment.apps/netops scaled
[root@netlab6 etc]# kubectl -n demo get po -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
netops-f49b8bdb7-8d747 1/1 Running 0 8s 192.168.197.66 netlab7
netops-f49b8bdb7-8g6mt 1/1 Running 0 8s 192.168.191.66 netlab8
netops-f49b8bdb7-fwmzb 1/1 Running 0 17h 192.168.191.65 netlab8
netops-f49b8bdb7-gbx79 1/1 Running 0 121m 192.168.197.65 netlab7
[root@netlab7 ~]# docker inspect k8s_netops_netops-f49b8bdb7-8d747_demo_97fa310a-90cc-4f46-a5d4-fa3ece50511a_0 | jq -r '.[].State.Pid'
709675
[root@netlab7 ~]# ip netns identify 709675
[root@netlab7 ~]# cd /var/run/
[root@netlab7 run]# ln -s /var/run/docker/netns netns
[root@netlab7 run]# ip netns identify 709675
a2badaf582ef
[root@netlab7 run]# ip netns list
a2badaf582ef (id: 1)
a8c95b2f1d86 (id: 0)
default
[root@netlab7 run]# nsenter -t 709675 -n ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: tunl0@NONE: mtu 1480 qdisc noop state DOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if8: mtu 1480 qdisc noqueue state UP group default
link/ether de:94:bb:a8:4b:86 brd ff:ff:ff:ff:ff:ff link-netns default
inet 192.168.197.66/32 scope global eth0
valid_lft forever preferred_lft forever
#ip addr show (trimmed output)
...
7: cali7901bed189a@if4: mtu 1480 qdisc noqueue state UP group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns a8c95b2f1d86
inet6 fe80::ecee:eeff:feee:eeee/64 scope link
valid_lft forever preferred_lft forever
8: cali68a5797a837@if4: mtu 1480 qdisc noqueue state UP group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns a2badaf582ef
inet6 fe80::ecee:eeff:feee:eeee/64 scope link
valid_lft forever preferred_lft forever
...
Note:
Network namespace name : a2badaf582ef can be mapped against the link to understand the veth pair.
Also eth0@if8 within the docker container indicates that the ID is "8" and the iface id from rootns which has
"8" is cali68a5797a837
Another way to map it
[root@netlab7 run]# ip -j addr show | jq -cr '.[] | select (.link_index != null) | { ifindex: .ifindex, link_index: .link_index, ifname: .ifname}'
{"ifindex":7,"link_index":4,"ifname":"cali7901bed189a"}
{"ifindex":8,"link_index":4,"ifname":"cali68a5797a837"}
[root@netlab7 ~]# for container in $(docker container ls | egrep -v 'pause|kube-proxy|CONTAINER' | awk '{print $1}') ;do echo -n [ $container ]: ; docker exec -it $container bash -c 'ip -j addr show ' | jq -cr '.[] | select (.link_index != null) | { ifindex: .ifindex, link_index: .link_index, ifname: .ifname}' | jq -rc; done
[ adfe6a7cdedb ]:{"ifindex":4,"link_index":8,"ifname":"eth0"}
[ 7d1ac8e70bf5 ]:{"ifindex":4,"link_index":7,"ifname":"eth0"}
[ c1fcefbe3e73 ]:{"ifindex":7,"link_index":4,"ifname":"cali7901bed189a"}
{"ifindex":8,"link_index":4,"ifname":"cali68a5797a837"}
#Need to write a python script to consoldiate and list all container, if names with mapping.
[root@netlab6 ~]# kubectl -n demo create deployment netops-a --image=baburaj/netops
deployment.apps/netops-a created
[root@netlab6 ~]# kubectl -n demo create deployment netops-b --image=baburaj/netops
deployment.apps/netops-b created
[root@netlab6 ~]# kubectl -n demo scale deployment.apps/netops-a --replicas=2
deployment.apps/netops-a scaled
[root@netlab6 ~]# kubectl -n demo scale deployment.apps/netops-b --replicas=2
deployment.apps/netops-b scaled
Reference Notes*
kubeadm init --pod-network-cidr 192.168.0.0/16
[root@netlab6 ~]# calicoctl get wep --allow-version-mismatch -n demo -o wide
NAMESPACE NAME WORKLOAD NODE NETWORKS INTERFACE PROFILES NATS
demo netlab7-k8s-netops--a--9f477c8df--22zjj-eth0 netops-a-9f477c8df-22zjj netlab7 192.168.197.69/32 calibefa1121658 kns.demo,ksa.demo.default
demo netlab8-k8s-netops--a--9f477c8df--hnj9v-eth0 netops-a-9f477c8df-hnj9v netlab8 192.168.191.68/32 cali15d36023299 kns.demo,ksa.demo.default
demo netlab8-k8s-netops--b--78475694bb--nvf6s-eth0 netops-b-78475694bb-nvf6s netlab8 192.168.191.69/32 cali3cd86690e6f kns.demo,ksa.demo.default
demo netlab7-k8s-netops--b--78475694bb--xbl7m-eth0 netops-b-78475694bb-xbl7m netlab7 192.168.197.68/32 cali6e6629da0f5 kns.demo,ksa.demo.default
[root@netlab7 run]# ip route show
default via 192.168.122.1 dev enp1s0 proto dhcp metric 100
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
192.168.122.0/24 dev enp1s0 proto kernel scope link src 192.168.122.7 metric 100
192.168.158.0/26 via 192.168.122.6 dev tunl0 proto bird onlink
blackhole 192.168.197.64/26 proto bird
192.168.197.68 dev cali6e6629da0f5 scope link
192.168.197.69 dev calibefa1121658 scope link
[root@netlab7 run]# ip addr show
1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: enp1s0: mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 52:54:00:70:00:00 brd ff:ff:ff:ff:ff:ff
inet 192.168.122.7/24 brd 192.168.122.255 scope global dynamic noprefixroute enp1s0
valid_lft 3315sec preferred_lft 3315sec
inet6 2001:db8:ca2:2:1::7a/128 scope global dynamic noprefixroute
valid_lft 3140sec preferred_lft 3140sec
inet6 fe80::5fda:323d:d2d6:6bfe/64 scope link noprefixroute
valid_lft forever preferred_lft forever
3: docker0: mtu 1500 qdisc noqueue state DOWN group default
link/ether 02:42:b0:de:42:69 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
4: tunl0@NONE: mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
inet 192.168.197.64/32 scope global tunl0
valid_lft forever preferred_lft forever
10: cali6e6629da0f5@if4: mtu 1480 qdisc noqueue state UP group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns aae2b88920f5
inet6 fe80::ecee:eeff:feee:eeee/64 scope link
valid_lft forever preferred_lft forever
11: calibefa1121658@if4: mtu 1480 qdisc noqueue state UP group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns aadfc8999004
inet6 fe80::ecee:eeff:feee:eeee/64 scope link
valid_lft forever preferred_lft forever
Note:
Seems tunl0 is a IP-IP tunnel but the route has a blackhole to this route.
Why is dst=10.0.1.11 in conntrack output when a ping originated from 192.168.122.6 ?
[root@panchajanya ~]# virsh domifaddr netlab6 |grep vnet
vnet0 52:54:00:60:00:00 ipv4 192.168.122.6/24
[root@netlab6 ~]# ping -c1 baburaj.github.io
PING baburaj.github.io (185.199.110.153) 56(84) bytes of data.
64 bytes from cdn-185-199-110-153.github.com (185.199.110.153): icmp_seq=1 ttl=51 time=28.4 ms
--- baburaj.github.io ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 28.377/28.377/28.377/0.000 ms
[root@panchajanya ~]# conntrack -L -j
icmp 1 25 src=192.168.122.6 dst=185.199.110.153 type=8 code=0 id=61885 src=185.199.110.153 dst=10.0.1.11 type=0 code=0 id=61885 mark=0 use=1
conntrack v1.4.4 (conntrack-tools): 1 flow entries have been shown.
[root@panchajanya ~]# ip route show
default via 10.0.1.1 dev enp7s0 proto static metric 100
default via 192.168.2.1 dev enp8s0 proto static metric 101
10.0.1.0/24 dev enp7s0 proto kernel scope link src 10.0.1.11 metric 100
192.168.2.0/24 dev enp8s0 proto kernel scope link src 192.168.2.11 metric 101
192.168.3.0/24 via 192.168.2.1 dev enp8s0 proto static metric 101
192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1
There is a MASQUERADE rule for 192.168.122.0/24 and so the NAT'ing should
happen for the outgoing interface. Per routing table the first default route is
via 10.0.1.1 ! So this must be the reason why the conntrack is showing dst=10.0.1.1
[root@panchajanya ~]# iptables -t nat -L -n
Chain PREROUTING (policy ACCEPT)
target prot opt source destination
Chain INPUT (policy ACCEPT)
target prot opt source destination
Chain POSTROUTING (policy ACCEPT)
target prot opt source destination
RETURN all -- 192.168.122.0/24 224.0.0.0/24
RETURN all -- 192.168.122.0/24 255.255.255.255
MASQUERADE tcp -- 192.168.122.0/24 !192.168.122.0/24 masq ports: 1024-65535
MASQUERADE udp -- 192.168.122.0/24 !192.168.122.0/24 masq ports: 1024-65535
MASQUERADE all -- 192.168.122.0/24 !192.168.122.0/24
Chain OUTPUT (policy ACCEPT)
target prot opt source destination