K8S Networking Internals

Pause Container!

What is a pause container? This acts as a parent container to other containers in the POD and is responsible for bringing up the network namespace.


    [root@netlab6 etc]# kubectl create namespace demo
    namespace/demo created

    [root@netlab6 etc]# kubectl -n demo create deployment netops --image=baburaj/netops
    deployment.apps/netops created

    [root@netlab6 etc]# kubectl -n demo get po -o wide
    NAME                     READY   STATUS    RESTARTS   AGE   IP               NODE      NOMINATED NODE   READINESS GATES
    netops-f49b8bdb7-fwmzb   1/1     Running   0          29s   192.168.191.65   netlab8              

    [root@netlab8 ~]# docker ps  |grep netops
    e5a08108d0d5   baburaj/netops         "/usr/bin/nginx-star…"   5 minutes ago    Up 5 minutes              k8s_netops_netops-f49b8bdb7-fwmzb_demo_a65ea2e7-41c3-4be4-8cab-e40fd5955dca_0
    35281cd4a860   k8s.gcr.io/pause:3.6   "/pause"                 5 minutes ago    Up 5 minutes              k8s_POD_netops-f49b8bdb7-fwmzb_demo_a65ea2e7-41c3-4be4-8cab-e40fd5955dca_0
    
    [root@netlab8 ~]# docker inspect k8s_netops_netops-f49b8bdb7-fwmzb_demo_a65ea2e7-41c3-4be4-8cab-e40fd5955dca_0 | jq '.[].State.Pid'
    18484

    [root@netlab8 ~]# nsenter -t 18484 -a

    [root@netops-f49b8bdb7-fwmzb /]# ip addr show
    1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
        valid_lft forever preferred_lft forever
    2: tunl0@NONE:  mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if7:  mtu 1480 qdisc noqueue state UP group default
        link/ether 5e:bf:1f:df:67:ce brd ff:ff:ff:ff:ff:ff link-netnsid 0
        inet 192.168.191.65/32 scope global eth0
        valid_lft forever preferred_lft forever

        [root@netlab8 ~]# cd /var/run/

        [root@netlab8 run]# ln -s /var/run/docker/netns netns

        [root@netlab8 run]# ip netns list
        c09b7f2fae29 (id: 0)
        default

        [root@netlab8 run]# ip link show | grep -B1 c09b7f2fae29
        7: cali861b4e7784e@if4:  mtu 1480 qdisc noqueue state UP mode DEFAULT group default
            link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns c09b7f2fae29
            

k8s (Identify Network Namespace for the PODs)


    [root@netlab6 etc]# kubectl -n demo create deployment netops --image=baburaj/netops
    deployment.apps/netops created

    [root@netlab6 etc]# kubectl -n demo scale deployment.apps/netops --replicas=4
    deployment.apps/netops scaled

    [root@netlab6 etc]# kubectl -n demo get po -o wide
    NAME                     READY   STATUS    RESTARTS   AGE    IP               NODE      NOMINATED NODE   READINESS GATES
    netops-f49b8bdb7-8d747   1/1     Running   0          8s     192.168.197.66   netlab7              
    netops-f49b8bdb7-8g6mt   1/1     Running   0          8s     192.168.191.66   netlab8              
    netops-f49b8bdb7-fwmzb   1/1     Running   0          17h    192.168.191.65   netlab8              
    netops-f49b8bdb7-gbx79   1/1     Running   0          121m   192.168.197.65   netlab7              

    [root@netlab7 ~]# docker inspect k8s_netops_netops-f49b8bdb7-8d747_demo_97fa310a-90cc-4f46-a5d4-fa3ece50511a_0 |  jq -r '.[].State.Pid'
    709675

    [root@netlab7 ~]# ip netns identify 709675
    
    [root@netlab7 ~]# cd /var/run/
    [root@netlab7 run]# ln -s /var/run/docker/netns netns
    
    [root@netlab7 run]# ip netns identify 709675
    a2badaf582ef    

    [root@netlab7 run]# ip netns list
    a2badaf582ef (id: 1)
    a8c95b2f1d86 (id: 0)
    default

    [root@netlab7 run]# nsenter -t 709675 -n ip a
    1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
        valid_lft forever preferred_lft forever
    2: tunl0@NONE:  mtu 1480 qdisc noop state DOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
    4: eth0@if8:  mtu 1480 qdisc noqueue state UP group default
        link/ether de:94:bb:a8:4b:86 brd ff:ff:ff:ff:ff:ff link-netns default
        inet 192.168.197.66/32 scope global eth0
        valid_lft forever preferred_lft forever
       
    #ip addr show (trimmed output)

    ...
    7: cali7901bed189a@if4:  mtu 1480 qdisc noqueue state UP group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns a8c95b2f1d86
    inet6 fe80::ecee:eeff:feee:eeee/64 scope link
       valid_lft forever preferred_lft forever
    8: cali68a5797a837@if4:  mtu 1480 qdisc noqueue state UP group default
        link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns a2badaf582ef
        inet6 fe80::ecee:eeff:feee:eeee/64 scope link
        valid_lft forever preferred_lft forever
    ...

    Note:
    Network namespace name : a2badaf582ef  can be mapped against the link to understand the veth pair.

    Also eth0@if8 within the docker container indicates that the ID is "8" and the iface id from rootns which has
    "8" is cali68a5797a837

    Another way to map it 
    [root@netlab7 run]# ip -j addr show  | jq -cr '.[] | select (.link_index != null) | { ifindex: .ifindex, link_index: .link_index, ifname: .ifname}'
    {"ifindex":7,"link_index":4,"ifname":"cali7901bed189a"}
    {"ifindex":8,"link_index":4,"ifname":"cali68a5797a837"}

    [root@netlab7 ~]# for container in $(docker container ls | egrep -v 'pause|kube-proxy|CONTAINER'  | awk '{print $1}') ;do echo -n [ $container ]: ; docker exec -it $container bash -c 'ip -j addr show ' | jq -cr '.[] | select (.link_index != null) | { ifindex: .ifindex, link_index: .link_index, ifname: .ifname}' | jq -rc; done
    [ adfe6a7cdedb ]:{"ifindex":4,"link_index":8,"ifname":"eth0"}
    [ 7d1ac8e70bf5 ]:{"ifindex":4,"link_index":7,"ifname":"eth0"}
    [ c1fcefbe3e73 ]:{"ifindex":7,"link_index":4,"ifname":"cali7901bed189a"}
    {"ifindex":8,"link_index":4,"ifname":"cali68a5797a837"}    

    #Need to write a python script to consoldiate and list all container, if names with mapping.


Calico - Networking (Details of POD/Interfaces)

    [root@netlab6 ~]# kubectl -n demo create deployment netops-a --image=baburaj/netops
    deployment.apps/netops-a created
    [root@netlab6 ~]# kubectl -n demo create deployment netops-b --image=baburaj/netops
    deployment.apps/netops-b created
    [root@netlab6 ~]# kubectl -n demo scale deployment.apps/netops-a --replicas=2
    deployment.apps/netops-a scaled
    [root@netlab6 ~]# kubectl -n demo scale deployment.apps/netops-b --replicas=2
    deployment.apps/netops-b scaled

    Reference Notes*
    kubeadm init --pod-network-cidr 192.168.0.0/16

    [root@netlab6 ~]# calicoctl get wep --allow-version-mismatch -n demo -o wide
    NAMESPACE   NAME                                            WORKLOAD                    NODE      NETWORKS            INTERFACE         PROFILES                    NATS
    demo        netlab7-k8s-netops--a--9f477c8df--22zjj-eth0    netops-a-9f477c8df-22zjj    netlab7   192.168.197.69/32   calibefa1121658   kns.demo,ksa.demo.default
    demo        netlab8-k8s-netops--a--9f477c8df--hnj9v-eth0    netops-a-9f477c8df-hnj9v    netlab8   192.168.191.68/32   cali15d36023299   kns.demo,ksa.demo.default
    demo        netlab8-k8s-netops--b--78475694bb--nvf6s-eth0   netops-b-78475694bb-nvf6s   netlab8   192.168.191.69/32   cali3cd86690e6f   kns.demo,ksa.demo.default
    demo        netlab7-k8s-netops--b--78475694bb--xbl7m-eth0   netops-b-78475694bb-xbl7m   netlab7   192.168.197.68/32   cali6e6629da0f5   kns.demo,ksa.demo.default    

    [root@netlab7 run]# ip route show
    default via 192.168.122.1 dev enp1s0 proto dhcp metric 100
    172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
    192.168.122.0/24 dev enp1s0 proto kernel scope link src 192.168.122.7 metric 100
    192.168.158.0/26 via 192.168.122.6 dev tunl0 proto bird onlink
    blackhole 192.168.197.64/26 proto bird
    192.168.197.68 dev cali6e6629da0f5 scope link
    192.168.197.69 dev calibefa1121658 scope link

    [root@netlab7 run]# ip addr show
    1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
        link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
        inet 127.0.0.1/8 scope host lo
        valid_lft forever preferred_lft forever
        inet6 ::1/128 scope host
        valid_lft forever preferred_lft forever
    2: enp1s0:  mtu 1500 qdisc fq_codel state UP group default qlen 1000
        link/ether 52:54:00:70:00:00 brd ff:ff:ff:ff:ff:ff
        inet 192.168.122.7/24 brd 192.168.122.255 scope global dynamic noprefixroute enp1s0
        valid_lft 3315sec preferred_lft 3315sec
        inet6 2001:db8:ca2:2:1::7a/128 scope global dynamic noprefixroute
        valid_lft 3140sec preferred_lft 3140sec
        inet6 fe80::5fda:323d:d2d6:6bfe/64 scope link noprefixroute
        valid_lft forever preferred_lft forever
    3: docker0:  mtu 1500 qdisc noqueue state DOWN group default
        link/ether 02:42:b0:de:42:69 brd ff:ff:ff:ff:ff:ff
        inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
        valid_lft forever preferred_lft forever
    4: tunl0@NONE:  mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000
        link/ipip 0.0.0.0 brd 0.0.0.0
        inet 192.168.197.64/32 scope global tunl0
        valid_lft forever preferred_lft forever
    10: cali6e6629da0f5@if4:  mtu 1480 qdisc noqueue state UP group default
        link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns aae2b88920f5
        inet6 fe80::ecee:eeff:feee:eeee/64 scope link
        valid_lft forever preferred_lft forever
    11: calibefa1121658@if4:  mtu 1480 qdisc noqueue state UP group default
        link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netns aadfc8999004
        inet6 fe80::ecee:eeff:feee:eeee/64 scope link
        valid_lft forever preferred_lft forever

    Note:
    Seems tunl0 is a IP-IP tunnel but the route has a blackhole to this route.


NAT/Routing from K8S VMs/Hosts

Why is dst=10.0.1.11 in conntrack output when a ping originated from 192.168.122.6 ?


    [root@panchajanya ~]# virsh domifaddr netlab6 |grep vnet
    vnet0      52:54:00:60:00:00    ipv4         192.168.122.6/24
    
    [root@netlab6 ~]# ping -c1 baburaj.github.io
    PING baburaj.github.io (185.199.110.153) 56(84) bytes of data.
    64 bytes from cdn-185-199-110-153.github.com (185.199.110.153): icmp_seq=1 ttl=51 time=28.4 ms

    --- baburaj.github.io ping statistics ---
    1 packets transmitted, 1 received, 0% packet loss, time 0ms
    rtt min/avg/max/mdev = 28.377/28.377/28.377/0.000 ms
    
    [root@panchajanya ~]# conntrack -L  -j
    icmp     1 25 src=192.168.122.6 dst=185.199.110.153 type=8 code=0 id=61885 src=185.199.110.153 dst=10.0.1.11 type=0 code=0 id=61885 mark=0 use=1
    conntrack v1.4.4 (conntrack-tools): 1 flow entries have been shown.

    [root@panchajanya ~]# ip route show
    default via 10.0.1.1 dev enp7s0 proto static metric 100
    default via 192.168.2.1 dev enp8s0 proto static metric 101
    10.0.1.0/24 dev enp7s0 proto kernel scope link src 10.0.1.11 metric 100
    192.168.2.0/24 dev enp8s0 proto kernel scope link src 192.168.2.11 metric 101
    192.168.3.0/24 via 192.168.2.1 dev enp8s0 proto static metric 101
    192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1

    There is a MASQUERADE rule for 192.168.122.0/24 and so the NAT'ing should 
    happen for the outgoing interface. Per routing table the first default route is
    via 10.0.1.1 ! So this must be the reason why the conntrack is showing dst=10.0.1.1
    

    [root@panchajanya ~]# iptables -t nat -L -n
    Chain PREROUTING (policy ACCEPT)
    target     prot opt source               destination

    Chain INPUT (policy ACCEPT)
    target     prot opt source               destination

    Chain POSTROUTING (policy ACCEPT)
    target     prot opt source               destination
    RETURN     all  --  192.168.122.0/24     224.0.0.0/24
    RETURN     all  --  192.168.122.0/24     255.255.255.255
    MASQUERADE  tcp  --  192.168.122.0/24    !192.168.122.0/24     masq ports: 1024-65535
    MASQUERADE  udp  --  192.168.122.0/24    !192.168.122.0/24     masq ports: 1024-65535
    MASQUERADE  all  --  192.168.122.0/24    !192.168.122.0/24

    Chain OUTPUT (policy ACCEPT)
    target     prot opt source               destination