3. Keepalived

Keepalived static pod is deployed in the controllers using unicast over srvc0 interface. The virtual ip is constructed using Hex(FabricId) + Hex(virt) + MAC (last 3 octets).

Keepalived

3.1. Prerequisites

3.2. Get the code and build the image

Note

Update etc/keepalived.conf

git clone https://gitlab.stroila.ca/ocp/keepalived.git
cd keepalived
make build

3.3. Configuration

The configuration of the keepalived daemon is recorded in /etc/keepalived/keepalived.conf. The configuration blocks in the file control notification settings, the VIPs to manage, and the health checks to use to test the services that rely on VIPs.

  • Each node in the HA setup needs its own copy of the configuration file, with values for the priority, unicast_src_ip, and unicast_peer directives that are appropriate to the node’s role (primary or backup).

  • The priority directive controls which host becomes the primary.

  • The notify directive names the notification script included in the distribution, which can be used to generate syslog messages (or other notifications) when a state transition or fault occurs.

  • The value for the virtual_router_id directive in the vrrp_instance VI_1 block is necessary to be unique in your environment.

  • If you have multiple pairs of keepalived instances (or other VRRP instances) running in your local network, create a vrrp_instance block for each one, with a unique name and virtual_router_id number.

Keepailved config template:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
global_defs {
    enable_script_security
}

vrrp_script chk_api_service {
    script  "/usr/bin/curl -s -k https://localhost:6443/healthz -o /dev/null"
    interval 20
    timeout  5
    rise     1
    fall     1
    user     root
}

vrrp_script chk_kubelet_service {
    script  "/usr/bin/curl -s -k http://localhost:10248/healthz -o /dev/null"
    interval 5
    timeout  5
    rise     1
    fall     1
    user     root
}

vrrp_instance VI_1 {
    interface         {{VIF1}}
    state             BACKUP
    priority          {{P1}}
    virtual_router_id 10
    advert_int        1
    unicast_src_ip {{unicast_src}}

    unicast_peer {
        {{PEER1}}
        {{PEER2}}
    }

    virtual_ipaddress {
        {{VIP1}}/64
    }

    track_script {
        chk_api_service
        #chk_kubelet_service
    }

    # notify "/opt/ocp/bin/keepalived/api-ha-notify"
}

vrrp_instance VI_2 {
    interface         {{VIF2}}
    state             BACKUP
    priority          {{P2}}
    virtual_router_id 20
    advert_int        1
    unicast_src_ip {{unicast_src}}

    unicast_peer {
        {{PEER1}}
        {{PEER2}}
    }

    virtual_ipaddress {
        {{VIP2}}/64
    }

    track_script {
        chk_api_service
        #chk_kubelet_service
    }

    # notify "/opt/ocp/bin/keepalived/api-ha-notify"
}

vrrp_instance VI_3 {
    interface         {{VIF3}}
    state             BACKUP
    priority          {{P3}}
    virtual_router_id 30
    advert_int        1
    unicast_src_ip {{unicast_src}}

    unicast_peer {
        {{PEER1}}
        {{PEER2}}
    }

    virtual_ipaddress {
        {{VIP3}}/64
    }

    track_script {
        chk_api_service
        #chk_kubelet_service
    }

    # notify "/opt/ocp/bin/keepalived/api-ha-notify"
}

Example of an Active-Active-Active configuration using FabricId=666

Node 1

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
#vrrp_script chk_api_service {
#    script  "/usr/bin/curl -s -k https://localhost:6443/healthz -o /dev/null"
#    interval 20
#    timeout  5
#    rise     1
#    fall     1
#    user     root
#}

vrrp_script chk_kubelet_service {
    script  "/usr/bin/curl -s -k http://localhost:10248/healthz -o /dev/null"
    interval 5
    timeout  5
    rise     1
    fall     1
    user     root
}

vrrp_instance VI_1 {
    interface         svc0
    state             BACKUP
    priority          200
    virtual_router_id 10
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::90b1:1c26:6aa2

    unicast_peer {
        fd02:9a01::1866:dae6:4674
        fd02:9a01::549f:3509:8eac
    }

    virtual_ipaddress {
        fd02:9a01::1/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

vrrp_instance VI_2 {
    interface         svc0
    state             BACKUP
    priority          150
    virtual_router_id 20
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::90b1:1c26:6aa2

    unicast_peer {
        fd02:9a01::1866:dae6:4674
        fd02:9a01::549f:3509:8eac
    }

    virtual_ipaddress {
        fd02:9a01::2/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

vrrp_instance VI_3 {
    interface         svc0
    state             BACKUP
    priority          50
    virtual_router_id 30
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::90b1:1c26:6aa2

    unicast_peer {
        fd02:9a01::1866:dae6:4674
        fd02:9a01::549f:3509:8eac
    }

    virtual_ipaddress {
        fd02:9a01::3/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

Node 2

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
#vrrp_script chk_api_service {
#    script  "/usr/bin/curl -s -k https://localhost:6443/healthz -o /dev/null"
#    interval 20
#    timeout  5
#    rise     1
#    fall     1
#    user     root
#}

vrrp_script chk_kubelet_service {
    script  "/usr/bin/curl -s -k http://localhost:10248/healthz -o /dev/null"
    interval 5
    timeout  5
    rise     1
    fall     1
    user     root
}

vrrp_instance VI_1 {
    interface         svc0
    state             BACKUP
    priority          50
    virtual_router_id 10
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::549f:3509:8eac 

    unicast_peer {
        fd02:9a01::90b1:1c26:6aa2
        fd02:9a01::1866:dae6:4674
    }

    virtual_ipaddress {
        fd02:9a01::1/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

vrrp_instance VI_2 {
    interface         svc0
    state             BACKUP
    priority          200
    virtual_router_id 20
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::549f:3509:8eac

    unicast_peer {
        fd02:9a01::90b1:1c26:6aa2
        fd02:9a01::1866:dae6:4674
    }

    virtual_ipaddress {
        fd02:9a01::2/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

vrrp_instance VI_3 {
    interface         svc0
    state             BACKUP
    priority          150
    virtual_router_id 30
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::549f:3509:8eac

    unicast_peer {
        fd02:9a01::90b1:1c26:6aa2
        fd02:9a01::1866:dae6:4674
    }

    virtual_ipaddress {
        fd02:9a01::3/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

Node 3

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
#vrrp_script chk_api_service {
#    script  "/usr/bin/curl -s -k https://localhost:6443/healthz -o /dev/null"
#    interval 20
#    timeout  5
#    rise     1
#    fall     1
#    user     root
#}

vrrp_script chk_kubelet_service {
    script  "/usr/bin/curl -s -k http://localhost:10248/healthz -o /dev/null"
    interval 5
    timeout  5
    rise     1
    fall     1
    user     root
}

vrrp_instance VI_1 {
    interface         svc0
    state             BACKUP
    priority          150
    virtual_router_id 10
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::1866:dae6:4674

    unicast_peer {
        fd02:9a01::90b1:1c26:6aa2
        fd02:9a01::549f:3509:8eac
    }

    virtual_ipaddress {
        fd02:9a01::1/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

vrrp_instance VI_2 {
    interface         svc0
    state             BACKUP
    priority          50
    virtual_router_id 20
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::1866:dae6:4674

    unicast_peer {
        fd02:9a01::90b1:1c26:6aa2
        fd02:9a01::549f:3509:8eac
    }

    virtual_ipaddress {
        fd02:9a01::2/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

vrrp_instance VI_3 {
    interface         svc0
    state             BACKUP
    priority          200
    virtual_router_id 30
    advert_int        1
    authentication {
        auth_type PASS
        auth_pass cGFzc3dvcmQK
    }
    unicast_src_ip fd02:9a01::1866:dae6:4674

    unicast_peer {
        fd02:9a01::90b1:1c26:6aa2
        fd02:9a01::549f:3509:8eac
    }

    virtual_ipaddress {
        fd02:9a01::3/64
    }

    track_script {
        #chk_api_service
        chk_kubelet_service
    }

    # notify "/usr/local/bin/keepalived/api-ha-notify"
}

3.4. Deployment

On each participating node set nonlocal bind to 1, which allows processes to bind() to non-local IPv6 addresses, which is needed by vrrp.

sysctl -w net.ipv4.ip_nonlocal_bind=1
sysctl -w net.ipv6.ip_nonlocal_bind=1

Copy the keepalived manifest file in /etc/kubernetes/manifests on each participating node.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
apiVersion: v1
kind: Pod
metadata:
  name: keepalived
  namespace: fabric
spec:
  containers:
  - name: keepalived
    image: {{REG}}/ocp/keepalived:latest
    imagePullPolicy: IfNotPresent
    args:
      - --vrrp
      - --log-detail
      - --dump-conf
      - --use-file=/etc/keepalived/keepalived.conf
    volumeMounts:
    - mountPath: /etc/localtime
      name: host-localtime
    - mountPath: /etc/keepalived/keepalived.conf
      name: config
    securityContext:
      privileged: true
      capabilities:
        add:
          - NET_ADMIN
  hostNetwork: true
  priorityClassName: system-node-critical
  restartPolicy: Always
  volumes:
  - hostPath:
      path: /etc/localtime
    name: host-localtime
  - hostPath:
      path: /etc/keepalived/keepalived.conf
    name: config
  status: {}

Node 1

srvc0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 9000
        inet6 fd02:9a01::90b1:1c26:6aa2  prefixlen 64  scopeid 0x0<global>
        inet6 fd02:9a01::1  prefixlen 64  scopeid 0x0<global>
        inet6 fe80::c4b6:57ff:fe33:3017  prefixlen 64  scopeid 0x20<link>
        ether c6:b6:57:33:30:17  txqueuelen 1000  (Ethernet)
        RX packets 103589  bytes 8143914 (7.7 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 105139  bytes 12430874 (11.8 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

Node 2

srvc0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 9000
        inet6 fe80::a4ee:baff:fea2:d780  prefixlen 64  scopeid 0x20<link>
        inet6 fd02:9a01::549f:3509:8eac  prefixlen 64  scopeid 0x0<global>
        inet6 fd02:9a01::2  prefixlen 64  scopeid 0x0<global>
        ether a6:ee:ba:a2:d7:80  txqueuelen 1000  (Ethernet)
        RX packets 106466  bytes 8755192 (8.3 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 109836  bytes 12536384 (11.9 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

Node 3

srvc0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 9000
        inet6 fd02:9a01::3  prefixlen 64  scopeid 0x0<global>
        inet6 fd02:9a01::1866:dae6:4674  prefixlen 64  scopeid 0x0<global>
        inet6 fe80::14ed:bff:fecd:51de  prefixlen 64  scopeid 0x20<link>
        ether 16:ed:0b:cd:51:de  txqueuelen 1000  (Ethernet)
        RX packets 189387  bytes 22707506 (21.6 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 181485  bytes 14359254 (13.6 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

3.5. Reference