[CKA] Apiserver Crash

Apiserver Crash

문제. K8s API-Server 장애를 복구한다.

답.

로그 확인하는 것

# journalctl 로 보기

journalctl -r (역순)

f (front)

b (before)로 커서 이동

 

# cri 상태 확인

crictl ps

controlplane $ crictl ps
CONTAINER           IMAGE               CREATED             STATE               NAME                      ATTEMPT             POD ID              POD
9cffaa1a8bd39       03c7f74a90d48       31 minutes ago      Running             local-path-provisioner    1                   e0597184df554       local-path-provisioner-6c5cff8948-2x89z
34deabd072a0e       f9c3c1813269c       31 minutes ago      Running             calico-kube-controllers   1                   8e75eae807d83       calico-kube-controllers-94fb6bc47-rxh7x
9bc43ac558359       cbb01a7bd410d       31 minutes ago      Running             coredns                   1                   ec298ce7a93f0       coredns-57888bfdc7-bh4rd
1b4d77eafe054       cbb01a7bd410d       31 minutes ago      Running             coredns                   1                   496c376744413       coredns-57888bfdc7-clhw7
5f9d9b529658c       e6ea68648f0cd       31 minutes ago      Running             kube-flannel              1                   775ead6526552       canal-ln25l
3cf8c9b259a52       75392e3500e36       31 minutes ago      Running             calico-node               1                   775ead6526552       canal-ln25l
2b81d1a72d7c1       ad83b2ca7b09e       32 minutes ago      Running             kube-proxy                1                   e876c01fa11b9       kube-proxy-2mfwz
e2383e91aaa3f       2e96e5913fc06       32 minutes ago      Running             etcd                      1                   6d51673349a93       etcd-controlplane
32f2ab185eed8       1766f54c897f0       32 minutes ago      Running             kube-scheduler            1                   26d667c5a22c3       kube-scheduler-controlplane
985600806f4ae       045733566833c       32 minutes ago      Running             kube-controller-manager   1                   782a52947c30a       kube-controller-manager-controlplane
c3bccff3704d3       604f5db92eaa8       32 minutes ago      Running             kube-apiserver            1                   0149770208492       kube-apiserver-controlplane

crictl logs

controlplane $ crictl logs
FATA[0000] ID cannot be empty

 

cd /var/log/pods

cd /var/log/containers

 

api server에 잘못된 인자를 주입하고 복구하자.

# 경로로 이동
cd /etc/kubernates/manifests

# 설정 파일 백업
cp ./kube-apiserver.yaml ./kube-apiserver.yaml.bak

# --this-is-wrong을 입력한다.
vi ./kube-apiserver.yaml

spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=172.30.1.2
    - --allow-privileged=true
    - --secure-port=6443
    - --this-is-wrong  # 중간에 넣는다.


# 변경한 config 적용
kubectl apply -f ./kube-apiserver.yaml

# 적용이 되지 않는다면 기존 pod를 삭제하고 생성한다.
controlplane $ kubectl delete pod kube-apiserver -n kube-system
pod "kube-apiserver" deleted

# 다시 생성
controlplane $ kubectl apply -f kube-apiserver.yaml 
pod/kube-apiserver created

 

 

 

controlplane $ kubectl get pod -n kube-system
NAME                                      READY   STATUS             RESTARTS      AGE
calico-kube-controllers-94fb6bc47-rxh7x   1/1     Running            2 (90s ago)   5d17h
canal-ln25l                               2/2     Running            2 (39m ago)   5d17h
coredns-57888bfdc7-bh4rd                  1/1     Running            1 (39m ago)   5d17h
coredns-57888bfdc7-clhw7                  1/1     Running            1 (39m ago)   5d17h
etcd-controlplane                         1/1     Running            1 (39m ago)   5d17h
kube-apiserver                            0/1     CrashLoopBackOff   3 (10s ago)   66s
kube-apiserver-controlplane               0/1     Running            0             2m41s
kube-controller-manager-controlplane      0/1     Running            3 (26s ago)   5d17h
kube-proxy-2mfwz                          1/1     Running            1 (39m ago)   5d17h
kube-scheduler-controlplane               0/1     Running            3 (27s ago)   5d17h


# 상태가 crash로 변경되었다. 잘못된 인자 덕분
kube-apiserver                            0/1     CrashLoopBackOff