kubeadm部署kubernetes

1、环境规划

HostName IPV4 系统版本
k8smaster1 10.0.1.200/24 OpenEuler22.03-SP2
k8smaster2 10.0.1.201/24 OpenEuler22.03-SP2
k8smaster3 10.0.1.202/24 OpenEuler22.03-SP2
k8snode1 10.0.1.203/24 OpenEuler22.03-SP2
k8snode2 10.0.1.204/24 OpenEuler22.03-SP2
k8snode3 10.0.1.205/24 OpenEuler22.03-SP2

K8S版本：1.27.4
docker版本：18.09.0
cri-dockerd版本：0.3.4
cni-plugins版本：1.2.0

2、环境准备

2.1 关闭防火墙、SELINUX

所有节点

1
2
3
4
systemctl stop firewalld && systemctl disable firewalld
sed -i 's/enforcing/disabled/g' /etc/sysconfig/selinux
sed -i 's/enforcing/disabled/g' /etc/selinux/config
setenforce 0

2.2 配置互信

Master到所有节点

1
2
3
4
5
6
7
8
ssh-keygen (一路回车)

ssh-copy-id root@10.0.1.200
ssh-copy-id root@10.0.1.201
ssh-copy-id root@10.0.1.202
ssh-copy-id root@10.0.1.203
ssh-copy-id root@10.0.1.204
ssh-copy-id root@10.0.1.205

2.3 配置Host文件

所有节点

1
2
3
4
5
6
7
8
vim /etc/hosts

10.0.1.200 node01
10.0.1.201 node02
10.0.1.202 node03
10.0.1.203 node04
10.0.1.204 node05
10.0.1.205 node06

2.4 配置iptable管理ipv4/6请求

所有节点

1
2
3
4
5
6
7
8
modprobe br_netfilter

cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl --system

2.5 安装Docker

所有节点
{.is-success}

1
2
3
dnf install -y docker

systemctl enable docker && systemctl start docker

配置镜像加速、cgroup driver

1
2
3
4
5
6
7
8
9
cat > /etc/docker/daemon.json <<EOF
{
  "registry-mirrors": ["https://yywkvob3.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

systemctl daemon-reload
systemctl restart docke

2.6 安装cri-docker

所有节点

1
2
3
4
5
6
7
8
9
10
地址：https://github.com/Mirantis/cri-dockerd
可执行文件：tar -xvf cri-dockerd-0.x.x.amd64.tgz
cd cri-dockerd-0.x.x
install -o root -g root -m 0755 cri-dockerd /usr/local/bin/cri-dockerd
源码：tar -xvf cri-dockerd-0.x.x.tar.gz
cd cri-dockerd
install packaging/systemd/* /etc/systemd/system
sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service
systemctl daemon-reload
systemctl enable --now cri-docker.socket

2.7 设置kubernetes源

所有节点

1
2
3
4
5
6
7
8
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

3、安装kubernetes

所有节点

1
2
3
dnf install -y kubelet kubeadm kubectl

systemctl enable kubelet && systemctl start kubelet

3.1 下载镜像

所有节点

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
vim image.sh

url=registry.aliyuncs.com/google_containers
version=v1.27.4
images=(`kubeadm config images list --kubernetes-version=$version|awk -F '/' '{print $2}'`)
for imagename in ${images[@]} ; do
  docker pull $url/$imagename
  docker tag $url/$imagename registry.k8s.io/$imagename
  docker rmi -f $url/$imagename
done

docker pull registry.aliyuncs.com/google_containers/pause:3.6
docker pull registry.aliyuncs.com/google_containers/coredns:v1.10.1
docker tag registry.aliyuncs.com/google_containers/pause:3.6 registry.k8s.io/pause:3.6
docker tag registry.aliyuncs.com/google_containers/coredns:v1.10.1 registry.k8s.io/coredns/coredns:v1.10.1
docker rmi -f registry.aliyuncs.com/google_containers/coredns:v1.10.1
docker rmi -f registry.aliyuncs.com/google_containers/pause:3.6
docker rmi -f registry.k8s.io/coredns:latest

3.2 修改kubelet配置默认cgroup driver

Master节点

1
2
3
4
5
6
7
8
9
10
mkdir -p /var/lib/kubelet/

cat > /var/lib/kubelet/config.yaml <<EOF
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
EOF

# 需要重新启动 kubelet
[root@master ~]# systemctl restart kubelet

3.3 初始化集群

1
2
3
4
5
6
kubeadm init  --kubernetes-version=v1.27.4 \
  --pod-network-cidr=10.244.0.0/16 \
  --apiserver-advertise-address=10.0.1.200 \
  --cri-socket unix:///var/run/cri-dockerd.sock \
  --image-repository registry.k8s.io \
  -v5

3.4 安装Flannel

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
vim flannel.yaml

---
kind: Namespace
apiVersion: v1
metadata:
  name: kube-flannel
  labels:
    k8s-app: flannel
    pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
- apiGroups:
  - networking.k8s.io
  resources:
  - clustercidrs
  verbs:
  - list
  - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-flannel
---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: flannel
  name: flannel
  namespace: kube-flannel
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
  labels:
    tier: node
    k8s-app: flannel
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan",
        "Directrouting": true
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-flannel
  labels:
    tier: node
    app: flannel
    k8s-app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugin
        image: docker.io/flannel/flannel-cni-plugin:v1.2.0
        command:
        - cp
        args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
        image: docker.io/flannel/flannel:v0.22.1
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: docker.io/flannel/flannel:v0.22.1
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
        - name: xtables-lock
          mountPath: /run/xtables.lock
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni-plugin
        hostPath:
          path: /opt/cni/bin
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
1
kubectl apply -f kube-flannel.yml

安装cni-plugins

所有节点

1
2
地址：https://github.com/containernetworking/plugins
tar -xvf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/

检验网络是否正常

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
circtl info 

{
  "status": {
    "conditions": [
      {
        "type": "RuntimeReady",
        "status": true,
        "reason": "",
        "message": ""
      },
      {
        "type": "NetworkReady",
        "status": true,
        "reason": "",
        "message": ""
      }
    ]
  },
  "config": {
    "sandboxImage": "registry.k8s.io/pause:3.6"
  }
}

4、部署Kubernetes仪表板

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
vi dashboard.yaml

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: Namespace
metadata:
  name: kubernetes-dashboard

---

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 443
      targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-certs
  namespace: kubernetes-dashboard
type: Opaque

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-csrf
  namespace: kubernetes-dashboard
type: Opaque
data:
  csrf: ""

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-key-holder
  namespace: kubernetes-dashboard
type: Opaque

---

kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-settings
  namespace: kubernetes-dashboard

---

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
rules:
  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
    verbs: ["get", "update", "delete"]
    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["kubernetes-dashboard-settings"]
    verbs: ["get", "update"]
    # Allow Dashboard to get metrics.
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["heapster", "dashboard-metrics-scraper"]
    verbs: ["proxy"]
  - apiGroups: [""]
    resources: ["services/proxy"]
    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
    verbs: ["get"]

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
rules:
  # Allow Metrics Scraper to get metrics from the Metrics server
  - apiGroups: ["metrics.k8s.io"]
    resources: ["pods", "nodes"]
    verbs: ["get", "list", "watch"]

---

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
    spec:
      securityContext:
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: kubernetes-dashboard
          image: kubernetesui/dashboard:v2.7.0
          imagePullPolicy: Always
          ports:
            - containerPort: 8443
              protocol: TCP
          args:
            - --auto-generate-certificates
            - --namespace=kubernetes-dashboard
            # Uncomment the following line to manually specify Kubernetes API server Host
            # If not specified, Dashboard will attempt to auto discover the API server and connect
            # to it. Uncomment only if the default does not work.
            # - --apiserver-host=http://my-address:port
          volumeMounts:
            - name: kubernetes-dashboard-certs
              mountPath: /certs
              # Create on-disk volume to store exec logs
            - mountPath: /tmp
              name: tmp-volume
          livenessProbe:
            httpGet:
              scheme: HTTPS
              path: /
              port: 8443
            initialDelaySeconds: 30
            timeoutSeconds: 30
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      volumes:
        - name: kubernetes-dashboard-certs
          secret:
            secretName: kubernetes-dashboard-certs
        - name: tmp-volume
          emptyDir: {}
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 8000
      targetPort: 8000
  selector:
    k8s-app: dashboard-metrics-scraper

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: dashboard-metrics-scraper
  template:
    metadata:
      labels:
        k8s-app: dashboard-metrics-scraper
    spec:
      securityContext:
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: dashboard-metrics-scraper
          image: kubernetesui/metrics-scraper:v1.0.8
          ports:
            - containerPort: 8000
              protocol: TCP
          livenessProbe:
            httpGet:
              scheme: HTTP
              path: /
              port: 8000
            initialDelaySeconds: 30
            timeoutSeconds: 30
          volumeMounts:
          - mountPath: /tmp
            name: tmp-volume
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      volumes:
        - name: tmp-volume
          emptyDir: {}

执行以下命令部署

1
kubectl apply -f dashboard.yaml

查看运行情况

1
2
3
4
[root@node01 ~]# kubectl get pod -o wide -n kubernetes-dashboard
NAME                                         READY   STATUS    RESTARTS   AGE     IP           NODE     NOMINATED NODE   READINESS GATES
dashboard-metrics-scraper-5cb4f4bb9c-kcwhb   1/1     Running   0          4m23s   10.244.3.2   node04   <none>           <none>
kubernetes-dashboard-6967859bff-2tc82        1/1     Running   0          4m23s   10.244.1.2   node02   <none>           <none>

Creating a Service Account

1
2
3
4
5
6
7
8
9
# 保存至dashboard-adminuser.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard

# 执行以下命令
kubectl apply -f dashboard-adminuser.yaml

Creating a ClusterRoleBinding

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# 保存至dashboard-clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard

# 执行以下命令
kubectl apply -f dashboard-clusterrolebinding.yaml

Getting a Bearer Token for ServiceAccount

1
kubectl -n kubernetes-dashboard create token admin-user

输出如下：

1
eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLXY1N253Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiIwMzAzMjQzYy00MDQwLTRhNTgtOGE0Ny04NDllZTliYTc5YzEiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.Z2JrQlitASVwWbc-s6deLRFVk5DWD3P_vjUFXsqVSY10pbjFLG4njoZwh8p3tLxnX_VBsr7_6bwxhWSYChp9hwxznemD5x5HLtjb16kI9Z7yFWLtohzkTwuFbqmQaMoget_nYcQBUC5fDmBHRfFvNKePh_vSSb2h_aYXa8GV5AcfPQpY7r461itme1EXHQJqv-SN-zUnguDguCTjD80pFZ_CmnSE1z9QdMHPB8hoB4V68gtswR1VLa6mSYdgPwCHauuOobojALSaMc3RH7MmFUumAgguhqAkX3Omqd3rJbYOMRuMjhANqd08piDC3aIabINX6gP5-Tuuw2svnV6NYQ

本机器访问需要执行如下命令

1
kubectl proxy

外部机器访问
方法一：端口转发模式：
监听所有IP地址，并将8080转发至443https端口访问。

1
kubectl port-forward -n kubernetes-dashboard --address 0.0.0.0 service/kubernetes-dashboard 8080:443

这时在外部机器浏览器输入，（注意必须是 https），对，没错就是这么短的连接即可访问：

1
https://192.168.152.100:8080/

方法二：NodePort：
编辑命令空间kubernetes-dashboard中的kubernetes-dashboard服务

1
kubectl -n kubernetes-dashboard edit service kubernetes-dashboard

打开后，将type: ClusterIP 改为 type: NodePort

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
apiVersion: v1
kind: Service
...
...
  ports:
  - nodePort: 30169
    port: 443
    protocol: TCP
    targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard
  sessionAffinity: None
  type: NodePort    #修改这一行即可，原为type: ClusterIP
status:
  loadBalancer: {}

重新查看命令空间kubernetes-dashboard中的kubernetes-dashboard服务的端口地址。

1
kubectl -n kubernetes-dashboard get service kubernetes-dashboard

显示如下，外部暴露端口自动为30169

1
2
3
ubuntu@k8s-master:~$ kubectl -n kubernetes-dashboard get service kubernetes-dashboard
NAME                   TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
kubernetes-dashboard   NodePort   10.106.68.110   <none>        443:30169/TCP   112m

这时在外部机器浏览器输入IP加30169，（注意必须是 https）即可访问：

1
https://192.168.152.100:30169/

方法三：API Server：

这种方法仅适用于在浏览器中安装用户证书时才可用，可自行研究，这里不深究了。
如果没有安装证书，显示“检测到不安全的访问。无法登陆。通过 HTTPS 或使用 localhost 安全访问 Dashboard”

设置API server接收所有主机的请求：

1
kubectl proxy --address='0.0.0.0'  --accept-hosts='^*$'

浏览器访问命令为：

1
https://<master-ip>:<apiserver-port>/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/

5、FAQ

5.1 crictl查看信息报错

1
2
3
4
[root@node02 ~]# crictl info
WARN[0000] runtime connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock unix:///var/run/cri-dockerd.sock]. As the default settings are now deprecated, you should set the endpoint instead. 
E0802 21:31:57.009964   12229 remote_runtime.go:616] "Status from runtime service failed" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial unix /var/run/dockershim.sock: connect: no such file or directory\""
FATA[0000] getting status of runtime: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial unix /var/run/dockershim.sock: connect: no such file or directory"

修改运行时解决

1
[root@node02 ~]# crictl config runtime-endpoint unix:///var/run/cri-dockerd.sock

5.2 crictl状态不正常

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@node02 ~]# crictl info
{
  "status": {
    "conditions": [
      {
        "type": "RuntimeReady",
        "status": true,
        "reason": "",
        "message": ""
      },
      {
        "type": "NetworkReady",
        "status": false,
        "reason": "NetworkPluginNotReady",
        "message": "docker: network plugin is not ready: cni config uninitialized"
      }
    ]
  },
  "config": {
    "sandboxImage": "registry.k8s.io/pause:3.6"
  }
}

查看状态

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@node01 ~]# systemctl status cri-docker
● cri-docker.service - CRI Interface for Docker Application Container Engine
     Loaded: loaded (/etc/systemd/system/cri-docker.service; enabled; vendor preset: disabled)
     Active: active (running) since Thu 2023-08-03 00:47:52 CST; 8s ago
TriggeredBy: ● cri-docker.socket
       Docs: https://docs.mirantis.com
   Main PID: 31073 (cri-dockerd)
      Tasks: 12
     Memory: 16.4M
     CGroup: /system.slice/cri-docker.service
             └─ 31073 /usr/local/bin/cri-dockerd --container-runtime-endpoint fd://

Aug 03 00:47:52 node01 cri-dockerd[31073]: time="2023-08-03T00:47:52+08:00" level=info msg="Loaded network plugin cni"
Aug 03 00:47:52 node01 cri-dockerd[31073]: time="2023-08-03T00:47:52+08:00" level=info msg="Docker cri networking managed by network plugin cni"
Aug 03 00:47:52 node01 cri-dockerd[31073]: time="2023-08-03T00:47:52+08:00" level=error msg="Error validating CNI config list ({\n  \"name\": \"cbr0\",\n  \"cniVersion\": \"1.2.0\",\n  \"plugins\": [\n    {\n      \"type\": \"flannel\",\n      \"delegate\": {\n        \"hairpinMode\": tru>
Aug 03 00:47:52 node01 cri-dockerd[31073]: time="2023-08-03T00:47:52+08:00" level=info msg="Docker Info: &{ID:65TR:ONZL:5TSQ:L7JN:EX6C:2AJP:U3VP:DLJW:TN37:N3YG:FAUJ:LB3Q Containers:14 ContainersRunning:12 ContainersPaused:0 ContainersStopped:2 Images:10 Driver:overlay2 DriverStatus:[[Back>
Aug 03 00:47:52 node01 cri-dockerd[31073]: time="2023-08-03T00:47:52+08:00" level=info msg="Setting cgroupDriver systemd"
Aug 03 00:47:52 node01 cri-dockerd[31073]: time="2023-08-03T00:47:52+08:00" level=info msg="Docker cri received runtime config &RuntimeConfig{NetworkConfig:&NetworkConfig{PodCidr:,},}"
Aug 03 00:47:52 node01 cri-dockerd[31073]: time="2023-08-03T00:47:52+08:00" level=info msg="Starting the GRPC backend for the Docker CRI interface."
Aug 03 00:47:52 node01 cri-dockerd[31073]: time="2023-08-03T00:47:52+08:00" level=info msg="Start cri-dockerd grpc backend"
Aug 03 00:47:52 node01 systemd[1]: Started CRI Interface for Docker Application Container Engine.
Aug 03 00:47:57 node01 cri-dockerd[31073]: time="2023-08-03T00:47:57+08:00" level=error msg="Error validating CNI config list ({\n  \"name\": \"cbr0\",\n  \"cniVersion\": \"1.2.0\",\n  \"plugins\": [\n    {\n      \"type\": \"flannel\",\n      \"delegate\": {\n        \"hairpinMode\": tru>

出现“Error validating CNI config list”错误，是配置文件校验不通过
查看配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
vim /etc/cni/net.d/10-flannel.conflist
{
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }

查看版本是否兼容

1
2
3
[root@node01 ~]# /opt/cni/bin/flannel 
CNI Plugin flannel version v1.2.0 (linux/amd64) commit 6464faac built on 2023-07-21T15:07:42Z
CNI protocol versions supported: 0.1.0, 0.2.0, 0.3.0, 0.3.1, 0.4.0, 1.0.0

出现”failed to find plugin "portmap" in path [/opt/cni/bin]”错误，是未配置cni plugin

下载cni plugin并解压到/opt/cni/bin

1
[root@node01 ~]# tar -xvf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/

5.3 初始化超时、失败

1
2
3
4
5
6
I0802 21:34:55.072491    2121 waitcontrolplane.go:83] [wait-control-plane] Waiting for the API server to be healthy
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.

Unfortunately, an error has occurred:
	timed out waiting for the condition

查看日志

1
2
3
4
5
6
7
8
journalctl -xeu kubelet
有如下报错
Aug 02 22:18:51 node01 kubelet[2279]: E0802 22:18:51.106004    2279 remote_runtime.go:176] "RunPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed pulling image \"registry.k8s.io/pause:3.6\": Error response from daemon: Get \"https://asia-east1-docker.pkg.>
Aug 02 22:18:51 node01 kubelet[2279]: E0802 22:18:51.106060    2279 kuberuntime_sandbox.go:72] "Failed to create sandbox for pod" err="rpc error: code = Unknown desc = failed pulling image \"registry.k8s.io/pause:3.6\": Error response from daemon: Get \"https://asia-east1-docker.pkg.dev/v>
Aug 02 22:18:51 node01 kubelet[2279]: E0802 22:18:51.106091    2279 kuberuntime_manager.go:1122] "CreatePodSandbox for pod failed" err="rpc error: code = Unknown desc = failed pulling image \"registry.k8s.io/pause:3.6\": Error response from daemon: Get \"https://asia-east1-docker.pkg.dev/>
Aug 02 22:18:51 node01 kubelet[2279]: E0802 22:18:51.106158    2279 pod_workers.go:1294] "Error syncing pod, skipping" err="failed to \"CreatePodSandbox\" for \"kube-scheduler-node01_kube-system(65a3cbca3914ccd9af1a70265906ca9a)\" with CreatePodSandboxError: \"Failed to create sandbox for>
failed, error" error="failed to get sandbox image \"k8s.gcr.io/pause:3.6\": failed to pull image \"k8s.gcr.io/pause:3.6\": failed to pull and unpack image \"k8s.gcr.io/pause:3.6\": failed to resolve reference \"k8s.gcr.io/pause:3.6\": failed to do request: Head \"https://k8s.gcr.io/v2/pause/manifests/3.6\": dial tcp 108.177.125.82:443: i/o timeout"
Jul 05 19:08:30 k8s-testing01-190 containerd[13788]: time="2022-07-05T19:08:30.696324518+08:00" level=info msg="trying next host" error="failed to do request: Head \"https://k8s.gcr.io/v2/pause/manifests/3.6\": dial tcp 108.177.125.82:443: i/o timeout" host=k8s.gcr.io

网络问题，将镜像下载到本地，重新按照上面步骤打Tag

5.4 初始化完成后出现以下问题

1
2
3
4
5
6
7
8
9
10
11
12
13
1、执行kubectl get cs，scheduler和controller-manager为Unhealthy状态！             --- --- 禁用了--port
如下：
$ kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                       ERROR
scheduler            Unhealthy   Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused   
controller-manager   Unhealthy   Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused   
etcd-2               Healthy     {"health":"true"}                                                                             
etcd-1               Healthy     {"health":"true"}                                                                             
etcd-0               Healthy     {"health":"true"} 

2、执行kubectl get node，node是NotReady状态                                       --- --- 没有安装网络，参考3.4
3、执行kubectl get po，pod中coredns是Pending状态！                                 --- --- 没有安装网络，参考3.4

出现问题1，是/etc/kubernetes/manifests/下的kube-controller-manager.yaml和kube-scheduler.yaml设置的默认端口是0导致的，解决方式是注释掉对应的port即可，操作如下：

1
2
3
[root@k8s-master ~]# cd /etc/kubernetes/manifests/
[root@k8s-master manifests]# ls
kube-apiserver.yaml  kube-controller-manager.yaml  kube-scheduler.yaml

修改kube-controller-manager.yaml文件：注释掉27行

1
2
[root@k8s-master manifests]# vim kube-controller-manager.yaml 
#    - --port=0

修改kube-scheduler.yaml文件：注释掉19行,- –port=0

1
2
[root@k8s-master manifests]# vim kube-scheduler.yaml 
#    - --port=0

在master节点上重启kubelet

1
[root@k8s-master manifests]# systemctl restart kubelet.service
1
2
3
4
5
6
[root@node01 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE            ERROR
controller-manager   Healthy   ok        
scheduler            Healthy   ok        
etcd-0               Healthy  {"health":"true"}

5.5 找不到主节点

1
2
3
kubelet.go:2466] "Error getting node" err="node \"master\" not found"
或
dial tcp 10.0.1.200:6443: connect: connection refused

先检查本机的/ etc / hosts中的主机名和ip的配置是否有误，其次检查kubeadm-config.yaml文件中的advertise_ip是否外网IP，如是，改成内网试试，最后检查防火墙设置，在 / etc / sysctl.conf文件中添加如下内容：

1
2
3
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-arptables = 1

执行sysctl -p /etc/sysctl.d/k8s.conf

Linux
#kubernetes
kubeadm部署kubernetes
https://www.zhoumx.net/kubeadm部署kubernetes.html
作者
阿星
发布于
2023年8月22日
许可协议