Post

kk创建k8s

Posted
By
31 min read
kk创建k8s

规划磁盘情况

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

#!/bin/bash

# Print CSV header
echo "Filesystem,Size,Used,Available,Use%,Mounted on"

# Parse the filesystem information and print CSV
df -h --output=source,size,used,avail,pcent,target | tail -n +2 | while read -r line; do
filesystem=$(echo "$line" | awk '{print $1}')
size=$(echo "$line" | awk '{print $2}')
used=$(echo "$line" | awk '{print $3}')
available=$(echo "$line" | awk '{print $4}')
use_percent=$(echo "$line" | awk '{print $5}')
mounted_on=$(echo "$line" | awk '{print $6}')

    # Print CSV line
    echo "\"$filesystem\",$size,$used,$available,\"$use_percent\",$mounted_on"
done

部署参考

使用 KubeKey 快速离线部署 K8s 与 KubeSphere

自定义 StorageClass

1
2
3
4
5
6
7
8
9
10

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: local-storage
provisioner: rancher.io/local-path
parameters:
  type: 'local'
  path: '/data/custom-path'  # 指定你的自定义路径
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer

1

kubectl apply -f custom-storageclass.yaml

配置 NFS 作为默认存储类

使用 KubeKey (kk) 创建 Kubernetes 集群时，可以在配置文件中预定义 NFS 作为默认的存储类。以下是具体步骤，展示如何在使用 KubeKey 部署 Kubernetes 时配置 NFS 作为默认存储类。

确保 NFS 服务器已经设置并正在运行

确保所有 Kubernetes 节点安装了 NFS 客户端：

1
2

sudo apt-get update
sudo apt-get install nfs-common

配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

apiVersion: kubekey.kubesphere.io/v1alpha2
kind: Cluster
metadata:
  name: sample
spec:
  hosts:
  - {name: master, address: 192.168.1.2, internalAddress: 192.168.1.2, user: root, password: "your_password"}
  - {name: worker, address: 192.168.1.3, internalAddress: 192.168.1.3, user: root, password: "your_password"}
  roleGroups:
    etcd:
    - master
    master:
    - master
    worker:
    - worker
  controlPlaneEndpoint:
    domain: lb.kubesphere.local
    address: ""
    port: 6443
  kubernetes:
    version: v1.21.5
  network:
    plugin: calico
  storage:
    defaultStorageClass: nfs-client
    nfs:
      server: <NFS_SERVER_IP>    # 替换为你的 NFS 服务器 IP
      path: /path/to/nfs/share   # 替换为你的 NFS 共享目录
  addons:
  - name: nfs-client-provisioner
    namespace: kube-system
    sources:
      chart:
        name: nfs-subdir-external-provisioner
        repo: https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
        version: 4.0.14
        values:
          storageClass:
            name: nfs-client
            defaultClass: true
          nfs:
            server: <NFS_SERVER_IP>    # 替换为你的 NFS 服务器 IP
            path: /path/to/nfs/share   # 替换为你的 NFS 共享目录

1

kk create cluster -f config-sample.yaml

测试文件

1
2
3
4
5
6
7
8
9
10

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: test-claim
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi

卸载环境

1

./kk delete clusters --cluster -f config-sample.yaml

指定 StorageClass 路径

在 KubeKey 安装过程中指定 StorageClass 路径

创建一个 KubeKey 配置文件

参考文档

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

apiVersion: kubekey.kubesphere.io/v1alpha2
kind: Cluster
metadata:
  name: sample
spec:
  hosts:
    - {name: master, address: 192.168.0.2, internalAddress: 192.168.0.2, user: root, password: "your_password"}
    - {name: worker, address: 192.168.0.3, internalAddress: 192.168.0.3, user: root, password: "your_password"}
  roleGroups:
    etcd:
      - master
    master:
      - master
    worker:
      - worker
  kubernetes:
    version: v1.23.6
    clusterName: cluster.local
  storage:
    localVolume:
      storageClass:
        isDefaultClass: true
        name: local-path
        hostPaths:
          - path: "/data/k8s-storage" # 指定存储路径
            node: "worker" # 指定节点名称
  network:
    plugin: calico
  registry:
    privateRegistry: ""

在上面的配置中，通过 hostPaths 字段为 local-path StorageClass 指定了存储路径 /data/k8s-storage，并且该存储路径位于 worker 节点上。

验证 StorageClass

1

kubectl get storageclass

应该能够看到名为 local-path 的 StorageClass，并且它被设置为默认 StorageClass。

指定runtime是docker还是containerd

1

源集群中导出制品 artifact
使用 KubeKey 快速离线部署 KubeSphere 集群
  
1
2
3
4
5
6

./kk create manifest

# 源集群中导出制品 artifact
$ export KKZONE=cn
$ ./kk artifact export -m manifest-sample.yaml -o kubesphere.tar.gz
#默认tar包的名字是kubekey-artifact.tar.gz，可通过-o参数自定义包名

离线环境安装集群
执行脚本创建 harbor 项目
  
1
2
3
4
5

$ curl https://github.com/kubesphere/ks-installer/blob/master/scripts/create_project_harbor.sh
$ vim create_project_harbor.sh
# TODO 修改url的值为https://dockerhub.kubekey.local
$ chmod +x create_project_harbor.sh
$ ./create_project_harbor.sh

使用KK安装镜像仓库
  • config-sample.yaml(离线环境集群的配置文件)
  • kubesphere.tar.gz（源集群打包出来的 tar 包镜像）
  • harbor 安装文件在 /opt/harbor , 如需运维 harbor，可至该目录下。
  
1

$ ./kk init registry -f config-sample.yaml -a kubesphere.tar.gz

安装kubesphere集群
  
1

$ ./kk create cluster -f config-sample1.yaml -a kubesphere.tar.gz --with-kubernetes v1.21.5 --with-kubesphere v3.2.1 --with-packages

–with-packages（必须添加否则 ios 依赖安装失败）
查看集群状态
  
1

$ kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f

查看需要的资源
  
1
2
3
4
5
6
7
8
9
10
11

[root@harbor kubesphere]# ls -lt /data/kubesphere/
总用量 13103464
drwxr-xr-x 26 root root        4096 8月  16 2023 kubekey
-rw-r--r--  1 root root        6178 8月  16 2023 config-sample.yaml
-rw-r--r--  1 root root       11616 4月   4 2023 mainfest_b.json
-rw-r--r--  1 root root        5433 4月   3 2023 config-sample.yaml.bak
-rw-r--r--  1 root root   155678720 3月   2 2023 ubuntu-20.04-amd64.iso
-rw-r--r--  1 root root   316667904 3月   2 2023 centos-7-amd64.iso
-rwxr-xr-x  1 root root        1267 2月  23 2023 create_project_harbor.sh
-rw-r--r--  1 root root 12865907909 2月  23 2023 kubesphere.tar.gz
-rwxr-xr-x  1 root root    79648122 2月  23 2023 kk

修改containerd的数据目录
参考链接
sudo vim /etc/containerd/config.toml
遇到的问题
containerd下载错误
  
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

08:31:57 CST message: [worker1]
pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
E0611 08:26:42.011960   53893 remote_image.go:238] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to resolve reference \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to do request: Head \"https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8\": x509: certificate signed by unknown authority" image="dockerhub.kubekey.local/kubesphereio/pause:3.8"
FATA[0000] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to resolve reference "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to do request: Head "https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8": x509: certificate signed by unknown authority: Process exited with status 1

08:31:58 CST message: [worker9]
pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
WARN[0000] image connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock]. As the default settings are now deprecated, you should set the endpoint instead.
ERRO[0002] connect endpoint 'unix:///var/run/dockershim.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
ERRO[0004] connect endpoint 'unix:///run/containerd/containerd.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
FATA[0006] connect: connect endpoint 'unix:///run/crio/crio.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded: Process exited with status 1
08:31:58 CST retry: [worker9]
08:31:59 CST message: [worker7]
downloading image: dockerhub.kubekey.local/kubesphereio/pause:3.8
08:32:03 CST message: [worker9]
downloading image: dockerhub.kubekey.local/kubesphereio/pause:3.8
08:32:09 CST message: [worker9]
pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
WARN[0000] image connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock]. As the default settings are now deprecated, you should set the endpoint instead.
ERRO[0002] connect endpoint 'unix:///var/run/dockershim.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
ERRO[0004] connect endpoint 'unix:///run/containerd/containerd.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
FATA[0006] connect: connect endpoint 'unix:///run/crio/crio.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded: Process exited with status 1
08:32:15 CST message: [worker7]
pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
WARN[0000] image connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock]. As the default settings are now deprecated, you should set the endpoint instead.
ERRO[0002] connect endpoint 'unix:///var/run/dockershim.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
FATA[0015] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to resolve reference "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to do request: Head https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8: x509: certificate signed by unknown authority: Process exited with status 1
08:32:15 CST success: [harbor]
08:32:15 CST failed: [master1]
08:32:15 CST failed: [worker11]
08:32:15 CST failed: [worker3]
08:32:15 CST failed: [worker1]
08:32:15 CST failed: [worker9]
08:32:15 CST failed: [worker7]
error: Pipeline[CreateClusterPipeline] execute failed: Module[PullModule] exec failed:
failed: [master1] [PullImages] exec failed after 3 retries: pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
E0611 08:26:36.265921   42910 remote_image.go:238] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to resolve reference \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to do request: Head \"https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8\": x509: certificate signed by unknown authority" image="dockerhub.kubekey.local/kubesphereio/pause:3.8"
FATA[0000] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to resolve reference "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to do request: Head "https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8": x509: certificate signed by unknown authority: Process exited with status 1
failed: [worker11] [PullImages] exec failed after 3 retries: pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
E0611 08:30:11.308202   45576 remote_image.go:238] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to resolve reference \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to do request: Head \"https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8\": x509: certificate signed by unknown authority" image="dockerhub.kubekey.local/kubesphereio/pause:3.8"
FATA[0000] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to resolve reference "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to do request: Head "https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8": x509: certificate signed by unknown authority: Process exited with status 1
failed: [worker3] [PullImages] exec failed after 3 retries: pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
E0611 08:26:36.476116   51599 remote_image.go:238] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to resolve reference \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to do request: Head \"https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8\": x509: certificate signed by unknown authority" image="dockerhub.kubekey.local/kubesphereio/pause:3.8"
FATA[0000] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to resolve reference "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to do request: Head "https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8": x509: certificate signed by unknown authority: Process exited with status 1
failed: [worker1] [PullImages] exec failed after 3 retries: pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
E0611 08:26:42.011960   53893 remote_image.go:238] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to resolve reference \"dockerhub.kubekey.local/kubesphereio/pause:3.8\": failed to do request: Head \"https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8\": x509: certificate signed by unknown authority" image="dockerhub.kubekey.local/kubesphereio/pause:3.8"
FATA[0000] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to resolve reference "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to do request: Head "https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8": x509: certificate signed by unknown authority: Process exited with status 1
failed: [worker9] [PullImages] exec failed after 3 retries: pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
WARN[0000] image connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock]. As the default settings are now deprecated, you should set the endpoint instead.
ERRO[0002] connect endpoint 'unix:///var/run/dockershim.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
ERRO[0004] connect endpoint 'unix:///run/containerd/containerd.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
FATA[0006] connect: connect endpoint 'unix:///run/crio/crio.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded: Process exited with status 1
failed: [worker7] [PullImages] exec failed after 3 retries: pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
WARN[0000] image connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock]. As the default settings are now deprecated, you should set the endpoint instead.
ERRO[0002] connect endpoint 'unix:///var/run/dockershim.sock', make sure you are running as root and the endpoint has been started: context deadline exceeded
FATA[0015] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to resolve reference "dockerhub.kubekey.local/kubesphereio/pause:3.8": failed to do request: Head https://dockerhub.kubekey.local/v2/kubesphereio/pause/manifests/3.8: x509: certificate signed by unknown authority: Process exited with status 1
您在 /var/spool/mail/root 中有新邮件

解决办法
解决办法2
etcd error
  
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

15:11:34 CST message: [master1]
etcd health check failed: Failed to exec command: sudo -E /bin/bash -c "export ETCDCTL_API=2;export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-master1.pem';export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-master1-key.pem';export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';/usr/local/bin/etcdctl --endpoints=https://10.130.9.31:2379 cluster-health | grep -q 'cluster is healthy'"
Error:  client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 10.130.9.31:2379: connect: connection refused

error #0: dial tcp 10.130.9.31:2379: connect: connection refused: Process exited with status 1
15:11:34 CST retry: [master1]
15:11:39 CST message: [master1]
etcd health check failed: Failed to exec command: sudo -E /bin/bash -c "export ETCDCTL_API=2;export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-master1.pem';export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-master1-key.pem';export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';/usr/local/bin/etcdctl --endpoints=https://10.130.9.31:2379 cluster-health | grep -q 'cluster is healthy'"
Error:  client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 10.130.9.31:2379: connect: connection refused

error #0: dial tcp 10.130.9.31:2379: connect: connection refused: Process exited with status 1
15:11:39 CST retry: [master1]
15:11:44 CST message: [master1]
etcd health check failed: Failed to exec command: sudo -E /bin/bash -c "export ETCDCTL_API=2;export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-master1.pem';export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-master1-key.pem';export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';/usr/local/bin/etcdctl --endpoints=https://10.130.9.31:2379 cluster-health | grep -q 'cluster is healthy'"
Error:  client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 10.130.9.31:2379: connect: connection refused

error #0: dial tcp 10.130.9.31:2379: connect: connection refused: Process exited with status 1
15:11:44 CST retry: [master1]
15:11:49 CST message: [master1]
etcd health check failed: Failed to exec command: sudo -E /bin/bash -c "export ETCDCTL_API=2;export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-master1.pem';export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-master1-key.pem';export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';/usr/local/bin/etcdctl --endpoints=https://10.130.9.31:2379 cluster-health | grep -q 'cluster is healthy'"
Error:  client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 10.130.9.31:2379: connect: connection refused

error #0: dial tcp 10.130.9.31:2379: connect: connection refused: Process exited with status 1
15:11:49 CST retry: [master1]

如果是用的kk 2.1之后的版本，试试在config文件中，配置.etcd.type为kubeadm。
https://github.com/kubesphere/kubekey/blob/master/docs/config-example.md
download image error
  
1
2
3
4

downloading image: dockerhub.kubekey.local/kubesphereio/k8s-dns-node-cache:1.15.12
16:40:40 CST message: [worker11]
pull image failed: Failed to exec command: sudo -E /bin/bash -c "env PATH=$PATH crictl pull dockerhub.kubekey.local/kubesphereio/pause:3.8 --platform amd64"
FATA[0005] unable to determine image API version: rpc error: code = DeadlineExceeded desc = context deadline exceeded: Process exited with status 1

通过拷贝证书, 并且重启containerd来解决这个问题：
ks-isntall还是安装的互联网的内容
手动可以处理, 怎么自动化解决
kubeadm init error
  
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

[root@master1 kubernetes]# kubeadm reset
W0611 17:18:16.527346   40848 preflight.go:55] [reset] WARNING: Changes made to this host by 'kubeadm init' or 'kubeadm join' will be reverted.
[reset] Are you sure you want to proceed? [y/N]: y
W0611 17:18:17.715669   40848 removeetcdmember.go:85] [reset] No kubeadm config, using etcd pod spec to get data directory
[reset] No etcd config found. Assuming external etcd
[reset] Please, manually reset etcd to prevent further issues
[reset] Stopping the kubelet service
[reset] Unmounting mounted directories in "/var/lib/kubelet"
[reset] Deleting contents of directories: [/etc/kubernetes/manifests /etc/kubernetes/pki]
[reset] Deleting files: [/etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/bootstrap-kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf]
[reset] Deleting contents of stateful directories: [/var/lib/kubelet]

The reset process does not clean CNI configuration. To do so, you must remove /etc/cni/net.d

The reset process does not reset or clean up iptables rules or IPVS tables.
If you wish to reset iptables, you must do so manually by using the "iptables" command.

If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar)
to reset your system's IPVS tables.

The reset process does not clean your kubeconfig files and you must remove them manually.
Please, check the contents of the $HOME/.kube/config file.
[root@master1 kubernetes]# echo $?
0
[root@master1 kubernetes]# sudo rm -rf /var/lib/etcd /var/lib/kubelet /etc/kubernetes/pki /etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/bootstrap-kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf
[root@master1 kubernetes]#

有脏数据的情况, 可能会出现这种情况
重新制作tar包
新增下列到tar的制作yaml文件, 重新制作
  • registry.cn-beijing.aliyuncs.com/kubesphereio/pause
  • registry.cn-beijing.aliyuncs.com/kubesphereio/coredns:1.9.3
  • dockerhub.kubekey.local/kubesphereio/pod2daemon-flexvol:v3.26.1
  • dockerhub.kubekey.local/kubesphereio/haproxy:2.3
  • dockerhub.kubekey.local/kubesphereio/kubectl:v1.22.0
  • dockerhub.kubekey.local/kubesphereio/scope:1.13.0
  • dockerhub.kubekey.local/kubesphereio/kube-controllers:v3.23.2
  • dockerhub.kubekey.local/kubesphereio/node:v3.23.2
部署调试成功归档
存储tar包/kk/文件
最起码保证文件存在
制作的文件
  
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

130 17:20:55 192.168.10.233:/mnt/mydata/kk $ cat manifest-sample-0605.yaml
apiVersion: kubekey.kubesphere.io/v1alpha2
kind: Manifest
metadata:
  name: sample
spec:
  arches:
  - amd64
  operatingSystems:
  - arch: amd64
    type: linux
    id: centos
    version: "7"
    osImage: CentOS Linux 7 (Core)
    repository:
      iso:
        localPath: /mnt/mydata/kk/centos-7-amd64.iso
        url:
  kubernetesDistributions:
  - type: kubernetes
    version: v1.25.3
  components:
    helm:
      version: v3.6.3
    cni:
      version: v0.9.1
    etcd:
      version: v3.4.13
    ## For now, if your cluster container runtime is containerd, KubeKey will add a docker 20.10.8 container runtime in the below list.
    ## The reason is KubeKey creates a cluster with containerd by installing a docker first and making kubelet connect the socket file of containerd which docker contained.
    containerRuntimes:
    - type: containerd
      version: 1.6.4
    - type: docker
      version: 20.10.8
    crictl:
      version: v1.22.0
    docker-registry:
      version: "2"
    harbor:
      version: v2.5.3
    docker-compose:
      version: v2.2.2
  images:
  - dockerhub.kubekey.local/kubesphereio/k8s-dns-node-cache:1.15.12
  - dockerhub.kubekey.local/kubesphereio/kafka:3.6.0
  - dockerhub.kubekey.local/kubesphereio/ks-apiserver:v3.4.1
  - dockerhub.kubekey.local/kubesphereio/ks-console:v3.4.0
  - dockerhub.kubekey.local/kubesphereio/ks-console:v3.4.3
  - dockerhub.kubekey.local/kubesphereio/ks-controller-manager:v3.4.0
  - dockerhub.kubekey.local/kubesphereio/ks-installer:v3.4.0
  - dockerhub.kubekey.local/kubesphereio/kube-apiserver:v1.25.3
  - dockerhub.kubekey.local/kubesphereio/kube-controller-manager:v1.25.3
  - dockerhub.kubekey.local/kubesphereio/kube-controllers:v3.26.1
  - dockerhub.kubekey.local/kubesphereio/kube-proxy:v1.25.3
  - dockerhub.kubekey.local/kubesphereio/kube-rbac-proxy:v0.11.0
  - dockerhub.kubekey.local/kubesphereio/kube-scheduler:v1.25.3
  - dockerhub.kubekey.local/kubesphereio/mongodb:7.0.4-debian-11-r2
  - dockerhub.kubekey.local/kubesphereio/node-exporter:v1.3.1
  - dockerhub.kubekey.local/kubesphereio/node:v3.26.1
  - dockerhub.kubekey.local/kubesphereio/prometheus:v2.39.1
  - dockerhub.kubekey.local/kubesphereio/provisioner-localpv:3.3.0
  - dockerhub.kubekey.local/monitor/kube-state-metrics:2.6.0
  - dockerhub.kubekey.local/wwmonitor/clean:3.1.6
  - dockerhub.kubekey.local/wwmonitor/clean:3.1.5
  - dockerhub.kubekey.local/wwmonitor/clean:3.1.7
  - dockerhub.kubekey.local/wwmonitor/community-operator:8.4.0-2.1.3
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-adservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-cartservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-checkoutservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-currencyservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-emailservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-frauddetectionservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-frontend
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-frontendproxy
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-kafka
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-loadgenerator
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-paymentservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-quoteservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-recommendationservice
  - dockerhub.kubekey.local/wwmonitor/demo:1.9.0-shippingservice
  - dockerhub.kubekey.local/wwmonitor/elasticsearch:7.17.3
  - dockerhub.kubekey.local/wwmonitor/elasticsearch:8.5.1
  - dockerhub.kubekey.local/wwmonitor/flagd:v0.9.0
  - dockerhub.kubekey.local/wwmonitor/grafana:10.4.0
  - dockerhub.kubekey.local/wwmonitor/jaeger-operator:1.54.0
  - dockerhub.kubekey.local/wwmonitor/mcs-api-server:v1.0
  - dockerhub.kubekey.local/wwmonitor/mcs-data-server:v1.0
  - dockerhub.kubekey.local/wwmonitor/mcs-file-server:v1.0
  - dockerhub.kubekey.local/wwmonitor/mcs-task-server:v1.0
  - dockerhub.kubekey.local/wwmonitor/minio:2023.9.20-debian-11-r0
  - dockerhub.kubekey.local/wwmonitor/monitor-alert:1.0.42
  - dockerhub.kubekey.local/wwmonitor/monitor-api:1.0.17
  - dockerhub.kubekey.local/wwmonitor/monitor-auth:1.0.7
  - dockerhub.kubekey.local/wwmonitor/monitor-gateway:1.0.6
  - dockerhub.kubekey.local/wwmonitor/monitor-mcs:1.0.4
  - dockerhub.kubekey.local/wwmonitor/monitor-vue:1.1.7
  - dockerhub.kubekey.local/wwmonitor/monitor-web:1.1.10
  - dockerhub.kubekey.local/wwmonitor/mysql:5.7
  - dockerhub.kubekey.local/wwmonitor/mysql:8.0.37-debian-12-r2
  - dockerhub.kubekey.local/wwmonitor/nginx:1.26.0-debian-12-r1
  - dockerhub.kubekey.local/wwmonitor/opensearch:2.13.0
  - dockerhub.kubekey.local/wwmonitor/opensearch:2.14.0
  - dockerhub.kubekey.local/wwmonitor/opentelemetry-collector-contrib:0.97.0
  - dockerhub.kubekey.local/wwmonitor/prometheus:latest
  - dockerhub.kubekey.local/wwmonitor/prometheus:v2.51.1
  - dockerhub.kubekey.local/wwmonitor/redis:6.2
  - dockerhub.kubekey.local/wwmonitor/redis:7.2-alpine
  - dockerhub.kubekey.local/wwmonitor/redis:7.2.1
  - dockerhub.kubekey.local/wwmonitor/unify:v1.0.13
  - dockerhub.kubekey.local/wwmonitor/xxl-job-admin:2.4.0
  - registry.cn-beijing.aliyuncs.com/kubesphereio/cni:v3.26.1
  registry:
    auths: {}

  
1

./kkproxy artifact export -m manifest-sample-0605.yaml -o kubesphere0605.tar.gz

部署的文件
  
1
2
3
4
5
6
7
8
9
10
11

[root@harbor kk]# pwd
/data/kk


[root@harbor kk]# ls -lt
总用量 10288940
-rw-r--r--  1 root root        5979 6月  12 10:54 config-sample.yaml
drwxr-xr-x 22 root root        4096 6月  11 18:05 kubekey
-rwxr-xr-x  1 root root    79993860 6月   7 15:29 kk
-rw-r--r--  1 root root 10455859665 6月   7 15:10 kubesphere0605.tar.gz


  
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246

apiVersion: kubekey.kubesphere.io/v1alpha2
kind: Cluster
metadata:
  name: sample
spec:
  hosts:
  - {name: harbor, address: 10.130.9.89, internalAddress: 10.130.9.89, user: root, password: "vrNM6@k1b#N"}
  - {name: master1, address: 10.130.9.31, internalAddress: 10.130.9.31, user: root, password: "%u2YVj3u"}
  - {name: worker1, address: 10.130.9.32, internalAddress: 10.130.9.32, user: root, password: "S!J45#r3"}
  # - {name: worker2, address: 10.130.9.33, internalAddress: 10.130.9.33, user: root, password: "a!sXZqnL"}
  - {name: worker3, address: 10.130.9.34, internalAddress: 10.130.9.34, user: root, password: "^Tt#$^VT"}
  - {name: worker5, address: 10.130.9.36, internalAddress: 10.130.9.36, user: root, password: "nseqY#*t"}
  # - {name: worker7, address: 10.130.9.86, internalAddress: 10.130.9.86, user: root, password: "vrNM6@k1b#N"}
  # - {name: worker9, address: 10.130.9.88, internalAddress: 10.130.9.88, user: root, password: "vrNM6@k1b#N"}
  # - {name: harbor, address: 10.130.9.89, internalAddress: 10.130.9.89, user: root, password: "vrNM6@k1b#N"}
  # - {name: worker11, address: 10.130.9.30, internalAddress: 10.130.9.30, user: root, password: "RXjcmHfc"}
  roleGroups:
    etcd:
    - master1
    control-plane:
    - master1
    worker:
    - master1
    - worker1
    - worker3
    # - worker2
    #- worker3
    # - worker5
    # - worker7
    # - worker9
    # - worker11
    registry:
    - harbor
  controlPlaneEndpoint:
    domain: lb.kubesphere.local
    address: "10.130.9.31"
    port: 6443
  kubernetes:
    version: v1.25.3
    clusterName: cluster.local
    autoRenewCerts: true
    # containerManager: docker
    containerManager: containerd
  storage:
    openebs:
      basePath: /data/openebs/local # 默认没有的新增配置，base path of the local PV provisioner
  etcd:
    type: kubekey
    # type: kubeadm
  network:
    plugin: calico
    kubePodsCIDR: 10.233.64.0/18
    kubeServiceCIDR: 10.233.0.0/18
    ## multus support. https://github.com/k8snetworkplumbingwg/multus-cni
    multusCNI:
      enabled: false
  registry:
    type: harbor
    auths:
      "dockerhub.kubekey.local":
        username: admin
        password: Harbor12345
    privateRegistry: "dockerhub.kubekey.local"
    namespaceOverride: "kubesphereio"
    registryMirrors: []
    insecureRegistries: ["dockerhub.kubekey.local"]
  addons: []



---
apiVersion: installer.kubesphere.io/v1alpha1
kind: ClusterConfiguration
metadata:
  name: ks-installer
  namespace: kubesphere-system
  labels:
    version: v3.4.0
spec:
  persistence:
    storageClass: ""
  authentication:
    jwtSecret: ""
  zone: ""
  local_registry: ""
  namespace_override: ""
  # dev_tag: ""
  etcd:
    monitoring: false
    endpointIps: localhost
    port: 2379
    tlsEnable: true
  common:
    core:
      console:
        enableMultiLogin: true
        port: 30880
        type: NodePort
    # apiserver:
    #  resources: {}
    # controllerManager:
    #  resources: {}
    redis:
      enabled: false
      volumeSize: 2Gi
    openldap:
      enabled: false
      volumeSize: 2Gi
    minio:
      volumeSize: 20Gi
    monitoring:
      # type: external
      endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090
      GPUMonitoring:
        enabled: false
    gpu:
      kinds:
      - resourceName: "nvidia.com/gpu"
        resourceType: "GPU"
        default: true
    es:
      # master:
      #   volumeSize: 4Gi
      #   replicas: 1
      #   resources: {}
      # data:
      #   volumeSize: 20Gi
      #   replicas: 1
      #   resources: {}
      logMaxAge: 7
      elkPrefix: logstash
      basicAuth:
        enabled: false
        username: ""
        password: ""
      externalElasticsearchHost: ""
      externalElasticsearchPort: ""
  alerting:
    enabled: false
    # thanosruler:
    #   replicas: 1
    #   resources: {}
  auditing:
    enabled: false
    # operator:
    #   resources: {}
    # webhook:
    #   resources: {}
  devops:
    enabled: false
    # resources: {}
    jenkinsMemoryLim: 8Gi
    jenkinsMemoryReq: 4Gi
    jenkinsVolumeSize: 8Gi
  events:
    enabled: false
    # operator:
    #   resources: {}
    # exporter:
    #   resources: {}
    # ruler:
    #   enabled: true
    #   replicas: 2
    #   resources: {}
  logging:
    enabled: false
    logsidecar:
      enabled: true
      replicas: 2
      # resources: {}
  metrics_server:
    enabled: false
  monitoring:
    storageClass: ""
    node_exporter:
      port: 9100
      # resources: {}
    # kube_rbac_proxy:
    #   resources: {}
    # kube_state_metrics:
    #   resources: {}
    # prometheus:
    #   replicas: 1
    #   volumeSize: 20Gi
    #   resources: {}
    #   operator:
    #     resources: {}
    # alertmanager:
    #   replicas: 1
    #   resources: {}
    # notification_manager:
    #   resources: {}
    #   operator:
    #     resources: {}
    #   proxy:
    #     resources: {}
    gpu:
      nvidia_dcgm_exporter:
        enabled: false
        # resources: {}
  multicluster:
    clusterRole: none
  network:
    networkpolicy:
      enabled: false
    ippool:
      type: none
    topology:
      type: none
  openpitrix:
    store:
      enabled: false
  servicemesh:
    enabled: false
    istio:
      components:
        ingressGateways:
        - name: istio-ingressgateway
          enabled: false
        cni:
          enabled: false
  edgeruntime:
    enabled: false
    kubeedge:
      enabled: false
      cloudCore:
        cloudHub:
          advertiseAddress:
            - ""
        service:
          cloudhubNodePort: "30000"
          cloudhubQuicNodePort: "30001"
          cloudhubHttpsNodePort: "30002"
          cloudstreamNodePort: "30003"
          tunnelNodePort: "30004"
        # resources: {}
        # hostNetWork: false
      iptables-manager:
        enabled: true
        mode: "external"
        # resources: {}
      # edgeService:
      #   resources: {}
  terminal:
    timeout: 600


  
1

./kk create cluster -f config-sample.yaml -a kubesphere0605.tar.gz --with-kubernetes v1.25.3 --with-kubesphere v3.4.0 --with-packages

核心脚本步骤
  
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311

08:27:40 CST [ConfigureOSModule] Get OS release
08:27:40 CST success: [master1]
08:27:40 CST success: [worker9]
08:27:40 CST success: [worker1]
08:27:40 CST success: [harbor]
08:27:40 CST success: [worker11]
08:27:40 CST success: [worker7]
08:27:40 CST success: [worker3]
08:27:40 CST [ConfigureOSModule] Prepare to init OS
08:27:48 CST success: [master1]
08:27:48 CST success: [worker9]
08:27:48 CST success: [worker1]
08:27:48 CST success: [worker11]
08:27:48 CST success: [harbor]
08:27:48 CST success: [worker3]
08:27:48 CST success: [worker7]
08:27:48 CST [ConfigureOSModule] Generate init os script
08:27:49 CST success: [master1]
08:27:49 CST success: [worker9]
08:27:49 CST success: [worker3]
08:27:49 CST success: [worker1]
08:27:49 CST success: [worker7]
08:27:49 CST success: [harbor]
08:27:49 CST success: [worker11]
08:27:49 CST [ConfigureOSModule] Exec init os script
08:27:51 CST stdout: [worker3]
setenforce: SELinux is disabled
Disabled
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
vm.max_map_count = 262144
vm.swappiness = 0
fs.inotify.max_user_instances = 524288
kernel.pid_max = 65535
net.ipv4.route.flush = 1
net.core.netdev_max_backlog = 65535
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432
net.core.somaxconn = 32768
net.ipv4.tcp_max_syn_backlog = 1048576
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 2048
net.ipv4.neigh.default.gc_thresh3 = 4096
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_max_tw_buckets = 1048576
net.ipv4.tcp_max_orphans = 65535
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.arp_accept = 1
net.ipv4.conf.default.arp_accept = 1
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.default.arp_ignore = 1
vm.overcommit_memory = 0
fs.inotify.max_user_watches = 524288
fs.pipe-max-size = 4194304
fs.aio-max-nr = 262144
kernel.watchdog_thresh = 5
kernel.hung_task_timeout_secs = 5
08:27:51 CST stdout: [worker11]
setenforce: SELinux is disabled
Disabled
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
vm.max_map_count = 262144
vm.swappiness = 0
fs.inotify.max_user_instances = 524288
kernel.pid_max = 65535
net.ipv4.route.flush = 1
net.core.netdev_max_backlog = 65535
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432
net.core.somaxconn = 32768
net.ipv4.tcp_max_syn_backlog = 1048576
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 2048
net.ipv4.neigh.default.gc_thresh3 = 4096
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_max_tw_buckets = 1048576
net.ipv4.tcp_max_orphans = 65535
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.arp_accept = 1
net.ipv4.conf.default.arp_accept = 1
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.default.arp_ignore = 1
vm.overcommit_memory = 0
fs.inotify.max_user_watches = 524288
fs.pipe-max-size = 4194304
fs.aio-max-nr = 262144
kernel.watchdog_thresh = 5
kernel.hung_task_timeout_secs = 5
08:28:02 CST stdout: [master1]
setenforce: SELinux is disabled
Disabled
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
vm.max_map_count = 262144
vm.swappiness = 0
fs.inotify.max_user_instances = 524288
kernel.pid_max = 65535
net.ipv4.route.flush = 1
net.core.netdev_max_backlog = 65535
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432
net.core.somaxconn = 32768
net.ipv4.tcp_max_syn_backlog = 1048576
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 2048
net.ipv4.neigh.default.gc_thresh3 = 4096
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_max_tw_buckets = 1048576
net.ipv4.tcp_max_orphans = 65535
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.arp_accept = 1
net.ipv4.conf.default.arp_accept = 1
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.default.arp_ignore = 1
vm.overcommit_memory = 0
fs.inotify.max_user_watches = 524288
fs.pipe-max-size = 4194304
fs.aio-max-nr = 262144
kernel.watchdog_thresh = 5
kernel.hung_task_timeout_secs = 5
08:28:07 CST stdout: [harbor]
setenforce: SELinux is disabled
Disabled
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
vm.max_map_count = 262144
vm.swappiness = 0
fs.inotify.max_user_instances = 524288
kernel.pid_max = 65535
net.core.netdev_max_backlog = 65535
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432
net.core.somaxconn = 32768
net.ipv4.tcp_max_syn_backlog = 1048576
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 2048
net.ipv4.neigh.default.gc_thresh3 = 4096
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_max_tw_buckets = 1048576
net.ipv4.tcp_max_orphans = 65535
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.arp_accept = 1
net.ipv4.conf.default.arp_accept = 1
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.default.arp_ignore = 1
vm.overcommit_memory = 0
fs.inotify.max_user_watches = 524288
fs.pipe-max-size = 4194304
fs.aio-max-nr = 262144
kernel.watchdog_thresh = 5
kernel.hung_task_timeout_secs = 5
08:28:10 CST stdout: [worker9]
setenforce: SELinux is disabled
Disabled
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
vm.max_map_count = 262144
vm.swappiness = 0
fs.inotify.max_user_instances = 524288
kernel.pid_max = 65535
net.core.netdev_max_backlog = 65535
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432
net.core.somaxconn = 32768
net.ipv4.tcp_max_syn_backlog = 1048576
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 2048
net.ipv4.neigh.default.gc_thresh3 = 4096
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_max_tw_buckets = 1048576
net.ipv4.tcp_max_orphans = 65535
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.arp_accept = 1
net.ipv4.conf.default.arp_accept = 1
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.default.arp_ignore = 1
vm.overcommit_memory = 0
fs.inotify.max_user_watches = 524288
fs.pipe-max-size = 4194304
fs.aio-max-nr = 262144
kernel.watchdog_thresh = 5
kernel.hung_task_timeout_secs = 5
08:28:13 CST stdout: [worker1]
setenforce: SELinux is disabled
Disabled
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
vm.max_map_count = 262144
vm.swappiness = 0
fs.inotify.max_user_instances = 524288
kernel.pid_max = 65535
net.ipv4.route.flush = 1
net.core.netdev_max_backlog = 65535
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432
net.core.somaxconn = 32768
net.ipv4.tcp_max_syn_backlog = 1048576
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 2048
net.ipv4.neigh.default.gc_thresh3 = 4096
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_max_tw_buckets = 1048576
net.ipv4.tcp_max_orphans = 65535
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.arp_accept = 1
net.ipv4.conf.default.arp_accept = 1
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.default.arp_ignore = 1
vm.overcommit_memory = 0
fs.inotify.max_user_watches = 524288
fs.pipe-max-size = 4194304
fs.aio-max-nr = 262144
kernel.watchdog_thresh = 5
kernel.hung_task_timeout_secs = 5





08:29:33 CST stdout: [worker7]
setenforce: SELinux is disabled
Disabled
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
vm.max_map_count = 262144
vm.swappiness = 0
fs.inotify.max_user_instances = 524288
kernel.pid_max = 65535
net.core.netdev_max_backlog = 65535
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432
net.core.somaxconn = 32768
net.ipv4.tcp_max_syn_backlog = 1048576
net.ipv4.neigh.default.gc_thresh1 = 512
net.ipv4.neigh.default.gc_thresh2 = 2048
net.ipv4.neigh.default.gc_thresh3 = 4096
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_max_tw_buckets = 1048576
net.ipv4.tcp_max_orphans = 65535
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.arp_accept = 1
net.ipv4.conf.default.arp_accept = 1
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.default.arp_ignore = 1
vm.overcommit_memory = 0
fs.inotify.max_user_watches = 524288
fs.pipe-max-size = 4194304
fs.aio-max-nr = 262144
kernel.watchdog_thresh = 5
kernel.hung_task_timeout_secs = 5
08:29:33 CST success: [worker3]
08:29:33 CST success: [worker11]
08:29:33 CST success: [master1]
08:29:33 CST success: [harbor]
08:29:33 CST success: [worker9]
08:29:33 CST success: [worker1]
08:29:33 CST success: [worker7]
08:29:33 CST [ConfigureOSModule] configure the ntp server for each node
08:29:33 CST skipped: [harbor]
08:29:33 CST skipped: [master1]
08:29:33 CST skipped: [worker1]
08:29:33 CST skipped: [worker7]
08:29:33 CST skipped: [worker3]
08:29:33 CST skipped: [worker11]
08:29:33 CST skipped: [worker9]
08:29:33 CST [KubernetesStatusModule] Get kubernetes cluster status
08:29:36 CST success: [master1]
08:29:36 CST [InstallContainerModule] Sync containerd binaries

  云原生, k8s
  k8s storageclass kubekey
 This post is licensed under  CC BY 4.0  by the author.