Rocky9.2 部署高可用K8S + containerd集群

系统环境h2

[root@localhost ~]# cat /etc/redhat-release
Rocky Linux release 9.2 (Blue Onyx)
[root@localhost ~]# uname -a
Linux localhost 5.14.0-284.11.1.el9_2.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 9 17:09:15 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

安装工具h2

yum install epel-release -y
yum clean all
yum makecache
yum install tmux net-tools vnstat htop nvtop wget proxychains-ng vim git  epel-release -y

配置系统环境h2

配置时间同步h3

yum install chrony -y
cp /etc/chrony.conf /etc/chrony.conf.bak

cat <<EOF > /etc/chrony.conf
server ntp.tuna.tsinghua.edu.cn iburst
server time1.cloud.tencent.com   iburst
server ntp.aliyun.com iburst
keyfile /etc/chrony.keys
ntsdumpdir /var/lib/chrony
leapsectz right/UTC
logdir /var/log/chrony
EOF


timedatectl set-timezone Asia/Shanghai
timedatectl
systemctl enable chronyd
systemctl restart chronyd
chronyc sources
chronyc -a makestep

关闭SELinuxh3

setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

关闭防火墙h3

systemctl stop firewalld
systemctl disable firewalld

关闭swaph3

swapoff -a
sed -i '/swap/d' /etc/fstab

优化内核参数h3

cp -a /etc/sysctl.conf /etc/sysctl.conf.bak

cat << EOF >  /etc/sysctl.conf
# https://www.kernel.org/doc/Documentation/sysctl/
#############################################################################################
# 调整虚拟内存
#############################################################################################

# Default: 30
# 0 - 任何情况下都不使用swap。
# 1 - 除非内存不足（OOM），否则不使用swap。
vm.swappiness = 0

# 内存分配策略
#0 - 表示内核将检查是否有足够的可用内存供应用进程使用；如果有足够的可用内存，内存申请允许；否则，内存申请失败，并把错误返回给应用进程。
#1 - 表示内核允许分配所有的物理内存，而不管当前的内存状态如何。
#2 - 表示内核允许分配超过所有物理内存和交换空间总和的内存
vm.overcommit_memory=1

# OOM时处理
# 1关闭，等于0时，表示当内存耗尽时，内核会触发OOM killer杀掉最耗内存的进程。
vm.panic_on_oom=0

# vm.dirty_background_ratio 用于调整内核如何处理必须刷新到磁盘的脏页。
# Default value is 10.
# 该值是系统内存总量的百分比，在许多情况下将此值设置为5是合适的。
# 此设置不应设置为零。
vm.dirty_background_ratio = 5

# 内核强制同步操作将其刷新到磁盘之前允许的脏页总数
# 也可以通过更改 vm.dirty_ratio 的值（将其增加到默认值30以上（也占系统内存的百分比））来增加
# 推荐 vm.dirty_ratio 的值在60到80之间。
vm.dirty_ratio = 60

# vm.max_map_count 计算当前的内存映射文件数。
# mmap 限制（vm.max_map_count）的最小值是打开文件的ulimit数量（cat /proc/sys/fs/file-max）。
# 每128KB系统内存 map_count应该大约为1。 因此，在32GB系统上，max_map_count为262144。
# Default: 65530
vm.max_map_count = 2097152

#############################################################################################
# 调整文件
#############################################################################################

fs.may_detach_mounts = 1

# 增加文件句柄和inode缓存的大小，并限制核心转储。
fs.file-max = 2097152
fs.nr_open = 2097152
fs.suid_dumpable = 0

# 文件监控
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=524288
fs.inotify.max_queued_events=16384

#############################################################################################
# 调整网络设置
#############################################################################################

# 为每个套接字的发送和接收缓冲区分配的默认内存量。
net.core.wmem_default = 25165824
net.core.rmem_default = 25165824

# 为每个套接字的发送和接收缓冲区分配的最大内存量。
net.core.wmem_max = 25165824
net.core.rmem_max = 25165824

# 除了套接字设置外，发送和接收缓冲区的大小
# 必须使用net.ipv4.tcp_wmem和net.ipv4.tcp_rmem参数分别设置TCP套接字。
# 使用三个以空格分隔的整数设置这些整数，分别指定最小，默认和最大大小。
# 最大大小不能大于使用net.core.wmem_max和net.core.rmem_max为所有套接字指定的值。
# 合理的设置是最小4KiB，默认64KiB和最大2MiB缓冲区。
net.ipv4.tcp_wmem = 20480 12582912 25165824
net.ipv4.tcp_rmem = 20480 12582912 25165824

# 增加最大可分配的总缓冲区空间
# 以页为单位（4096字节）进行度量
net.ipv4.tcp_mem = 65536 25165824 262144
net.ipv4.udp_mem = 65536 25165824 262144

# 为每个套接字的发送和接收缓冲区分配的最小内存量。
net.ipv4.udp_wmem_min = 16384
net.ipv4.udp_rmem_min = 16384

# 启用TCP窗口缩放，客户端可以更有效地传输数据，并允许在代理方缓冲该数据。
net.ipv4.tcp_window_scaling = 1

# 提高同时接受连接数。
net.ipv4.tcp_max_syn_backlog = 10240

# 将net.core.netdev_max_backlog的值增加到大于默认值1000
# 可以帮助突发网络流量，特别是在使用数千兆位网络连接速度时，
# 通过允许更多的数据包排队等待内核处理它们。
net.core.netdev_max_backlog = 65536

# 增加选项内存缓冲区的最大数量
net.core.optmem_max = 25165824

# 被动TCP连接的SYNACK次数。
net.ipv4.tcp_synack_retries = 2

# 允许的本地端口范围。
net.ipv4.ip_local_port_range = 2048 65535

# 防止TCP时间等待
# Default: net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_rfc1337 = 1

# 减少tcp_fin_timeout连接的时间默认值
net.ipv4.tcp_fin_timeout = 15

# 积压套接字的最大数量。
# Default is 128.
net.core.somaxconn = 32768

# 打开syncookies以进行SYN洪水攻击保护。
net.ipv4.tcp_syncookies = 1

# 避免Smurf攻击
# 发送伪装的ICMP数据包，目的地址设为某个网络的广播地址，源地址设为要攻击的目的主机，
# 使所有收到此ICMP数据包的主机都将对目的主机发出一个回应，使被攻击主机在某一段时间内收到成千上万的数据包
net.ipv4.icmp_echo_ignore_broadcasts = 1

# 为icmp错误消息打开保护
net.ipv4.icmp_ignore_bogus_error_responses = 1

# 启用自动缩放窗口。
# 如果延迟证明合理，这将允许TCP缓冲区超过其通常的最大值64K。
net.ipv4.tcp_window_scaling = 1

# 打开并记录欺骗，源路由和重定向数据包
net.ipv4.conf.all.log_martians = 1
net.ipv4.conf.default.log_martians = 1

# 告诉内核有多少个未附加的TCP套接字维护用户文件句柄。 万一超过这个数字，
# 孤立的连接会立即重置，并显示警告。
# Default: net.ipv4.tcp_max_orphans = 65536
net.ipv4.tcp_max_orphans = 65536

# 不要在关闭连接时缓存指标
net.ipv4.tcp_no_metrics_save = 1

# 启用RFC1323中定义的时间戳记：
# Default: net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_timestamps = 1

# 启用选择确认。
# Default: net.ipv4.tcp_sack = 1
net.ipv4.tcp_sack = 1

# 增加 tcp-time-wait 存储桶池大小，以防止简单的DOS攻击。
# net.ipv4.tcp_tw_recycle 已从Linux 4.12中删除。请改用net.ipv4.tcp_tw_reuse。
net.ipv4.tcp_max_tw_buckets = 14400
net.ipv4.tcp_tw_reuse = 1

# accept_source_route 选项使网络接口接受设置了严格源路由（SSR）或松散源路由（LSR）选项的数据包。
# 以下设置将丢弃设置了SSR或LSR选项的数据包。
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# 打开反向路径过滤
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1

# 禁用ICMP重定向接受
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0

# 禁止发送所有IPv4 ICMP重定向数据包。
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0

# 开启IP转发.
net.ipv4.ip_forward = 1

# 禁止IPv6
net.ipv6.conf.lo.disable_ipv6=1
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1

# 要求iptables不对bridge的数据进行处理
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1

# arp缓存
# 存在于 ARP 高速缓存中的最少层数，如果少于这个数，垃圾收集器将不会运行。缺省值是 128
net.ipv4.neigh.default.gc_thresh1=2048
# 保存在 ARP 高速缓存中的最多的记录软限制。垃圾收集器在开始收集前，允许记录数超过这个数字 5 秒。缺省值是 512
net.ipv4.neigh.default.gc_thresh2=4096
# 保存在 ARP 高速缓存中的最多记录的硬限制，一旦高速缓存中的数目高于此，垃圾收集器将马上运行。缺省值是 1024
net.ipv4.neigh.default.gc_thresh3=8192

# 持久连接
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10

# conntrack表
net.nf_conntrack_max=1048576
net.netfilter.nf_conntrack_max=1048576
net.netfilter.nf_conntrack_buckets=262144
net.netfilter.nf_conntrack_tcp_timeout_fin_wait=30
net.netfilter.nf_conntrack_tcp_timeout_time_wait=30
net.netfilter.nf_conntrack_tcp_timeout_close_wait=15
net.netfilter.nf_conntrack_tcp_timeout_established=300

#############################################################################################
# 调整内核参数
#############################################################################################

# 地址空间布局随机化（ASLR）是一种用于操作系统的内存保护过程，可防止缓冲区溢出攻击。
# 这有助于确保与系统上正在运行的进程相关联的内存地址不可预测，
# 因此，与这些流程相关的缺陷或漏洞将更加难以利用。
# Accepted values: 0 = 关闭, 1 = 保守随机化, 2 = 完全随机化
kernel.randomize_va_space = 2

# 调高 PID 数量
kernel.pid_max = 65536
kernel.threads-max=30938

# coredump
kernel.core_pattern=core

# 决定了检测到soft lockup时是否自动panic，缺省值是0
kernel.softlockup_all_cpu_backtrace=1
kernel.softlockup_panic=1
EOF

sysctl -p

优化系统参数h3

cp /etc/security/limits.conf /etc/security/limits.conf.bak

cat <<EOF >> /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
EOF

配置hostsh3

cat <<EOF >> /etc/hosts
192.168.153.48 k8s-master-48
192.168.153.42 k8s-master-42
192.168.153.82 k8s-master-82
EOF

配置hostnameh3

ipseg="192.168"
ip4=$(ip a | grep $ipseg | awk '{print $2}' | cut -d '/' -f 1 | awk -F '.' '{print $NF}')
hostnamectl set-hostname k8s-master-${ip4}


ip4=$(ip a | grep $ipseg | awk '{print $2}' | cut -d '/' -f 1 | awk -F '.' '{print $NF}')
hostnamectl set-hostname k8s-worker-${ip4}

配置ipvsh2

yum install ipvsadm ipset  bridge-utils -y


cat <<EOF >> /etc/modules-load.d/bridge.conf
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
br_netfilter
EOF


modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack
modprobe br_netfilter

配置Dockerh2

Docker和containerd 二选一就行

curl -SsL get.docker.com  | bash -s docker --mirror Aliyun

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.16/cri-dockerd-0.3.16.amd64.tgz
tar -xf cri-dockerd-0.3.16.amd64.tgz
mv cri-dockerd/cri-dockerd /usr/local/bin/


wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.socket
wget  https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.service
mv cri-docker.socket cri-docker.service /etc/systemd/system/

sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service

systemctl enable cri-docker.service
systemctl enable --now cri-docker.socket
systemctl restart  cri-docker.socket

部署containerdh2

二进制方式安装

mkdir /opt/src
cd /opt/src

# 下载containerd
wget https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz
tar -C /usr/local -xzvf containerd-2.0.4-linux-amd64.tar.gz

mkdir /etc/containerd ;containerd  config default > /etc/containerd/config.toml


# /etc/systemd/system/containerd.service
cat <<EOF >>  /etc/systemd/system/containerd.service
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target dbus.service

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity

[Install]
WantedBy=multi-user.target
EOF


systemctl daemon-reload
systemctl enable containerd
systemctl restart containerd
systemctl status containerd

wget https://github.com/opencontainers/runc/releases/download/v1.2.6/runc.amd64
install -m 755 runc.amd64 /usr/local/sbin/runc

wget https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz
mkdir -p   /opt/cni/bin
tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

wget https://github.com/containerd/nerdctl/releases/download/v2.0.3/nerdctl-2.0.3-linux-amd64.tar.gz

tar Cxzvf  /usr/local/bin/ nerdctl-2.0.3-linux-amd64.tar.gz

偷懒小tip: 剩下两台机器的containerd配置和启动过程是一样的，所以可以复制过去

scp /etc/systemd/system/containerd.service root@192.168.153.42:/etc/systemd/system/
scp /etc/systemd/system/containerd.service root@192.168.153.48:/etc/systemd/system/

scp -r /usr/local/bin/* root@192.168.153.42:/usr/local/bin/
scp -r /usr/local/bin/* root@192.168.153.48:/usr/local/bin/

scp -r /etc/containerd root@192.168.153.42:/etc/
scp -r /etc/containerd root@192.168.153.48:/etc/

scp -r  /usr/local/sbin/runc     root@192.168.153.42:/usr/local/sbin/
scp -r  /usr/local/sbin/runc     root@192.168.153.48:/usr/local/sbin/


scp -r /opt/cni root@192.168.153.42:/opt/
scp -r /opt/cni root@192.168.153.48:/opt/

scp -r /usr/local/bin/nerdctl root@192.168.153.42:/usr/local/bin/
scp -r /usr/local/bin/nerdctl root@192.168.153.48:/usr/local/bin/

systemctl daemon-reload
systemctl enable containerd
systemctl start containerd
systemctl status containerd

配置内部harbor(自签发ssl)h3

tips 按需

# pwd
/etc/containerd
# tree
.
├── certs.d
│   └── harbor.com
│       ├── ca.crt
│       ├── harbor.com.cert
│       ├── harbor.com.key
│       └── hosts.toml
└── config.toml


cat << EOF > certs.d/harbor.com/hosts.toml
server = "https://harbor.com"

[host."https://harbor.com"]
  ca = "/etc/containerd/certs.d/harbor.com/ca.crt"
EOF

vim /etc/containerd/config.toml
+ [plugins."io.containerd.grpc.v1.cri".registry]
+     config_path = "/etc/containerd/certs.d"

#配置系统信任本地 CA 证书
cp -a ca.crt /etc/pki/ca-trust/source/anchors/
update-ca-trust

部署负载均衡h2

你可以单独配置2或3台机器作为负载均衡器, 也可以在master or node节点上部署我这里直接在master节点上部署了

haproxyh3

yum install -y haproxy


cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

cat <<EOF > /etc/haproxy/haproxy.cfg
global
  log 127.0.0.1 local0
  log 127.0.0.1 local1 notice
  tune.ssl.default-dh-param 2048

defaults
  log global
  mode http
  option dontlognull
  timeout connect 5000ms
  timeout client 600000ms
  timeout server 600000ms

listen stats
  bind :9090
  mode http
  balance
  stats uri /haproxy_stats
  stats auth admin:admin123
  stats admin if TRUE

frontend apiserver_front
    mode tcp
    bind *:16443
    default_backend apiserver_back

backend apiserver_back
    mode tcp
    balance roundrobin
    stick-table type ip size 200k expire 30m
    stick on src
    server k8s-master-48 192.168.153.48:6443 check
    server k8s-master-42 192.168.153.42:6443 check
    server k8s-master-82 192.168.153.82:6443 check
EOF

systemctl restart haproxy
systemctl enable haproxy
systemctl status haproxy

keepalivedh3

一共3个节点, 这里是Master节点

yum install -y keepalived


cat <<EOF > /etc/keepalived/keepalived.conf
global_defs {
    router_id HAProxy_HA
}

vrrp_script chk_haproxy {
    script "killall -0 haproxy"
    interval 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth1
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass secret
    }
    virtual_ipaddress {
        192.168.153.109
    }
    track_script {
        chk_haproxy
    }
}
EOF


systemctl enable keepalived
systemctl restart keepalived
systemctl status keepalived

其他两个节点配置一样, 只需要把state改为BACKUP, priority改为80

cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
cat <<EOF > /etc/keepalived/keepalived.conf
global_defs {
    router_id HAProxy_HA
}

vrrp_script chk_haproxy {
    script "killall -0 haproxy"
    interval 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth1
    virtual_router_id 51
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass secret
    }
    virtual_ipaddress {
        192.168.153.109
    }
    track_script {
        chk_haproxy
    }
}
EOF
systemctl enable keepalived
systemctl restart keepalived
systemctl status keepalived

部署master节点h2

cp /etc/yum.repos.d/kubernetes.repo /etc/yum.repos.d/kubernetes.repo.bak

cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/rpm/
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/rpm/repodata/repomd.xml.key
EOF

yum list --showduplicates  kubectl

KUBE_VERSION=1.31.3-150500.1.1
yum install   -y kubeadm-$KUBE_VERSION kubelet-$KUBE_VERSION kubectl-$KUBE_VERSION



cat << EOF > /usr/lib/systemd/system/kubelet.service.d/11-cgroup.conf
[Service]
CPUAccounting=true
MemoryAccounting=true
BlockIOAccounting=true
ExecStartPre=/usr/bin/bash -c '/usr/bin/mkdir -p /sys/fs/cgroup/{cpuset,memory,systemd,pids,"cpu,cpuacct"}/{system,kube,kubepods}.slice'
Slice=kube.slice
EOF


systemctl daemon-reload
systemctl restart kubelet

cat << EOF > /etc/kubernetes/audit-policy.yaml
# Log all requests at the Metadata level.
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: Metadata
EOF


cat <<EOF > /etc/kubernetes/kubeadm-init.yaml
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock

---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
ipvs:
  minSyncPeriod: 5s
  syncPeriod: 5s
  scheduler: wrr

---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: 1.31.0
# 这里使用的是keepalived的虚拟ip
controlPlaneEndpoint: 192.168.153.109:16443
networking:
  dnsDomain: cluster.local
  podSubnet: 172.20.0.0/16
  serviceSubnet: 172.21.0.0/16
imageRepository: harbor.com/library
apiServer:
  certSANs:
    - 127.0.0.1
    - apiserver.cluster.local
    - 192.168.153.109
    - 192.168.153.42
    - 192.168.153.82
    - 192.168.153.48
  extraArgs:
    - name: event-ttl
      value: 720h
    - name: service-node-port-range
      value: 30000-50000
    - name: audit-log-maxage
      value: '20'
    - name: audit-log-maxbackup
      value: '10'
    - name: audit-log-maxsize
      value: '100'
    - name: audit-log-path
      value: /var/log/kube-audit/audit.log
    - name: audit-policy-file
      value: /etc/kubernetes/audit-policy.yaml
  extraVolumes:
    - name: audit-config
      hostPath: /etc/kubernetes/audit-policy.yaml
      mountPath: /etc/kubernetes/audit-policy.yaml
      readOnly: true
      pathType: File
    - name: audit-log
      hostPath: /var/log/kube-audit
      mountPath: /var/log/kube-audit
      pathType: DirectoryOrCreate
    - name: localtime
      hostPath: /etc/localtime
      mountPath: /etc/localtime
      readOnly: true
      pathType: File
controllerManager:
  extraArgs:
    - name: node-cidr-mask-size
      value: '24'
  extraVolumes:
    - hostPath: /etc/localtime
      mountPath: /etc/localtime
      name: localtime
      readOnly: true
      pathType: File

scheduler:
  extraVolumes:
    - hostPath: /etc/localtime
      mountPath: /etc/localtime
      name: localtime
      readOnly: true
      pathType: File
EOF

echo '127.0.0.1 apiserver.cluster.local' >> /etc/hosts

kubeadm config images pull --config=/etc/kubernetes/kubeadm-init.yaml

# 修改pause image Tag
# 即便配置了imageRepository, kubeadm init的时候默认使用的还会是registry.k8s.io/pause, 所以要提前给她换上
# 网上有办法吧默认的registry.k8s.io/pause 改为其他pause地址的方法, 我感觉没必要, 直接改个tag解了
ctr -n k8s.io images tag harbor.com/library/pause:3.10 registry.k8s.io/pause:3.10

kubeadm init --config=/etc/kubernetes/kubeadm-init.yaml  --upload-certs

  kubeadm join 192.168.153.109:16443  --token jfq8up.i60667hfo3w48ec4 \
        --discovery-token-ca-cert-hash sha256:d96a3743120c9d6c6907c0929d590e6db597a4a6b4d15850c6cf643cec3db82b \
        --control-plane --certificate-key 426efe3d2f11b0eaf0a08dd3c9114467c8bd329dfb56681dc4bfb40a454b53a7 --v=5

部署node节点h2

kubeadm join 192.168.153.109:16443 --token  6y9qsh.dt288jafd981xu3g --discovery-token-ca-cert-hash sha256:96332189e02febe17ff66748b118c8c6f78d1e999550742a4256fd745309edf4   --cri-socket unix:///var/run/cri-dockerd.sock --v=5


#将 worker 节点的 role 标签设置为 worker
kubectl get node --selector='!node-role.kubernetes.io/master' | grep '<none>' | awk '{print "kubectl label node " $1 " node-role.kubernetes.io/worker= --overwrite" }' | bash

token 过期后, 重新生成tokenh3

# 查看token
kubeadm token list

# 重新生成token
kubeadm token create --print-join-command


# discovery-token-ca-cert-hash  生成
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

# 删除token
#kubeadm token delete <token>

# worker节点加入集群
kubeadm join 192.168.153.109:16443 --token  vncj8h.xg6ftued8loa05u4 --discovery-token-ca-cert-hash sha256:7a3ccb4c3c98895d1c8b455833ac36f2ecf1f1425412850b969f461e286cd097    --v=5

配置网络插件calicoh2

wget https://raw.githubusercontent.com/projectcalico/calico/refs/heads/release-v3.29/manifests/calico.yaml

kubectl apply -f calico.yaml

watch kubectl get pods -n calico-system