[root@localhost ~]# cat /etc/redhat-release
Rocky Linux release 9.2 (Blue Onyx)
[root@localhost ~]# uname -a
Linux localhost 5.14.0-284.11.1.el9_2.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 9 17:09:15 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

yum install epel-release -y
yum clean all
yum makecache
yum install tmux net-tools vnstat htop nvtop wget proxychains-ng vim git epel-release -y

yum install chrony -y
cp /etc/chrony.conf /etc/chrony.conf.bak

cat <<EOF > /etc/chrony.conf
server ntp.tuna.tsinghua.edu.cn iburst
server time1.cloud.tencent.com   iburst
server ntp.aliyun.com iburst
keyfile /etc/chrony.keys
ntsdumpdir /var/lib/chrony
leapsectz right/UTC
logdir /var/log/chrony
EOF

timedatectl set-timezone Asia/Shanghai
timedatectl
systemctl enable chronyd
systemctl restart chronyd
chronyc sources
chronyc -a makestep

setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

systemctl stop firewalld
systemctl disable firewalld

swapoff -a
sed -i '/swap/d' /etc/fstab

cp -a /etc/sysctl.conf /etc/sysctl.conf.bak

cat << EOF > /etc/sysctl.conf
# https://www.kernel.org/doc/Documentation/sysctl/
#############################################################################################
# 调整虚拟内存
#############################################################################################

# Default: 30
# 0 - 任何情况下都不使用 swap。
# 1 - 除非内存不足（OOM），否则不使用 swap。
vm.swappiness = 0

# 内存分配策略
# 0 - 表示内核将检查是否有足够的可用内存供应用进程使用；如果有足够的可用内存，内存申请允许；否则，内存申请失败，并把错误返回给应用进程。
# 1 - 表示内核允许分配所有的物理内存，而不管当前的内存状态如何。
# 2 - 表示内核允许分配超过所有物理内存和交换空间总和的内存
vm.overcommit_memory=1

# OOM 时处理
# 1 关闭，等于 0 时，表示当内存耗尽时，内核会触发 OOM killer 杀掉最耗内存的进程。
vm.panic_on_oom=0

# vm.dirty_background_ratio 用于调整内核如何处理必须刷新到磁盘的脏页。
# Default value is 10.
# 该值是系统内存总量的百分比，在许多情况下将此值设置为 5 是合适的。
# 此设置不应设置为零。
vm.dirty_background_ratio = 5

# 内核强制同步操作将其刷新到磁盘之前允许的脏页总数
# 也可以通过更改 vm.dirty_ratio 的值（将其增加到默认值 30 以上（也占系统内存的百分比））来增加
# 推荐 vm.dirty_ratio 的值在 60 到 80 之间。
vm.dirty_ratio = 60

# vm.max_map_count 计算当前的内存映射文件数。
# mmap 限制（vm.max_map_count）的最小值是打开文件的 ulimit 数量（cat /proc/sys/fs/file-max）。
# 每 128KB 系统内存 map_count 应该大约为 1。因此，在 32GB 系统上，max_map_count 为 262144。
# Default: 65530
vm.max_map_count = 2097152

#############################################################################################
# 调整文件
#############################################################################################

fs.may_detach_mounts = 1

# 增加文件句柄和 inode 缓存的大小，并限制核心转储。
fs.file-max = 2097152
fs.nr_open = 2097152
fs.suid_dumpable = 0

# 文件监控
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=524288
fs.inotify.max_queued_events=16384

#############################################################################################
# 调整网络设置
#############################################################################################

# 为每个套接字的发送和接收缓冲区分配的默认内存量。
net.core.wmem_default = 25165824
net.core.rmem_default = 25165824

# 为每个套接字的发送和接收缓冲区分配的最大内存量。
net.core.wmem_max = 25165824
net.core.rmem_max = 25165824

# 除了套接字设置外，发送和接收缓冲区的大小
# 必须使用 net.ipv4.tcp_wmem 和 net.ipv4.tcp_rmem 参数分别设置 TCP 套接字。
# 使用三个以空格分隔的整数设置这些整数，分别指定最小、默认和最大大小。
# 最大大小不能大于使用 net.core.wmem_max 和 net.core.rmem_max 为所有套接字指定的值。
# 合理的设置是最小 4 KiB，默认 64 KiB 和最大 2 MiB 缓冲区。
net.ipv4.tcp_wmem = 20480 12582912 25165824
net.ipv4.tcp_rmem = 20480 12582912 25165824

# 增加最大可分配的总缓冲区空间
# 以页为单位（4096 字节）进行度量
net.ipv4.tcp_mem = 65536 25165824 262144
net.ipv4.udp_mem = 65536 25165824 262144

# 为每个套接字的发送和接收缓冲区分配的最小内存量。
net.ipv4.udp_wmem_min = 16384
net.ipv4.udp_rmem_min = 16384

# 启用 TCP 窗口缩放，客户端可以更有效地传输数据，并允许在代理方缓冲该数据。
net.ipv4.tcp_window_scaling = 1

# 提高同时接受连接数。
net.ipv4.tcp_max_syn_backlog = 10240

# 将 net.core.netdev_max_backlog 的值增加到大于默认值 1000
# 可以帮助突发网络流量，特别是在使用数千兆位网络连接速度时，
# 通过允许更多的数据包排队等待内核处理它们。
net.core.netdev_max_backlog = 65536

# 增加选项内存缓冲区的最大数量
net.core.optmem_max = 25165824

# 被动 TCP 连接的 SYNACK 次数。
net.ipv4.tcp_synack_retries = 2

# 允许的本地端口范围。
net.ipv4.ip_local_port_range = 2048 65535

# 防止 TCP 时间等待
# Default: net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_rfc1337 = 1

# 减少 tcp_fin_timeout 连接的时间默认值
net.ipv4.tcp_fin_timeout = 15

# 积压套接字的最大数量。
# Default is 128.
net.core.somaxconn = 32768

# 打开 syncookies 以进行 SYN 洪水攻击保护。
net.ipv4.tcp_syncookies = 1

# 避免 Smurf 攻击
# 发送伪装的 ICMP 数据包，目的地址设为某个网络的广播地址，源地址设为要攻击的目的主机，
# 使所有收到此 ICMP 数据包的主机都将对目的主机发出一个回应，使被攻击主机在某一段时间内收到成千上万的数据包
net.ipv4.icmp_echo_ignore_broadcasts = 1

# 为 icmp 错误消息打开保护
net.ipv4.icmp_ignore_bogus_error_responses = 1

# 启用自动缩放窗口。
# 如果延迟证明合理，这将允许 TCP 缓冲区超过其通常的最大值 64K。
net.ipv4.tcp_window_scaling = 1

# 打开并记录欺骗、源路由和重定向数据包
net.ipv4.conf.all.log_martians = 1
net.ipv4.conf.default.log_martians = 1

# 告诉内核有多少个未附加的 TCP 套接字维护用户文件句柄。万一超过这个数字，
# 孤立的连接会立即重置，并显示警告。
# Default: net.ipv4.tcp_max_orphans = 65536
net.ipv4.tcp_max_orphans = 65536

# 不要在关闭连接时缓存指标
net.ipv4.tcp_no_metrics_save = 1

# 启用 RFC1323 中定义的时间戳记：
# Default: net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_timestamps = 1

# 启用选择确认。
# Default: net.ipv4.tcp_sack = 1
net.ipv4.tcp_sack = 1

# 增加 tcp-time-wait 存储桶池大小，以防止简单的 DOS 攻击。
# net.ipv4.tcp_tw_recycle 已从 Linux 4.12 中删除。请改用 net.ipv4.tcp_tw_reuse。
net.ipv4.tcp_max_tw_buckets = 14400
net.ipv4.tcp_tw_reuse = 1

# accept_source_route 选项使网络接口接受设置了严格源路由（SSR）或松散源路由（LSR）选项的数据包。
# 以下设置将丢弃设置了 SSR 或 LSR 选项的数据包。
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# 打开反向路径过滤
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1

# 禁用 ICMP 重定向接受
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0

# 禁止发送所有 IPv4 ICMP 重定向数据包。
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0

# 开启 IP 转发。
net.ipv4.ip_forward = 1

# 禁止 IPv6
net.ipv6.conf.lo.disable_ipv6=1
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1

# 要求 iptables 不对 bridge 的数据进行处理
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1

# arp 缓存
# 存在于 ARP 高速缓存中的最少层数，如果少于这个数，垃圾收集器将不会运行。缺省值是 128
net.ipv4.neigh.default.gc_thresh1=2048
# 保存在 ARP 高速缓存中的最多的记录软限制。垃圾收集器在开始收集前，允许记录数超过这个数字 5 秒。缺省值是 512
net.ipv4.neigh.default.gc_thresh2=4096
# 保存在 ARP 高速缓存中的最多记录的硬限制，一旦高速缓存中的数目高于此，垃圾收集器将马上运行。缺省值是 1024
net.ipv4.neigh.default.gc_thresh3=8192

# 持久连接
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10

# conntrack 表
net.nf_conntrack_max=1048576
net.netfilter.nf_conntrack_max=1048576
net.netfilter.nf_conntrack_buckets=262144
net.netfilter.nf_conntrack_tcp_timeout_fin_wait=30
net.netfilter.nf_conntrack_tcp_timeout_time_wait=30
net.netfilter.nf_conntrack_tcp_timeout_close_wait=15
net.netfilter.nf_conntrack_tcp_timeout_established=300

#############################################################################################
# 调整内核参数
#############################################################################################

# 地址空间布局随机化（ASLR）是一种用于操作系统的内存保护过程，可防止缓冲区溢出攻击。
# 这有助于确保与系统上正在运行的进程相关联的内存地址不可预测，
# 因此，与这些流程相关的缺陷或漏洞将更加难以利用。
# Accepted values: 0 = 关闭, 1 = 保守随机化, 2 = 完全随机化
kernel.randomize_va_space = 2

# 调高 PID 数量
kernel.pid_max = 65536
kernel.threads-max=30938

# coredump
kernel.core_pattern=core

# 决定了检测到 soft lockup 时是否自动 panic，缺省值是 0
kernel.softlockup_all_cpu_backtrace=1
kernel.softlockup_panic=1
EOF

sysctl -p

cp /etc/security/limits.conf /etc/security/limits.conf.bak

cat <<EOF >> /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
EOF

cat <<EOF >> /etc/hosts
192.168.153.48 k8s-master-48
192.168.153.42 k8s-master-42
192.168.153.82 k8s-master-82
EOF

ipseg="192.168"
ip4=$(ip a | grep $ipseg | awk '{print $2}' | cut -d '/' -f 1 | awk -F '.' '{print $NF}')
hostnamectl set-hostname k8s-master-${ip4}

# 如果是 worker 节点，执行以下命令：
# ip4=$(ip a | grep $ipseg | awk '{print $2}' | cut -d '/' -f 1 | awk -F '.' '{print $NF}')
# hostnamectl set-hostname k8s-worker-${ip4}

yum install ipvsadm ipset bridge-utils -y

cat <<EOF >> /etc/modules-load.d/bridge.conf
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
br_netfilter
EOF

modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack
modprobe br_netfilter

curl -SsL get.docker.com | bash -s docker --mirror Aliyun

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.16/cri-dockerd-0.3.16.amd64.tgz
tar -xf cri-dockerd-0.3.16.amd64.tgz
mv cri-dockerd/cri-dockerd /usr/local/bin/

wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.socket
wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.service
mv cri-docker.socket cri-docker.service /etc/systemd/system/

sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service

systemctl enable cri-docker.service
systemctl enable --now cri-docker.socket
systemctl restart cri-docker.socket

mkdir /opt/src
cd /opt/src

# 下载 containerd
wget https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz
tar -C /usr/local -xzvf containerd-2.0.4-linux-amd64.tar.gz

mkdir /etc/containerd
containerd config default > /etc/containerd/config.toml

# 创建 systemd service 文件
cat <<EOF >> /etc/systemd/system/containerd.service
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target dbus.service

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable containerd
systemctl restart containerd
systemctl status containerd

wget https://github.com/opencontainers/runc/releases/download/v1.2.6/runc.amd64
install -m 755 runc.amd64 /usr/local/sbin/runc

wget https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz
mkdir -p /opt/cni/bin
tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

wget https://github.com/containerd/nerdctl/releases/download/v2.0.3/nerdctl-2.0.3-linux-amd64.tar.gz
tar Cxzvf /usr/local/bin/ nerdctl-2.0.3-linux-amd64.tar.gz

# 将配置文件复制到其他节点（示例 IP 为 192.168.153.42 和 192.168.153.48）
scp /etc/systemd/system/containerd.service root@192.168.153.42:/etc/systemd/system/
scp /etc/systemd/system/containerd.service root@192.168.153.48:/etc/systemd/system/

scp -r /usr/local/bin/* root@192.168.153.42:/usr/local/bin/
scp -r /usr/local/bin/* root@192.168.153.48:/usr/local/bin/

scp -r /etc/containerd root@192.168.153.42:/etc/
scp -r /etc/containerd root@192.168.153.48:/etc/

scp -r /usr/local/sbin/runc root@192.168.153.42:/usr/local/sbin/
scp -r /usr/local/sbin/runc root@192.168.153.48:/usr/local/sbin/

scp -r /opt/cni root@192.168.153.42:/opt/
scp -r /opt/cni root@192.168.153.48:/opt/

scp -r /usr/local/bin/nerdctl root@192.168.153.42:/usr/local/bin/
scp -r /usr/local/bin/nerdctl root@192.168.153.48:/usr/local/bin/

systemctl daemon-reload
systemctl enable containerd
systemctl start containerd
systemctl status containerd

# 假设当前目录为 /etc/containerd
# 目录结构如下：
# .
# ├── certs.d
# │   └── example.com
# │       ├── ca.crt
# │       ├── example.com.cert
# │       ├── example.com.key
# │       └── hosts.toml
# └── config.toml

cat << EOF > certs.d/example.com/hosts.toml
server = "https://example.com"

[host."https://example.com"]
  ca = "/etc/containerd/certs.d/example.com/ca.crt"
EOF

# 编辑 /etc/containerd/config.toml，添加或修改以下内容：
# [plugins."io.containerd.grpc.v1.cri".registry]
#     config_path = "/etc/containerd/certs.d"

# 配置系统信任本地 CA 证书
cp -a ca.crt /etc/pki/ca-trust/source/anchors/
update-ca-trust

yum install -y haproxy

cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

cat <<EOF > /etc/haproxy/haproxy.cfg
global
  log 127.0.0.1 local0
  log 127.0.0.1 local1 notice
  tune.ssl.default-dh-param 2048

defaults
  log global
  mode http
  option dontlognull
  timeout connect 5000ms
  timeout client 600000ms
  timeout server 600000ms

listen stats
  bind :9090
  mode http
  balance
  stats uri /haproxy_stats
  stats auth admin:******
  stats admin if TRUE

frontend apiserver_front
    mode tcp
    bind *:16443
    default_backend apiserver_back

backend apiserver_back
    mode tcp
    balance roundrobin
    stick-table type ip size 200k expire 30m
    stick on src
    server k8s-master-48 192.168.153.48:6443 check
    server k8s-master-42 192.168.153.42:6443 check
    server k8s-master-82 192.168.153.82:6443 check
EOF

systemctl restart haproxy
systemctl enable haproxy
systemctl status haproxy

yum install -y keepalived

cat <<EOF > /etc/keepalived/keepalived.conf
global_defs {
    router_id HAProxy_HA
}

vrrp_script chk_haproxy {
    script "killall -0 haproxy"
    interval 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth1
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass ******
    }
    virtual_ipaddress {
        192.168.153.109
    }
    track_script {
        chk_haproxy
    }
}
EOF

systemctl enable keepalived
systemctl restart keepalived
systemctl status keepalived

cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
cat <<EOF > /etc/keepalived/keepalived.conf
global_defs {
    router_id HAProxy_HA
}

vrrp_script chk_haproxy {
    script "killall -0 haproxy"
    interval 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth1
    virtual_router_id 51
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass ******
    }
    virtual_ipaddress {
        192.168.153.109
    }
    track_script {
        chk_haproxy
    }
}
EOF
systemctl enable keepalived
systemctl restart keepalived
systemctl status keepalived

cp /etc/yum.repos.d/kubernetes.repo /etc/yum.repos.d/kubernetes.repo.bak

cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/rpm/
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/rpm/repodata/repomd.xml.key
EOF

yum list --showduplicates kubectl

KUBE_VERSION=1.31.3-150500.1.1
yum install -y kubeadm-$KUBE_VERSION kubelet-$KUBE_VERSION kubectl-$KUBE_VERSION

cat << EOF > /usr/lib/systemd/system/kubelet.service.d/11-cgroup.conf
[Service]
CPUAccounting=true
MemoryAccounting=true
BlockIOAccounting=true
ExecStartPre=/usr/bin/bash -c '/usr/bin/mkdir -p /sys/fs/cgroup/{cpuset,memory,systemd,pids,"cpu,cpuacct"}/{system,kube,kubepods}.slice'
Slice=kube.slice
EOF

systemctl daemon-reload
systemctl restart kubelet

cat << EOF > /etc/kubernetes/audit-policy.yaml
# Log all requests at the Metadata level.
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: Metadata
EOF

cat <<EOF > /etc/kubernetes/kubeadm-init.yaml
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock

---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
ipvs:
  minSyncPeriod: 5s
  syncPeriod: 5s
  scheduler: wrr

---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: 1.31.0
# 这里使用的是 keepalived 的虚拟 IP
controlPlaneEndpoint: 192.168.153.109:16443
networking:
  dnsDomain: cluster.local
  podSubnet: 172.20.0.0/16
  serviceSubnet: 172.21.0.0/16
imageRepository: example.com/library
apiServer:
  certSANs:
    - 127.0.0.1
    - apiserver.cluster.local
    - 192.168.153.109
    - 192.168.153.42
    - 192.168.153.82
    - 192.168.153.48
  extraArgs:
    - name: event-ttl
      value: 720h
    - name: service-node-port-range
      value: 30000-50000
    - name: audit-log-maxage
      value: '20'
    - name: audit-log-maxbackup
      value: '10'
    - name: audit-log-maxsize
      value: '100'
    - name: audit-log-path
      value: /var/log/kube-audit/audit.log
    - name: audit-policy-file
      value: /etc/kubernetes/audit-policy.yaml
  extraVolumes:
    - name: audit-config
      hostPath: /etc/kubernetes/audit-policy.yaml
      mountPath: /etc/kubernetes/audit-policy.yaml
      readOnly: true
      pathType: File
    - name: audit-log
      hostPath: /var/log/kube-audit
      mountPath: /var/log/kube-audit
      pathType: DirectoryOrCreate
    - name: localtime
      hostPath: /etc/localtime
      mountPath: /etc/localtime
      readOnly: true
      pathType: File
controllerManager:
  extraArgs:
    - name: node-cidr-mask-size
      value: '24'
  extraVolumes:
    - hostPath: /etc/localtime
      mountPath: /etc/localtime
      name: localtime
      readOnly: true
      pathType: File

scheduler:
  extraVolumes:
    - hostPath: /etc/localtime
      mountPath: /etc/localtime
      name: localtime
      readOnly: true
      pathType: File
EOF

echo '127.0.0.1 apiserver.cluster.local' >> /etc/hosts

kubeadm config images pull --config=/etc/kubernetes/kubeadm-init.yaml

# 修改 pause 镜像 Tag
# 即使配置了 imageRepository，kubeadm init 默认仍会使用 registry.k8s.io/pause，因此需要提前替换标签
ctr -n k8s.io images tag example.com/library/pause:3.10 registry.k8s.io/pause:3.10

kubeadm init --config=/etc/kubernetes/kubeadm-init.yaml --upload-certs

kubeadm join 192.168.153.109:16443 --token ****** \
        --discovery-token-ca-cert-hash sha256:****** \
        --control-plane --certificate-key ****** --v=5

kubeadm join 192.168.153.109:16443 --token ****** --discovery-token-ca-cert-hash sha256:****** --cri-socket unix:///var/run/cri-dockerd.sock --v=5

kubectl get node --selector='!node-role.kubernetes.io/master' | grep '<none>' | awk '{print "kubectl label node " $1 " node-role.kubernetes.io/worker= --overwrite" }' | bash

# 查看 token
kubeadm token list

# 重新生成 token 并打印加入命令
kubeadm token create --print-join-command

# 获取 discovery-token-ca-cert-hash
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

# 删除 token（如果需要）
# kubeadm token delete <token>

# worker 节点重新加入
kubeadm join 192.168.153.109:16443 --token ****** --discovery-token-ca-cert-hash sha256:****** --v=5

wget https://raw.githubusercontent.com/projectcalico/calico/refs/heads/release-v3.29/manifests/calico.yaml
kubectl apply -f calico.yaml
watch kubectl get pods -n calico-system