边缘集群 | Mutter

基于K3S + KubeEdge的边缘计算

边缘集群

May 8, 2026

4 mins

866 words

Loading views

REC linux

控制面h2

安装k3sh3

mkdir /etc/sysctl.d

cat > /etc/sysctl.d/99-k3s.conf <<EOF
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.core.somaxconn = 65535
net.ipv4.ip_forward = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 30
fs.file-max = 1000000
fs.inotify.max_user_watches = 524288
vm.swappiness = 10


net.ipv4.tcp_keepalive_time = 30  # 降低保活时间，快速检测断连
net.ipv4.tcp_keepalive_intvl = 10
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_retries2 = 5  # 减少重试次数，适应高延迟
net.ipv4.tcp_syn_retries = 3
net.core.somaxconn = 1024  # 增加连接队列
net.ipv4.ip_forward = 1  # 启用转发（ZeroTier需要）
vm.overcommit_memory = 1  # 允许内存过分配
EOF


sysctl -p /etc/sysctl.d/99-k3s.conf

cat > /etc/security/limits.conf <<EOF
* soft nofile 65536
* hard nofile 65536
* soft nproc 65536
* hard nproc 65536
root soft nofile 65536
root hard nofile 65536

EOF

VERSION='v1.32.10+k3s1'


if [[ "`uname -m`" != "x86_64" ]];then
    K3S_BIN='k3s-arm64'
    K3S_IMAGE_BIN='k3s-airgap-images-arm64.tar'
else
    K3S_BIN='k3s'
    K3S_IMAGE_BIN='k3s-airgap-images-amd64.tar'
fi

curl https://get.k3s.io  -SsL > install.sh
aria2c -s 10 -x 10  -c   https://serv00.0197011.xyz/https://github.com/k3s-io/k3s/releases/download/${VERSION}/${K3S_BIN}
aria2c -s 10 -x 10  -c  https://serv00.0197011.xyz/https://github.com/k3s-io/k3s/releases/download/${VERSION}/${K3S_IMAGE_BIN}





apt-get update && apt-get install -y ipset ipvsadm conntrack

cat > /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack

lsmod | grep ip_vs

EASYTIER_ETH=es0
EASYTIER_IP=$(ip -4 addr show ${EASYTIER_ETH} | grep -oP '(?<=inet\s)\d+(\.\d+){3}')

if [ -z "$EASYTIER_IP" ]; then
  echo "Error: Cannot find IP for interface $EASYTIER_ETH"
  exit 1
fi

echo "Deploying K3s on $EASYTIER_ETH with IP $EASYTIER_IP..."

mkdir -p /var/lib/rancher/k3s/agent/images/
chmod +x install.sh
cp -a k3s-airgap-images-*  /var/lib/rancher/k3s/agent/images/
cp -a k3s /usr/local/bin/
chmod +x /usr/local/bin/k3s

# 3. 执行安装
INSTALL_K3S_DEBUG=true INSTALL_K3S_SKIP_DOWNLOAD=true  INSTALL_K3S_EXEC=" \
  server \
  --flannel-iface=${EASYTIER_ETH} \
  --flannel-backend="vxlan" \
  --node-external-ip=${EASYTIER_IP} \
  --node-ip=${EASYTIER_IP} \
  --bind-address=${EASYTIER_IP} \
  --advertise-address=${EASYTIER_IP} \
  --tls-san=${EASYTIER_IP} \
  --tls-san=127.0.0.1 \
  --data-dir=/var/lib/rancher/k3s \
  --disable=traefik,servicelb,metrics-server,local-storage \
  --cluster-cidr=10.42.0.0/16 \
  --service-cidr=10.43.0.0/16 \
  --disable-network-policy \
  --write-kubeconfig-mode=644 \
  --kube-proxy-arg=proxy-mode=ipvs \
  --kube-proxy-arg=ipvs-scheduler=rr \
" ./install.sh

部署edge cloudh3

# 稳定版的 x86 架构包
ARCH=$([ "$(uname -m)" = "x86_64" ] && echo "amd64" || echo "arm64")

aria2c -s 10 -x 10  -c    https://serv00.0197011.xyz/https://github.com/kubeedge/kubeedge/releases/download/v1.23.0/keadm-v1.23.0-linux-${ARCH}.tar.gz

tar -zxvf keadm-v1.23.0-linux-${ARCH}.tar.gz
cp keadm-v1.23.0-linux-${ARCH}/keadm/keadm /usr/local/bin/
keadm init  \
  --advertise-address="100.64.100.253,127.0.0.1,192.168.10.31" \
  --kubeedge-version="1.23.0" \
  --kube-config=/root/.kube/config

keadm gettoken

Edge面h2

部署containerdh3

ARCH=$([ "$(uname -m)" = "x86_64" ] && echo "amd64" || echo "arm64")

# 下载 containerd 2.1.5 (x86_64)
aria2c -s 10 -x 10  -c  https://github.com/containerd/containerd/releases/download/v2.1.5/containerd-2.1.5-linux-${ARCH}.tar.gz

# 解压到 /usr/local
tar Cxzvf /usr/local containerd-2.1.5-linux-${ARCH}.tar.gz



wget   https://raw.githubusercontent.com/containerd/containerd/main/containerd.service -O /etc/systemd/system/containerd.service
systemctl daemon-reload


mkdir -p /etc/containerd
containerd config default |  tee /etc/containerd/config.toml
# 将 SystemdCgroup 设置为 true
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
systemctl restart containerd
systemctl enable containerd


aria2c -s 10 -x 10  -c  https://serv00.0197011.xyz/https://github.com/opencontainers/runc/releases/download/v1.4.2/runc.${ARCH}
install -m 755 runc.${ARCH} /usr/local/bin/runc


# 1. 创建 CNI 目录并下载插件 (注意：这里默认是 arm64 版本，如果是 x86 节点请把 arm64 改为 amd64)
mkdir -p /opt/cni/bin
aria2c -s 10 -x 10  -c    https://serv00.0197011.xyz/https://github.com/containernetworking/plugins/releases/download/v1.9.1/cni-plugins-linux-${ARCH}-v1.9.1.tgz
tar Cxzvf /opt/cni/bin cni-plugins-linux-${ARCH}-v1.9.1.tgz
ln -snf /opt/cni/bin /usr/lib/cni

# 这个配置文件等下还要修改
mkdir -p /etc/cni/net.d
cat <<EOF |  tee /etc/cni/net.d/10-containerd-net.conflist
{
  "cniVersion": "1.0.0",
  "name": "containerd-net",
  "plugins": [
    {
      "type": "bridge",
      "bridge": "cni0",
      "isGateway": true,
      "ipMasq": true,
      "promiscMode": true,
      "ipam": {
        "type": "host-local",
        "ranges": [
          [{
            "subnet": "10.42.42.0/24"
          }]
        ],
        "routes": [
          { "dst": "0.0.0.0/0" }
        ]
      }
    },
    {
      "type": "portmap",
      "capabilities": {"portMappings": true}
    }
  ]
}
EOF

部署edge coreh3

keadm join \
--cloudcore-ipport="100.64.100.253:10000" \
--token="0d56163c2b1e9cb6a0d0827af32d3a85307399fe8834bf34ba99e781af3542de.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3Nzc2MDQwNDN9.-np_7wUvlCzRi6gbC13kxcOivb-7s6Vvp-aNhf957ps" \
--kubeedge-version="v1.23.0" \
--remote-runtime-endpoint="unix:///run/containerd/containerd.sock" \
--cgroupdriver="systemd"



systemctl restart edgecore containerd

完善集群h2

kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.podCIDR}{"\n"}{end}'
# 根据这个输出修改
/etc/cni/net.d/10-containerd-net.conflist > ranges."subnet": "10.42.42.0/24"

kubectl label nodes  bj-aliyun-160 location=dom
kubectl label nodes  bj-hw-net  location=dom
kubectl label nodes  sd-aliyun-228 location=dom


kubectl label nodes usa-oracle-241 location=ini
kubectl label nodes jp-oracle-4 location=ini

网络互通h3

(base) root@sd-aliyun-228:~# ip -4 addr show cni0
4: cni0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    inet 10.42.2.1/24 brd 10.42.2.255 scope global cni0
       valid_lft forever preferred_lft forever

/opt/easytier/easytier-core --hostname sd-aliyun-228 --network-name vpc-lan --network-secret qFmkqF8aI3Imdo206LFDiIgYSeIVKZ -p tcp://fs.0197011.xyz:12399 --dev-name es0 -i 100.64.100.228 -n 10.42.2.0/24





# 中继服务器
/opt/easytier/easytier-core --private-mode true --network-name vpc-lan --network-secret qFmkqF8aI3Imdo206LFDiIgYSeIVKZ --relay-all-peer-rpc -l tcp://0.0.0.0:12399 -l udp://0.0.0.0:12398 -l ring://0.0.0.0:12397 -l ws://0.0.0.0:12396 -l wss://0.0.0.0:12395



# 网络测试
(base) root@sd-aliyun-228:~# cat <<EOF | while read line; do
10.42.0.4
10.42.6.4
10.42.1.4
10.42.2.4
10.42.4.3
EOF
echo "$line `curl $line -is | grep HTTP`"
 done
10.42.0.4 HTTP/1.1 200 OK
10.42.6.4 HTTP/1.1 200 OK
10.42.1.4 HTTP/1.1 200 OK
10.42.2.4 HTTP/1.1 200 OK
10.42.4.3 HTTP/1.1 200 OK

域名解析h3

apt install dnsmasq -y

cp -a /etc/dnsmasq.conf /etc/dnsmasq.conf.bak
cat > /etc/dnsmasq.conf <<EOF
# 监听本地地址
listen-address=127.0.0.1

# 不读取 /etc/hosts
no-hosts

# .local 域名走  coredns
server=/cluster.local/10.43.0.10

# 其他域名走 8.8.8.8
server=8.8.8.8
server=8.8.4.4

# 缓存大小
cache-size=500
EOF

echo "nohook resolv.conf" >> /etc/dhcpcd.conf


cat > /etc/resolv.conf << 'EOF'
nameserver 127.0.0.1
EOF

# 防止其他程序修改
chattr +i /etc/resolv.conf

systemctl enable dnsmasq --now
systemctl restart dnsmasq

其他h2

nerdctlh3

wget https://serv00.0197011.xyz/https://github.com/containerd/nerdctl/releases/download/v2.2.2/nerdctl-2.2.2-linux-amd64.tar.gz ; tar -xf nerdctl-2.2.2-linux-amd64.tar.gz ; mv  nerdctl /usr/local/bin/


mkdir /etc/nerdctl

echo 'namespace = "k8s.io"' > /etc/nerdctl/nerdctl.toml

 helm  upgrade --install f5  nginx-ingress/   --set controller.service.type=NodePort -n  f5 --create-namespace

控制面h2

安装k3sh3

部署edge cloudh3

Edge面h2

部署containerdh3

部署edge coreh3

完善集群h2

网络互通h3

域名解析h3

其他h2

nerdctlh3

Comments