Logo
Posts Projects Photos

文档详细介绍了基于K3s和KubeEdge构建混合云边缘计算集群的完整步骤，包括系统配置、容器运行时和网络互通方案。

边缘集群
3 mins
691 words
Loading views
REC K3sKubeEdgeContainerd边缘计算混合云

控制面h2

安装k3sh3

mkdir /etc/sysctl.d


cat > /etc/sysctl.d/99-k3s.conf <<EOF
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.core.somaxconn = 65535
net.ipv4.ip_forward = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 30
fs.file-max = 1000000
fs.inotify.max_user_watches = 524288
vm.swappiness = 10


net.ipv4.tcp_keepalive_time = 30
net.ipv4.tcp_keepalive_intvl = 10
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_retries2 = 5
net.ipv4.tcp_syn_retries = 3
net.core.somaxconn = 1024
net.ipv4.ip_forward = 1
vm.overcommit_memory = 1
EOF


sysctl -p /etc/sysctl.d/99-k3s.conf


cat > /etc/security/limits.conf <<EOF
* soft nofile 65536
* hard nofile 65536
* soft nproc 65536
* hard nproc 65536
root soft nofile 65536
root hard nofile 65536
EOF
VERSION='v1.32.10+k3s1'


if [[ "`uname -m`" != "x86_64" ]];then
    K3S_BIN='k3s-arm64'
    K3S_IMAGE_BIN='k3s-airgap-images-arm64.tar'
else
    K3S_BIN='k3s'
    K3S_IMAGE_BIN='k3s-airgap-images-amd64.tar'
fi


curl https://get.k3s.io -SsL > install.sh
aria2c -s 10 -x 10 -c https://serv00.0197011.xyz/https://github.com/k3s-io/k3s/releases/download/${VERSION}/${K3S_BIN}
aria2c -s 10 -x 10 -c https://serv00.0197011.xyz/https://github.com/k3s-io/k3s/releases/download/${VERSION}/${K3S_IMAGE_BIN}


apt-get update && apt-get install -y ipset ipvsadm conntrack


cat > /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack


lsmod | grep ip_vs
/usr/local/bin/k3s-uninstall.sh
EASYTIER_ETH=es0
EASYTIER_IP=$(ip -4 addr show ${EASYTIER_ETH} | grep -oP '(?<=inet\s)\d+(\.\d+){3}')


if [ -z "$EASYTIER_IP" ]; then
  echo "Error: Cannot find IP for interface $EASYTIER_ETH"
  exit 1
fi


echo "Deploying K3s on $EASYTIER_ETH with IP $EASYTIER_IP..."


mkdir -p /var/lib/rancher/k3s/agent/images/
chmod +x install.sh
cp -a k3s-airgap-images-* /var/lib/rancher/k3s/agent/images/
cp -a k3s /usr/local/bin/
chmod +x /usr/local/bin/k3s


INSTALL_K3S_DEBUG=true INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC=" \
  server \
  --flannel-iface=${EASYTIER_ETH} \
  --flannel-backend="vxlan" \
  --node-external-ip=${EASYTIER_IP} \
  --node-ip=${EASYTIER_IP} \
  --bind-address=${EASYTIER_IP} \
  --advertise-address=${EASYTIER_IP} \
  --tls-san=${EASYTIER_IP} \
  --tls-san=127.0.0.1 \
  --data-dir=/var/lib/rancher/k3s \
  --disable=traefik,servicelb,metrics-server,local-storage \
  --cluster-cidr=10.42.0.0/16 \
  --service-cidr=10.43.0.0/16 \
  --disable-network-policy \
  --write-kubeconfig-mode=644 \
  --kube-proxy-arg=proxy-mode=ipvs \
  --kube-proxy-arg=ipvs-scheduler=rr \
" ./install.sh

部署edge cloudh3

ARCH=$([ "$(uname -m)" = "x86_64" ] && echo "amd64" || echo "arm64")


aria2c -s 10 -x 10 -c https://serv00.0197011.xyz/https://github.com/kubeedge/kubeedge/releases/download/v1.23.0/keadm-v1.23.0-linux-${ARCH}.tar.gz


tar -zxvf keadm-v1.23.0-linux-${ARCH}.tar.gz
cp keadm-v1.23.0-linux-${ARCH}/keadm/keadm /usr/local/bin/
keadm init \
  --advertise-address="100.64.100.253,127.0.0.1,192.168.10.31" \
  --kubeedge-version="1.23.0" \
  --kube-config=/root/.kube/config


keadm gettoken

Edge面h2

部署containerdh3

ARCH=$([ "$(uname -m)" = "x86_64" ] && echo "amd64" || echo "arm64")


aria2c -s 10 -x 10 -c https://github.com/containerd/containerd/releases/download/v2.1.5/containerd-2.1.5-linux-${ARCH}.tar.gz


tar Cxzvf /usr/local containerd-2.1.5-linux-${ARCH}.tar.gz


wget https://raw.githubusercontent.com/containerd/containerd/main/containerd.service -O /etc/systemd/system/containerd.service
systemctl daemon-reload


mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
systemctl restart containerd
systemctl enable containerd


aria2c -s 10 -x 10 -c https://serv00.0197011.xyz/https://github.com/opencontainers/runc/releases/download/v1.4.2/runc.${ARCH}
install -m 755 runc.${ARCH} /usr/local/bin/runc


mkdir -p /opt/cni/bin
aria2c -s 10 -x 10 -c https://serv00.0197011.xyz/https://github.com/containernetworking/plugins/releases/download/v1.9.1/cni-plugins-linux-${ARCH}-v1.9.1.tgz
tar Cxzvf /opt/cni/bin cni-plugins-linux-${ARCH}-v1.9.1.tgz
ln -snf /opt/cni/bin /usr/lib/cni
mkdir -p /etc/cni/net.d
cat <<EOF | tee /etc/cni/net.d/10-containerd-net.conflist
{
  "cniVersion": "1.0.0",
  "name": "containerd-net",
  "plugins": [
    {
      "type": "bridge",
      "bridge": "cni0",
      "isGateway": true,
      "ipMasq": true,
      "promiscMode": true,
      "ipam": {
        "type": "host-local",
        "ranges": [
          [{
            "subnet": "10.42.42.0/24"
          }]
        ],
        "routes": [
          { "dst": "0.0.0.0/0" }
        ]
      }
    },
    {
      "type": "portmap",
      "capabilities": {"portMappings": true}
    }
  ]
}
EOF

部署edge coreh3

keadm join \
--cloudcore-ipport="100.64.100.253:10000" \
--token="******" \
--kubeedge-version="v1.23.0" \
--remote-runtime-endpoint="unix:///run/containerd/containerd.sock" \
--cgroupdriver="systemd"


systemctl restart edgecore containerd

完善集群h2

kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.podCIDR}{"\n"}{end}'
# 根据这个输出修改 /etc/cni/net.d/10-containerd-net.conflist > ranges."subnet": "10.42.42.0/24"
kubectl label nodes bj-aliyun-160 location=dom
kubectl label nodes bj-hw-net location=dom
kubectl label nodes sd-aliyun-228 location=dom


kubectl label nodes usa-oracle-241 location=ini
kubectl label nodes jp-oracle-4 location=ini

网络互通h3

Terminal window
(base) root@sd-aliyun-228:~# ip -4 addr show cni0
4: cni0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    inet 10.42.2.1/24 brd 10.42.2.255 scope global cni0
       valid_lft forever preferred_lft forever


/opt/easytier/easytier-core --hostname sd-aliyun-228 --network-name vpc-lan --network-secret "******" -p tcp://fs.0197011.xyz:12399 --dev-name es0 -i 100.64.100.228 -n 10.42.2.0/24
# 中继服务器
/opt/easytier/easytier-core --private-mode true --network-name vpc-lan --network-secret "******" --relay-all-peer-rpc -l tcp://0.0.0.0:12399 -l udp://0.0.0.0:12398 -l ring://0.0.0.0:12397 -l ws://0.0.0.0:12396 -l wss://0.0.0.0:12395
# 网络测试
(base) root@sd-aliyun-228:~# cat <<EOF | while read line; do
10.42.0.4
10.42.6.4
10.42.1.4
10.42.2.4
10.42.4.3
EOF
echo "$line `curl $line -is | grep HTTP`"
 done
10.42.0.4 HTTP/1.1 200 OK
10.42.6.4 HTTP/1.1 200 OK
10.42.1.4 HTTP/1.1 200 OK
10.42.2.4 HTTP/1.1 200 OK
10.42.4.3 HTTP/1.1 200 OK

域名解析h3

apt install dnsmasq -y


cp -a /etc/dnsmasq.conf /etc/dnsmasq.conf.bak
cat > /etc/dnsmasq.conf <<EOF
listen-address=127.0.0.1
no-hosts
server=/cluster.local/10.43.0.10
server=8.8.8.8
server=8.8.4.4
cache-size=500
EOF


echo "nohook resolv.conf" >> /etc/dhcpcd.conf


cat > /etc/resolv.conf << 'EOF'
nameserver 127.0.0.1
EOF


chattr +i /etc/resolv.conf


systemctl enable dnsmasq --now
systemctl restart dnsmasq

其他h2

nerdctlh3

wget https://serv00.0197011.xyz/https://github.com/containerd/nerdctl/releases/download/v2.2.2/nerdctl-2.2.2-linux-amd64.tar.gz
tar -xf nerdctl-2.2.2-linux-amd64.tar.gz
mv nerdctl /usr/local/bin/


mkdir /etc/nerdctl
echo 'namespace = "k8s.io"' > /etc/nerdctl/nerdctl.toml
helm upgrade --install f5 nginx-ingress/ --set controller.service.type=NodePort -n f5 --create-namespace