K8s高可用集群部署：基于某银行电子商城实战全记录

                    ┌─────────────────────────┐
                    │    Keepalived + HAProxy  │  ← VIP: 192.168.1.100
                    │    (API Server 负载均衡)   │
                    └───────────┬─────────────┘
                                │
            ┌───────────────────┼───────────────────┐
            ▼                   ▼                   ▼
    ┌───────────────┐   ┌───────────────┐   ┌───────────────┐
    │  Master-01    │   │  Master-02    │   │  Master-03    │
    │  kube-apiserver│   │  kube-apiserver│   │  kube-apiserver│
    │  etcd          │   │  etcd          │   │  etcd          │
    │  scheduler     │   │  scheduler     │   │  scheduler     │
    │  controller-mgr│   │  controller-mgr│   │  controller-mgr│
    └───────────────┘   └───────────────┘   └───────────────┘
            │                   │                   │
            └───────────────────┼───────────────────┘
                                │
                    ┌───────────┴───────────┐
                    │    Calico CNI 网络     │  ← Pod跨节点通信
                    └───────────┬───────────┘
                                │
        ┌───────────┬───────────┼───────────┬───────────┐
        ▼           ▼           ▼           ▼           ▼
   ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐
   │Worker-01│ │Worker-02│ │Worker-03│ │Worker-04│ │Worker-05│
   │ kubelet │ │ kubelet │ │ kubelet │ │ kubelet │ │ kubelet │
   │ kube-proxy│ │ kube-proxy│ │ kube-proxy│ │ kube-proxy│ │ kube-proxy│
   └─────────┘ └─────────┘ └─────────┘ └─────────┘ └─────────┘

bash# ====== 1. 修改主机名 ======
hostnamectl set-hostname k8s-master-01  # 按节点修改

# ====== 2. 配置hosts ======
cat >> /etc/hosts << EOF
192.168.1.11 k8s-master-01
192.168.1.12 k8s-master-02
192.168.1.13 k8s-master-03
192.168.1.21 k8s-node-01
192.168.1.22 k8s-node-02
192.168.1.23 k8s-node-03
192.168.1.24 k8s-node-04
192.168.1.25 k8s-node-05
EOF

# ====== 3. 关闭Swap（K8s强制要求） ======
swapoff -a
sed -ri 's/.*swap.*/#&/' /etc/fstab

# ====== 4. 关闭防火墙 ======
systemctl stop firewalld && systemctl disable firewalld

# ====== 5. 禁用SELinux ======
sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config
setenforce 0

# ====== 6. 时间同步（精度到秒级，证书验证依赖） ======
yum install -y chrony
systemctl enable --now chronyd
# 配置阿里云NTP
cat > /etc/chrony.conf << EOF
server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
allow 192.168.0.0/16
local stratum 10
EOF
systemctl restart chronyd
chronyc sources -v  # 验证同步状态

# ====== 7. 内核参数优化（开启桥接转发） ======
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
fs.inotify.max_user_watches = 1048576
EOF
sysctl --system

# 加载网桥模块
modprobe br_netfilter
modprobe overlay

bash# ====== 安装Docker（v20.10+） ======
yum install -y yum-utils
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce docker-ce-cli containerd.io

# 配置Docker（阿里云镜像加速 + systemd cgroup）
cat > /etc/docker/daemon.json << EOF
{
  "registry-mirrors": ["https://b9pmyelo.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {"max-size": "100m"}
}
EOF
systemctl enable --now docker

# ====== 配置K8s阿里云yum源 ======
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# ====== 安装kubeadm/kubelet/kubectl（固定版本，避免兼容问题） ======
yum install -y kubelet-1.23.17 kubeadm-1.23.17 kubectl-1.23.17
systemctl enable --now kubelet

bashkubeadm init \
  --apiserver-advertise-address=192.168.1.11 \
  --image-repository registry.aliyuncs.com/google_containers \
  --kubernetes-version v1.23.17 \
  --service-cidr=10.96.0.0/12 \
  --pod-network-cidr=10.244.0.0/16 \
  --ignore-preflight-errors=all

bashmkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

bash# 在master-01上生成新的join命令（token有效期24小时，过期需重新生成）
kubeadm token create --print-join-command

# 输出类似：
# kubeadm join 192.168.1.11:6443 --token xxx.yyy \
#   --discovery-token-ca-cert-hash sha256:zzz \
#   --control-plane --certificate-key <新生成的key>

# 在master-02、master-03上执行上述命令，添加 --control-plane 标志

bashkubectl get nodes
# NAME              STATUS   ROLES                  AGE     VERSION
# k8s-master-01     Ready    control-plane,master   5m      v1.23.17
# k8s-master-02     Ready    control-plane,master   2m      v1.23.17
# k8s-master-03     Ready    control-plane,master   1m      v1.23.17

bashyum install -y haproxy keepalived

conffrontend kube_apiserver_frontend
    bind *:6443
    mode tcp
    default_backend kube_apiserver_backend

backend kube_apiserver_backend
    mode tcp
    balance roundrobin
    option httpchk GET /healthz HTTP/1.1\r\nHost:\ kube-apiserver
    server k8s-master-01 192.168.1.11:6443 check ssl verify none
    server k8s-master-02 192.168.1.12:6443 check ssl verify none
    server k8s-master-03 192.168.1.13:6443 check ssl verify none

confvrrp_instance VI_1
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass k8sHA
    }
    virtual_ipaddress {
        192.168.1.100/24    # VIP地址
    }

bash# 在每个Worker节点上执行kubeadm join命令（从master-01输出的那条）
kubeadm join 192.168.1.11:6443 --token xxx.yyy \
  --discovery-token-ca-cert-hash sha256:zzz

# 验证
kubectl get nodes
# 5个Worker节点状态应为 Ready

bashwget https://docs.projectcalico.org/manifests/calico.yaml
# 修改CALICO_IPV4POOL_CIDR为与--pod-network-cidr一致
sed -i 's|192.168.0.0/16|10.244.0.0/16|' calico.yaml

kubectl apply -f calico.yaml

# 验证
kubectl get pods -n kube-system | grep calico
# calico-node        Running   0/1     1    2m
# calico-kube-controllers   Running   0/1     1    2m

bash# 部署Rook Operator
git clone --single-branch --branch v1.11.2 https://github.com/rook/rook.git
cd rook/deploy/examples
kubectl create -f crds.yaml -f common.yaml -f operator.yaml

# 创建Ceph集群
kubectl create -f cluster.yaml

# 验证
kubectl get pods -n rook-ceph

bash# 查看证书过期时间
kubeadm certs check-expiration

# 手动更新核心证书（有效期延长至10年）
kubeadm certs renew all
systemctl restart kubelet