See: https://kubernetes.io/zh-cn/docs/setup/production-environment/container-runtimes/#containerd

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
​
sudo modprobe overlay
sudo modprobe br_netfilter
​
# 设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.core.somaxconn = 32768
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-arptables = 1
net.ipv4.ip_forward                 = 1
net.ipv4.tcp_syncookies = 0
net.ipv4.conf.all.rp_filter = 1
net.ipv4.neigh.default.gc_thresh1 = 80000
net.ipv4.neigh.default.gc_thresh2 = 90000
net.ipv4.neigh.default.gc_thresh3 = 100000
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.all.forwarding = 1
fs.inotify.max_user_watches=2099999999
fs.inotify.max_user_instances=2099999999
fs.inotify.max_queued_events=2099999999
fs.file-max = 1000000
vm.swappiness = 0
vm.max_map_count = 262144
EOF

# TiDB
cat <<EOF >>  /etc/security/limits.conf
root        soft        nofile        1048576
root        hard        nofile        1048576
root        soft        stack         10240
EOF

sysctl --system

sudo sed -i.bak 's/^deb http:\/\/deb\.debian\.org\/debian\//deb https:\/\/mirrors\.tuna\.tsinghua\.edu\.cn\/debian\//g' /etc/apt/sources.list
sudo sed -i.bak 's/^deb http:\/\/security\.debian\.org\/debian-security\//deb https:\/\/mirrors\.tuna\.tsinghua\.edu\.cn\/debian-security\//g' /etc/apt/sources.list
​
apt update
apt install gnupg2 git -y
​
curl http://mirrors.ustc.edu.cn/docker-ce/linux/debian/gpg | apt-key add -
​
echo "deb [arch=amd64] https://mirrors.ustc.edu.cn/docker-ce/linux/debian bookworm stable" >> /etc/apt/sources.list
​
apt-get update
​
apt-get install -y apt-transport-https containerd.io

curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/deb /" | tee /etc/apt/sources.list.d/kubernetes.list

apt-get update
apt-get install -y kubelet kubeadm kubectl

# 生成默认 containerd 配置文件
containerd config default > /etc/containerd/config.toml

# install helm
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# 拉取镜像
kubeadm config images pull --image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers

kubeadm init --image-repository registry.aliyuncs.com/google_containers --kubernetes-version 1.30.0  --pod-network-cidr=10.244.0.0/16 --v=5

镜像源

[plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://jvyajhe1.mirror.aliyuncs.com"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
          endpoint = ["registry.aliyuncs.com/google_containers"]

Flannel

kubectl apply -f https://github.sakurapuare.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

禁用 Swap

到 /etc/fstab 中禁用

swapoff -a

systemctl --type swap
systemctl mask  "dev-*.swap"
nano /etc/initramfs-tools/conf.d/resume

CNI

# cni
mkdir -p /opt/cni/bin
curl -O -L https://files-1302216700.cos.ap-shanghai.myqcloud.com/cni-plugins-linux-amd64-v1.2.0.tgz
tar -C /opt/cni/bin -xzf cni-plugins-linux-amd64-v1.2.0.tgz

# 删除 K8s
echo y | kubeadm reset
ipvsadm --clear
rm -rf $HOME/.kube/config
rm -rf /etc/cni/net.d

生成新的 join token

kubeadm token create --print-join-command

添加另一个 control-plane

 kubectl -n kube-system edit cm kubeadm-config
 ```
kind: ClusterConfiguration
kubernetesVersion: v1.18.0
controlPlaneEndpoint: 172.16.64.2:6443
```
 
 kubeadm init phase upload-certs --upload-certs
 
 kubeadm join 192.168.81.150:6443 --token cekl5n.i1vs4q36a4fb2ysj --discovery-token-ca-cert-hash sha256:8fadcd4b3ef9c885522bf941c650ea8ae19d7a326b98699b68876057b3d701eb  --control-plane --v=5 --certificate-key 获取的 Cert key

安装 Krew

apt install git -y

(
  set -x; cd "$(mktemp -d)" &&
  OS="$(uname | tr '[:upper:]' '[:lower:]')" &&
  ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" &&
  KREW="krew-${OS}_${ARCH}" &&
  curl -fsSLO "https://github.sakurapuare.com/kubernetes-sigs/krew/releases/latest/download/${KREW}.tar.gz" &&
  tar zxvf "${KREW}.tar.gz" &&
  ./"${KREW}" install krew
  
  echo 'export PATH="${PATH}:${HOME}/.krew/bin"' >> ~/.bashrc
  source ~/.bashrc
)

Ceph 准备

modprobe rbd

echo rbd >> /etc/modules-load.d/ceph.conf
apt install lvm2 -y

Longhorn 需要的一些 module

apt install uuid-dev nvme-cli nfs-common
echo nvme-tcp >> /etc/modules-load.d/k8s.conf
echo uio >> /etc/modules-load.d/k8s.conf
echo uio_pci_generic >> /etc/modules-load.d/k8s.conf
modprobe uio
modprobe uio_pci_generic
modprobe nvme-tcp

在应用 longhorn-spdk-setup.yaml 后，可能会遇到 bus error 的问题。这个是 longhorn 的 spdk 的设置问题，会添加一个 hugepage 参数到 sysctl.conf 中，值为 512。

https://longhorn.io/docs/1.5.6/advanced-resources/rwx-workloads/#configuring-volume-mount-options

bash 补全

echo 'source <(kubectl completion bash)' >>~/.bashrc
source  <(kubectl completion bash)

ulimit

cat <<EOF >>  /etc/security/limits.conf
root        soft        nofile        1048576
root        hard        nofile        1048576
root        soft        stack         10240
EOF

sysctl --system

containerd config.toml

wget -O /etc/containerd/config.toml https://ivampiresp.com/files/containerd.toml

version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  tcp_address = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[ttrpc]
  address = ""
  uid = 0
  gid = 0

[debug]
  address = ""
  uid = 0
  gid = 0
  level = ""

[metrics]
  address = ""
  grpc_histogram = false

[cgroup]
  path = ""

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[plugins]
  [plugins."io.containerd.gc.v1.scheduler"]
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
    startup_delay = "100ms"
  [plugins."io.containerd.grpc.v1.cri"]
    disable_tcp_service = true
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    stream_idle_timeout = "4h0m0s"
    enable_selinux = false
    selinux_category_range = 1024
    sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"
    stats_collect_period = 10
    systemd_cgroup = false
    enable_tls_streaming = false
    max_container_log_line_size = 16384
    disable_cgroup = false
    disable_apparmor = false
    restrict_oom_score_adj = false
    max_concurrent_downloads = 3
    disable_proc_mount = false
    unset_seccomp_profile = ""
    tolerate_missing_hugetlb_controller = true
    disable_hugetlb_controller = true
    ignore_image_defined_volumes = false
    [plugins."io.containerd.grpc.v1.cri".containerd]
      snapshotter = "overlayfs"
      default_runtime_name = "runc"
      no_pivot = false
      disable_snapshot_annotations = true
      discard_unpacked_layers = false
      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
        base_runtime_spec = ""
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v2"
          runtime_engine = ""
          runtime_root = ""
          privileged_without_host_devices = false
          base_runtime_spec = ""
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
                SystemdCgroup = true
    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      max_conf_num = 1
      conf_template = ""
    [plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://jvyajhe1.mirror.aliyuncs.com"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
          endpoint = ["registry.aliyuncs.com/google_containers"]
    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = ""
    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""
  [plugins."io.containerd.internal.v1.opt"]
    path = "/opt/containerd"
  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"
  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"
  [plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false
  [plugins."io.containerd.runtime.v1.linux"]
    shim = "containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
    shim_debug = false
  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]
  [plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]
  [plugins."io.containerd.snapshotter.v1.devmapper"]
    root_path = ""
    pool_name = ""
    base_image_size = ""
    async_remove = false

karmada

vim /lib/systemd/system/containerd.service
Environment="HTTP_PROXY=http://192.168.81.105:7890/"
Environment="HTTPS_PROXY=http://192.168.81.105:7890/"
Environment="NO_PROXY=10.0.0.0/16,127.0.0.1,192.168.0.0/16,localhost"

systemctl daemon-reload
systemctl restart containerd

# image
export http_proxy=http://192.168.81.105:7890 &&  export https_proxy=http://192.168.81.105:7890

ctr image pull registry.k8s.io/kube-apiserver:v1.25.4
ctr image pull registry.k8s.io/etcd:3.5.3-0
ctr image pull docker.io/karmada/karmada-aggregated-apiserver:v1.6.0
ctr image pull registry.k8s.io/kube-controller-manager:v1.25.4
ctr image pull docker.io/karmada/karmada-controller-manager:v1.6.0
ctr image pull docker.io/karmada/karmada-scheduler:v1.6.0



# discovery token ca hash
kubectl karmada register 192.168.81.150:32443 --token disern.ywtrps34now4gpmr --discovery-token-ca-cert-hash sha256:8fadcd4b3ef9c885522bf941c650ea8ae19d7a326b98699b68876057b3d701eb


# uninstall & clean
 rm -rf /etc/karmada/
 rm -rf /var/lib/karmada-etcd