参考文档：https://ee.docs.rancher.cn/docs/installation/kubernetes-cluster-setup/high-availability%20/rke2-ha-install-for-nginx
部署架构图：
创建 RKE2 集群的时候需要在 TLS Alternate Names 中添加 Nginx 的 IP or FQDN：
部署 Nginx：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
| cat <<EOF > nginx.conf
worker_processes 1;
events {
worker_connections 8192;
}
stream {
upstream kube_apiservers {
server 172.16.16.120:6443 max_fails=5 fail_timeout=8s;
server 172.16.16.121:6443 max_fails=5 fail_timeout=8s;
server 172.16.16.122:6443 max_fails=5 fail_timeout=8s;
}
upstream backend {
server 172.16.16.120:9345 max_fails=5 fail_timeout=8s;
server 172.16.16.121:9345 max_fails=5 fail_timeout=8s;
server 172.16.16.122:9345 max_fails=5 fail_timeout=8s;
}
server {
listen 443;
proxy_pass kube_apiservers;
proxy_connect_timeout 5s;
proxy_timeout 300s;
}
server {
listen 9345;
proxy_pass backend;
proxy_connect_timeout 5s;
proxy_timeout 300s;
}
}
EOF
docker run -d --name nginx --restart=always -v $(pwd)/nginx.conf:/etc/nginx/nginx.conf -p 443:443 -p 9345:9345 harbor.warnerchen.com/library/nginx:mainline
|
尝试在 Rancher 设置 Direct Mode 访问 RKE2：
如果要给 Nginx 配置 HTTPS，需要注意不能使用自定义的 CA 下发证书，这是因为后端 kube-apiserver 是 HTTPS，而 kube-apiserver 启用的是双向认证，如果 Nginx 使用了非 KUBE CA 生成的证书，那么就会导致认证失败。
在 RKE2，可以使用下面两个文件生成证书：
1
2
| /var/lib/rancher/rke2/server/tls/server-ca.crt
/var/lib/rancher/rke2/server/tls/server-ca.key
|
生成证书命令可以参考：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
| cat <<EOF > openssl.cnf
[ req ]
default_bits = 2048
default_keyfile = privkey.pem
distinguished_name = req_distinguished_name
req_extensions = v3_req
[ req_distinguished_name ]
countryName = Country Name (2 letter code)
countryName_default = CN
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_default = Guangdong
localityName = Locality Name (eg, city)
localityName_default = Shenzhen
organizationName = Organization Name (eg, company)
organizationName_default = SUSE
commonName = Common Name (eg, fully qualified host name)
commonName_default = rke2-cilium.warnerchen.com
[ v3_req ]
subjectAltName = @alt_names
[ alt_names ]
DNS.1 = rke2-cilium.warnerchen.com
IP.1 = 172.16.16.141
EOF
openssl req -new -key /var/lib/rancher/rke2/server/tls/server-ca.key -out server.csr -config openssl.cnf
openssl x509 -req -in server.csr -CA /var/lib/rancher/rke2/server/tls/server-ca.crt -CAkey /var/lib/rancher/rke2/server/tls/server-ca.key -CAcreateserial -out server.crt -days 365 -extensions v3_req -extfile openssl.cnf
|
将生成的证书文件挂载到 Nginx 容器中：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
| cat <<EOF > nginx.conf
worker_processes 1;
events {
worker_connections 1024;
}
http {
upstream kube_apiservers {
server 172.16.16.120:6443 max_fails=5 fail_timeout=8s;
server 172.16.16.121:6443 max_fails=5 fail_timeout=8s;
server 172.16.16.122:6443 max_fails=5 fail_timeout=8s;
}
upstream backend {
server 172.16.16.120:9345 max_fails=5 fail_timeout=8s;
server 172.16.16.121:9345 max_fails=5 fail_timeout=8s;
server 172.16.16.122:9345 max_fails=5 fail_timeout=8s;
}
server {
listen 443 ssl;
server_name rke2-cilium.warnerchen.com;
# 使用之前生成的证书
ssl_certificate /etc/nginx/certs/server.crt;
# 使用 RKE2 Control Plane 节点的 /var/lib/rancher/rke2/server/tls/server-ca.key
ssl_certificate_key /etc/nginx/certs/server.key;
location / {
proxy_pass https://kube_apiservers;
proxy_ssl_verify off;
# proxy_ssl_trusted_certificate /etc/nginx/certs/server-ca.crt;
# proxy_ssl_name rke2-cilium.warnerchen.com;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 300;
proxy_connect_timeout 60;
proxy_send_timeout 300;
}
}
server {
listen 9345 ssl;
server_name rke2-cilium.warnerchen.com;
ssl_certificate /etc/nginx/certs/server.crt;
ssl_certificate_key /etc/nginx/certs/server.key;
location / {
proxy_pass https://backend;
proxy_ssl_verify off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 300;
proxy_connect_timeout 60;
proxy_send_timeout 300;
}
}
}
EOF
docker run -d --name nginx --restart=always -v $(pwd)/nginx.conf:/etc/nginx/nginx.conf -v $(pwd)/certs:/etc/nginx/certs -p 443:443 harbor.warnerchen.com/library/nginx:mainline
|