#AWS EKS 创建k8s生产环境实例
- 在AWS部署海外节点, 图简单使用web控制台创建VPC和k8s集群出错(k8s), 使用cli命令行工具创建成功
- 本实例为复盘, 记录aws命令行工具创建eks, 安装efs驱动、LBS、ingress-nginx,使用ECR镜像储存等
#安装命令行工具
| #安装aws cli |
| cd /tmp |
| curl -kL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" |
| unzip awscliv2.zip |
| sudo ./aws/install |
| aws --version |
| #配置aws key |
| aws configure |
| #查看配置 |
| aws configure list |
| |
| #安装kubectl |
| curl -o kubectl https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl |
| chmod +x ./kubectl |
| mv kubectl /usr/local/bin |
| kubectl version --short --client |
| |
| #安装eksctl |
| curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp |
| sudo mv /tmp/eksctl /usr/local/bin |
| eksctl version |
| |
#创建VPC网络和子网
#创建VPC网络和子网已单独发帖
aws命令行cli创建VPC网络、公有子网、私有子网、nat网关+EIP - Elvin™ - 博客园
#创建k8s集群
| #env |
| k8s_name=aws-k8s |
| Region=ap-southeast-1 #新加坡 |
| #获取aws账户id |
| OwnerId=$(aws ec2 describe-vpcs --region ${Region} |jq -r ".Vpcs[0].OwnerId") |
| #使用已有子网 |
| private-subnets-id="subnet-lan-a-xxx,subnet-lan-b-xxx" |
| public-subnets-id="subnet-public-a-xxx,subnet-public-b-xxx" |
| # k8s cluster |
| eksctl create cluster \ |
| --region ${Region} \ |
| --name ${k8s_name} \ |
| --version 1.22 \ |
| --vpc-private-subnets ${private-subnets-id} \ |
| --vpc-public-subnets ${public-subnets-id} \ |
| --managed \ |
| --without-nodegroup \ |
| --dry-run |
| |
| # 查看 |
| eksctl get cluster --name ${k8s_name} --region ${Region} |
| |
| # 出错或不要了,可删除 |
| # eksctl delete cluster --name=${k8s_name} |
| |
| # --dry-run 试运行,正式创建时去掉 |
| # --without-nodegroup 不创建node节点 |
| # --vpc-xx 添加已有网络,若不指定会自动创建 |
| # 建议使用多个可用区网络,k8s集群创建后无法更改 |
| # eksctl create cluster --help #查看帮助 |
#创建k8s计算节点组
| #创建b区k8s节点 |
| #k8s nodegroup test |
| eksctl create nodegroup \ |
| --region ${Region} \ |
| --cluster ${k8s_name} \ |
| --name k8s-work-test \ |
| --node-type m5.large \ |
| --nodes 1 \ |
| --nodes-min 1 \ |
| --nodes-max 10 \ |
| --instance-name test-node-b \ |
| --node-ami-family Ubuntu2004 \ |
| --node-private-networking \ |
| --node-zones ${Region}b \ |
| --node-security-groups sg-xxxxxxx \ |
| --ssh-access \ |
| --ssh-public-key aws-bastion \ |
| --full-ecr-access \ |
| --managed \ |
| --dry-run |
| |
| # --nodes 1 创建1个node节点, 规格 m5.large 2核8G |
| # --node-ami-family Ubuntu2004 操作系统Ubuntu20.04 |
| # --node-private-networking 使用私有子网 |
| # --node-zones 可用区 |
| # --node-security-groups 使用已创建的安全组 |
| # --full-ecr-access ECR镜像仓库权限,一定要 |
| # eksctl create nodegroup --help #查看帮助 |
| |
| #节点扩容 |
| eksctl scale nodegroup --region ${Region} \ |
| --cluster ${k8s_name} --nodes=2 --name k8s-work-test |
| |
| # 测试正常就可以删除, 创建配置更高的正式节点 |
| # delete node |
| # eksctl delete nodegroup --cluster=${k8s_name} --name=k8s-work-test |
| |
| #创建b区正式节点组 |
| eksctl create nodegroup \ |
| --region ${Region} \ |
| --cluster ${k8s_name} \ |
| --name k8s-work-b \ |
| --node-type m5.4xlarge \ |
| --nodes 2 \ |
| --nodes-min 1 \ |
| --nodes-max 10 \ |
| --instance-name k8s-node-b \ |
| --max-pods-per-node 110 \ |
| --node-ami-family Ubuntu2004 \ |
| --node-private-networking \ |
| --node-zones ${Region}b \ |
| --node-security-groups sg-xxxxxxx \ |
| --ssh-access \ |
| --ssh-public-key aws-bastion \ |
| --full-ecr-access \ |
| --external-dns-access \ |
| --managed \ |
| --dry-run |
| |
| #规格m5.4xlarge 16核64G |
| #node-zones创建多区,可用于高可用 |
#为k8s集群创建IAM OIDC提供商
| # IAM OIDC即 AWS Identity and Access Management (IAM) OpenID Connect (OIDC) |
| # 创建IMA权限角色时,需要此功能开启 |
| |
| #查看是否有OIDC,没有则创建 |
| oidc_id=$(aws eks describe-cluster --name ${k8s_name} --query "cluster.identity.oidc.issuer" --output text |cut -d'/' -f 5) |
| if [ $(aws iam list-open-id-connect-providers | grep $oidc_id | wc -l ) -eq 0 ]; then |
| eksctl utils associate-iam-oidc-provider --cluster ${k8s_name} --approve |
| fi |
#eks安装efs csi驱动
- k8s使用AWS EFS储存时用到csi驱动
- efs可使用nfs协议挂载,但k8s节点默认没安装nfs客户端
| #创建IAM policy和角色 |
| curl -o iam-policy-efs.json \ |
| https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/master/docs/iam-policy-example.json |
| |
| aws iam create-policy \ |
| --policy-name EKS_EFS_CSI_Driver_Policy \ |
| --policy-document file://iam-policy-efs.json |
| |
| #创建权限 |
| eksctl create iamserviceaccount \ |
| --cluster ${k8s_name} \ |
| --namespace kube-system \ |
| --name efs-csi-controller-sa \ |
| --attach-policy-arn arn:aws:iam::${OwnerId}:policy/EKS_EFS_CSI_Driver_Policy \ |
| --approve \ |
| --region ${Region} |
| |
| # 更新kubeconfig ~/.kube/config |
| aws eks update-kubeconfig --region ${Region} --name ${k8s_name} |
| |
| #下载yaml文件 |
| kubectl kustomize \ |
| "github.com/kubernetes-sigs/aws-efs-csi-driver/deploy/kubernetes/overlays/stable/?ref=release-1.4" > aws-eks-efs-csi.1.4.yaml |
| |
| # vim aws-eks-efs-csi.1.4.yaml |
| # 手动删除如下部分 |
| apiVersion: v1 |
| kind: ServiceAccount |
| metadata: |
| labels: |
| app.kubernetes.io/name: aws-efs-csi-driver |
| name: efs-csi-controller-sa |
| namespace: kube-system |
| --- |
| |
| #部署efs csi |
| kubectl apply -f aws-eks-efs-csi.1.4.yaml |
#使用efs创建pvc实例
| apiVersion: v1 |
| kind: PersistentVolume |
| metadata: |
| name: aws-efs-test |
| spec: |
| capacity: |
| storage: 2000Gi |
| accessModes: |
| - ReadWriteMany |
| persistentVolumeReclaimPolicy: Retain |
| csi: |
| driver: efs.csi.aws.com |
| volumeHandle: fs-xxx:/data |
| --- |
| apiVersion: v1 |
| kind: PersistentVolumeClaim |
| metadata: |
| name: aws-efs-test |
| spec: |
| accessModes: |
| - ReadWriteMany |
| resources: |
| requests: |
| storage: 2000Gi |
| |
| # fs-xxx 为efs实例id,需要单独创建 |
| # 创建efs后需添加子网和安全组,否则无法访问 |
#安装AWS LB Controller
- AWS LoadBalancer默认使用Classic Load Balancer模式
- 使用NLB、ALB模式的负载均衡器,和绑定EIP(绑定固定IP),必须安装LB controller
| #创建IAM角色 |
| curl -o iam_lbs_v2.4.2.json \ |
| https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.4.2/docs/install/iam_policy.json |
| |
| aws iam create-policy \ |
| --policy-name iam_lbs_v2.4.2 \ |
| --policy-document file://iam_lbs_v2.4.2.json |
| |
| eksctl create iamserviceaccount \ |
| --cluster=${k8s_name} \ |
| --namespace=kube-system \ |
| --name=aws-load-balancer-controller \ |
| --role-name "AmazonEKSLoadBalancerControllerRole" \ |
| --attach-policy-arn=arn:aws:iam::${OwnerId}:policy/iam_lbs_v2.4.2 \ |
| --approve |
| |
| #安装cert-manager |
| kubectl apply \ |
| --validate=false \ |
| -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml |
| |
| #下载yaml |
| curl -Lo aws-load-balancer-controller_2.4.2.yaml \ |
| https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.4.2/v2_4_2_full.yaml |
| |
| #更改k8s集群名称 |
| sed -i.bak -e "s|your-cluster-name|${k8s_name}|" aws-load-balancer-controller_2.4.2.yaml |
| |
| #手动删除如下部分 |
| apiVersion: v1 |
| kind: ServiceAccount |
| metadata: |
| labels: |
| app.kubernetes.io/component: controller |
| app.kubernetes.io/name: aws-load-balancer-controller |
| name: aws-load-balancer-controller |
| namespace: kube-system |
| --- |
| |
| #部署lbs |
| kubectl apply -f aws-load-balancer-controller_2.4.2.yaml |
| |
| #查看 |
| kubectl get deployment -n kube-system aws-load-balancer-controller |
| |
#安装ingress-nginx-controller
| #下载yaml |
| curl -o aws-ingress-nginx.nlb.v1.3.0.yml \ |
| https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.3.0/deploy/static/provider/aws/deploy.yaml |
| |
| #增加spec.ipFamilyPolicy: SingleStack |
#修改LoadBalancer部分的Service如下
| --- |
| apiVersion: v1 |
| kind: Service |
| metadata: |
| annotations: |
| #负载均衡器自定义名称 |
| service.beta.kubernetes.io/aws-load-balancer-name: k8s-ingress-slb |
| #负载均衡 NLB模式 |
| service.beta.kubernetes.io/aws-load-balancer-type: "external" |
| service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" |
| #使用EIP,互联网模式 |
| service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" |
| #public子网 |
| service.beta.kubernetes.io/aws-load-balancer-subnets: subnet-axxx, subnet-bxxx |
| #弹性IP地址 |
| service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-axxx, eipalloc-bxxx |
| #获取客户端真事IP |
| service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: preserve_client_ip.enabled=true |
| labels: |
| app.kubernetes.io/component: controller |
| app.kubernetes.io/instance: ingress-nginx |
| app.kubernetes.io/name: ingress-nginx |
| app.kubernetes.io/part-of: ingress-nginx |
| app.kubernetes.io/version: 1.3.0 |
| name: ingress-nginx-controller |
| namespace: ingress-nginx |
| spec: |
| type: LoadBalancer |
| # externalTrafficPolicy: Local |
| ipFamilyPolicy: SingleStack |
| ipFamilies: |
| - IPv4 |
| ports: |
| - appProtocol: http |
| name: http |
| port: 80 |
| protocol: TCP |
| targetPort: http |
| - appProtocol: https |
| name: https |
| port: 443 |
| protocol: TCP |
| targetPort: https |
| selector: |
| app.kubernetes.io/component: controller |
| app.kubernetes.io/instance: ingress-nginx |
| app.kubernetes.io/name: ingress-nginx |
| |
| #部署 |
| kubectl apply -f aws-ingress-nginx.nlb.v1.3.0.yml |
| |
| #查看,获得得到EXTERNAL-IP地址 |
| kubectl get svc ingress-nginx-controller -n ingress-nginx |
| |
| #ping测试EXTERNAL-IP地址ip是否为自己的EIP地址 |
| ping k8s-ingress-slb-xxx.elb.${Region}.amazonaws.com |
| |
| #访问测试 |
| curl -I k8s-ingress-slb-xxx.elb.${Region}.amazonaws.com |
#使用私有镜像仓库,并部署服务测试
| #创建存储库nginx |
| aws ecr create-repository \ |
| --repository-name nginx \ |
| --region $Region |
| |
| #登录储存库(缓存的登录凭证有效期12小时) |
| aws ecr get-login-password --region $Region \ |
| | docker login --username AWS --password-stdin ${OwnerId}.dkr.ecr.${Region}.amazonaws.com |
| |
| #下载公共镜像, 改tag为私有储存库地址 |
| docker pull public.ecr.aws/nginx/nginx:alpine |
| docker tag public.ecr.aws/nginx/nginx:alpine \ |
| ${OwnerId}.dkr.ecr.${Region}.amazonaws.com/nginx:alpine |
| |
| #push镜像到新建的储存库 |
| docker push ${OwnerId}.dkr.ecr.${Region}.amazonaws.com/nginx:alpine |
| |
| #deploy test |
| kubectl create deployment nginx --port=80 \ |
| --image=${OwnerId}.dkr.ecr.${Region}.amazonaws.com/nginx:alpine |
| |
| #查看 |
| kubectl get pod |
| |
| #生命周期策略示例，保持5个镜像版本(tag) |
| cat >aws-ecr-policy.json < |
| { |
| "rules": [ |
| { |
| "rulePriority": 1, |
| "description": "Keep only 3 image", |
| "selection": { |
| "tagStatus": "any", |
| "countType": "imageCountMoreThan", |
| "countNumber": 3 |
| }, |
| "action": { |
| "type": "expire" |
| } |
| } |
| ] |
| } |
| EOF |
| #创建策略 |
| aws ecr put-lifecycle-policy --region $Region \ |
| --repository-name nginx \ |
| --lifecycle-policy-text file://aws-ecr-policy.json |
| |
| #删除清理pod |
| kubectl delete deploy/nginx |
| |
| #删除存储库 |
| aws ecr delete-repository \ |
| --region $Region --force \ |
| --repository-name nginx |
- k8s有pull私有镜像仓库权限,是因为创建参数--full-ecr-access
- AWS ECR镜像储存服务不支持目录,只能分别给每个镜像创建储存库
- aws ecr get-login-password生成的凭证有效期12小时,可使用定时任务每天登录2次解决