사전 작업

• 이전 실습 내용 정리

  • weave scope 삭제
    • kubectl delete -f https://github.com/weaveworks/scope/releases/download/v1.13.2/k8s-scope.yaml

• K8S Cluster 생성

  • $ terraform apply --auto-approve

• Terraform 코드 내용 변경

  • 노드 리소스 증가
    • Auto Scaling Group의 Desired Capacity를 변경 (2→3)
    • t3.medium으로 노드 스케일 업
    • capacity type - SPOT
  • EKS Add-ons 및 Add-on의 실행 권한에 맞는 Role 추가
    • aws-ebs-csi-driver, vpc-cni

  provider "aws" {
    region = local.region
  }
  
  provider "kubernetes" {
    host                   = module.eks.cluster_endpoint
    cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
    token                  = data.aws_eks_cluster_auth.this.token
  }
  
  data "aws_eks_cluster_auth" "this" {
    name = module.eks.cluster_name
  }
  
  data "aws_availability_zones" "available" {}
  
  locals {
    name   = basename(path.cwd)
    region = "ap-northeast-2"
  
    vpc_cidr = "10.0.0.0/16"
    azs      = slice(data.aws_availability_zones.available.names, 0, 3)
  
    tags = {
      Cluster  = local.name
    }
  }
  
  ################################################################################
  # Cluster
  ################################################################################
  
  module "eks" {
    source  = "terraform-aws-modules/eks/aws"
    version = "~> 19.16"
  
    cluster_name                   = local.name
    cluster_version                = "1.27"
    cluster_endpoint_public_access = true
  
    vpc_id     = module.vpc.vpc_id
    subnet_ids = module.vpc.private_subnets
  
    eks_managed_node_groups = {
      default_node_group = {
        instance_types = ["t3.medium"]
        capacity_type  = "SPOT"
        
        min_size     = 2
        max_size     = 5
        desired_size = 2
  
      }
    }
    
    node_security_group_additional_rules = {
      ingress_cluster_api_ephemeral_ports_tcp = {
        description   = "Cluster API to kubeseal services"
        protocol      = "tcp"
        from_port     = 8080
        to_port       = 8080
        type          = "ingress"
        source_cluster_security_group = true
      }
    }
    
    cluster_addons = {
      aws-ebs-csi-driver = {
        service_account_role_arn = module.ebs_csi_addon_irsa_role.iam_role_arn
      }
      vpc-cni = {
        resolve_conflicts = "OVERWRITE"
        service_account_role_arn = module.vpc_cni_addon_irsa_role.iam_role_arn
      }
    }
  
    tags = local.tags
  }
  
  module "load_balancer_controller_irsa_role" {
    source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
  
    role_name                              = "load-balancer-controller"
    attach_load_balancer_controller_policy = true
  
    oidc_providers = {
      ex = {
        provider_arn               = module.eks.oidc_provider_arn
        namespace_service_accounts = ["kube-system:aws-load-balancer-controller"]
      }
    }
  
    tags = local.tags
  }
  
  module "ebs_csi_addon_irsa_role" {
    source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
  
    role_name             = "ebs-csi-controller"
    attach_ebs_csi_policy = true
  
    oidc_providers = {
      ex = {
        provider_arn               = module.eks.oidc_provider_arn
        namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
      }
    }
  
    tags = local.tags
  }
  
  module "vpc_cni_addon_irsa_role" {
    source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
  
    role_name             = "vpc-cni"
    attach_vpc_cni_policy = true
    vpc_cni_enable_ipv4   = true
  
    oidc_providers = {
      main = {
        provider_arn               = module.eks.oidc_provider_arn
        namespace_service_accounts = ["kube-system:aws-node"]
      }
    }
  
    tags = local.tags
  }
  
  ################################################################################
  # Supporting Resoruces
  ################################################################################
  
  module "vpc" {
    source  = "terraform-aws-modules/vpc/aws"
    version = "~> 5.0"
  
    name = local.name
    cidr = local.vpc_cidr
  
    azs             = local.azs
    public_subnets   = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)]
    private_subnets  = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 3)]
    database_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 6)]
  
    create_database_subnet_group = true
  
    enable_nat_gateway = true
    single_nat_gateway = true
  
    public_subnet_tags = {
      "kubernetes.io/role/elb" = 1
    }
  
    private_subnet_tags = {
      "kubernetes.io/role/internal-elb" = 1
    }
  
    tags = local.tags
  }
  

• Ingress Controller 설치 (Grafana 외부 접속 시 사용)

  • https://github.com/attachnplay/fastcampus-pay-bootstrap/blob/main/install_ingress_controller.md

Prometheus + Grafana 설치

• Prometheus Operator 사용

  • https://github.com/attachnplay/kube-prometheus
  • Physical Volume 관련 설정 추가함

• Prometheus 설치

  • https://prometheus-operator.dev/docs/user-guides/getting-started/

  git clone [email protected]:attachnplay/kube-prometheus.git
  
  cd kube-prometheus
  
  # Create the namespace and CRDs, and then wait for them to be availble before creating the remaining resources
  kubectl create -f manifests/setup
  
  # Wait until the "servicemonitors" CRD is created. The message "No resources found" means success in this context.
  until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
  
  kubectl create -f manifests/
  

• NLB 주소로 로그인

  • 초기 접속정보: admin / admin

• 대시보드에서 Node 정보 확인

• Alert 정보 확인

Loki 설치