diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 7e2196a..a01cdf8 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -6,7 +6,7 @@ on: push: branches: - main - - feat/week11 + - cvt/traefik paths: - 'url-shortener/app/**' - 'url-shortener/Dockerfile' diff --git a/README.md b/README.md index c8014b2..f1f7e09 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ FastAPI 애플리케이션을 AWS EKS에 배포하기 위한 인프라, CI/CD, G ## 아키텍처 +### 멀티 AZ 인프라 아키텍처 +AIDrawing_260901_9c28db43-9288-4fec-9dae-449eb76fbda3_0_MiriCanvas + +### nginx Ingress 기반 트래픽 흐름 구조 + image diff --git a/cluster-addons/ingress-nginx/application.yaml b/cluster-addons/ingress-nginx/application.yaml deleted file mode 100644 index b954dd9..0000000 --- a/cluster-addons/ingress-nginx/application.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# ingress-nginx-controller 설치 Application in Kubernetes Cluster by ArgoCD -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: ingress-nginx - namespace: argocd -spec: - project: default - source: - repoURL: https://kubernetes.github.io/ingress-nginx - chart: ingress-nginx - targetRevision: 4.15.x - helm: - values: | - controller: - replicaCount: 2 - - # TODO: Metric 여는 설정 추가하기 - - # ingress class resource 를 어떻게 생성할 것인가 - ingressClassResource: - name: nginx - enabled: true - default: false # true 로 설정하면 ingress-nginx-controller 가 ingressClassName 을 쓰지 않은 Ingress 도 처리할 수 있음 - - ingressClass: nginx # ingress class: ingress-nginx-controller 가 처리할 Ingress (class) 이름 - # ingressClassResource 와 ingressClass 를 모두 설정하면 ingressClassResource 가 우선순위를 가짐 -> 이 경우 ingressClassResource 가 우선순위를 가짐 - # ingressClassResource 가 설정되어 있지 않으면 ingressClass 가 설정되어 있어도 처리하지 않음 -> ingressClass 가 우선순위를 가짐 - - # service(svc): ingress-nginx-controller 가 네트워크로 노출할 Service 설정 - service: - type: LoadBalancer - externalTrafficPolicy: Local - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "external" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol: "http" - service.beta.kubernetes.io/aws-load-balancer-healthcheck-path: "/healthz" - service.beta.kubernetes.io/aws-load-balancer-healthcheck-port: "10254" - - ports: # 이 svc 가 외부에서 접근 가능한 포트 설정 - http: 80 - https: 443 - targetPorts: # 받은 요청을 ingress-nginx-controller pod로 전달 - http: http - https: https - destination: - server: https://kubernetes.default.svc - namespace: ingress-nginx - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/cluster-addons/prometheus/values.yaml b/cluster-addons/prometheus/values.yaml index a0f7101..8f9c961 100644 --- a/cluster-addons/prometheus/values.yaml +++ b/cluster-addons/prometheus/values.yaml @@ -55,6 +55,10 @@ alertmanager: group_wait: 0s # 알림을 묶어서 보낼 대기 시간 repeat_interval: 1h # 해결 안 된 알림 재발송 간격 receiver: slack-notifications # 알림 수신자 + routes: + - matchers: + - alertname="Watchdog" + receiver: "null" receivers: - name: "null" # Watchdog 등 무시할 알림용 no-op receiver diff --git a/cluster-addons/traefik/application.yaml b/cluster-addons/traefik/application.yaml new file mode 100644 index 0000000..826ae4f --- /dev/null +++ b/cluster-addons/traefik/application.yaml @@ -0,0 +1,110 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: traefik + namespace: argocd +spec: + project: default + source: + repoURL: https://traefik.github.io/charts + chart: traefik + targetRevision: 41.0.2 + helm: + values: | + deployment: + replicas: 2 + + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 1000m + memory: 512Mi + + podDisruptionBudget: + enabled: true + minAvailable: 1 + + # DoNotSchedule: 두 번째 replica를 다른 AZ/노드에 둘 수 없으면 Pending으로 두어 + # 단일 AZ 장애를 Ready replica로 숨기지 않는다. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: traefik + app.kubernetes.io/instance: traefik-traefik + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: traefik + app.kubernetes.io/instance: traefik-traefik + + ingressClass: + enabled: true + isDefaultClass: false # true 로 설정하면 traefik-ingress-controller 가 ingressClassName 을 쓰지 않은 Ingress 도 처리할 수 있음 + name: traefik + + # 라우팅 설정을 어디서 읽어 올지 결정하는 설정 + providers: + kubernetesCRD: + enabled: true + ingressClass: traefik + kubernetesIngress: + enabled: true + ingressClass: traefik + publishedService: + enabled: true + + service: + enabled: true + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "external" + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" + service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: "preserve_client_ip.enabled=true" + spec: + type: LoadBalancer + + ports: + web: + exposedPort: 80 + # http: + # redirections: + # entryPoint: + # to: websecure + # scheme: https + # permanent: true + websecure: + exposedPort: 443 + + log: + level: INFO + accessLog: + enabled: true + + metrics: + prometheus: + addEntryPointsLabels: true + addRoutersLabels: true + addServicesLabels: true + buckets: "0.01,0.025,0.05,0.1,0.25,0.5,1.0,2.0,5.0" + service: + enabled: true + serviceMonitor: + enabled: true + additionalLabels: + release: kube-prometheus-stack + destination: + server: https://kubernetes.default.svc + namespace: traefik + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/resume_url_shortener_project_draft.md b/resume_url_shortener_project_draft.md deleted file mode 100644 index e779cf9..0000000 --- a/resume_url_shortener_project_draft.md +++ /dev/null @@ -1,102 +0,0 @@ -# Url-Shortener-EKS-Platform Resume Draft - -## 작성 기준 - -- 기존 이력서 형식: Summary, Education, Experience, Skills, Talent의 2컬럼형 구조 -- 기존 Experience 문체: 프로젝트 설명 1문장 + 성과/구현 중심 bullet 5~6개 -- 이번 프로젝트 포지셔닝: Backend 중심보다 DevOps / Cloud Infrastructure / Platform Engineering 역량 강조 -- 민감 정보: AWS Account ID, 실제 RDS Endpoint, Secret 값, Webhook URL 등은 이력서 본문에서 제외 - ---- - -## Header - -**박종현** -DevOps Engineer -Seoul, South Korea -jounghyeon123@gmail.com -https://github.com/joungGo - ---- - -## Summary - -AWS EKS 기반 애플리케이션 운영 환경을 Terraform, Kubernetes, Helm, ArgoCD로 구성하며 인프라 자동화와 GitOps 배포 흐름을 실습했습니다. FastAPI 애플리케이션을 ECR, EKS, RDS, Redis, Prometheus/Grafana, Loki, Karpenter와 연동해 배포, 관측, 알림, 오토스케일링, 장애 대응까지 이어지는 운영 관점의 전체 흐름을 구축한 경험이 있습니다. - -### English Version - -Built an AWS EKS-based application platform using Terraform, Kubernetes, Helm, and ArgoCD. Designed an end-to-end DevOps workflow covering container image delivery, GitOps deployment, RDS primary/replica connectivity, Redis caching, Prometheus/Grafana observability, Loki log analysis, Karpenter-based node scaling, and operational runbooks. - ---- - -## Experience - -### Url-Shortener-EKS-Platform - -**DevOps / Backend Infrastructure Project** -**Personal Project** -**2026.04 - 2026.05** - -FastAPI 기반 URL Shortener API를 AWS EKS 위에 배포하고, Terraform IaC, GitHub Actions, ArgoCD GitOps, RDS, Redis, Prometheus/Grafana, Loki, Karpenter를 연동하여 클라우드 네이티브 운영 환경을 구축한 프로젝트 - -- Terraform module 기반으로 VPC, public/private subnet, NAT Gateway, EKS, ECR, RDS Primary/Read Replica, IRSA Role, Karpenter Role, GitHub Actions OIDC Role을 코드화 -- GitHub Actions에서 AWS OIDC 인증을 사용해 Access Key 없이 ECR 이미지 빌드/푸시 후 Helm `values.prod.yaml`의 이미지 태그를 자동 갱신하는 CI/CD 파이프라인 구성 -- ArgoCD Application과 Helm Chart를 이용해 애플리케이션, Metrics Server, ALB Controller, Redis, Prometheus/Grafana, Karpenter 등 클러스터 리소스를 GitOps 방식으로 배포 및 동기화 -- FastAPI 애플리케이션에 SQLAlchemy 기반 Write/Read DB 세션을 분리하여 RDS Primary에는 쓰기, Read Replica에는 조회 트래픽이 연결되도록 구성하고 `/items/_db` 진단 엔드포인트로 검증 -- Redis Cache-Aside 패턴을 적용해 단건/목록 조회 캐시, TTL, 캐시 무효화, Redis 장애 시 DB 직접 조회로 전환되는 graceful degradation 구조 구현 -- Prometheus `/metrics`, ServiceMonitor, PrometheusRule을 구성하여 HTTP 5xx 비율, 캐시 히트율, DB P99 latency 지표를 수집하고 Alertmanager Slack 알림 경로 설계 -- Grafana, Loki/Grafana Alloy 기반으로 메트릭과 로그를 함께 확인하는 장애 분석 흐름을 문서화하고 Alert 발생 시점과 로그 타임라인을 대조하는 운영 런북 작성 -- HPA와 Karpenter NodePool/EC2NodeClass를 구성하여 CPU 기반 Pod scale-out과 Spot/On-demand 노드 프로비저닝 구조를 설계하고, k6 기반 100 -> 1,000 -> 5,000 RPS 부하 테스트 시나리오 작성 - -### 압축형 버전 - -FastAPI 기반 URL Shortener API를 AWS EKS에 배포하고 Terraform, Helm, ArgoCD, GitHub Actions, RDS, Redis, Prometheus/Grafana, Loki, Karpenter를 연동해 GitOps 기반 운영 환경을 구축한 프로젝트 - -- Terraform으로 VPC, EKS, ECR, RDS Primary/Read Replica, IRSA, Karpenter, GitHub Actions OIDC Role을 IaC로 구성 -- GitHub Actions OIDC 인증, Docker Buildx, ECR Push, Helm values 이미지 태그 자동 갱신 기반 CI/CD 파이프라인 구축 -- ArgoCD Application과 Helm Chart로 애플리케이션 및 클러스터 애드온을 GitOps 방식으로 배포하고 self-heal/prune 정책 적용 -- RDS Primary/Replica write-read 분리, Redis Cache-Aside, cache hit/miss metric, 장애 시 graceful degradation 구조 구현 -- Prometheus/Grafana/Alertmanager/Loki 기반 관측 환경과 Slack 알림, 장애 분석 런북, k6 부하 테스트 및 HPA/Karpenter 검증 시나리오 작성 - ---- - -## Skills - -### Programming Skills - -Python, FastAPI, SQLAlchemy, Pydantic - -### Databases & Cloud Services - -AWS, EKS, ECR, RDS PostgreSQL, RDS Read Replica, Redis, VPC, IAM, OIDC, IRSA - -### Infrastructure & Tools - -Terraform, Docker, Kubernetes, Helm, ArgoCD, GitHub Actions, Karpenter, AWS Load Balancer Controller, Metrics Server, cert-manager - -### Observability & Operations - -Prometheus, Grafana, Alertmanager, Loki, Grafana Alloy, ServiceMonitor, PrometheusRule, k6, HPA, Runbook - ---- - -## Talent - -### Operational Thinking - -장애가 발생한 뒤의 복구만이 아니라, 메트릭/로그/알림/런북을 통해 문제를 빠르게 좁혀갈 수 있는 운영 흐름을 함께 설계합니다. - -### Documentation - -인프라 설치, 검증, 장애 주입, 정리 절차를 단계별 문서로 남겨 재현 가능한 실습과 운영 기준을 만드는 데 강점이 있습니다. - ---- - -## 면접에서 강조할 포인트 - -- 단순히 EKS에 앱을 띄운 프로젝트가 아니라, 배포 자동화, GitOps, 관측성, 알림, 오토스케일링, 장애 대응까지 연결한 운영형 프로젝트임 -- GitHub Actions에서 Access Key 대신 OIDC 기반 IAM Role Assume 방식을 사용한 점 -- RDS Primary/Replica, Redis Cache-Aside, Prometheus metric, Loki log를 애플리케이션 코드와 인프라에 같이 녹인 점 -- Karpenter와 HPA를 별도 키워드로 나열하는 데 그치지 않고, k6 부하 테스트와 운영 검증 흐름까지 설계한 점 -- Secret을 Git에 직접 넣지 않고 Kubernetes Secret, IRSA, values 분리 전략을 고려한 점 - diff --git a/terraform/github_actions.tf b/terraform/github_actions.tf index f2fae35..80e5fbb 100644 --- a/terraform/github_actions.tf +++ b/terraform/github_actions.tf @@ -16,16 +16,17 @@ module "github_actions_role" { version = "~> 6.0" name = "${var.project_name}-github-actions-role" - use_name_prefix = false # 타임스탬프 suffix 방지 + use_name_prefix = false # 타임스탬프 suffix 방지 enable_github_oidc = true # ← 추가 (GitHub OIDC 신뢰 관계 자동 구성) # 보안: 특정 레포지토리의 main 브랜치만 허용 # pull_request 이벤트까지 허용하려면 "repo:org/repo:*"으로 변경 # TODO: dev, prod 환경 구분하기 + # 현재 저장소의 모든 브랜치와 pull_request 이벤트 허용 oidc_wildcard_subjects = [ - "repo:f-lab-edu/Url-Shortener-EKS-Platform:ref:refs/heads/*", # 모든 브랜치 허용 - "repo:f-lab-edu/Url-Shortener-EKS-Platform:pull_request" + "repo:f-lab-edu/F-Lab-DevOps:ref:refs/heads/*", + "repo:f-lab-edu/F-Lab-DevOps:pull_request" ] policies = { diff --git a/terraform/karpenter.tf b/terraform/karpenter.tf index 5899d3b..54adf50 100644 --- a/terraform/karpenter.tf +++ b/terraform/karpenter.tf @@ -1,5 +1,10 @@ +# Karpenter가 Spot 인스턴스를 생성할 때 사용하는 AWS 관리형 서비스 연결 역할 +resource "aws_iam_service_linked_role" "ec2_spot" { + aws_service_name = "spot.amazonaws.com" +} + # Karpenter 인프라 — terraform-aws-modules/eks karpenter 서브모듈 사용 -# Controller IRSA + Node Role + Instance Profile + SQS + EventBridge를 한 번에 생성 +# Controller Pod Identity + Node Role + Instance Profile + SQS + EventBridge를 한 번에 생성 module "karpenter" { source = "terraform-aws-modules/eks/aws//modules/karpenter" diff --git a/terraform/main.tf b/terraform/main.tf index f6b0227..3480053 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -310,7 +310,7 @@ module "eks" { eks_managed_node_groups = { main = { name = "${var.project_name}-nodegroup" - use_name_prefix = false # 타임스탬프 suffix 방지 - 항상 동일한 이름 유지 + use_name_prefix = true # 교체 시 새 노드 그룹을 먼저 생성할 수 있도록 이름 충돌 방지 iam_role_name = "${var.project_name}-nodegroup-role" iam_role_use_name_prefix = false diff --git a/terraform/variables.tf b/terraform/variables.tf index b5a7b53..e06bc95 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -27,13 +27,13 @@ variable "availability_zones" { variable "eks_cluster_version" { description = "EKS Kubernetes 버전" type = string - default = "1.34" + default = "1.35" } variable "node_instance_type" { description = "EKS Worker Node EC2 인스턴스 타입" type = string - default = "t3.medium" + default = "t3.large" } # terraform apply 시 생성되는 노드 수 @@ -61,4 +61,4 @@ variable "db_password" { description = "RDS master(user=postgres) password (manage_master_user_password=false 사용 시 필수)" type = string sensitive = true -} \ No newline at end of file +} diff --git a/url-shortener/app/api/routes/health.py b/url-shortener/app/api/routes/health.py index f96ae3e..9300592 100644 --- a/url-shortener/app/api/routes/health.py +++ b/url-shortener/app/api/routes/health.py @@ -1,16 +1,35 @@ -from fastapi import HTTPException -from fastapi import APIRouter +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from app.core.database import get_write_db router = APIRouter() +WriteDb = Annotated[Session, Depends(get_write_db)] + @router.get("/healthz", tags=["health"]) def healthcheck(): """ 서버 상태를 확인하는 헬스체크 엔드포인트. """ - return {"status": "ok", "version": "v33"} + return {"status": "ok", "version": "v42"} + + +@router.get("/readyz", tags=["health"]) +def readiness(db: WriteDb): + """트래픽을 받을 준비가 됐는지 Primary DB 연결까지 확인한다.""" + try: + db.execute(text("select 1")) + except SQLAlchemyError as exc: + raise HTTPException(status_code=503, detail="primary database unavailable") from exc + return {"status": "ready"} + @router.get("/error-test") def error_test(): """[테스트용] 강제로 500 에러 발생""" - raise HTTPException(status_code=500, detail="의도적 에러 — Aleㅛrt 테스트용") \ No newline at end of file + raise HTTPException(status_code=500, detail="의도적 에러 — Aleㅛrt 테스트용") diff --git a/url-shortener/app/api/routes/item.py b/url-shortener/app/api/routes/item.py index b5ba404..096ae92 100644 --- a/url-shortener/app/api/routes/item.py +++ b/url-shortener/app/api/routes/item.py @@ -1,17 +1,21 @@ import json import logging import time +from typing import Annotated -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, Header, HTTPException from pydantic import BaseModel +from redis.exceptions import RedisError from sqlalchemy import text from sqlalchemy.orm import Session from app.core.cache import get_redis -from app.core.database import get_write_db, get_read_db +from app.core.config import settings +from app.core.database import get_read_db, get_write_db from app.core.metrics import ( cache_hit_total, cache_miss_total, + cache_operation_total, db_query_latency_seconds, ) from app.models.item import Item @@ -19,12 +23,48 @@ router = APIRouter(prefix="/items", tags=["items"]) logger = logging.getLogger(__name__) -ITEM_TTL = 300 # 단건 조회 캐시 TTL: 5분 -LIST_TTL = 60 # 목록 조회 캐시 TTL: 1분 (변경 가능성 높아 짧게) -LIST_KEY = "items:all" +ITEM_TTL = 300 # 단건 조회 캐시 TTL: 5분 +LIST_TTL = 60 # 목록 조회 캐시 TTL: 1분 (변경 가능성 높아 짧게) +LIST_KEY = "items:all" + +WriteDb = Annotated[Session, Depends(get_write_db)] +ReadDb = Annotated[Session, Depends(get_read_db)] +Week12BypassCache = Annotated[ + bool | None, + Header(alias="X-Week12-Bypass-Cache"), +] +Week12ReadDelayMs = Annotated[ + int | None, + Header(alias="X-Week12-Read-Delay-Ms"), +] + + +def _week12_fault_options( + bypass_cache: bool | None, + read_delay_ms: int | None, +) -> tuple[bool, int]: + """비운영 Week 12 실습용 옵션을 안전한 범위로 제한한다.""" + if not settings.ENABLE_FAULT_INJECTION: + return False, 0 + + bounded_delay_ms = min( + max(read_delay_ms or 0, 0), + max(settings.MAX_FAULT_DELAY_MS, 0), + ) + return bool(bypass_cache), bounded_delay_ms + + +def _inject_read_delay(db: Session, delay_ms: int) -> None: + """애플리케이션이 사용하는 Read DB 세션 안에서 지연을 발생시킨다.""" + if delay_ms > 0: + db.execute( + text("select pg_sleep(:delay_seconds)"), + {"delay_seconds": delay_ms / 1000}, + ) # ── 스키마 ──────────────────────────────────────────────────── + class ItemCreate(BaseModel): name: str description: str | None = None @@ -81,7 +121,7 @@ def _probe_db(db: Session) -> DbProbe: # ── POST: 아이템 생성 — 목록 캐시 무효화 ────────────────────── @router.post("", response_model=ItemResponse, status_code=201) -def create_item(body: ItemCreate, db: Session = Depends(get_write_db)): +def create_item(body: ItemCreate, db: WriteDb): """[Primary] 아이템 생성 — 목록 캐시 무효화.""" start = time.perf_counter() @@ -90,6 +130,8 @@ def create_item(body: ItemCreate, db: Session = Depends(get_write_db)): db.commit() db.refresh(record) + logger.info(f"db_route=primary operation=insert item_id={record.id} name={body.name}") + db_query_latency_seconds.labels(operation="insert").observe( time.perf_counter() - start ) @@ -99,7 +141,7 @@ def create_item(body: ItemCreate, db: Session = Depends(get_write_db)): if cache: try: cache.delete(LIST_KEY) - except Exception as e: + except RedisError as e: logger.warning(f"캐시 무효화 실패 (무시): {e}") return ItemResponse.from_orm_custom(record) @@ -107,34 +149,53 @@ def create_item(body: ItemCreate, db: Session = Depends(get_write_db)): # ── GET 목록: Cache-Aside ──────────────────────────────────────── @router.get("", response_model=list[ItemResponse]) -def list_items(db: Session = Depends(get_read_db)): +def list_items( + db: ReadDb, + x_week12_bypass_cache: Week12BypassCache = None, + x_week12_read_delay_ms: Week12ReadDelayMs = None, +): """[Replica] 아이템 목록 — Cache-Aside (TTL: 1분).""" - cache = get_redis() + bypass_cache, read_delay_ms = _week12_fault_options( + x_week12_bypass_cache, + x_week12_read_delay_ms, + ) + cache = None if bypass_cache else get_redis() + + if bypass_cache: + cache_operation_total.labels(endpoint="list_items", result="bypass").inc() + elif cache is None: + cache_operation_total.labels(endpoint="list_items", result="unavailable").inc() if cache: try: cached = cache.get(LIST_KEY) if cached: cache_hit_total.labels(endpoint="list_items").inc() + cache_operation_total.labels(endpoint="list_items", result="hit").inc() logger.info("cache_hit endpoint=list_items") return [ItemResponse(**i) for i in json.loads(cached)] cache_miss_total.labels(endpoint="list_items").inc() + cache_operation_total.labels(endpoint="list_items", result="miss").inc() logger.info("cache_miss endpoint=list_items") - except Exception as e: + except (RedisError, json.JSONDecodeError) as e: + cache_operation_total.labels(endpoint="list_items", result="error").inc() logger.warning(f"캐시 조회 실패, DB 직접 조회: {e}") start = time.perf_counter() + _inject_read_delay(db, read_delay_ms) items = db.query(Item).all() db_query_latency_seconds.labels(operation="select_all").observe( time.perf_counter() - start ) + logger.info(f"db_route=replica operation=select_all count={len(items)}") + result = [ItemResponse.from_orm_custom(i) for i in items] if cache: try: cache.setex(LIST_KEY, LIST_TTL, json.dumps([r.model_dump() for r in result])) - except Exception as e: + except RedisError as e: logger.warning(f"캐시 저장 실패 (무시): {e}") return result @@ -144,8 +205,8 @@ def list_items(db: Session = Depends(get_read_db)): # /{item_id} 보다 먼저 등록해야 라우트 충돌 방지 @router.get("/_db", response_model=DbProbeResponse) def probe_db( - write_db: Session = Depends(get_write_db), - read_db: Session = Depends(get_read_db), + write_db: WriteDb, + read_db: ReadDb, ): """ [진단] write/read 세션이 각각 Primary/Replica로 붙는지 확인. @@ -160,24 +221,42 @@ def probe_db( # ── GET 단건: Cache-Aside ──────────────────────────────────────── @router.get("/{item_id}", response_model=ItemResponse) -def get_item(item_id: int, db: Session = Depends(get_read_db)): +def get_item( + item_id: int, + db: ReadDb, + x_week12_bypass_cache: Week12BypassCache = None, + x_week12_read_delay_ms: Week12ReadDelayMs = None, +): """[Replica] 아이템 단건 조회 — Cache-Aside (TTL: 5분).""" - cache = get_redis() + bypass_cache, read_delay_ms = _week12_fault_options( + x_week12_bypass_cache, + x_week12_read_delay_ms, + ) + cache = None if bypass_cache else get_redis() cache_key = f"item:{item_id}" + if bypass_cache: + cache_operation_total.labels(endpoint="get_item", result="bypass").inc() + elif cache is None: + cache_operation_total.labels(endpoint="get_item", result="unavailable").inc() + if cache: try: cached = cache.get(cache_key) if cached: cache_hit_total.labels(endpoint="get_item").inc() + cache_operation_total.labels(endpoint="get_item", result="hit").inc() logger.info(f"cache_hit endpoint=get_item item_id={item_id}") return ItemResponse(**json.loads(cached)) cache_miss_total.labels(endpoint="get_item").inc() + cache_operation_total.labels(endpoint="get_item", result="miss").inc() logger.info(f"cache_miss endpoint=get_item item_id={item_id}") - except Exception as e: + except (RedisError, json.JSONDecodeError) as e: + cache_operation_total.labels(endpoint="get_item", result="error").inc() logger.warning(f"캐시 조회 실패, DB 직접 조회: {e}") start = time.perf_counter() + _inject_read_delay(db, read_delay_ms) record = db.query(Item).filter(Item.id == item_id).first() db_query_latency_seconds.labels(operation="select_one").observe( time.perf_counter() - start @@ -186,12 +265,14 @@ def get_item(item_id: int, db: Session = Depends(get_read_db)): if not record: raise HTTPException(status_code=404, detail=f"id={item_id} 아이템을 찾을 수 없습니다.") + logger.info(f"db_route=replica operation=select_one item_id={item_id}") + result = ItemResponse.from_orm_custom(record) if cache: try: cache.setex(cache_key, ITEM_TTL, json.dumps(result.model_dump())) - except Exception as e: + except RedisError as e: logger.warning(f"캐시 저장 실패 (무시): {e}") return result @@ -199,7 +280,7 @@ def get_item(item_id: int, db: Session = Depends(get_read_db)): # ── DELETE: 캐시 무효화 필수 ──────────────────────────────────── @router.delete("/{item_id}", status_code=204) -def delete_item(item_id: int, db: Session = Depends(get_write_db)): +def delete_item(item_id: int, db: WriteDb): """[Primary] 아이템 삭제 — 단건 + 목록 캐시 무효화.""" start = time.perf_counter() record = db.query(Item).filter(Item.id == item_id).first() @@ -210,6 +291,8 @@ def delete_item(item_id: int, db: Session = Depends(get_write_db)): db.delete(record) db.commit() + logger.info(f"db_route=primary operation=delete item_id={item_id}") + db_query_latency_seconds.labels(operation="delete").observe( time.perf_counter() - start ) @@ -217,7 +300,7 @@ def delete_item(item_id: int, db: Session = Depends(get_write_db)): cache = get_redis() if cache: try: - cache.delete(f"item:{item_id}") # 단건 캐시 - cache.delete(LIST_KEY) # 목록 캐시 - except Exception as e: + cache.delete(f"item:{item_id}") # 단건 캐시 + cache.delete(LIST_KEY) # 목록 캐시 + except RedisError as e: logger.warning(f"캐시 무효화 실패 (무시): {e}") diff --git a/url-shortener/app/core/cache.py b/url-shortener/app/core/cache.py index 066c8ca..4fdd798 100644 --- a/url-shortener/app/core/cache.py +++ b/url-shortener/app/core/cache.py @@ -1,6 +1,8 @@ -import redis import logging +import redis +from redis.exceptions import RedisError + from app.core.config import settings logger = logging.getLogger(__name__) @@ -25,16 +27,16 @@ def get_redis() -> redis.Redis | None: try: client = redis.from_url( settings.REDIS_URL, - decode_responses=True, # bytes 대신 str 반환 - socket_connect_timeout=2, # 연결 타임아웃 2초 + decode_responses=True, # bytes 대신 str 반환 + socket_connect_timeout=2, # 연결 타임아웃 2초 socket_timeout=2, ) - client.ping() # 실제 연결 테스트 (실패 시 except로 이동) + client.ping() # 실제 연결 테스트 (실패 시 except로 이동) _redis_client = client - except Exception as e: + except (RedisError, ValueError) as e: # Redis 연결 실패 시 None 반환 → 캐시 없이 DB 직접 조회 # _redis_client는 None으로 유지 → 다음 요청에서 재시도 logger.warning(f"Redis 연결 실패, 캐시 비활성화: {e}") return None - return _redis_client \ No newline at end of file + return _redis_client diff --git a/url-shortener/app/core/config.py b/url-shortener/app/core/config.py index 67c4284..1808b18 100644 --- a/url-shortener/app/core/config.py +++ b/url-shortener/app/core/config.py @@ -23,10 +23,18 @@ class Settings(BaseSettings): # 디버그 모드 여부 DEBUG: bool = False + # Week 12 비운영 실습에서만 사용하는 읽기 지연/캐시 우회 기능. + # 운영 중에는 반드시 false로 유지한다. + ENABLE_FAULT_INJECTION: bool = False + MAX_FAULT_DELAY_MS: int = 2000 + class Config: # 이 파일과 같은 경로의 .env 를 자동으로 읽습니다 env_file = ".env" env_file_encoding = "utf-8" + # docker-compose용 POSTGRES_* 등 애플리케이션 외 변수가 같은 .env에 + # 있어도 Settings 초기화를 막지 않도록 합니다. + extra = "ignore" # 앱 전체에서 공유할 settings 인스턴스 diff --git a/url-shortener/app/core/database.py b/url-shortener/app/core/database.py index 310ad8d..19fa5b3 100644 --- a/url-shortener/app/core/database.py +++ b/url-shortener/app/core/database.py @@ -1,5 +1,5 @@ from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker, DeclarativeBase +from sqlalchemy.orm import DeclarativeBase, sessionmaker from app.core.config import settings @@ -12,10 +12,10 @@ write_engine = create_engine( settings.DATABASE_URL, pool_pre_ping=True, - pool_size=5, # 연결 풀 크기 — RDS t3.micro max_connections ≈ 85 - max_overflow=10, # pool_size 초과 시 임시 추가 연결 (최대 15개 총합) - pool_timeout=30, # 연결 획득 대기 최대 시간 (초) - pool_recycle=1800, # 30분마다 연결 재생성 — RDS 재시작·IAM토큰 만료 대비 + pool_size=5, # 연결 풀 크기 — RDS t3.micro max_connections ≈ 85 + max_overflow=10, # pool_size 초과 시 임시 추가 연결 (최대 15개 총합) + pool_timeout=30, # 연결 획득 대기 최대 시간 (초) + pool_recycle=1800, # 30분마다 연결 재생성 — RDS 재시작·IAM토큰 만료 대비 ) # ── Read Engine (Replica) ───────────────────────────────────── @@ -23,15 +23,15 @@ _read_url = settings.DATABASE_READ_URL or settings.DATABASE_URL read_engine = create_engine( _read_url, - pool_pre_ping=True, # 연결 유지 확인 (비활성화 시 연결 끊김 가능성 증가) - pool_size=10, # Read Replica: 읽기 트래픽이 많으므로 pool 크게 - max_overflow=20, # pool_size 초과 시 임시 추가 연결 (최대 30개 총합) - pool_timeout=30, # 연결 획득 대기 최대 시간 (초) - pool_recycle=1800, # Replica 재시작 시 stale connection 방지 + pool_pre_ping=True, # 연결 유지 확인 (비활성화 시 연결 끊김 가능성 증가) + pool_size=10, # Read Replica: 읽기 트래픽이 많으므로 pool 크게 + max_overflow=20, # pool_size 초과 시 임시 추가 연결 (최대 30개 총합) + pool_timeout=30, # 연결 획득 대기 최대 시간 (초) + pool_recycle=1800, # Replica 재시작 시 stale connection 방지 ) WriteSession = sessionmaker(bind=write_engine, autocommit=False, autoflush=False) -ReadSession = sessionmaker(bind=read_engine, autocommit=False, autoflush=False) +ReadSession = sessionmaker(bind=read_engine, autocommit=False, autoflush=False) class Base(DeclarativeBase): @@ -57,4 +57,4 @@ def get_read_db(): # 하위 호환: 기존 get_db() 참조 코드가 있다면 write로 연결 -get_db = get_write_db \ No newline at end of file +get_db = get_write_db diff --git a/url-shortener/app/core/metrics.py b/url-shortener/app/core/metrics.py index cdc6ba2..4ccaa2e 100644 --- a/url-shortener/app/core/metrics.py +++ b/url-shortener/app/core/metrics.py @@ -26,6 +26,24 @@ ["endpoint"], ) +# hit/miss 외에 Redis 오류와 의도적인 cache bypass를 구분한다. +# cache_miss_total은 "Redis는 정상이나 key가 없음"만 의미하므로 장애 지표로 +# 사용하지 않는다. +cache_operation_total = Counter( + "cache_operation_total", + "캐시 조회 결과 총 수", + ["endpoint", "result"], # hit / miss / error / unavailable / bypass +) + +# ── HTTP 응답 레이턴시 ──────────────────────────────────────── +# 실제 URL 대신 route template(/items/{item_id})을 label로 사용해 cardinality를 제한한다. +http_request_duration_seconds = Histogram( + "http_request_duration_seconds", + "HTTP 요청 처리 시간 (초)", + ["method", "route", "status_code"], + buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0], +) + # ── DB 쿼리 레이턴시 Histogram ──────────────────────────────── # buckets: 1ms ~ 1s 구간으로 P50/P95/P99 측정 # labels: operation (select_one / select_all / insert / delete) diff --git a/url-shortener/app/main.py b/url-shortener/app/main.py index 71801a7..3c2864d 100644 --- a/url-shortener/app/main.py +++ b/url-shortener/app/main.py @@ -1,4 +1,5 @@ import logging +import time from contextlib import asynccontextmanager from fastapi import FastAPI, Request @@ -6,7 +7,7 @@ from app.api.routes import health, item from app.core.database import Base, write_engine -from app.core.metrics import http_request_total +from app.core.metrics import http_request_duration_seconds, http_request_total logging.basicConfig(level=logging.INFO) @@ -40,17 +41,30 @@ async def lifespan(app: FastAPI): # 모든 요청에 대해 http_request_total 카운터 증가 @app.middleware("http") async def record_http_metrics(request: Request, call_next): - response = await call_next(request) + start = time.perf_counter() + status_code = "500" - # /metrics 자체 요청은 카운터에서 제외 (무한 루프 방지) - if request.url.path != "/metrics": - http_request_total.labels( - method=request.method, - path=request.url.path, - status_code=str(response.status_code), - ).inc() - - return response + try: + response = await call_next(request) + status_code = str(response.status_code) + return response + finally: + # /metrics와 /metrics/ scrape 자체는 애플리케이션 지표에서 제외한다. + if not request.url.path.startswith("/metrics"): + # 라우팅이 끝난 뒤 scope에 들어온 템플릿 경로를 사용한다. + # 일치하지 않은 요청은 실제 path 대신 "unmatched"로 묶는다. + route = request.scope.get("route") + route_path = getattr(route, "path", "unmatched") + http_request_total.labels( + method=request.method, + path=route_path, + status_code=status_code, + ).inc() + http_request_duration_seconds.labels( + method=request.method, + route=route_path, + status_code=status_code, + ).observe(time.perf_counter() - start) app.include_router(health.router) # 헬스체크 엔드포인트 diff --git a/url-shortener/app/models/item.py b/url-shortener/app/models/item.py index 188c716..2836a9b 100644 --- a/url-shortener/app/models/item.py +++ b/url-shortener/app/models/item.py @@ -1,6 +1,8 @@ from datetime import datetime -from sqlalchemy import Integer, String, DateTime, func + +from sqlalchemy import DateTime, Integer, String, func from sqlalchemy.orm import Mapped, mapped_column + from app.core.database import Base diff --git a/url-shortener/k8s/argocd/application-traefik-routing.yaml b/url-shortener/k8s/argocd/application-traefik-routing.yaml new file mode 100644 index 0000000..b0a5d5e --- /dev/null +++ b/url-shortener/k8s/argocd/application-traefik-routing.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: url-shortener-traefik-routing + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/f-lab-edu/F-Lab-DevOps.git + targetRevision: cvt/traefik + path: url-shortener/k8s/traefik + destination: + server: https://kubernetes.default.svc + namespace: url-shortener + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + prune: true + selfHeal: true diff --git a/url-shortener/k8s/argocd/application.yaml b/url-shortener/k8s/argocd/application.yaml index d180e93..458b0f2 100644 --- a/url-shortener/k8s/argocd/application.yaml +++ b/url-shortener/k8s/argocd/application.yaml @@ -14,12 +14,12 @@ spec: source: # HTTPS vs SSH: HTTPS는 토큰 인증, SSH는 키 인증 # Public repo: 인증 불필요. Private repo: Secret 설정 필요 (아래 참고) - repoURL: https://github.com/f-lab-edu/Url-Shortener-EKS-Platform + repoURL: https://github.com/f-lab-edu/F-Lab-DevOps.git # HEAD: 브랜치의 최신 커밋을 추적 # main: main 브랜치 고정 # v1.0.0: 특정 태그 고정 (운영 환경 권장) - targetRevision: feat/week11 + targetRevision: cvt/traefik # Helm chart가 있는 경로 (repo root 기준) path: url-shortener/url-shortener-chart @@ -46,4 +46,4 @@ spec: automated: prune: true # Git에서 삭제된 리소스를 클러스터에서도 자동 삭제 selfHeal: true # 클러스터 상태가 Git과 다르면 자동으로 되돌림 - # (kubectl로 직접 수정해도 ArgoCD가 Git 상태로 복구) \ No newline at end of file + # (kubectl로 직접 수정해도 ArgoCD가 Git 상태로 복구) diff --git a/url-shortener/k8s/cert-manager/certificate.yaml b/url-shortener/k8s/cert-manager/certificate.yaml deleted file mode 100644 index 23d04b9..0000000 --- a/url-shortener/k8s/cert-manager/certificate.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# 특정 도메인(api.bidflow.cloud)에 대한 TLS 인증서를 발급해 달라고 cert-manager에 요청하는 리소스 - -apiVersion: cert-manager.io/v1 -kind: Certificate # cert-manager에게 실제 인증서 발급을 요청하는 리소스 -metadata: - name: url-shortener-cert # 이 인증서 요청 리소스의 이름 - namespace: url-shortener # 인증서와 Secret이 생성될 네임스페이스 -spec: - secretName: url-shortener-tls # 발급된 인증서와 개인키를 저장할 Kubernetes Secret 이름 - issuerRef: - name: letsencrypt-prod # 어떤 발급자(issuer)를 사용할지 지정 - kind: ClusterIssuer # 위 이름의 리소스 타입이 ClusterIssuer임을 명시 - dnsNames: - - api.bidflow.cloud # 인증서를 발급받을 실제 도메인 이름 diff --git a/url-shortener/k8s/cert-manager/clusterissuer.yaml b/url-shortener/k8s/cert-manager/clusterissuer.yaml deleted file mode 100644 index e10d4c2..0000000 --- a/url-shortener/k8s/cert-manager/clusterissuer.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# cert-manager가 Let's Encrypt를 이용해 인증서를 발급할 때 사용할 “발급자(issuer)” 설정 - -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: letsencrypt-prod # 이 발급자 이름. Certificate에서 issuerRef.name으로 참조함 -spec: - acme: # ACME 프로토콜 기반 인증서 발급 설정 (Let's Encrypt가 대표적) - email: jounghyeon123@gmail.com # 인증서 만료/문제 알림을 받을 이메일 주소 - server: https://acme-v02.api.letsencrypt.org/directory # Let's Encrypt 운영(production) 서버 주소 - privateKeySecretRef: - name: letsencrypt-prod # ACME 계정용 개인키를 저장할 Kubernetes Secret 이름 - solvers: - - http01: # 도메인 소유권 검증 방식으로 HTTP-01 사용 - ingress: - class: nginx # HTTP-01 검증 요청을 ingress-nginx가 처리하도록 지정 diff --git a/url-shortener/k8s/karpenter/nodepool.yaml b/url-shortener/k8s/karpenter/nodepool.yaml index 98e921b..0a6c509 100644 --- a/url-shortener/k8s/karpenter/nodepool.yaml +++ b/url-shortener/k8s/karpenter/nodepool.yaml @@ -29,9 +29,7 @@ spec: - key: node.kubernetes.io/instance-type operator: In values: - - t3.medium # 2 vCPU, 4 GB (기본) - t3.large # 2 vCPU, 8 GB - - t3a.medium # t3.medium의 AMD 버전 (약 5% 저렴) - t3a.large # t3.large의 AMD 버전 # 사용 가능한 가용 영역 diff --git a/url-shortener/k8s/load-test/karpenter-inflate.yaml b/url-shortener/k8s/load-test/karpenter-inflate.yaml new file mode 100644 index 0000000..6fb1ad4 --- /dev/null +++ b/url-shortener/k8s/load-test/karpenter-inflate.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karpenter-inflate + namespace: url-shortener + labels: + app: karpenter-inflate + purpose: week12-load-test +spec: + replicas: 0 + selector: + matchLabels: + app: karpenter-inflate + template: + metadata: + labels: + app: karpenter-inflate + purpose: week12-load-test + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: pause + image: registry.k8s.io/pause:3.10 + resources: + requests: + cpu: 900m + memory: 512Mi + limits: + cpu: 900m + memory: 512Mi diff --git a/url-shortener/k8s/load-test/week12-alert-test.yaml b/url-shortener/k8s/load-test/week12-alert-test.yaml new file mode 100644 index 0000000..3fa73ca --- /dev/null +++ b/url-shortener/k8s/load-test/week12-alert-test.yaml @@ -0,0 +1,20 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: week12-end-to-end-alert-test + namespace: url-shortener + labels: + release: kube-prometheus-stack +spec: + groups: + - name: week12.test.rules + interval: 15s + rules: + - alert: Week12EndToEndTest + expr: vector(1) + for: 30s + labels: + severity: warning + annotations: + summary: "Week 12 Alert 전송 경로 테스트" + description: "Prometheus → Alertmanager → Slack end-to-end 검증용 임시 Alert" diff --git a/url-shortener/k8s/traefik/clusterissuer-traefik.yaml b/url-shortener/k8s/traefik/clusterissuer-traefik.yaml new file mode 100644 index 0000000..00b5fd8 --- /dev/null +++ b/url-shortener/k8s/traefik/clusterissuer-traefik.yaml @@ -0,0 +1,16 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-traefik + annotations: + argocd.argoproj.io/sync-wave: "-1" +spec: + acme: + email: jounghyeon123@gmail.com + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: letsencrypt-traefik + solvers: + - http01: + ingress: + class: traefik diff --git a/url-shortener/k8s/ingress.yaml b/url-shortener/k8s/traefik/ingress-http-redirect.yaml similarity index 50% rename from url-shortener/k8s/ingress.yaml rename to url-shortener/k8s/traefik/ingress-http-redirect.yaml index ea5220c..cc1ba64 100644 --- a/url-shortener/k8s/ingress.yaml +++ b/url-shortener/k8s/traefik/ingress-http-redirect.yaml @@ -1,17 +1,20 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: url-shortener-ingress + name: url-shortener-traefik-http-redirect + namespace: url-shortener + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web spec: - ingressClassName: nginx + ingressClassName: traefik rules: - - host: url-shortener.local + - host: api.bidservice.store http: paths: - path: / pathType: Prefix backend: service: - name: url-shortener-svc + name: redirect-308 port: number: 80 diff --git a/url-shortener/k8s/traefik/ingress.yaml b/url-shortener/k8s/traefik/ingress.yaml new file mode 100644 index 0000000..011a810 --- /dev/null +++ b/url-shortener/k8s/traefik/ingress.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: url-shortener-traefik-ingress + namespace: url-shortener + annotations: + cert-manager.io/cluster-issuer: letsencrypt-traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.middlewares: > + url-shortener-url-shortener-ratelimit@kubernetescrd, + url-shortener-url-shortener-ipallowlist@kubernetescrd + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + tls: + - hosts: + - api.bidservice.store + secretName: url-shortener-tls + rules: + - host: api.bidservice.store + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: url-shortener-svc + port: + number: 80 diff --git a/url-shortener/k8s/traefik/middleware-ipallowlist.yaml b/url-shortener/k8s/traefik/middleware-ipallowlist.yaml new file mode 100644 index 0000000..8c9dd9b --- /dev/null +++ b/url-shortener/k8s/traefik/middleware-ipallowlist.yaml @@ -0,0 +1,10 @@ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: url-shortener-ipallowlist + namespace: url-shortener +spec: + ipAllowList: + sourceRange: + - "61.72.0.87/32" + - "121.128.27.156/32" diff --git a/url-shortener/k8s/traefik/middleware-ratelimit.yaml b/url-shortener/k8s/traefik/middleware-ratelimit.yaml new file mode 100644 index 0000000..b4de008 --- /dev/null +++ b/url-shortener/k8s/traefik/middleware-ratelimit.yaml @@ -0,0 +1,9 @@ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: url-shortener-ratelimit + namespace: url-shortener +spec: + rateLimit: + average: 5 + burst: 15 diff --git a/url-shortener/k8s/traefik/redirect-308.yaml b/url-shortener/k8s/traefik/redirect-308.yaml new file mode 100644 index 0000000..52acfb2 --- /dev/null +++ b/url-shortener/k8s/traefik/redirect-308.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: redirect-308-config + namespace: url-shortener +data: + default.conf: | + server { + listen 8080; + server_name _; + return 308 https://$host$request_uri; + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redirect-308 + namespace: url-shortener + labels: + app: redirect-308 +spec: + replicas: 2 + selector: + matchLabels: + app: redirect-308 + template: + metadata: + labels: + app: redirect-308 + spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 8080 + volumeMounts: + - name: config + mountPath: /etc/nginx/conf.d/default.conf + subPath: default.conf + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + volumes: + - name: config + configMap: + name: redirect-308-config +--- +apiVersion: v1 +kind: Service +metadata: + name: redirect-308 + namespace: url-shortener + labels: + app: redirect-308 +spec: + ports: + - port: 80 + targetPort: 8080 + name: http + selector: + app: redirect-308 diff --git a/url-shortener/load-tests/week12-load.js b/url-shortener/load-tests/week12-load.js new file mode 100644 index 0000000..9bfaf41 --- /dev/null +++ b/url-shortener/load-tests/week12-load.js @@ -0,0 +1,114 @@ +import http from "k6/http"; +import { check } from "k6"; + +const RAW_BASE = __ENV.TARGET_BASE_URL; +const PROFILE = __ENV.PROFILE || "ramp"; +const TEST_ITEM_ID = __ENV.TEST_ITEM_ID || "1"; +const INGRESS_CONTROLLER = __ENV.INGRESS_CONTROLLER || "unknown"; +const BYPASS_CACHE = (__ENV.BYPASS_CACHE || "false").toLowerCase() === "true"; + +if (!RAW_BASE) { + throw new Error("TARGET_BASE_URL is required"); +} + +const BASE = RAW_BASE.replace(/\/+$/, ""); + +function numberEnv(name, fallback) { + const value = Number(__ENV[name] || fallback); + if (!Number.isFinite(value) || value < 0) { + throw new Error(`${name} must be a non-negative number`); + } + return value; +} + +const preAllocatedVUs = numberEnv("PRE_ALLOCATED_VUS", 500); +const READ_DELAY_MS = numberEnv("READ_DELAY_MS", 0); + +const rampScenario = { + executor: "ramping-arrival-rate", + startRate: numberEnv("RPS_LOW", 100), + timeUnit: "1s", + preAllocatedVUs, + stages: [ + { target: numberEnv("RPS_LOW", 100), duration: __ENV.LOW_RAMP_DURATION || "3m" }, + { target: numberEnv("RPS_LOW", 100), duration: __ENV.LOW_HOLD_DURATION || "2m" }, + { target: numberEnv("RPS_MID", 1000), duration: __ENV.MID_RAMP_DURATION || "5m" }, + { target: numberEnv("RPS_MID", 1000), duration: __ENV.MID_HOLD_DURATION || "3m" }, + { target: numberEnv("RPS_HIGH", 5000), duration: __ENV.HIGH_RAMP_DURATION || "5m" }, + { target: numberEnv("RPS_HIGH", 5000), duration: __ENV.HIGH_HOLD_DURATION || "5m" }, + { target: 0, duration: __ENV.RAMP_DOWN_DURATION || "2m" }, + ], + gracefulStop: "30s", +}; + +const steadyScenario = { + executor: "constant-arrival-rate", + rate: numberEnv("STEADY_RPS", 500), + timeUnit: "1s", + duration: __ENV.STEADY_DURATION || "12m", + preAllocatedVUs, + gracefulStop: "30s", +}; + +if (!["ramp", "steady"].includes(PROFILE)) { + throw new Error("PROFILE must be ramp or steady"); +} + +export const options = { + discardResponseBodies: true, + tags: { + ingress_controller: INGRESS_CONTROLLER, + test_id: __ENV.TEST_ID || "week12-manual", + profile: PROFILE, + }, + scenarios: { + requests: PROFILE === "ramp" ? rampScenario : steadyScenario, + }, + thresholds: { + http_req_failed: ["rate<0.05"], + checks: ["rate>0.95"], + dropped_iterations: ["count==0"], + "http_req_duration{endpoint:items_list}": ["p(95)<2000", "p(99)<5000"], + "http_req_duration{endpoint:item_get}": ["p(95)<2000", "p(99)<5000"], + }, +}; + +function dataRequestParams(endpoint, name) { + const headers = {}; + if (BYPASS_CACHE) { + headers["X-Week12-Bypass-Cache"] = "true"; + } + if (READ_DELAY_MS > 0) { + headers["X-Week12-Read-Delay-Ms"] = String(READ_DELAY_MS); + } + + return { + headers, + tags: { endpoint, name }, + }; +} + +export default function () { + const choice = Math.random(); + let response; + + if (choice < 0.2) { + response = http.get(`${BASE}/healthz`, { + tags: { endpoint: "healthz", name: "GET /healthz" }, + }); + } else if (choice < 0.7) { + response = http.get( + `${BASE}/items`, + dataRequestParams("items_list", "GET /items"), + ); + } else { + response = http.get( + `${BASE}/items/${TEST_ITEM_ID}`, + dataRequestParams("item_get", "GET /items/{item_id}"), + ); + } + + check(response, { + "status is 200": (result) => result.status === 200, + }); +} diff --git a/url-shortener/url-shortener-chart/templates/deployment.yaml b/url-shortener/url-shortener-chart/templates/deployment.yaml index 7bd5c0a..dc8576b 100644 --- a/url-shortener/url-shortener-chart/templates/deployment.yaml +++ b/url-shortener/url-shortener-chart/templates/deployment.yaml @@ -15,6 +15,21 @@ spec: labels: app: {{ .Release.Name }}-api spec: + {{- if .Values.api.topologySpread.enabled }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: {{ .Values.api.topologySpread.whenUnsatisfiable }} + labelSelector: + matchLabels: + app: {{ .Release.Name }}-api + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: {{ .Values.api.topologySpread.whenUnsatisfiable }} + labelSelector: + matchLabels: + app: {{ .Release.Name }}-api + {{- end }} containers: - name: api image: {{ .Values.api.image.repository }}:{{ .Values.api.image.tag }} @@ -43,17 +58,22 @@ spec: value: {{ .Values.api.env.appEnv }} - name: DEBUG value: {{ .Values.api.env.debug | quote }} - # TODO: path를 /readyz로 변경 + failureThreshold 값을 1~3으로 변경 + - name: ENABLE_FAULT_INJECTION + value: {{ .Values.api.env.enableFaultInjection | default false | quote }} + - name: MAX_FAULT_DELAY_MS + value: {{ .Values.api.env.maxFaultDelayMs | default 2000 | quote }} readinessProbe: httpGet: - path: /healthz + path: /readyz port: {{ .Values.api.port }} initialDelaySeconds: {{ .Values.api.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.api.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.api.readinessProbe.failureThreshold | default 3 }} livenessProbe: httpGet: path: /healthz port: {{ .Values.api.port }} initialDelaySeconds: {{ .Values.api.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.api.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.api.livenessProbe.failureThreshold | default 3 }} resources: {{- toYaml .Values.api.resources | nindent 12 }} diff --git a/url-shortener/url-shortener-chart/templates/hpa.yaml b/url-shortener/url-shortener-chart/templates/hpa.yaml index d3cb2ba..ed8e10d 100644 --- a/url-shortener/url-shortener-chart/templates/hpa.yaml +++ b/url-shortener/url-shortener-chart/templates/hpa.yaml @@ -19,4 +19,8 @@ spec: target: type: Utilization averageUtilization: {{ .Values.hpa.targetCPUUtilizationPercentage }} -{{- end }} \ No newline at end of file + {{- with .Values.hpa.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/url-shortener/url-shortener-chart/templates/ingress.yaml b/url-shortener/url-shortener-chart/templates/ingress.yaml index c6204dc..1f654be 100644 --- a/url-shortener/url-shortener-chart/templates/ingress.yaml +++ b/url-shortener/url-shortener-chart/templates/ingress.yaml @@ -1,5 +1,6 @@ -{{- if .Values.ingress.enabled }} # values.yaml에서 ingress.enabled가 true일 때만 이 리소스를 생성 - # dev 환경에서 Ingress 없이 port-forward로 테스트하는 경우에 좋음 +{{- if .Values.ingress.enabled }} +# values.yaml에서 ingress.enabled가 true일 때만 이 리소스를 생성 +# dev 환경에서 Ingress 없이 port-forward로 테스트하는 경우에 좋음 apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/url-shortener/url-shortener-chart/templates/pdb.yaml b/url-shortener/url-shortener-chart/templates/pdb.yaml new file mode 100644 index 0000000..e21a36b --- /dev/null +++ b/url-shortener/url-shortener-chart/templates/pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .Release.Name }}-api-pdb + labels: + app: {{ .Release.Name }}-api +spec: + minAvailable: {{ .Values.pdb.minAvailable }} + selector: + matchLabels: + app: {{ .Release.Name }}-api +{{- end }} diff --git a/url-shortener/url-shortener-chart/templates/prometheusrule.yaml b/url-shortener/url-shortener-chart/templates/prometheusrule.yaml index 291d337..f5a11c3 100644 --- a/url-shortener/url-shortener-chart/templates/prometheusrule.yaml +++ b/url-shortener/url-shortener-chart/templates/prometheusrule.yaml @@ -15,9 +15,13 @@ spec: # ── 높은 에러율 ─────────────────────────────────────── - alert: HighErrorRate expr: | - sum(rate(http_request_total{status_code=~"5.."}[5m])) - / - sum(rate(http_request_total[5m])) > 0.05 + ( + sum(rate(http_request_total{status_code=~"5.."}[5m])) + / + clamp_min(sum(rate(http_request_total[5m])), 0.001) + ) > 0.05 + and + sum(rate(http_request_total[5m])) > 1 for: 2m # 2분 동안 지속될 때만 알림 발송 labels: severity: warning @@ -28,10 +32,16 @@ spec: # ── 캐시 히트율 저하 ───────────────────────────────── - alert: CacheHitRateLow expr: | - sum(rate(cache_hit_total[5m])) - / - (sum(rate(cache_hit_total[5m])) + sum(rate(cache_miss_total[5m]))) - < 0.5 + ( + sum(rate(cache_hit_total[5m])) + / + clamp_min( + sum(rate(cache_hit_total[5m])) + sum(rate(cache_miss_total[5m])), + 0.001 + ) + ) < 0.5 + and + (sum(rate(cache_hit_total[5m])) + sum(rate(cache_miss_total[5m]))) > 1 for: 5m labels: severity: warning @@ -39,12 +49,33 @@ spec: summary: "캐시 히트율 저하 ({{` {{ $value | humanizePercentage }}`}})" description: "캐시 히트율이 50% 미만입니다. Redis 상태 및 TTL 설정을 확인하세요." + # ── Redis 캐시 접근 오류/사용 불가 ───────────────────── + - alert: CacheOperationErrorHigh + expr: | + ( + sum(rate(cache_operation_total{result=~"error|unavailable"}[5m])) + / + clamp_min(sum(rate(cache_operation_total[5m])), 0.001) + ) > 0.05 + and + sum(rate(cache_operation_total[5m])) > 1 + for: 2m + labels: + severity: warning + annotations: + summary: "Redis 캐시 접근 오류 증가 ({{` {{ $value | humanizePercentage }}`}})" + description: "캐시 요청 중 Redis 오류 또는 사용 불가 비율이 5%를 초과했습니다." + # ── DB 쿼리 지연 ───────────────────────────────────── - alert: DBQueryLatencyHigh expr: | - histogram_quantile(0.99, - sum(rate(db_query_latency_seconds_bucket[5m])) by (le, operation) - ) > 0.5 + ( + histogram_quantile(0.99, + sum(rate(db_query_latency_seconds_bucket[5m])) by (le, operation) + ) > 0.5 + ) + and on (operation) + sum(rate(db_query_latency_seconds_count[5m])) by (operation) > 1 for: 3m labels: severity: critical @@ -52,6 +83,25 @@ spec: summary: "DB 쿼리 P99 레이턴시 높음 ({{` {{ $value }}`}}s)" description: "operation={{` {{ $labels.operation }}`}}의 P99 레이턴시가 500ms를 초과했습니다." + # ── 사용자 관점 API 응답 지연 ─────────────────────────── + - alert: APIRequestLatencyHigh + expr: | + ( + histogram_quantile(0.95, + sum(rate(http_request_duration_seconds_bucket{route!~"/healthz|/readyz"}[5m])) + by (le, route) + ) > 1 + ) + and on (route) + sum(rate(http_request_duration_seconds_count{route!~"/healthz|/readyz"}[5m])) + by (route) > 1 + for: 3m + labels: + severity: warning + annotations: + summary: "API P95 응답 지연 ({{` {{ $value }}`}}s)" + description: "route={{` {{ $labels.route }}`}}의 P95 응답시간이 1초를 초과했습니다." + # Watchdog (DeadManSwitch) — Prometheus 자체 정상 동작 확인용 # 항상 발화하는 알림 → AlertManager → 외부 모니터링 시스템에 heartbeat 전송 # 이 알림이 멈추면 Prometheus 또는 AlertManager 자체에 문제 발생 diff --git a/url-shortener/url-shortener-chart/values.prod.yaml b/url-shortener/url-shortener-chart/values.prod.yaml index 5031283..44665da 100644 --- a/url-shortener/url-shortener-chart/values.prod.yaml +++ b/url-shortener/url-shortener-chart/values.prod.yaml @@ -2,11 +2,16 @@ api: replicas: 2 # prod는 고가용성을 위해 2개 image: repository: 716174522908.dkr.ecr.ap-northeast-2.amazonaws.com/urlshortener - tag: sha-e1480e9 + tag: sha-c3071c9 pullPolicy: IfNotPresent env: appEnv: production debug: 'false' # prod는 디버그 비활성화 + enableFaultInjection: false # Week 12 DB 지연 실습 때만 GitOps로 true 전환 + maxFaultDelayMs: 2000 + topologySpread: + enabled: true + whenUnsatisfiable: ScheduleAnyway resources: requests: cpu: 200m @@ -19,6 +24,9 @@ hpa: minReplicas: 2 maxReplicas: 10 targetCPUUtilizationPercentage: 50 +pdb: + enabled: true + minAvailable: 1 postgres: enabled: false # 운영 환경 database: urldb @@ -27,31 +35,9 @@ postgres: databaseUrl: "" storage: 10Gi # prod는 스토리지 넉넉하게 storageClass: gp2 -# ALB Ingress 활성화 (internet-facing = 인터넷에서 접근 가능) +# Ingress 설정 (Traefik으로 이전 완료되어 차트 기본 Ingress 비활성화) ingress: - enabled: true - className: nginx - host: "api.bidflow.cloud" # ALB DNS 직접 사용 시 비워야 함 - pathType: Prefix - tls: - enabled: true - secretName: url-shortener-tls - annotations: - # 아래는 ingress-nginx Ingress 설정 (prod 전용) - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - # Rate Limit 설정 - nginx.ingress.kubernetes.io/limit-rps: "5" # 1초당 5개의 요청을 허용 - nginx.ingress.kubernetes.io/limit-burst-multiplier: "3" # 1초당 5개의 요청을 허용하고 3배의 요청을 허용 - # Ip 제한 설정 - nginx.ingress.kubernetes.io/limit-ip-whitelist: "127.0.0.1" # 127.0.0.1 주소만 허용 - nginx.ingress.kubernetes.io/whitelist-source-range: "203.0.113.10/32,10.0.0.0/8" - nginx.ingress.kubernetes.io/denylist-source-range: "198.51.100.0/24" # 198.51.100.0/24 주소만 허용하지 않음 - # 아래는 ALB Ingress 설정 (prod 전용) - # kubernetes.io/ingress.class: alb - # alb.ingress.kubernetes.io/scheme: internet-facing # 인터넷에서 직접 접근 - # alb.ingress.kubernetes.io/target-type: ip # Pod IP로 직접 포워딩 (권장) - # alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}]' + enabled: false redis: enabled: true # prod: Redis 캐시 활성화 # --- ServiceMonitor 설정 ──────────────────────────────────────── diff --git a/url-shortener/url-shortener-chart/values.yaml b/url-shortener/url-shortener-chart/values.yaml index 112c2dd..316109b 100644 --- a/url-shortener/url-shortener-chart/values.yaml +++ b/url-shortener/url-shortener-chart/values.yaml @@ -9,12 +9,19 @@ api: env: appEnv: production debug: 'false' + enableFaultInjection: false + maxFaultDelayMs: 2000 + topologySpread: + enabled: false + whenUnsatisfiable: ScheduleAnyway readinessProbe: initialDelaySeconds: 5 periodSeconds: 10 + failureThreshold: 3 livenessProbe: initialDelaySeconds: 15 periodSeconds: 20 + failureThreshold: 3 # HPA가 CPU 메트릭을 사용하기 위함 -> 컨테이너에도 적용 resources: @@ -54,6 +61,28 @@ hpa: minReplicas: 2 maxReplicas: 2 targetCPUUtilizationPercentage: 50 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + selectPolicy: Max + policies: + - type: Percent + value: 100 + periodSeconds: 60 + - type: Pods + value: 4 + periodSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 300 + selectPolicy: Max + policies: + - type: Percent + value: 50 + periodSeconds: 60 + +pdb: + enabled: false + minAvailable: 1 # ── RDS 설정 (prod 전용) ──────────────────────────────────────── rds: @@ -68,4 +97,4 @@ redis: # --- ServiceMonitor 설정 ──────────────────────────────────────── serviceMonitor: enabled: false - interval: 30s \ No newline at end of file + interval: 30s