""" test_quality_evaluator.py — task-2421 IDS Phase 1 단위 테스트 silent corruption (단조 그라데이션+박스+텍스트 1종 패턴) 영구 차단 검증 작성자: 카구야 (단위 테스트 담당) 관련 태스크: task-2421 (task-2389 한글 깨짐, task-2401 단조 그라데이션 silent pass 재발 방지) task-2421-G2 (로키 G2 적대적 평가 4건 약점 회귀 차단) 구현 의존: quality_evaluator.py (벤자이텐 작성) - check_visual_diversity: STD_THRESHOLD=25.0, UNIQUE_THRESHOLD=1000, SPATIAL_COHERENCE_DIFF_MAX=25.0 (TV-static 차단) - check_brand_color_match: design_md["primary"] 키, Delta-E < 30, BRAND_AREA_RATIO_MIN=0.10 (면적 비율 검증) - check_hybrid_pattern: 각 패턴별 알고리즘 (Sobel 기반) - check_font_size: pytesseract 미설치/예외 시 BLOCKED 통일 (passed=True, blocked=True, score=0) - check_ocr_confidence: pytesseract 없으면 BLOCKED, OCR_KOREAN_RATIO_MIN=0.5 (한글 비율 검증) - evaluate_image: 5개 check 통합, retry_hints = 각 check의 retry_hint 병합 """ from __future__ import annotations import colorsys import sys from pathlib import Path import numpy as np import pytest from PIL import Image, ImageDraw # --------------------------------------------------------------------------- # Import 처리 # --------------------------------------------------------------------------- sys.path.insert(0, str(Path("/home/jay/workspace/skills/satori-cardnews"))) from scripts.quality_evaluator import ( EvalResult, check_brand_color_match, check_font_size, check_hybrid_pattern, check_ocr_confidence, check_visual_diversity, evaluate_image, ) # --------------------------------------------------------------------------- # 공통 상수 # --------------------------------------------------------------------------- FONT_BOLD = "/home/jay/.local/share/fonts/Pretendard-Bold.otf" FONT_REGULAR = "/home/jay/.local/share/fonts/Pretendard-Regular.otf" FONT_NOTO = "/home/jay/.local/share/fonts/NotoSansCJKKR.otf" SIZE = (1080, 1080) # --------------------------------------------------------------------------- # PIL-기반 이미지 생성 헬퍼 # --------------------------------------------------------------------------- def _make_monotone_gradient(size: tuple = SIZE) -> Image.Image: """단조 수직 회색 그라데이션 — silent corruption 대표 케이스 (task-2401).""" arr = np.zeros((size[1], size[0], 3), dtype=np.uint8) for x in range(size[0]): v = int(0x60 + (0xA0 - 0x60) * x / size[0]) arr[:, x, :] = v return Image.fromarray(arr) def _make_solid_pink(size: tuple = SIZE) -> Image.Image: """단색 분홍 이미지 — std_mean 0, unique_colors 1.""" return Image.new("RGB", size, "#FF69B4") def _make_box_text_only(size: tuple = SIZE) -> Image.Image: """흰색 배경 + 밝은 회색 박스만 — unique_colors 극소.""" img = Image.new("RGB", size, "#FFFFFF") draw = ImageDraw.Draw(img) draw.rectangle([100, 100, 980, 980], fill="#EEEEEE", outline="#CCCCCC", width=4) draw.rectangle([200, 200, 880, 880], fill="#EEEEEE") return img def _make_random_noise(size: tuple = SIZE) -> Image.Image: """랜덤 픽셀 노이즈 — Sobel edge density 0.9+ → h1_photo_card.""" arr = np.random.randint(0, 255, (size[1], size[0], 3), dtype=np.uint8) return Image.fromarray(arr) def _make_hsv_illustration(size: tuple = SIZE) -> Image.Image: """ HSV 전 색상 + 채도 변화 그라디언트 — unique_colors > 5000, 평균 채도 > 0.4. → h2_illustration_card. """ arr = np.zeros((size[1], size[0], 3), dtype=np.uint8) for y in range(size[1]): for x in range(0, size[0], 2): hue = (x / size[0] + y / size[1] * 0.5) % 1.0 sat = 0.6 + 0.4 * ((x + y) % 10) / 10.0 r, g, b = colorsys.hsv_to_rgb(hue, sat, 0.9) arr[y, x:x+2, 0] = int(r * 255) arr[y, x:x+2, 1] = int(g * 255) arr[y, x:x+2, 2] = int(b * 255) return Image.fromarray(arr) def _make_gpt_style_blocks(size: tuple = SIZE) -> Image.Image: """ 100px 컬러 블록 + 흰 블록 교번 — edge_density 0.03~0.08, sat_std > 0.1. → h3_gpt_style_card. """ block = 100 arr = np.zeros((size[1], size[0], 3), dtype=np.uint8) idx = 0 for y in range(0, size[1], block): for x in range(0, size[0], block): if idx % 2 == 0: hue = (idx * 0.15) % 1.0 r, g, b = colorsys.hsv_to_rgb(hue, 0.95, 0.95) arr[y:y+block, x:x+block, 0] = int(r * 255) arr[y:y+block, x:x+block, 1] = int(g * 255) arr[y:y+block, x:x+block, 2] = int(b * 255) else: arr[y:y+block, x:x+block] = 240 # near-white idx += 1 return Image.fromarray(arr) def _make_smooth_multicolor_gradient(size: tuple = SIZE) -> Image.Image: """ 2D RG 그라디언트 (R: x축, G: y축) — smoothness < 5 AND unique_colors >= 1000. → h4_gradient_card PASS, visual_diversity도 PASS (2D이므로 샘플링 다양성 확보). """ w, h = size y_idx = np.arange(h).reshape(-1, 1) x_idx = np.arange(w).reshape(1, -1) arr = np.zeros((h, w, 3), dtype=np.uint8) arr[:, :, 0] = (255 * x_idx / (w - 1)).astype(np.uint8) arr[:, :, 1] = (255 * y_idx / (h - 1)).astype(np.uint8) arr[:, :, 2] = 128 return Image.fromarray(arr) def _make_jpeg_noise(size: tuple = SIZE) -> Image.Image: """ JPEG-like 자연 노이즈 (1~15 범위 픽셀 차이) — noise_ratio > 0.15. → h5_user_photo_card. """ base = np.random.randint(80, 180, (size[1], size[0], 3), dtype=np.uint8).astype(np.int16) noise = np.random.randint(-40, 40, (size[1], size[0], 3)) arr = np.clip(base + noise, 0, 255).astype(np.uint8) return Image.fromarray(arr) def _make_supabase_teal(size: tuple = SIZE) -> Image.Image: """Supabase primary #3ECF8E 청록색 dominant 이미지.""" arr = np.full((size[1], size[0], 3), [0x3E, 0xCF, 0x8E], dtype=np.uint8).astype(np.int16) noise = np.random.randint(-8, 8, (size[1], size[0], 3)) arr = np.clip(arr + noise, 0, 255).astype(np.uint8) return Image.fromarray(arr) def _make_solid_gray(size: tuple = SIZE) -> Image.Image: """완전 회색 — brand color 불일치용.""" return Image.new("RGB", size, "#888888") def _save(img: Image.Image, tmp_path: Path, name: str) -> Path: p = tmp_path / name img.save(p) return p # --------------------------------------------------------------------------- # A. silent corruption 검출 (3개) # --------------------------------------------------------------------------- class TestSilentCorruptionDetection: """A. silent corruption 검출 — task-2401 재발 방지 핵심.""" def test_silent_corruption_monotone_gradient_box_fails(self): """ A-1: 단조 회색 그라데이션 PNG → passed=False, reason에 'visual_diversity' 포함. """ img = _make_monotone_gradient() result = check_visual_diversity(img) assert result["passed"] is False, ( f"단조 그라데이션은 visual_diversity FAIL이어야 함. " f"std_mean={result['std_mean']}, unique_colors={result['unique_colors']}" ) assert result["reason"] is not None, "FAIL 시 reason 필드가 있어야 함" def test_silent_corruption_solid_color_fails(self): """ A-2: 단색 분홍 PNG → passed=False, std_mean 낮음. """ img = _make_solid_pink() result = check_visual_diversity(img) assert result["passed"] is False, ( f"단색 분홍은 visual_diversity FAIL이어야 함. std_mean={result['std_mean']}" ) assert result["std_mean"] < 25.0, ( f"단색 이미지의 std_mean은 STD_THRESHOLD(25.0) 미만이어야 함. 실제: {result['std_mean']}" ) def test_silent_corruption_low_unique_colors_fails(self): """ A-3: 단색 박스+텍스트만 PNG → unique_colors < 1000, passed=False. """ img = _make_box_text_only() result = check_visual_diversity(img) assert result["unique_colors"] < 1000, ( f"박스+텍스트만 있는 이미지는 unique_colors < 1000이어야 함. " f"실제: {result['unique_colors']}" ) assert result["passed"] is False, ( "unique_colors < 1000 이미지는 visual_diversity FAIL이어야 함" ) # --------------------------------------------------------------------------- # B. 5 hybrid 패턴 분화 (5개) # --------------------------------------------------------------------------- class TestHybridPatternDiversity: """B. 5 hybrid 패턴 분화 — 5가지 패턴이 각각 고유한 특징으로 구분됨을 검증.""" def test_h1_photo_card_pattern_pass(self): """ B-4: 랜덤 노이즈 PNG → Sobel edge density > 0.05 → h1_photo_card PASS. (실제 임계값: magnitude > 30 픽셀 비율 > 0.05) """ img = _make_random_noise() result = check_hybrid_pattern(img, "h1_photo_card") assert result["passed"] is True, ( f"랜덤 노이즈는 h1_photo_card PASS여야 함. " f"edge_density={result['feature_value']}" ) assert result["feature_value"] > 0.05, ( f"h1_photo_card: edge_density > 0.05 필요. 실제: {result['feature_value']}" ) def test_h2_illustration_pattern_pass(self): """ B-5: HSV 전색상+채도변화 PNG → unique_colors > 5000, 평균채도 > 0.4 → h2_illustration_card PASS. """ img = _make_hsv_illustration() result = check_hybrid_pattern(img, "h2_illustration_card") assert result["passed"] is True, ( f"HSV 일러스트는 h2_illustration_card PASS여야 함. " f"unique_colors(feature_value)={result['feature_value']}, reason={result['reason']}" ) assert result["feature_value"] > 5000, ( f"h2_illustration_card: unique_colors > 5000 필요. 실제: {result['feature_value']}" ) def test_h3_gpt_style_pattern_pass(self): """ B-6: 100px 컬러+흰 블록 교번 PNG → edge_density 0.03~0.08, sat_std > 0.1 → h3_gpt_style_card PASS. """ img = _make_gpt_style_blocks() result = check_hybrid_pattern(img, "h3_gpt_style_card") assert result["passed"] is True, ( f"컬러+흰 블록은 h3_gpt_style_card PASS여야 함. " f"edge_density={result['feature_value']}, reason={result['reason']}" ) assert 0.03 <= result["feature_value"] <= 0.08, ( f"h3: edge_density 0.03~0.08 필요. 실제: {result['feature_value']}" ) def test_h4_gradient_pattern_pass(self): """ B-7: 부드러운 다색 선형 그라디언트 PNG → smoothness < 5 → h4_gradient_card PASS. visual_diversity 와 별도 검사: multi-color 이므로 unique_colors >= 1000. """ img = _make_smooth_multicolor_gradient() hp_result = check_hybrid_pattern(img, "h4_gradient_card") vd_result = check_visual_diversity(img) assert hp_result["passed"] is True, ( f"부드러운 다색 그라디언트는 h4_gradient_card PASS여야 함. " f"smoothness={hp_result['feature_value']}, reason={hp_result['reason']}" ) # multi-color gradient는 unique_colors >= 1000 (단조 그라데이션과 구분) assert vd_result["unique_colors"] >= 1000, ( f"다색 그라디언트는 unique_colors >= 1000 필요. 실제: {vd_result['unique_colors']}" ) def test_h5_user_photo_pattern_pass(self): """ B-8: JPEG-like 노이즈 PNG → noise_ratio > 0.15 → h5_user_photo_card PASS. """ img = _make_jpeg_noise() result = check_hybrid_pattern(img, "h5_user_photo_card") assert result["passed"] is True, ( f"JPEG noise 이미지는 h5_user_photo_card PASS여야 함. " f"noise_ratio={result['feature_value']}" ) assert result["feature_value"] > 0.15, ( f"h5_user_photo_card: noise_ratio > 0.15 필요. 실제: {result['feature_value']}" ) # --------------------------------------------------------------------------- # C. design-md 브랜드 색 매칭 (2개) # --------------------------------------------------------------------------- class TestBrandColorMatch: """C. 브랜드 색상 매칭 — design_md["primary"] 키 사용, Delta-E < 30.""" def test_brand_color_match_supabase_teal_pass(self): """ C-9: Supabase primary #3ECF8E(청록) dominant PNG → passed=True, min_delta_e < 30. """ img = _make_supabase_teal() design_md = {"primary": "#3ECF8E"} result = check_brand_color_match(img, design_md) assert result["passed"] is True, ( f"Supabase 청록 dominant 이미지는 brand_color_match PASS여야 함. " f"min_delta_e={result['min_delta_e']}" ) assert result["min_delta_e"] < 30.0, ( f"min_delta_e는 30 미만이어야 함. 실제: {result['min_delta_e']}" ) def test_brand_color_match_wrong_color_fails(self): """ C-10: design_md primary=#3ECF8E(청록), PNG는 회색만 → passed=False, min_delta_e > 30. """ img = _make_solid_gray() design_md = {"primary": "#3ECF8E"} result = check_brand_color_match(img, design_md) assert result["passed"] is False, ( f"청록 primary + 회색 PNG는 brand_color_match FAIL이어야 함. " f"min_delta_e={result['min_delta_e']}" ) assert result["min_delta_e"] > 30.0, ( f"min_delta_e는 30 초과여야 함. 실제: {result['min_delta_e']}" ) # --------------------------------------------------------------------------- # D. OCR confidence (2개) # --------------------------------------------------------------------------- class TestOcrConfidence: """D. OCR confidence — task-2389 한글 깨짐 회귀 차단.""" def test_ocr_korean_high_confidence_pass(self, tmp_path): """ D-11: 한글 텍스트가 명확한 PNG → avg_confidence >= 70. pytesseract 없으면 skip (시스템 의존). """ try: import pytesseract as _pt # noqa: F401 except ImportError: pytest.skip("pytesseract not installed") img = Image.new("RGB", SIZE, "#FFFFFF") draw = ImageDraw.Draw(img) try: from PIL import ImageFont font = ImageFont.truetype(FONT_BOLD, 80) except Exception: try: from PIL import ImageFont font = ImageFont.truetype(FONT_NOTO, 80) except Exception: from PIL import ImageFont font = ImageFont.load_default() draw.text((100, 400), "안녕하세요 테스트", fill="#000000", font=font) draw.text((100, 520), "카드뉴스 품질 검사", fill="#000000", font=font) result = check_ocr_confidence(img) assert result["avg_confidence"] >= 70, ( f"명확한 한글 텍스트는 avg_confidence >= 70 필요. " f"실제: {result['avg_confidence']}" ) assert result["passed"] is True, "OCR passed 필드 True 필요" def test_ocr_garbled_text_low_confidence_fails(self): """ D-12: 깨진 문자(회색 사각형) PNG → avg_confidence < 70 또는 텍스트 미인식. pytesseract 없으면 skip. """ try: import pytesseract as _pt # noqa: F401 except ImportError: pytest.skip("pytesseract not installed") img = Image.new("RGB", SIZE, "#FFFFFF") draw = ImageDraw.Draw(img) for i in range(8): x = 100 + i * 90 draw.rectangle([x, 400, x + 70, 470], fill="#AAAAAA", outline="#888888") result = check_ocr_confidence(img) is_low_confidence = result["avg_confidence"] < 70 is_no_text = result["avg_confidence"] == 0.0 assert is_low_confidence or is_no_text, ( f"깨진 문자 이미지는 avg_confidence < 70 또는 no text여야 함. " f"실제: avg_confidence={result['avg_confidence']}" ) # --------------------------------------------------------------------------- # E. 통합 evaluate_image (1개) # --------------------------------------------------------------------------- class TestEvaluateImageIntegration: """E. 통합 evaluate_image — 5개 check 통합, EvalResult 반환.""" def test_evaluate_image_integration_normal_card_pass(self, tmp_path): """ E-13: 정상 카드뉴스 (랜덤 노이즈 h1_photo_card + #888888 brand) → passed=True, score >= 70. pytesseract 없으면 skip. """ pytest.importorskip("pytesseract", reason="pytesseract not installed") img = _make_random_noise() png_path = _save(img, tmp_path, "normal_card.png") design_md = {"primary": "#808080"} result = evaluate_image( png_path=png_path, design_md=design_md, hybrid_pattern="h1_photo_card", target_size=SIZE, ) assert isinstance(result, EvalResult), "반환값은 EvalResult 인스턴스여야 함" assert result.passed is True, ( f"정상 카드뉴스는 passed=True여야 함. " f"fail_reasons={result.fail_reasons}, score={result.score}" ) assert result.score >= 70, ( f"정상 카드뉴스는 score >= 70이어야 함. 실제: {result.score}" ) # --------------------------------------------------------------------------- # F. retry_hints 검증 (1개) # --------------------------------------------------------------------------- class TestRetryHints: """F. retry_hints 검증 — FAIL 시 retry_hint 병합 반환.""" def test_fail_returns_retry_hints(self, tmp_path): """ F-14: 단조 그라데이션 PNG (visual_diversity FAIL) → retry_hints 비어있지 않음, force_pattern_diversity 또는 suggested_seed 포함. """ img = _make_monotone_gradient() png_path = _save(img, tmp_path, "monotone_gradient.png") design_md = {"primary": "#3ECF8E"} # 청록 — 회색 이미지와 불일치 result = evaluate_image( png_path=png_path, design_md=design_md, hybrid_pattern="h4_gradient_card", target_size=SIZE, ) assert result.passed is False, ( f"단조 그라데이션은 passed=False여야 함. fail_reasons={result.fail_reasons}" ) assert len(result.retry_hints) > 0, ( f"FAIL 케이스는 retry_hints가 비어있으면 안 됨. 실제: {result.retry_hints}" ) has_diversity_hint = ( "force_pattern_diversity" in result.retry_hints or "suggested_seed" in result.retry_hints ) assert has_diversity_hint, ( f"retry_hints에 force_pattern_diversity 또는 suggested_seed 필요. " f"실제: {result.retry_hints}" ) # --------------------------------------------------------------------------- # G. 로키 G2 적대적 평가 4건 약점 회귀 차단 (4개) # --------------------------------------------------------------------------- class TestLokiG2RegressionBlock: """G. 로키 G2 적대적 평가 4건 약점 회귀 차단.""" def test_visual_diversity_tv_static_blocked(self): """ G-15 (CRITICAL): TV-static 노이즈 PNG → spatial_diff > 25.0 → passed=False. '공간 일관성' reason 포함 확인. """ np.random.seed(42) arr = np.full((1080, 1080, 3), 128, dtype=np.uint8).astype(np.int16) arr += np.random.randint(-60, 60, (1080, 1080, 3)) arr = np.clip(arr, 0, 255).astype(np.uint8) img = Image.fromarray(arr) result = check_visual_diversity(img) assert result["passed"] is False, ( f"TV-static 노이즈는 visual_diversity FAIL이어야 함. " f"spatial_diff={result['spatial_diff']}, std_mean={result['std_mean']}" ) assert result["spatial_diff"] > 25.0, ( f"TV-static의 spatial_diff는 SPATIAL_COHERENCE_DIFF_MAX(25.0) 초과여야 함. " f"실제: {result['spatial_diff']}" ) assert result["reason"] is not None and "공간 일관성" in result["reason"], ( f"FAIL reason에 '공간 일관성' 포함 필요. 실제: {result['reason']}" ) def test_brand_color_area_ratio_insufficient_fails(self): """ G-16 (MEDIUM): 99% 회색 + 1% 청록 PNG → matching_area_ratio < 0.10 → passed=False. """ arr = np.full((1080, 1080, 3), 128, dtype=np.uint8) arr[:108, :108] = [62, 207, 142] # 1% 영역만 supabase 청록 #3ECF8E img = Image.fromarray(arr) design_md = {"primary": "#3ECF8E", "name": "supabase"} result = check_brand_color_match(img, design_md) assert result["passed"] is False, ( f"1% 청록 영역은 brand_color_match FAIL이어야 함. " f"matching_area_ratio={result['matching_area_ratio']}" ) assert result["matching_area_ratio"] < 0.10, ( f"matching_area_ratio는 BRAND_AREA_RATIO_MIN(0.10) 미만이어야 함. " f"실제: {result['matching_area_ratio']}" ) assert "matching_area_ratio" in result, ( "반환 dict에 'matching_area_ratio' 키가 있어야 함" ) def test_check_font_size_ocr_exception_blocked(self): """ G-17 (HIGH): pytesseract 미설치 환경 → check_font_size가 BLOCKED 반환 확인. passed=True, blocked=True, score=0, reason에 'BLOCKED' 포함. """ from scripts.quality_evaluator import _TESSERACT_AVAILABLE img = Image.new("RGB", SIZE, "#FFFFFF") if not _TESSERACT_AVAILABLE: result = check_font_size(img, SIZE) assert result["passed"] is True, ( f"BLOCKED 상태에서 passed=True여야 함. 실제: {result['passed']}" ) assert result["blocked"] is True, ( f"BLOCKED 상태에서 blocked=True여야 함. 실제: {result.get('blocked')}" ) assert result["score"] == 0, ( f"BLOCKED 상태에서 score=0이어야 함. 실제: {result['score']}" ) assert "BLOCKED" in result["reason"], ( f"BLOCKED reason에 'BLOCKED' 문자열 포함 필요. 실제: {result['reason']}" ) else: pytest.skip("pytesseract installed — exception path not tested in this env") def test_ocr_english_only_fails_korean_ratio(self): """ G-18 (LOW): pytesseract 설치 환경에서 영문 텍스트만 그린 PNG → 한글 비율 < 50% → passed=False. pytesseract 미설치 시 skip. """ pytest.importorskip("pytesseract", reason="pytesseract not installed — skip korean_ratio test") img = Image.new("RGB", SIZE, "#FFFFFF") draw = ImageDraw.Draw(img) try: from PIL import ImageFont font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 80) except Exception: try: from PIL import ImageFont font = ImageFont.truetype(FONT_REGULAR, 60) except Exception: from PIL import ImageFont font = ImageFont.load_default() # 영문 텍스트만 그림 — 한글 비율 0% draw.text((100, 380), "ENGLISH ONLY TEXT", fill="#000000", font=font) draw.text((100, 480), "NO KOREAN HERE", fill="#000000", font=font) draw.text((100, 580), "ABCDEFGHIJ 12345", fill="#000000", font=font) result = check_ocr_confidence(img) # OCR이 텍스트를 아예 감지 못하거나(no text → passed=False) 한글 비율 부족으로 FAIL # avg_confidence == 0 (미감지) OR (korean_ratio < 0.5 → passed=False) if result.get("avg_confidence", 0) == 0.0 or result.get("avg_confidence") == -1: # OCR 미감지 케이스 — FAIL 이므로 통과 assert result["passed"] is False, ( "텍스트 미감지 케이스도 passed=False여야 함" ) else: assert result["passed"] is False, ( f"영문만 있는 이미지는 한글 비율 부족으로 passed=False여야 함. " f"korean_ratio={result.get('korean_ratio')}, reason={result.get('reason')}" ) assert result.get("reason") is not None and "한글" in result["reason"], ( f"reason에 '한글' 포함 필요. 실제: {result.get('reason')}" )