
    {i	+                       d Z ddlmZ ddlZddlmc mZ ddl	Z	ddl
Z
ddlmZ ddlZ ee      j                         j                   d   Zedz  Z ee      e
j(                  vr"e
j(                  j+                  d ee              e	j,                  d      Zg dZd	 Zej4                  j7                  d
g d      d        Zd Zd Zd Zd Z d Z!d Z"d Z#d Z$d Z%d Z&d Z'd Z(d Z)d Z*d Z+d Z,d Z-d Z.d Z/d  Z0d! Z1d" Z2y)#uo  IDS Phase 6 — natural-language routing regression tests.

Covers:
- intent classification per category (5 + ambiguous)
- 50-sample accuracy ≥ 90%
- routing matrix dual-version cross-use (satori vs threadauto_render)
- size/style extractors
- external API blocking gate
- confidence + needs_confirmation thresholds
- SLA P95 ≤ 2000ms
- diagnostic logging fields
    )annotationsN)Path   scriptsids_natural_routing)2)u.   인스타그램 카드뉴스 5장 만들어줘cardnews)u"   카드뉴스 supabase 스타일로r   )u   페이스북 카드뉴스 3장r   )u$   인포그래픽 한 장 만들어줘r   )u   카드 뉴스 자동 생성r   )u    instagram 카드뉴스 디자인r   )u    배너 만들어줘 인스타용r   )u   토스 스타일 카드뉴스r   )u-   threads에 올릴 스레드 카드뉴스 8장r   )zinfographic for instagramr   )"   supabase 스타일 보험 PPT 5장ppt)u%   PPT 발표 자료 매거진 스타일r
   )u!   덱 만들어줘 stripe 스타일r
   )u%   프레젠테이션 슬라이드 12장r
   )u   pitch deck 작성해줘r
   )u    발표자료 키노트 스타일r
   )u   회사 소개 PPT 만들어줘r
   )u"   매거진 PPT 보험 상품 소개r
   )u   슬라이드 덱 ppt 한글r
   )z presentation slides for chairmanr
   u'   iPhone 15 Pro 모바일 프로토타입mobile)u-   아이폰 시연용 모바일 프로토타입r   )u   Pixel 9 Pro 앱 시연 영상r   )u-   픽셀 모바일 프로토타입 만들어줘r   )u+   앱 화면 모바일 프로토타입 합성r   )u   iphone 베타테스트 시연r   )u-   모바일 프로토타입 신기능 보여줘r   )u$   프로토타입 iphone 가입 흐름r   )u   Pixel 앱 화면 시연r   )u   mobile prototype 회장 베타r   )u&   모션 카드뉴스 만들어줘 reelsmotion)u   동영상 카드뉴스 30초r   )u   MP4 카드뉴스 fade 효과r   )u   쇼츠용 모션 카드뉴스r   )u    리얼스 동영상 카드뉴스r   )u   html to video 변환r   )u    threads 모션 카드뉴스 3초r   )u    애니메이션 카드뉴스 BGMr   )zvideo for instagram reelsr   )u'   모션 카드뉴스 슬라이드 효과r   )u,   광고 포토 이미지 한글 카피 가득image)$   광고 사진 포토리얼 인스타r   )u   광고 이미지 facebook용r   )u   ad image 보험 상품 광고r   )u$   포토리얼 광고 이미지 페북r   )u   실사 배경 광고 이미지r   )zad photo for facebook insurancer   )u(   배경 이미지 광고용 사진 느낌r   )u/   광고 포토 이미지 텍스트 많은 카피r   )u$   photoreal 광고 이미지 instagramr   c                 <   t        t              } d}| |k(  }|st        j                  d|fd| |f      dt	        j
                         v st        j                  t               rt        j                  t               nddt	        j
                         v st        j                  t              rt        j                  t              ndt        j                  |       t        j                  |      dz  }dd|iz  }t        t        j                  |            d x} x}}y )	N2   ==)z0%(py3)s
{%(py3)s = %(py0)s(%(py1)s)
} == %(py6)slenSAMPLES)py0py1py3py6assert %(py8)spy8)
r   r   
@pytest_ar_call_reprcompare@py_builtinslocals_should_repr_global_name	_safereprAssertionError_format_explanation)@py_assert2@py_assert5@py_assert4@py_format7@py_format9s        A/home/jay/workspace/tests/dev6/test_ids_phase6_natural_routing.pytest_sample_countr*   ]   s    w<2<2<233ww<2    zprompt,expected))u!   인스타그램 카드뉴스 5장r   )   supabase 스타일 PPT 5장r
   r   )u   MP4 모션 카드뉴스 reelsr   )   광고 포토 이미지 페북r   )u   React 랜딩 페이지 코드codec                ^   t         j                  |       }|j                  }||k(  }|s t        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }t        j                  d| d|j                         dz   d	|iz  }t        t        j                  |            d x}}y )
Nr   )z.%(py2)s
{%(py2)s = %(py0)s.intent
} == %(py4)sdecisionexpectedr   py2py4zprompt=u    → 
>assert %(py6)sr   )
ids_routerrouteintentr   r   r   r   r    r!   _format_assertmsgr"   r#   )promptr1   r0   @py_assert1@py_assert3@py_format5r'   s          r)   test_intent_classificationr>   d   s     'H??R?h&RRR?hRRRRRR8RRR8RRR?RRRRRRhRRRhRRRR'&5@Q(RRRRRRRr+   c                 b   d} g }t         D ]M  \  }}t        j                  |      }|j                  |k(  r| dz  } 0|j	                  |||j                  f       O | t        t               z  }d}||k\  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }t        j                  d|d	d
|       dz   d|iz  }	t        t        j                  |	            d x}}y )Nr      g?>=)z%(py0)s >= %(py3)saccuracy)r   r   z	accuracy=z.2%z	; misses=z
>assert %(py5)spy5)r   r6   r7   r8   appendr   r   r   r   r   r    r!   r9   r"   r#   )
correctmissesr:   r1   drC   r$   r;   @py_format4@py_format6s
             r)    test_routing_accuracy_50_samplesrK   w   s    GF# 8V$88xqLGMM68QXX678 W%HH8tHHH8tHHHHHH8HHH8HHHtHHHy#ixHHHHHHHr+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )	Nr	   zmagazine-ppt-kor   z-%(py2)s
{%(py2)s = %(py0)s.skill
} == %(py5)srH   r   r3   rD   assert %(py7)spy7r6   r7   skillr   r   r   r   r    r!   r"   r#   rH   r;   r&   r<   rJ   @py_format8s         r)    test_route_ppt_to_magazine_skillrU      s    =>A77'''7'''''7'''''''1'''1'''7'''''''''''r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )	Nu.   iPhone 15 Pro 모바일 프로토타입 시연zmobile-prototype-kor   rM   rH   rN   rO   rP   rQ   rS   s         r)   +test_route_mobile_to_mobile_prototype_skillrW      s    IJA77+++7+++++7+++++++1+++1+++7+++++++++++r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )	Nu#   MP4 모션 카드뉴스 reels 30초zmotion-cardnews-kor   rM   rH   rN   rO   rP   rQ   rS   s         r)   !test_route_motion_to_motion_skillrY      s    >?A77***7*****7*******1***1***7***********r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )	Nu4   토스 스타일 React 랜딩 페이지 컴포넌트zfrontend-designr   rM   rH   rN   rO   rP   rQ   rS   s         r)   "test_route_code_to_frontend_designr[      s    OPA77'''7'''''7'''''''1'''1'''7'''''''''''r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  }d	}||k(  }|st        j                  d|fd
||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )Nu   인스타 카드뉴스 1장r   r   z.%(py2)s
{%(py2)s = %(py0)s.intent
} == %(py5)srH   rN   rO   rP   satori-cardnewsrM   threadauto_renderz6%(py2)s
{%(py2)s = %(py0)s.fallback_skill
} == %(py5)sr6   r7   r8   r   r   r   r   r    r!   r"   r#   rR   fallback_skillrS   s         r)   ,test_cardnews_short_body_uses_satori_primaryrc      sg   67A88!z!8z!!!!8z!!!!!!1!!!1!!!8!!!z!!!!!!!77'''7'''''7'''''''1'''1'''7'''''''''''2222222222222221222122222222222222r+   c                    d} t         j                  |       }|j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}|j                  }d	}||k(  }|st        j                  d|fd
||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}|j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )Nu   스레드에 올릴 카드뉴스 8장. 본문은 길게 — 보험 가입 흐름을 설명하고, 각 단계별 카피를 자세히 풀어서 작성해줘 supabase 스타일r   r   r]   rH   rN   rO   rP   r_   rM   r^   r`   ra   )long_promptrH   r;   r&   r<   rJ   rT   s          r)   4test_cardnews_long_or_thread_uses_threadauto_primaryrf      ss   	V  	%A88!z!8z!!!!8z!!!!!!1!!!1!!!8!!!z!!!!!!!77)))7)))))7)))))))1)))1)))7)))))))))))0000000000000001000100000000000000r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  }d	}||k(  }|st        j                  d|fd
||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )Nu6   광고 포토 이미지 한글 텍스트 많은 카피r   r   r]   rH   rN   rO   rP   hybrid-imagerM   gemini-imager`   ra   rS   s         r)   !test_image_heavy_text_uses_hybridrj      s_   QRA88w8w8w118w77$n$7n$$$$7n$$$$$$1$$$1$$$7$$$n$$$$$$$-~-~----~------1---1------~-------r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  }d	}||k(  }|st        j                  d|fd
||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )Nr   r   r   r]   rH   rN   rO   rP   ri   rM   rh   r`   ra   rS   s         r)   test_image_default_uses_geminirl      s_   ?@A88w8w8w118w77$n$7n$$$$7n$$$$$$1$$$1$$$7$$$n$$$$$$$-~-~----~------1---1------~-------r+   c                     t        j                  t        j                        5  t        j	                  dd       d d d        y # 1 sw Y   y xY w)Nr-   external_callercallerpytestraisesr6   RoutingErrorr7    r+   r)   ;test_block_direct_api_for_image_when_caller_not_design_teamrv      sB    	z..	/ U9BSTU U Us   AAc                     t        j                  t        j                        5  t        j	                  dd       d d d        t        j	                  dd       t        j	                  dd       y # 1 sw Y   8xY w)Nr
   dispatchro   design_teamr.   anything)rr   rs   r6   rt   block_direct_api_callru   r+   r)   !test_block_direct_api_call_helperr|      sh    	z..	/ C((z(BC $$U=$A$$VJ$?C Cs   A33A<c                    t         j                  d      } | j                  }t         j                  }||k\  }|st	        j
                  d|fd||f      dt        j                         v st	        j                  |       rt	        j                  |       ndt	        j                  |      dt        j                         v st	        j                  t               rt	        j                  t               ndt	        j                  |      dz  }dd|iz  }t        t	        j                  |            d x}x}}| j                  }d	}||u }|st	        j
                  d
|fd||f      dt        j                         v st	        j                  |       rt	        j                  |       ndt	        j                  |      t	        j                  |      dz  }dd|iz  }t        t	        j                  |            d x}x}}| j                  }d }||u }|st	        j
                  d
|fd||f      dt        j                         v st	        j                  |       rt	        j                  |       ndt	        j                  |      t	        j                  |      dz  }dd|iz  }t        t	        j                  |            d x}x}}y )NuD   supabase 스타일 매거진 발표자료 PPT 슬라이드 덱 12장rA   )zW%(py2)s
{%(py2)s = %(py0)s.confidence
} >= %(py6)s
{%(py6)s = %(py4)s.AUTO_CONFIDENCE
}rH   r6   )r   r3   r4   r   r   r   Fisz:%(py2)s
{%(py2)s = %(py0)s.needs_confirmation
} is %(py5)srN   rO   rP   )z7%(py2)s
{%(py2)s = %(py0)s.confirm_message
} is %(py5)s)r6   r7   
confidenceAUTO_CONFIDENCEr   r   r   r   r    r!   r"   r#   needs_confirmationconfirm_message)	rH   r;   r%   r<   r'   r(   r&   rJ   rT   s	            r)   /test_high_confidence_does_not_need_confirmationr      s   _`A<<5:555<55555<555555515551555<555555:555:55555555555(5(5((((5((((((1(((1((((((5((((((($$$$$$$$$$$$1$$$1$$$$$$$$$$$$$r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  }d	}||u }|st        j                  d
|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  }d }||u}|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )Nu   뭔가 만들어줘 멋있게	ambiguousr   r]   rH   rN   rO   rP   Tr~   r   )is not)z;%(py2)s
{%(py2)s = %(py0)s.confirm_message
} is not %(py5)s)r6   r7   r8   r   r   r   r   r    r!   r"   r#   r   r   rS   s         r)   .test_ambiguous_prompt_returns_ambiguous_intentr      sd   89A88"{"8{""""8{""""""1"""1"""8"""{"""""""'4'4''''4''''''1'''1''''''4'''''''(D(D((((D((((((1(((1((((((D(((((((r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )	Nu)   인스타그램 카드뉴스 만들어줘instagram_squarer   )z,%(py2)s
{%(py2)s = %(py0)s.size
} == %(py5)srH   rN   rO   rP   )r6   r7   sizer   r   r   r   r    r!   r"   r#   rS   s         r)   test_extracts_size_instagramr      s    DEA66'''6'''''6'''''''1'''1'''6'''''''''''r+   c                    t         j                  d      } | j                  }d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}y )	Nr,   supabaser   )z-%(py2)s
{%(py2)s = %(py0)s.style
} == %(py5)srH   rN   rO   rP   )r6   r7   styler   r   r   r   r    r!   r"   r#   rS   s         r)   test_extracts_style_brandr      s    67A77 j 7j    7j      1   1   7   j       r+   c                    g } t         D ]  \  }}t        j                  |      }| j                  |j                         |j
                  }|st        j                  d|d|j                   d      dz   dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }t        t        j                  |            d } | j                          | t        t!        |       dz           }t        j"                  }||k  }|st        j$                  d|fd	||f      d
t        j                         v st        j                  |      rt        j                  |      nd
dt        j                         v st        j                  t              rt        j                  t              ndt        j                  |      dz  }t        j                  d| dt        j"                   d      dz   d|iz  }	t        t        j                  |	            d x}}y )Nzsla_ok=False for z (zms)z,
>assert %(py2)s
{%(py2)s = %(py0)s.sla_ok
}rH   r   r3   gffffff?)<=)z4%(py0)s <= %(py4)s
{%(py4)s = %(py2)s.SLA_ROUTE_MS
}p95r6   r2   zP95=zms exceeds msr5   r   )r   r6   r7   rE   
elapsed_mssla_okr   r9   r   r   r    r!   r"   r#   sortintr   SLA_ROUTE_MSr   )
elapsedr:   _rH   r;   @py_format3r   r<   r=   r'   s
             r)   test_routing_sla_p95r      sc   G K	V$q||$xxJxJJ,VJbcJJJJJJJqJJJqJJJxJJJJJJK LLN
#c'lT)*
+C))]3))]]]3)]]]]]]3]]]3]]]]]]*]]]*]]])]]]T#k*BYBYAZZ\+]]]]]]]r+   c                    t         j                  d      } | j                  d   }d}||k(  }|slt        j                  d|fd||f      t        j
                  |      t        j
                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}d	}| j                  }||v }|st        j                  d
|fd||f      t        j
                  |      dt        j                         v st        j                  |       rt        j
                  |       ndt        j
                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}d}| j                  }||v }|st        j                  d
|fd||f      t        j
                  |      dt        j                         v st        j                  |       rt        j
                  |       ndt        j
                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}d}| j                  }||v }|st        j                  d
|fd||f      t        j
                  |      dt        j                         v st        j                  |       rt        j
                  |       ndt        j
                  |      dz  }dd|iz  }t        t        j                  |            d x}x}}| j                  d   }	t        |	t              }|sddt        j                         v st        j                  t              rt        j
                  t              ndt        j
                  |	      dt        j                         v st        j                  t              rt        j
                  t              ndt        j
                  |      dz  }t        t        j                  |            d x}	}y )Nu   PPT 발표자료 매거진phaser7   r   z%(py1)s == %(py4)sr   r4   assert %(py6)sr   model_qualityin)z2%(py1)s in %(py5)s
{%(py5)s = %(py3)s.diagnostic
}rH   r   r   rD   rO   rP   cli_constraintscores
body_charsz5assert %(py5)s
{%(py5)s = %(py0)s(%(py2)s, %(py3)s)
}
isinstancer   )r   r3   r   rD   )r6   r7   
diagnosticr   r   r!   r"   r#   r   r   r    r   r   )
rH   @py_assert0r<   r$   r=   r'   r&   rJ   rT   r;   s
             r)   test_diagnostic_fields_presentr     s7   56A<< +G+ G++++ G+++ +++G+++++++*all*?l****?l***?******a***a***l*******+q||+|++++|+++++++++q+++q+++|+++++++#q||#8|####8|###8######q###q###|#######ll<06:0#66666666:666:6660666666#666#6666666666r+   c                 p   d} t         j                  |       }|j                  }t        |      }d}||k(  }|s
t	        j
                  d|fd||f      dt        j                         v st	        j                  t              rt	        j                  t              nddt        j                         v st	        j                  |      rt	        j                  |      ndt	        j                  |      t	        j                  |      t	        j                  |      dz  }dd	|iz  }t        t	        j                  |            d x}x}x}}d
}|j                  }||v}|st	        j
                  d|fd||f      t	        j                  |      dt        j                         v st	        j                  |      rt	        j                  |      ndt	        j                  |      dz  }	dd|	iz  }
t        t	        j                  |
            d x}x}}d}|j                  }||v}|st	        j
                  d|fd||f      t	        j                  |      dt        j                         v st	        j                  |      rt	        j                  |      ndt	        j                  |      dz  }	dd|	iz  }
t        t	        j                  |
            d x}x}}|j                         }| |v}|st	        j
                  d|fd| |f      dt        j                         v st	        j                  |       rt	        j                  |       nddt        j                         v st	        j                  |      rt	        j                  |      nddz  }dd|iz  }t        t	        j                  |            d }y )Nu8   주민등록번호 901010-1234567 카드뉴스 인스타   r   )zQ%(py5)s
{%(py5)s = %(py0)s(%(py3)s
{%(py3)s = %(py1)s.prompt_hash
})
} == %(py8)sr   rH   r   r   r   rD   r   assert %(py10)spy10u   주민등록번호)not in)z7%(py1)s not in %(py5)s
{%(py5)s = %(py3)s.prompt_hash
}r   rO   rP   901010)z%(py0)s not in %(py2)sraw
serializedr   zassert %(py4)sr4   )r6   r7   prompt_hashr   r   r   r   r   r    r!   r"   r#   to_json)r   rH   r$   r&   @py_assert7@py_assert6r(   @py_format11r   rJ   rT   r   r;   r   r=   s                  r)   )test_prompt_hash_does_not_leak_raw_promptr     s   
DCA}}#3}############3###3######q###q###}#############4q}}4}4444}444444444q444q444}4444444(1==(8=((((8=(((8((((((1(((1(((=(((((((Jj    3j      3   3      j   j       r+   c                     t        j                  t        j                        5  t        j	                  d       d d d        y # 1 sw Y   y xY w)N rq   ru   r+   r)   test_empty_prompt_raisesr     s7    	z..	/   s   AAc                    t         j                  } t        |       }h d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  t              rt        j                  t              nddt        j                         v st        j                  t               rt        j                  t               ndt        j                  |       t        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d x} x}x}}y )	N>   r
   r.   r   r   r   r   r   r   )zM%(py5)s
{%(py5)s = %(py0)s(%(py3)s
{%(py3)s = %(py1)s.INTENTS
})
} == %(py8)ssetr6   r   r   r   )r6   INTENTSr   r   r   r   r   r    r!   r"   r#   )r$   r&   r   r   r(   r   s         r)   test_intent_set_completer     s   !! 3!"  ' " '    " '                        "    #   '       r+   c                    i } t         D ]&  \  }}| j                  |g       j                  |       ( | j                         D ]  \  }t	        fd|D              }t        |      }||z  }d}||k\  }|s`t        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      nddt        j                         v st        j                  t
              rt        j                  t
              nddt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      dz  }	t        j                  d	 d
| dt        |       d      dz   d|	iz  }
t        t        j                  |
            dx}x}x}} y)zGSmoke-check: each category has at least 80% accuracy in its sub-bucket.c              3  h   K   | ])  }t         j                  |      j                  k(  s&d  + yw)r@   N)r6   r7   r8   ).0pr8   s     r)   	<genexpr>z5test_routing_accuracy_per_category.<locals>.<genexpr>1  s)     QAJ,<,<Q,?,F,F&,PaQs   '22g?rA   )z<(%(py0)s / %(py4)s
{%(py4)s = %(py1)s(%(py2)s)
}) >= %(py8)srF   r   prompts)r   r   r3   r4   r   zintent=z per-bucket accuracy /z below 0.80z
>assert %(py10)sr   N)r   
setdefaultrE   itemssumr   r   r   r   r   r    r!   r9   r"   r#   )	by_intentr:   r1   r   rF   r<   r%   r   r   r(   r   r8   s              @r)   "test_routing_accuracy_per_categoryr   *  s    I# :Xr*11&9:$??, 
QQQW 	
w% 	
 	
%, 	
 	
 	
% 	
 	
	6	
 	
   	
 	
 		  	
 	
	6	
 	
   	
 	
 		  	
 	
	6	
 	
  % 	
 	
 		 % 	
 	
 		 & 	
 	
 		 *- 	
 	
  fX27)1S\N+V	
 	
 	
 	
 	
 	

r+   c                    dd l } t        j                  d      }| j                  |j	                               }|d   }d}||k(  }|slt        j                  d|fd||f      t        j                  |      t        j                  |      dz  }dd	|iz  }t        t        j                  |            d x}x}}|d
   }d}||v }|slt        j                  d|fd||f      t        j                  |      t        j                  |      dz  }dd	|iz  }t        t        j                  |            d x}x}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }dd|iz  }	t        t        j                  |	            d x}}y )Nr   u   인스타 카드뉴스r8   r   r   r   r   r   r   rR   )r^   r_   r   )z%(py1)s in %(py4)sr   )z%(py1)s in %(py3)sparsed)r   r   zassert %(py5)srD   )jsonr6   r7   loadsr   r   r   r!   r"   r#   r   r   r    )
r   rH   r   r   r<   r$   r=   r'   rI   rJ   s
             r)   test_to_json_round_tripr   7  s'   12AZZ		$F()z)z))))z))))))z)))))))'?FFF?FFFFF?FFFF?FFFFFFFFFFF!<6!!!!<6!!!<!!!!!!6!!!6!!!!!!!r+   )3__doc__
__future__r   builtinsr   _pytest.assertion.rewrite	assertionrewriter   	importlibsyspathlibr   rr   __file__resolveparentsROOTSCRIPTSstrpathinsertimport_moduler6   r   r*   markparametrizer>   rK   rU   rW   rY   r[   rc   rf   rj   rl   rv   r|   r   r   r   r   r   r   r   r   r   r   r   ru   r+   r)   <module>r      s.   #     
   H~''*

w<sxxHHOOAs7|$$Y$$%:;
8v 
S
S
I (
,
+
(31..U
@%)(
!^7!
	

"r+   