
    зi[                       d Z ddlmZ ddlZddlmc mZ ddl	Z	ddl
Z
ddlZddlZddlmZ  ed      ZddZ	 	 	 d	 	 	 	 	 	 	 	 	 ddZddZ	 	 d	 	 	 	 	 	 	 	 	 dd	Zdd d
Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z d Z!d Z"y)!u  
tests/test_watchdog_noise_elimination.py

task-2399 회귀 테스트 — false stalled 알람 0건 + 진짜 죽음만 검출
session-watchdog.sh 블랙박스 실행 기반 (옵션A: sed WORKSPACE 치환)

시나리오:
  1. test_relative_taskfile_resolved_to_absolute
  2. test_no_taskfile_and_stalled_alert_one_alarm_only
  3. test_escalate_marker_suppresses_alert
  4. test_escalate_acked_keeps_suppression_chairman_def  (task-2405 회장 정의 수정)
  5. test_design_heartbeat_30min_threshold
  6. test_code_heartbeat_10min_threshold
  7. test_progress_marker_codex_gate_keeps_alive
  8. test_progress_marker_pr_creating_keeps_alive
  9. test_alert_body_contains_debug_info
 10. test_recent_events_activity_keeps_alive
 11. test_pr_or_worktree_keeps_alive_skipped_if_unavailable
 12. test_no_double_push_for_same_task
 13. test_no_running_tasks_exits_clean  (보너스)
 14. test_grace_period_skips_recent_dispatch  (보너스)
    )annotationsN)Pathz//home/jay/workspace/scripts/session-watchdog.shc                
    d| iS )u   task-timers.json 포맷.tasks )r   s    </home/jay/workspace/tests/test_watchdog_noise_elimination.py_build_timersr	   '   s    U    c                    t        j                  dt        j                  t        j                          |z               }| |d||dddS )uI   status=running 태스크 항목. start_time은 현재 - start_offset 초.z%Y-%m-%dT%H:%M:%S.000000runningr      )task_idteam_idstatus
start_time	task_fileretry_count	max_retry)timestrftime	localtime)r   r   r   start_offsetstart_tss        r   _running_taskr   ,   sM     }}"tyy{\12H
  r
   c                   dD ]  }| |z  j                  dd        | dz  dz  j                  t        j                  |      d       | dz  j                  d	d       | dz  d
z  j                  dd       t        j                  d      }|j                  dd|  d      }| dz  dz  }|j                  |d       |j                  d       |S )u   
    tmp_path 아래 필수 디렉토리·파일을 생성하고
    WORKSPACE가 tmp_path를 가리키도록 패치된 스크립트를 반환.
    zmemory/eventszmemory/heartbeatszmemory/taskslogsscriptsTparentsexist_okmemorytask-timers.jsonutf-8encoding	.env.keysANU_BOT_TOKEN=dummy
task-timer.py#!/usr/bin/env python3
WORKSPACE="/home/jay/workspace"WORKSPACE=""r   session-watchdog.sh  )mkdir
write_textjsondumpsORIG_SCRIPT	read_textreplacechmod)tmp_pathtimersdorigpatchedscript_paths         r   setup_workspacer>   B   s   
 : 
ATD9: --99

6W : 
 ''(?''R ?*66"W 7 
   ' 2Dll)
hZq!G Y&)>>K7W5er
   c           	     H   t         j                  j                         }d|d<   |r|j                  |       t	        j
                  dt        |       gddt        |xs |      |d      }|dz  dz  }|j                         r|j                  d	
      nd}|j                  |fS )uZ   
    WATCHDOG_DRY_RUN=1로 스크립트 실행 후 (returncode, log_contents) 반환.
    1WATCHDOG_DRY_RUNbashT   capture_outputtextcwdenvtimeoutr   session-watchdog.logr$   r%    )
osenvironcopyupdate
subprocessrunstrexistsr5   
returncode)r=   r8   	extra_envrG   rH   resultlog_filelog_contents           r   run_watchdogrY   j   s     **//
C!C

9^^	[!"x F & #99H:B//:K($$g$6QSKk))r
   c                    | j                          |dkD  r0t        j                         |z
  }t        j                  | ||f       yy)u<   파일 생성 후 mtime을 (현재 - age_seconds)로 설정.r   N)touchr   rL   utime)pathage_secondsts      r   
touch_filer`      s9    JJLQIIK+%
1v r
   c                ^   d}d| d}t        |t        |d|      i      }t        | |      }| |z  }|j                  d| dd	       | d
z  dz  | dz  }t	        |d       t        || | dz        \  }}d}	||	k(  }
|
st        j                  d|
fd||	f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |	      dz  }t        j                  d|       dz   d|iz  }t        t        j                  |            dx}
}	g }
d}	|	|v}|}|s,|j                  d      d   j                  d      d   }||v}|}|snt        j                  d|fd|	|f      t        j                  |	      dt        j                         v st        j                  |      rt        j                  |      ndd z  }d!d"|iz  }|
j!                  |       |st        j                  dfd#|f      d$t        j                         v st        j                  |      rt        j                  |      nd$t        j                  |      d%z  }d&d'|iz  }|
j!                  |       t        j"                  |
d(      i z  }t        j                  d)      d*z   d+|iz  }t        t        j                  |            dx}x}
x}	x}x}}| d,}||v}	|	st        j                  d|	fd-||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      ndd.z  }t        j                  d/|       dz   d|iz  }t        t        j                  |            dx}}	y)0u   
    fix#1: task_file이 'memory/tasks/task-X.md' 상대 경로여도
    WORKSPACE 기준으로 정규화되어 파일을 찾는다.
    cwd를 scripts/ 서브디렉토리로 바꿔도 동일하게 alive 처리.
    z	task-9001zmemory/tasks/z.md	dev1-teamr   z# z
---
team: dev1-team
---
r$   r%   r"   
heartbeats
.heartbeat
   r^   r   )rG   r   ==z%(py0)s == %(py3)srcpy0py3u"   스크립트 비정상 종료: rc=
>assert %(py5)spy5Ntaskfile=no
not in)z%(py3)s not in %(py5)slogrn   rp   %(py7)spy7)z%(py9)s not in %(py12)stid)py9py12%(py14)spy14   uM   상대 경로 task_file이 정규화되지 않아 'taskfile=no' 오탐 발생
>assert %(py17)spy17(team=z%(py1)s not in %(py3)spy1rn   u0   alive 태스크가 stalled로 잘못 판정됨: )r	   r   r>   r1   r`   rY   
@pytest_ar_call_reprcompare@py_builtinslocals_should_repr_global_name	_saferepr_format_assertmsgAssertionError_format_explanationsplitappend_format_boolop)r8   rz   task_file_relr9   r=   task_file_abshbrk   rv   @py_assert2@py_assert1@py_format4@py_format6@py_assert4@py_assert0@py_assert11@py_assert10@py_format8@py_format13@py_format15@py_format16@py_format18s                         r   +test_relative_taskfile_resolved_to_absoluter      s9    C#C5,MCsK=!YZ[F "(F3K },Mr#&CDwW 
H	|	+Z.@	@Brr" ;h6JKGB=27===2======2===2======8=======X[ X>$ X399]3KA3N3T3TUY3Z[]3^ X3^(^ X XFWFWX> X XNWi  X XQWQWX X?W?W "% X XNWi "% X X XQWQWXQWXFWFWX3^ X XQWQWX X?W?W ), X XNWi ), X XNWi 4_ X X XQWQWXQWXIWX XFWFWWX X XDWDWX X X U&>^>$^^^>^^^>^^^^^^^^^^^^^(XY\X]&^^^^^^^r
   c                   d}t        |t        |dd      i      }t        | |      }t        ||       \  }}d}||k(  }|st	        j
                  d|fd||f      dt        j                         v st	        j                  |      rt	        j                  |      ndt	        j                  |      d	z  }d
d|iz  }	t        t	        j                  |	            dx}}|j                  | d      }
d}|
|k  }|st	        j
                  d|fd|
|f      dt        j                         v st	        j                  |
      rt	        j                  |
      ndt	        j                  |      d	z  }t	        j                  d|
 d      dz   d|iz  }	t        t	        j                  |	            dx}}|
dk(  r|j                  | d      }|||dz    }d}||v }|st	        j
                  d|fd||f      t	        j                  |      dt        j                         v st	        j                  |      rt	        j                  |      nddz  }t	        j                  d|       dz   d|iz  }	t        t	        j                  |	            dx}}|j                         D cg c]  }d|v sd|vsd|vs| }}t!        |      }d}||k(  }|st	        j
                  d|fd||f      dt        j                         v st	        j                  t               rt	        j                  t               ndd t        j                         v st	        j                  |      rt	        j                  |      nd t	        j                  |      t	        j                  |      d!z  }t	        j                  d"|       d#z   d$|iz  }t        t	        j                  |            dx}x}}yc c}w )%u   
    fix#1+#2: TASK_FILE 없어도 별도 'no-taskfile' 알람이 아닌
    stalled 알람 1건만 발생해야 한다 (taskfile=no 표시 포함).
    z	task-9002rb   rK   rc   r   rh   rj   rk   rl   assert %(py5)srp   Nr   r   <=z%(py0)s <= %(py3)sstalled_hitsu   동일 태스크 알람 u   건 (중복 발생)ro   i,  rq   inz%(py1)s in %(py3)ssnippetr   u   taskfile=no 표시 없음: zno-taskfilezstalled-no-taskfile)z0%(py3)s
{%(py3)s = %(py0)s(%(py1)s)
} == %(py6)slenlines_with_notaskfile)rm   r   rn   py6u)   별도 no-taskfile 알람 라인 존재: z
>assert %(py8)spy8)r	   r   r>   rY   r   r   r   r   r   r   r   r   countr   index
splitlinesr   )r8   rz   r9   r=   rk   rv   r   r   r   r   r   idxr   r   lr   @py_assert5r   @py_format7@py_format9s                       r   1test_no_taskfile_and_stalled_alert_one_alarm_onlyr      s   
 CCsK2!NOPF!(F3K;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNN 99uF^,LZ<1ZZZ<1ZZZZZZ<ZZZ<ZZZ1ZZZ 8FYZZZZZZZ qii3%v'cC#I&P}'PPP}PPP}PPPPPPPPPPPPP+Fwi)PPPPPPP
 ),(8  I1MQ<NShpqSqv~  GH  wHQ  I  I$%oo%*ooo%oooooo3ooo3oooooo$ooo$ooo%oooooo.WXmWn,oooooooo Is   "	O',O'1O'6O'c                   d}t        |t        |d      i      }t        | |      }t        | dz  dz  | dz         t	        ||       \  }}d}||k(  }|st        j                  d|fd||f      d	t        j                         v st        j                  |      rt        j                  |      nd	t        j                  |      d
z  }dd|iz  }	t        t        j                  |	            dx}}d}
|
|v }|st        j                  d|fd|
|f      t        j                  |
      dt        j                         v st        j                  |      rt        j                  |      nddz  }t        j                  d|       dz   d|iz  }	t        t        j                  |	            dx}
}| d}
|
|v}|st        j                  d|fd|
|f      t        j                  |
      dt        j                         v st        j                  |      rt        j                  |      nddz  }t        j                  d|       dz   d|iz  }	t        t        j                  |	            dx}
}y)u   
    fix#3: events/<tid>.escalate 있고 .escalate.acked 없으면
    알람 0건, 로그에 '알람 억제 (회장 승인 대기)' 포함.
    z	task-9003rb   r"   events	.escalater   rh   rj   rk   rl   r   rp   N   회장 승인 대기r   r   rv   r   u   escalate 억제 로그 없음: ro   r   rt   r   u-   escalate 상태에서 stalled 알람 발생: r	   r   r>   r`   rY   r   r   r   r   r   r   r   r   r   )r8   rz   r9   r=   rk   rv   r   r   r   r   r   s              r   %test_escalate_marker_suppresses_alertr      s   
 CCsK!@ABF!(F3K x("X-3%y0AAB;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNN!Q!S(QQQ!SQQQ!QQQQQQSQQQSQQQQ,KC5*QQQQQQQU&>[>$[[[>[[[>[[[[[[[[[[[[[(UVYUZ&[[[[[[[r
   c                   d}t        |t        |d      i      }t        | |      }| dz  dz  }t        || dz         t        || dz         | dz  dz  | dz  }t        |d	
       t	        ||       \  }}d}||k(  }	|	st        j                  d|	fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }
dd|
iz  }t        t        j                  |            dx}	}| d}||v}|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }
t        j                  d|       dz   d|
iz  }t        t        j                  |            dx}}g }	d}||v }|}|sd| }||v}|}|sqt        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }dd |iz  }|	j                  |       |st        j                  dfd!|f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      ndd"z  }d#d$|iz  }|	j                  |       t        j                  |	d%      i z  }t        j                  d&|       d'z   d(|iz  }t        t        j                  |            dx}x}	x}x}x}}y))u  
    task-2405 회장 정의: acked = 알람 그만.
    escalate + escalate.acked 둘 다 있으면 → 회장이 이미 인지 → skip (알람 0건).
    Fix A: should_skip_for_escalate()는 acked 유무와 무관하게 escalate/acked 어느 하나라도 있으면 skip.
    z	task-9004rb   r"   r   r   z.escalate.ackedrd   re     rg   r   rh   rj   rk   rl   r   rp   Nr   rt   r   rv   r   uJ   escalate+acked 상태에서 stalled 알람 발생 (회장 정의 위반): ro   r   u   검사 시작: r   z%(py3)s in %(py5)srw   rx   ry   )z%(py10)s not in %(py12)spy10r|   r}   r~   r   u$   acked 상태 skip 처리 미확인: r   r   )r	   r   r>   r`   rY   r   r   r   r   r   r   r   r   r   r   r   )r8   rz   r9   r=   
events_dirr   rk   rv   r   r   r   r   r   r   @py_assert9r   r   r   r   r   r   s                        r   2test_escalate_acked_keeps_suppression_chairman_defr      s    CCsK!@ABF!(F3K H$x/Jzse9--.zse?334 
H	|	+Z.@	@Brt$;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNNU&> [>$ [IZIZ[> [ [QZQZ  [ [TZTZ[ [BZBZ "% [ [QZQZ "% [ [IZIZ
TUXTYZ[ [ [GZGZ[ [5! 5!S( 5ocU,C 5,C3,N 5 5#4#45!S 5 5+49 " 5 5.4f5 544 &) 5 5+49 &) 5 5 5.4f5.45#4#45,C3 5 5+49 -D 5 5.4f5 544 LO 5 5+49 LO 5 5 5.4f5.45&4n5 5#4#4
.se45 5 5!4!45 5 5r
   c                	   d}t        |t        |d      i      }t        | |      }| dz  dz  | dz  }t        |d       t	        ||       \  }}d}||k(  }|st        j                  d	|fd
||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }	dd|	iz  }
t        t        j                  |
            dx}}| d}||v}|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}| dz  }|j                          dD ]  }||z  j                  dd        d}t        |t        |d      i      }|dz  dz  j                  t!        j"                  |      d       |dz  j                  d d       |dz  d!z  j                  d"       t$        j'                  d      }|j)                  d#d$| d%      }|d&z  d'z  }|j                  |       |j+                  d(       |dz  dz  | dz  }t        |d)       t,        j.                  j1                         }d*|d+<   t3        j4                  d,t7        |      gddt7        |      |d-.       |d/z  d0z  j9                         r|d/z  d0z  j'                         nd1}g }d2}||v }|}|s| d}||v }|}|sqt        j                  d3|fd4||f      t        j                  |      d5t        j                         v st        j                  |      rt        j                  |      nd5d6z  }
d7d8|
iz  }|j;                  |       |st        j                  d3fd9|f      t        j                  |      d5t        j                         v st        j                  |      rt        j                  |      nd5d:z  }d;d<|iz  }|j;                  |       t        j<                  |d=      i z  }t        j                  d>|       d?z   d@|iz  }t        t        j                  |            dx}x}x}x}x}}y)Au   
    fix#4: team_id="design", heartbeat 1500s ago → alive.
             heartbeat 1900s ago + 다른 진행 신호 없음 → stalled.
    z
task-9005adesignr"   rd   re   i  rg   r   rh   rj   rk   rl   r   rp   Nr   rt   r   rv   r   u)   1500s ago design task가 stalled 오탐: ro   ws2r   Tr   z
task-9005br#   r$   r%   r'   r(   r)   r*   r+   r,   r-   r   r.   r/   il  r@   rA   rB   rC   rD   r   rJ   rK   STALLEDr   r   log2rw   rx   ry   z%(py10)s in %(py12)sr   r}   r~   r   u4   1900s ago design task가 stalled로 판정 안 됨: r   r   )r	   r   r>   r`   rY   r   r   r   r   r   r   r   r   r   r0   r1   r2   r3   r4   r5   r6   r7   rL   rM   rN   rP   rQ   rR   rS   r   r   )r8   tid_ar9   r=   hb_ark   rv   r   r   r   r   r   tmp2r:   tid_btimers_br;   r<   sp2hb_brH   r   r   r   r   r   r   r   r   r   s                                 r   %test_design_heartbeat_30min_thresholdr     sQ    EE=#ABCF!(F3Kh-5'0DDDt&;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNNWFY3&YYY3YYYYYYYYY3YYY3YYYY*STWSX(YYYYYYY eDJJLV 6	56 Ee]5(%CDEH	H_))55djj6JU\5]	K##$;g#N	H_&223MN  ' 2Dll<D6QR>STG

2
2CNN7IIe(?\)ugZ,@@Dt&
**//
C!CNNFCH%d3t9Z]gijDH6MTjDjCrCrCtD6M22==?z|DF9 F9 F5' 0 F 0D 8 F F4E4EF9 F F<EI  F F?EvF F-E-E  F F<EI  F F F?EvF?EF4E4EF 0D F F<EI !1 F F?EvF F-E-E 59 F F<EI 59 F F F?EvF?EF7E~F F4E4E
>tfEF F F2E2EF F Fr
   c                	   d}t        |t        |d      i      }t        | |      }| dz  dz  | dz  }t        |d       t	        ||       \  }}d}||k(  }|st        j                  d	|fd
||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }	dd|	iz  }
t        t        j                  |
            dx}}g }d}||v }|}|s| d}||v }|}|sqt        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }
dd|
iz  }|j                  |       |st        j                  dfd|f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }dd|iz  }|j                  |       t        j                  |d      i z  }t        j                  d|       dz   d|iz  }t        t        j                  |            dx}x}x}x}x}}| d z  }|j!                          d!D ]  }||z  j!                  d"d"#        d$}t        |t        |d      i      }|dz  d%z  j#                  t%        j&                  |             |d&z  j#                  d'       |dz  d(z  j#                  d)       t(        j+                  d*+      }|d,z  d-z  }|j#                  |j-                  d.d/| d0             |j/                  d1       |dz  dz  | dz  }t        |d2       t0        j2                  j5                         }d3|d4<   t7        j8                  d5t;        |      gd"d"t;        |      |d67       |d8z  d9z  j=                         r|d8z  d9z  j+                         nd:}| d}||v}|st        j                  d;|fd<||f      t        j                  |      d=t        j                         v st        j                  |      rt        j                  |      nd=d>z  }	t        j                  d?|       d@z   d|	iz  }
t        t        j                  |
            dx}}y)Auf   
    fix#4: dev1-team, heartbeat 700s ago → stalled.
             heartbeat 500s ago → alive.
    z
task-9006srb   r"   rd   re   i  rg   r   rh   rj   rk   rl   r   rp   Nr   r   r   r   log_arw   rx   ry   r   r   r}   r~   r   u-   700s ago dev team task가 stalled 미감지: r   r   ws_code_aliver   Tr   z
task-9006ar#   r'   r(   r)   r*   r$   r%   r   r.   r+   r,   r-   r/   i  r@   rA   rB   rC   rD   r   rJ   rK   rt   r   log_br   u   500s ago dev task 오탐: ro   )r	   r   r>   r`   rY   r   r   r   r   r   r   r   r   r   r   r   r0   r1   r2   r3   r4   r5   r6   r7   rL   rM   rN   rP   rQ   rR   rS   )r8   	tid_staletimers_ascript_path_ahb_stalerk   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r:   	tid_aliver   r;   r   hb_aliverH   r   s                                r   #test_code_heartbeat_10min_thresholdr   M  sA    Iiy+)NOPH#Hh7M("\1yk4LLHxS)]H5IBN27NNN2NNNNNN2NNN2NNNNNNNNNN@9 @9 @I;f!5 @!5!> @ @.?.?@9 @ @6?i  @ @9?@ @'?'?  @ @6?i  @ @ @9?@9?@.?.?@!5 @ @6?i "6 @ @9?@ @'?'? :? @ @6?i :? @ @ @9?@9?@1?@ @.?.?
7w?@ @ @,?,?@ @ @ o%DJJLV 6	56 Iiy+)NOPH	H_))55djj6JK	K##$;<	H_&223MN  ' 2D

2
2CNN4<< A[QUPVVWCXYZIIeh-9+Z0HHHxS)
**//
C!CNNFCH%d3t9Z]gijEIF]UkEkDsDsDuTF]33>>@{}E[Ru,RRRuRRRRRRRRRuRRRuRRRR0J5'.RRRRRRRr
   c                   d}t        |t        |d      i      }t        | |      }| dz  dz  | dz  }t        |d       t        | dz  dz  | d	z         t	        ||       \  }}d
}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }	dd|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}| d}||v}|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}y)u   
    fix#7: events/<tid>.codex-gate 존재 + heartbeat 노후 → alive.
    로그에 '진행 마커 존재' + 'alive (long-running)' 포함.
    z	task-9007z	dev2-teamr"   rd   re   r   rg   r   z.codex-gater   rh   rj   rk   rl   r   rp   Nu   진행 마커 존재r   r   rv   r   u$   진행 마커 존재 로그 없음: ro   alive (long-running)u&   'alive (long-running)' 로그 없음: r   rt   r   u(   codex-gate 마커인데 stalled 오탐: r   r8   rz   r9   r=   r   rk   rv   r   r   r   r   r   s               r   +test_progress_marker_codex_gate_keeps_aliver     s   
 CCsK!@ABF!(F3K 
H	|	+Z.@	@Brt$ x("X-3%{0CCD;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNN!V!S(VVV!SVVV!VVVVVVSVVVSVVVV,PQTPU*VVVVVVV!X!S(XXX!SXXX!XXXXXXSXXXSXXXX,RSVRW*XXXXXXXU&>V>$VVV>VVV>VVVVVVVVVVVVV(PQTPU&VVVVVVVr
   c                   d}t        |t        |d      i      }t        | |      }| dz  dz  | dz  }t        |d       t        | dz  dz  | d	z         t	        ||       \  }}d
}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }	dd|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}| d}||v}|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}y)uW   
    fix#7: events/<tid>.pr-creating (신규 마커) + heartbeat 노후 → alive.
    z	task-9008z	dev3-teamr"   rd   re   r   rg   r   z.pr-creatingr   rh   rj   rk   rl   r   rp   Nr   r   r   rv   r   u$   pr-creating 마커 alive 미감지: ro   r   rt   r   u)   pr-creating 마커인데 stalled 오탐: r   r   s               r   ,test_progress_marker_pr_creating_keeps_aliver     s    CCsK!@ABF!(F3K	H	|	+Z.@	@Brt$x("X-3%|0DDE;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNN!V!S(VVV!SVVV!VVVVVVSVVVSVVVV,PQTPU*VVVVVVVU&>W>$WWW>WWW>WWWWWWWWWWWWW(QRUQV&WWWWWWWr
   c                Z	   d}t        |t        |d      i      }t        | |      }| dz  dz  | dz  }t        |d       t	        ||       \  }}d}||k(  }|st        j                  d	|fd
||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }	dd|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}y)uz   
    fix#8: stalled 알람 발생 시 로그 본문에
    taskfile=, escalate=, hb_age=, markers= 키 모두 포함.
    z	task-9009rb   r"   rd   re   r   rg   r   rh   rj   rk   rl   r   rp   NDRY_RUNr   r   rv   r   u*   DRY_RUN 로그 없음 (알람 미발생): ro   z	taskfile=u   taskfile= 키 없음: z	escalate=u   escalate= 키 없음: zhb_age=u   hb_age= 키 없음: zmarkers=u   markers= 키 없음: r   r   s               r   #test_alert_body_contains_debug_infor     s   
 CCsK!@ABF!(F3K 
H	|	+Z.@	@Brt$;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNNO9OOO9OOO9OOOOOOOOOOOOOI#OOOOOOO=;#===;#===;======#===#====!7u========;#===;#===;======#===#====!7u=======999999999999999999999993C59999999;:;;;:;;;:;;;;;;;;;;;;; 5cU;;;;;;;r
   c                   d}t        |t        |d      i      }t        | |      }| dz  dz  | dz  }t        |d       t        | dz  dz  | d	z  d
       t	        ||       \  }}d}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }	dd|	iz  }
t        t        j                  |
            dx}}d}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}| d}||v}|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }	t        j                  d|       dz   d|	iz  }
t        t        j                  |
            dx}}y)u[   
    fix#5/#6: heartbeat 노후지만 events/<tid>.qc-result mtime 100s ago → alive.
    z	task-9010rb   r"   rd   re   r   rg   r   z
.qc-resultd   r   rh   rj   rk   rl   r   rp   Nzrecent activityr   r   rv   r   u*   최근 events 활동 alive 로그 없음: ro   r   rt   r   u1   최근 events activity 태스크 stalled 오탐: r   r   s               r   'test_recent_events_activity_keeps_aliver     s    CCsK!@ABF!(F3K 
H	|	+Z.@	@Brt$ x("X-3%z0BBPST;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNNW#WWWWWWWWWWWWWWWWWWW'QRUQV%WWWWWWWU&>_>$___>___>_____________(YZ]Y^&_______r
   c                r   d}t        |t        |d      i      }t        | |      }| dz  dz  | dz  }t        |d       d}t        j
                  j                         }d	|d
<   ||d<   t        j                  dt        |      gddt        |       |d      }| dz  dz  }|j                         r|j                  d      nd}	|j                  }
d}|
|k(  }|st        j                  d|fd|
|f      dt        j                          v st        j"                  |      rt        j$                  |      ndt        j$                  |
      t        j$                  |      dz  }t        j&                  d|j                   d|j(                         dz   d|iz  }t+        t        j,                  |            dx}
x}}d}|	|k7  }
|
st        j                  d|
fd |	|f      d!t        j                          v st        j"                  |	      rt        j$                  |	      nd!t        j$                  |      d"z  }t        j&                  d#      d$z   d%|iz  }t+        t        j,                  |            dx}
}d&}||	v }|st        j                  d'|fd(||	f      t        j$                  |      d!t        j                          v st        j"                  |	      rt        j$                  |	      nd!d)z  }t        j&                  d*|	       d$z   d%|iz  }t+        t        j,                  |            dx}}y)+u   
    fix#9: gh/git PATH에서 제거 → 명령 실패해도 스크립트 정상 완료.
    false alert 방지: 오류 없이 stalled 판정으로 이어져야 한다.
    z	task-9011rb   r"   rd   re   r   rg   z/usr/bin:/binr@   rA   PATHrB   TrC   rD   r   rJ   r$   r%   rK   r   rh   )z2%(py2)s
{%(py2)s = %(py0)s.returncode
} == %(py5)srV   )rm   py2rp   u0   gh/git 없는 환경에서 비정상 종료: rc=z	, stderr=z
>assert %(py7)sry   N)!=)z%(py0)s != %(py3)srv   rl   u?   로그 파일 비어있음 — 스크립트가 조기 종료됨ro   rp   u   워치독 사이클 완료r   r   r   u    사이클 완료 로그 없음: )r	   r   r>   r`   rL   rM   rN   rP   rQ   rR   rS   r5   rT   r   r   r   r   r   r   r   stderrr   r   )r8   rz   r9   r=   r   minimal_pathrH   rV   rW   rv   r   r   @py_assert3r   r   r   r   r   s                     r   6test_pr_or_worktree_keeps_alive_skipped_if_unavailabler     s   
 CCsK!@ABF!(F3K 
H	|	+Z.@	@Brt$ #L
**//
C!CCK^^	[!"MF & #99H2://2C(

g

.C   B  B!  B  B  B  B  B  B  B  B  B6  B  B  B6  B  B  B  B  B  B  B  B  B%UV\VgVgUhhqrxrr  rA  $B  B  B  B  B  B  B  BW3"9WWW3"WWWWWW3WWW3WWW"WWWWWWWWWW'X'3.XXX'3XXX'XXXXXX3XXX3XXXX2RSVRW0XXXXXXXr
   c                   d}t        |t        |dd      i      }t        | |      }| dz  dz  | dz  }t        |d	       t	        ||       \  }}d
}||k(  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }	dd|	iz  }
t        t        j                  |
            dx}}|j                  | d      }d}||k  }|st        j                  d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dz  }	t        j                  d| d|       dz   d|	iz  }
t        t        j                  |
            dx}}y)u   
    fix#2: 단일 사이클에서 동일 task_id가 중복 stalled 등록되어도
    알람 본문에 task_id가 1번만 등장.
    z	task-9012rb   rK   rc   r"   rd   re   r   rg   r   rh   rj   rk   rl   r   rp   Nr   r   r   r   alert_occurrencesu   동일 태스크 알람 중복 u   건: ro   )r	   r   r>   r`   rY   r   r   r   r   r   r   r   r   r   r   )r8   rz   r9   r=   r   rk   rv   r   r   r   r   r   s               r   !test_no_double_push_for_same_taskr     s   
 CCsK2!NOPF!(F3K 
H	|	+Z.@	@Brt$;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNN 		SE.1 ! H! H6G6GH H HAGH H/G/G  H H>Gi  H H>Gi !" H H6G6G
)*;)<E#GH H H4G4GH Hr
   c                X   t        ddddddi      }t        | |      }t        ||       \  }}d}||k(  }|st        j                  d|fd||f      d	t        j                         v st        j                  |      rt        j                  |      nd	t        j                  |      d
z  }t        j                  d|       dz   d|iz  }t        t        j                  |            dx}}d}	|	|v }|st        j                  d|fd|	|f      t        j                  |	      dt        j                         v st        j                  |      rt        j                  |      nddz  }t        j                  d|       dz   d|iz  }t        t        j                  |            dx}	}y)uT   
    running 태스크 없음 → exit 0, 로그에 'running 태스크 없음'.
    z	task-donerb   	completedz2026-01-01T00:00:00.000000)r   r   r   r   r   rh   rj   rk   rl   u   비정상 종료: rc=ro   rp   Nu   running 태스크 없음r   r   rv   r   u*   'running 태스크 없음' 로그 없음: )r	   r>   rY   r   r   r   r   r   r   r   r   r   )
r8   r9   r=   rk   rv   r   r   r   r   r   s
             r   !test_no_running_tasks_exits_cleanr   5  s    ""!6	
 F "(F3K;1GB027000200000020002000000+B40000000%`%,```%```%`````````````0Z[^Z_.```````r
   c                   d}t        |dd      }t        ||i      }t        | |      }t        ||       \  }}d}||k(  }|st	        j
                  d|fd||f      dt        j                         v st	        j                  |      rt	        j                  |      ndt	        j                  |      d	z  }	d
d|	iz  }
t        t	        j                  |
            dx}}d}||v }|st	        j
                  d|fd||f      t	        j                  |      dt        j                         v st	        j                  |      rt	        j                  |      nddz  }	t	        j                  d|       dz   d|	iz  }
t        t	        j                  |
            dx}}| d}||v}|st	        j
                  d|fd||f      t	        j                  |      dt        j                         v st	        j                  |      rt	        j                  |      nddz  }	t	        j                  d|       dz   d|	iz  }
t        t	        j                  |
            dx}}y)uv   
    start_time이 5분(300s) 전인 태스크 → grace_period(600s) 이내 → 스킵.
    stalled 알람 0건.
    z	task-9014rb   i)r   r   rh   rj   rk   rl   r   rp   Nu   유예 기간r   r   rv   r   u   유예 기간 로그 없음: ro   r   rt   r   u+   grace period 내 태스크 stalled 오탐: )r   r	   r>   rY   r   r   r   r   r   r   r   r   r   )r8   rz   taskr9   r=   rk   rv   r   r   r   r   r   s               r   'test_grace_period_skips_recent_dispatchr   N  sd   
 Ck=DC;'F!(F3K ;1GBN27NNN2NNNNNN2NNN2NNNNNNNNNNH?c!HHH?cHHH?HHHHHHcHHHcHHHH%B3%#HHHHHHHU&>Y>$YYY>YYY>YYYYYYYYYYYYY(STWSX&YYYYYYYr
   )r   dictreturnr   )rb   rK   i)
r   rR   r   rR   r   rR   r   intr   r   )r8   r   r9   r   r   r   )NN)
r=   r   r8   r   rU   zdict | NonerG   zPath | Noner   ztuple[int, str])r   )r]   r   r^   r   r   None)#__doc__
__future__r   builtinsr   _pytest.assertion.rewrite	assertionrewriter   r2   rL   rP   r   pathlibr   r4   r	   r   r>   rY   r`   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   r   <module>r	     s  . #    	    DE 	  	
 
,%V "	*** * 
	*
 *8_LpF\05F.Fl+SfW:X2<:`6#YVH8a2Zr
   