
    i0#                     ^   d Z ddlZddlmc mZ ddlZddlZddl	Z	ddl
Z
ddlmZ ddlZdej                  vrej                  j                  dd       ddlmZmZmZmZ ej*                  d        Zdedefd	Zed
z   fdedefdZd Zd Zd Zd Zd Zd Zd Z d Z!d Z"d Z#d Z$d Z%d Z&y)u  task-2422 Fix B 회귀 테스트: 신호등 봇 실 PID 종합 판정

검증 포인트:
- evaluate_task_liveness 6 시나리오 (completed/no-hb/fresh-pid-alive/fresh-pid-dead/stale/escalated-status박제)
- _PidLivenessProvider override 동작
- get_running_tasks_by_team이 status=escalated여도 PID alive 시 작업중 표시 (사고 회귀 차단)
- task-2414/2417/2420 사고 재현: status가 escalated로 박제되어도 신호등 작업중 정확
    N)Pathz/home/jay/workspace)
DataLoaderHEARTBEAT_FRESH_SECONDS_PidLivenessProviderevaluate_task_livenessc               #   x   K   t        j                         5 } t        |        d d d        y # 1 sw Y   y xY ww)N)tempfileTemporaryDirectoryr   )tmps    B/home/jay/workspace/tests/dashboard/test_signal_pid_aggregation.pytmp_heartbeat_dirr      s1     		$	$	& #3i  s   :.	:7:hb_dirtask_idc                 6    | | dz  }|j                          |S )u'   방금 mtime의 heartbeat 파일 생성
.heartbeat)touch)r   r   fs      r   _touch_freshr   #   s!    G9J''AGGIH    <   c                     | | dz  }|j                          t        j                         |z
  }t        j                  |||f       |S )u*   오래된 mtime의 heartbeat 파일 생성r   )r   timeosutime)r   r   age_secr   pasts        r   _touch_staler   *   sC    G9J''AGGI99; DHHQtHr   c                    t        | d       t               }|j                  ddi       t        dddd| |      }d}||k(  }|st	        j
                  d|fd||f      d	t        j                         v st	        j                  |      rt	        j                  |      nd	t	        j                  |      d
z  }dd|iz  }t        t	        j                  |            dx}}y)u0   status=completed → idle (heartbeat/PID 무관)t1zsched-1T	completedstatusschedule_ididle==z%(py0)s == %(py3)sresultpy0py3assert %(py5)spy5Nr   r   set_overrider   
@pytest_ar_call_reprcompare@py_builtinslocals_should_repr_global_name	_safereprAssertionError_format_explanationr   pr(   @py_assert2@py_assert1@py_format4@py_format6s          r   +test_liveness_completed_status_returns_idler>   3   s    "D)ANNIt$%#Y?ARTUF 6V6V66Vr   c                    t               }|j                  ddi       t        dddd| |      }d}||k(  }|st        j                  d|fd||f      d	t        j                         v st        j                  |      rt        j                  |      nd	t        j                  |      d
z  }dd|iz  }t        t        j                  |            dx}}y)u0   status=running, heartbeat 파일 없음 → idlezsched-2Tt2runningr!   r$   r%   r'   r(   r)   r,   r-   N)r   r/   r   r0   r1   r2   r3   r4   r5   r6   r7   r8   s          r   /test_liveness_running_no_heartbeat_returns_idlerB   >   s    ANNIt$%#9=?PRSF 6V6V66Vr   c                    t        | d       t               }|j                  ddi       t        dddd| |      }d}||k(  }|st	        j
                  d|fd||f      d	t        j                         v st	        j                  |      rt	        j                  |      nd	t	        j                  |      d
z  }dd|iz  }t        t	        j                  |            dx}}y)u8   status=running + fresh heartbeat + PID alive → workingt3zsched-3TrA   r!   workingr%   r'   r(   r)   r,   r-   Nr.   r8   s          r   ?test_liveness_running_fresh_heartbeat_pid_alive_returns_workingrF   H   s    "D)ANNIt$%#9=?PRSF 6Y6Y66Yr   c                    t        | d       t               }|j                  i        t        dddd| |      }d}||k(  }|st	        j
                  d|fd||f      dt        j                         v st	        j                  |      rt	        j                  |      ndt	        j                  |      d	z  }d
d|iz  }t        t	        j                  |            dx}}y)u@   status=running + fresh heartbeat + PID dead → bot_suspect_deadt4rA   zsched-4r!   bot_suspect_deadr%   r'   r(   r)   r,   r-   Nr.   r8   s          r   Gtest_liveness_running_fresh_heartbeat_pid_dead_returns_bot_suspect_deadrJ   S   s    "D)ANN2#9=?PRSF ('6'''''6'''''''6'''6'''''''''''r   c                    t        | d       t               }|j                  ddi       t        dddd| |      }d}||k(  }|st	        j
                  d|fd||f      d	t        j                         v st	        j                  |      rt	        j                  |      nd	t	        j                  |      d
z  }dd|iz  }t        t	        j                  |            dx}}y)u;   status=running + stale heartbeat → idle (PID alive여도)t5zsched-5TrA   r!   r$   r%   r'   r(   r)   r,   r-   N)r   r   r/   r   r0   r1   r2   r3   r4   r5   r6   r7   r8   s          r   2test_liveness_running_stale_heartbeat_returns_idlerM   ^   s    "D)ANNIt$%#9=?PRSF 6V6V66Vr   c                    t        | d       t               }|j                  ddi       t        dddd| |      }d}||k(  }|st	        j
                  d|fd||f      d	t        j                         v st	        j                  |      rt	        j                  |      nd	t	        j                  |      d
z  }t	        j                  d|       dz   d|iz  }t        t	        j                  |            dx}}y)u   ★ 핵심: status=escalated여도 fresh heartbeat + PID alive → working (status박제 무시)

    task-2414/2417/2420 사고 회귀 차단의 핵심 시나리오.
    t6zsched-6T	escalatedr!   rE   r%   r'   r(   r)   u:   status박제에 휘둘림 (회귀): expected working, got 
>assert %(py5)sr-   N)r   r   r/   r   r0   r1   r2   r3   r4   r5   _format_assertmsgr6   r7   r8   s          r   Atest_liveness_escalated_status_with_alive_signals_returns_workingrS   i   s    
 "D)ANNIt$%#Y?ARTUF e6Yeee6Yeeeeee6eee6eeeYeee"\]c\d eeeeeeer   c                    t        | d       t               }|j                  ddi       t        dddd| |      }d}||k(  }|st	        j
                  d|fd||f      d	t        j                         v st	        j                  |      rt	        j                  |      nd	t	        j                  |      d
z  }dd|iz  }t        t	        j                  |            dx}}y)u+   status=foobar 등 알수없는 값 → idlet7zsched-7Tfoobarr!   r$   r%   r'   r(   r)   r,   r-   Nr.   r8   s          r   )test_liveness_unknown_status_returns_idlerW   w   s    "D)ANNIt$%#)<>OQRF 6V6V66Vr   c                  ,   t               } | j                  ddi       | j                  }d} ||      }d}||u }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      t        j                  |      t        j                  |      dz  }d	d
|iz  }t        t        j                  |            dx}x}x}x}}| j                  }d} ||      }d}||u }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      t        j                  |      t        j                  |      dz  }d	d
|iz  }t        t        j                  |            dx}x}x}x}}| j                  }d} ||      }d}||u }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      t        j                  |      t        j                  |      dz  }d	d
|iz  }t        t        j                  |            dx}x}x}x}}y)u,   override map에 없는 schedule_id → Falsezsched-ATzsched-BFiszN%(py6)s
{%(py6)s = %(py2)s
{%(py2)s = %(py0)s.is_alive
}(%(py4)s)
} is %(py9)sr9   r*   py2py4py6py9assert %(py11)spy11N r   r/   is_aliver0   r1   r2   r3   r4   r5   r6   r7   r9   r;   @py_assert3@py_assert5@py_assert8@py_assert7@py_format10@py_format12s           r   =test_pid_provider_override_returns_false_for_missing_schedulerm      s   ANNIt$%::)i):i )E) E)))) E))))))1)))1))):)))i))) )))E)))))))::"b":b>"U">U"""">U""""""1"""1""":"""b""">"""U"""""""::$d$:d$u$u$$$$u$$$$$$1$$$1$$$:$$$d$$$$$$u$$$$$$$$r   c                  T   t               } | j                  ddi       | j                  }d} ||      }d}||u }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      t        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d	x}x}x}x}}| j                  d	       | j                  }d} ||      }d
}||u }|st        j                  d|fd||f      dt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |      t        j                  |      t        j                  |      dz  }dd|iz  }t        t        j                  |            d	x}x}x}x}}y	)u@   set_override(None) → 실제 pgrep 모드 복귀 (캐시 reset)zsched-XTrY   r[   r9   r\   ra   rb   NFrd   rf   s           r   6test_pid_provider_clear_override_resets_to_real_lookupro      sT   ANNIt$%::(i(:i (D( D(((( D((((((1(((1(((:(((i((( (((D(((((((NN4::)i):i )E) E)))) E))))))1)))1))):)))i))) )))E))))))))r   c            
         t        j                         5 } t        |       }|dz  }|dz  j                  d       t	        |      }|dz  dz  j                          dddd	d
ddddii|_        |j                  j                  d
di       |j                         }d	}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }t        j                   d|       dz   d|iz  }t#        t        j$                  |            dx}}|d	   }	t'        |	      }
d}|
|k(  }|st        j                  d|fd|
|f      dt        j                         v st        j                  t&              rt        j                  t&              ndt        j                  |	      t        j                  |
      t        j                  |      dz  }dd|iz  }t#        t        j$                  |            dx}	x}
x}}|d	   d   }|d   }d}
||
k(  }|slt        j                  d|fd ||
f      t        j                  |      t        j                  |
      d!z  }d"d#|iz  }t#        t        j$                  |            dx}x}}
|j(                  }	d$}
 |	|
      }d%}||k(  }|st        j                  d|fd&||f      d't        j                         v st        j                  |      rt        j                  |      nd't        j                  |	      t        j                  |
      t        j                  |      t        j                  |      d(z  }d)d*|iz  }t#        t        j$                  |            dx}	x}
x}x}}ddd       y# 1 sw Y   yxY w)+uf   ★ 통합 테스트: status=escalated + fresh heartbeat + PID alive → 신호등에 작업중 표시memory
heartbeatsTparentsztask-9999.heartbeattasksz	task-9999rP   z	dev2-teamzsched-test-9999z	test taskz2026-05-01T00:00:00zbot-x)r"   team_idr#   description
start_timebotinz%(py1)s in %(py3)sr(   py1r+   u:   escalated + alive 봇이 신호등에서 누락 (회귀): rQ   r-   N   r%   )z0%(py4)s
{%(py4)s = %(py0)s(%(py2)s)
} == %(py7)slen)r*   r]   r^   py7zassert %(py9)sr`   r   r   z%(py1)s == %(py4)sr~   r^   assert %(py6)sr_   livenessrE   )zI%(py6)s
{%(py6)s = %(py2)s
{%(py2)s = %(py0)s.get
}(%(py4)s)
} == %(py9)sentryr\   ra   rb   )r	   r
   r   mkdirr   r   	task_datapid_providerr/   get_running_tasks_by_teamr0   r1   r5   r2   r3   r4   rR   r6   r7   r   get)r   	workspacerq   loaderr(   @py_assert0r:   r<   r=   r;   rg   @py_assert6rh   @py_format8rk   r   @py_format5@py_format7ri   rj   rl   s                        r   Gtest_get_running_tasks_by_team_with_escalated_status_includes_alive_botr      s   		$	$	& 2#I	X%	,	%%d%3I&	,	!6	6==? )*#4#."7"	
 	((*;T)BC113k{f$kkk{fkkk{kkkkkkfkkkfkkkk(bcibj&kkkkkkk+&,s&',1,'1,,,,'1,,,,,,s,,,s,,,&,,,',,,1,,,,,,,{#A&Y.;.;....;......;.......yy11y$1	1$	1111$	111111u111u111y111111$111	111111192 2 2s   N4OOc                  B   t        j                         5 } t        |       }|dz  }|dz  j                  d       t	        |      }|dz  dz  }|j                          t        j                         t        z
  dz
  }t        j                  |||f       ddd	d
dddii|_
        |j                  j                  ddi       |j                         }d
}||v}|st        j                  d|fd||f      t        j                   |      dt#        j$                         v st        j&                  |      rt        j                   |      nddz  }	t        j(                  d      dz   d|	iz  }
t+        t        j,                  |
            dx}}ddd       y# 1 sw Y   yxY w)uB   status=running이지만 heartbeat stale → 신호등에서 제외rq   rr   Trs   ztask-9998.heartbeatd   ru   z	task-9998rA   z	dev1-teamz
sched-9998z
stale taskr"   rv   r#   rw   F)not in)z%(py1)s not in %(py3)sr(   r}   u.   stale heartbeat task가 신호등에 포함됨rQ   r-   N)r	   r
   r   r   r   r   r   r   r   r   r   r   r/   r   r0   r1   r5   r2   r3   r4   rR   r6   r7   )r   r   rq   r   hbr   r(   r   r:   r<   r=   s              r   =test_get_running_tasks_by_team_excludes_running_with_dead_botr      sQ   		$	$	& [#I	X%	,	%%d%3I&l"%::

yy{44s:
dD\"'*#/#/		
 	((,)>?113Z{&(ZZZ{&ZZZ{ZZZZZZ&ZZZ&ZZZZ*ZZZZZZZ/[ [ [s   E7FFc                     t        j                         5 } t        |       }|dz  }|dz  j                  d       t	        |      }|dz  dz  j                          dddd	d
ddii|_        |j                  j                  i        |j                         }d	}||v }|st        j                  d|fd||f      t        j                  |      dt        j                         v st        j                  |      rt        j                  |      nddz  }dd|iz  }t!        t        j"                  |            dx}}|d	   d   d   }d}	||	k(  }|slt        j                  d|fd||	f      t        j                  |      t        j                  |	      dz  }
dd|
iz  }t!        t        j"                  |            dx}x}}	ddd       y# 1 sw Y   yxY w)u>   fresh heartbeat + PID dead → entry liveness=bot_suspect_deadrq   rr   Trs   ztask-7777.heartbeatru   z	task-7777rA   z	dev3-teamz
sched-7777zbot crashedr   rz   r|   r(   r}   r,   r-   Nr   r   rI   r%   r   r   r   r_   )r	   r
   r   r   r   r   r   r   r/   r   r0   r1   r5   r2   r3   r4   r6   r7   )r   r   rq   r   r(   r   r:   r<   r=   rg   r   r   s               r   5test_get_running_tasks_by_team_marks_bot_suspect_deadr      sv   		$	$	& H#I	X%	,	%%d%3I&	,	!6	6==?'*#/#0		
 	((,113${f$$$${f$$${$$$$$$f$$$f$$$$$$$k"1%j1G5GG15GGGGG15GGGG1GGG5GGGGGGGG)H H Hs   F-GGc                  	   t        j                         5 } t        t        |             }d}t	        ||      }|sddt        j                         v st        j                  t              rt        j                  t              nddt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      dz  }t        t        j                  |            dx}}|j                  }t        |t              }|sddt        j                         v st        j                  t              rt        j                  t              nddt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      d	t        j                         v st        j                  t              rt        j                  t              nd	t        j                  |      d
z  }t        t        j                  |            dx}}d}t	        ||      }|sddt        j                         v st        j                  t              rt        j                  t              nddt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      t        j                  |      dz  }t        t        j                  |            dx}}|j                  }t        |       }d}	||	z  }
d}|
|z  }||k(  }|s_t        j                   d|fd||f      dt        j                         v st        j                  |      rt        j                  |      ndt        j                  |      dt        j                         v st        j                  t              rt        j                  t              nddt        j                         v st        j                  |       rt        j                  |       ndt        j                  |      t        j                  |	      t        j                  |      dz  }dd|iz  }t        t        j                  |            dx}x}x}x}	x}
x}}ddd       y# 1 sw Y   yxY w)u5   DataLoader 인스턴스에 pid_provider 속성 존재r   z5assert %(py5)s
{%(py5)s = %(py0)s(%(py1)s, %(py3)s)
}hasattrr   )r*   r~   r+   r-   NzWassert %(py6)s
{%(py6)s = %(py0)s(%(py3)s
{%(py3)s = %(py1)s.pid_provider
}, %(py4)s)
}
isinstancer   )r*   r~   r+   r^   r_   heartbeat_dirrq   rr   r%   )zl%(py2)s
{%(py2)s = %(py0)s.heartbeat_dir
} == ((%(py7)s
{%(py7)s = %(py4)s(%(py5)s)
} / %(py9)s) / %(py12)s)r   r   )r*   r]   r^   r-   r   r`   py12zassert %(py15)spy15)r	   r
   r   r   r   r2   r3   r0   r4   r5   r6   r7   r   r   r   r   r1   )r   r   r:   @py_assert4r=   rh   r   r;   r   ri   @py_assert10@py_assert11@py_assert13rg   @py_format14@py_format16s                   r   )test_data_loader_initializes_pid_providerr      s   		$	$	& K#DI&-.wv~........w...w......v...v...~.......... --Dz-/CDDDDDDDDzDDDzDDDDDD&DDD&DDD-DDDDDD/CDDD/CDDDDDDDDDD./wv////////w///w//////v///v/////////////##JtCyJ8Jy8';JlJ';l'JJ#'JJJJJ#'JJJJJJJvJJJvJJJ#JJJJJJtJJJtJJJJJJCJJJCJJJyJJJ8JJJlJJJJJJJJK K Ks   SS##S,)'__doc__builtinsr2   _pytest.assertion.rewrite	assertionrewriter0   r   sysr	   r   pathlibr   pytestpathinsertdashboard.data_loaderr   r   r   r   fixturer   strr   r   r>   rB   rF   rJ   rM   rS   rW   rm   ro   r   r   r   r    r   r   <module>r      s     	 
     (HHOOA,-   
   6Mr5Q   (f%*2B[8H2Kr   