
    `gjN                    f   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ej.                  e      Ze
j4                  Zd	d	d
ddddddddddZded<   dZdZdZ ddZ!ddZ"ddZ#ddZ$ddZ%d dZ&ddZ' G d d      Z(g dZ)y)!a\  Supervisor for the dashboard compute-host child process.

The dashboard process owns sockets and JSON-RPC dispatch.  When
``dashboard.turn_isolation`` is enabled, agent turns move behind one persistent
``python -m tui_gateway.compute_host`` child so compute-heavy agent threads do
not contend with the serving process' event loop for the same GIL.
    )annotationsN)Callable)Path)Any)get_hermes_home)hermes_subprocess_envz	turn-pathzrun-concurrentz
idle-gated)zprompt.submitzsession.interrupt
reload.mcpzsession.compresszprompt.submit.truncatezslash.modelzslash.personalityzslash.promptzslash.compresszsession.resetzsession.history.reloadzslash.retryzdict[str, str]MUTATOR_ROUTE_TABLEzdashboard-compute-host.jsong     r@      $@c                   t        |       }|j                  j                  dd       |j                  d      r|n| d}|j	                  dd      }t        j                  t        |      t
        j                  t
        j                  z  t
        j                  z  d      }	 t        j                  ||       t        j                  |       y# t        j                  |       w xY w)	zCAppend one log record using O_APPEND and exactly one os.write call.Tparentsexist_ok
utf-8replaceerrorsi  N)r   parentmkdirendswithencodeosopenstrO_WRONLYO_CREATO_APPENDwriteclose)pathrecordptextdatafds         N/root/.hermes/venv/lib/python3.12/site-packages/tui_gateway/host_supervisor.pyappend_log_recordr(   3   s    T
AHHNN4$N/__T*66("D;;wy;1D	Qrzz1BKK?	GB
T
s   %C C(c                 V    t        t              j                         j                  d   S )N   )r   __file__resolver        r'   
_repo_rootr/   @   s     >!!#++A..r.   c                     	 t        j                  g dt        t                     dt         j                  d      j                         S # t        $ r Y yw xY w)N)gitz	rev-parseHEADT   )cwdr$   stderrtimeoutunknown)
subprocesscheck_outputr   r/   DEVNULLstrip	Exceptionr-   r.   r'   
_build_shar=   D   sS    	&&(JL!%%
 %'	  s   AA 	AAc                 *    t               dz  t        z  S )Nstate)r   _REGISTRY_NAMEr-   r.   r'   _default_registry_pathrA   Q   s    w&77r.   c                    | dk  ry	 t        j                  | d       y# t        $ r Y yt        $ r Y yt        $ r Y yw xY w)Nr   FT)r   killProcessLookupErrorPermissionErrorr<   )pids    r'   
_pid_aliverG   U   sI    
ax
Q   s    	????c                t   | dk  ryt        d      t        |       z  dz  }	 |j                         }|r#|j                  dd      j	                  dd	      S 	 	 t        j                  d
dt        |       ddgdt        j                  d      j                         S # t
        $ r Y Ow xY w# t
        $ r Y yw xY w)Nr    z/proccmdline        r   r   r   psz-pz-ozcommand=Tr3   )r$   r5   r6   )
r   r   
read_bytesr   decoder<   r8   r9   r:   r;   )rF   proc_cmdliner%   s      r'   _pid_commandrQ   c   s    
ax=3s8+i7L&&(<<.55gi5PP &&4S44%%	

 %'	    s$   4B AB+ 	B('B(+	B76B7c                     t        |       }d|v S )Ntui_gateway.compute_host)rQ   )rF   cmds     r'   is_compute_host_identityrU   y   s    
s
C%,,r.   c                  x   e Zd ZdZddddddddddd
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d&dZed'd       Zed(d	       Zd)d
Zd*dZ	d*dZ
d+dZdd	 	 	 	 	 d,dZddd-dZddd.dZdddd	 	 	 	 	 	 	 	 	 	 	 d/dZd0dZd*dZd*dZd*dZd1dZd2dZd2dZd1dZd1dZd2dZd3d Zd*d!Zd4d"Zed#d5d$Zd2d%Zy)6HostSupervisorz;Own one persistent compute-host child and relay its frames.N      T)
registry_pathargvr4   envrpc_sinkrespawn_maxheartbeat_secsexpected_build_shaexpected_hermes_home	autostartc       
           |t        |      n	t               | _        |xs t        j                  ddg| _        |t        |      n	t               | _        || _        |xs d | _	        t        dt        |            | _        t        dt        |            | _        ||n	t               | _        |	|	nt!        t#                     | _        t'        j(                         | _        d | _        d | _        d | _        d | _        t'        j4                         | _        i | _        d| _        d| _        g | _        i | _         i | _!        g | _"        d| _#        |
r| jI                          y y )Nz-mrS   c                     y Nr-   )_objs    r'   <lambda>z)HostSupervisor.__init__.<locals>.<lambda>   s    r.   r   r*   F)%r   rA   rZ   sys
executabler[   r/   r4   r\   r]   maxintr^   r_   r=   r`   r   r   ra   	threadingRLock_lock_proc_stdout_thread_stderr_thread_wait_threadEvent_hello_event_hello_closing_stopped_respawning_restart_times_pending_turns_pending_controls_stderr_tail_last_progress_counterstart)selfrZ   r[   r4   r\   r]   r^   r_   r`   ra   rb   s              r'   __init__zHostSupervisor.__init__   s=    5B4MT-0SiSkNS^^T3MN	 #49Z\ 7%6q#k"23!!S%898J8V"4\f\h<P<\$8befufwbx!__&
37
7;7;59%OO-&(#( +-TV?A')&'#JJL r.   c                T    | j                   }|t        |j                  xs d      S dS )Nr   )ro   rk   rF   r~   procs     r'   rF   zHostSupervisor.pid   s(    zz%)%5s488=q!<1<r.   c                ,    t        | j                        S re   )dictru   r~   s    r'   hellozHostSupervisor.hello   s    DKK  r.   c                h    | j                   }|d uxr! |j                         d u xr | j                   S re   )ro   pollrw   r   s     r'   
is_runningzHostSupervisor.is_running   s4    zz4XDIIK4$7X@X@X<XXr.   c                    | j                   5  | j                         r
	 d d d        y d| _        | j                          | j	                  d       d d d        y # 1 sw Y   y xY w)NFstartupreason)rn   r   rv   reconcile_startup_orphan_spawn_lockedr   s    r'   r}   zHostSupervisor.start   s[    ZZ 	1 	1 	1 "DM))+i0	1 	1 	1s   A)AA#c                   | j                   5  d| _        | j                  }d d d        y 	 |j                         U|j                  I| j                  ddt        j                         j                   d       |j                  t               | j                          y # 1 sw Y   xY w# t        $ r | j                  |       Y 9w xY w# | j                          w xY w)NTshutdownz	shutdown-)type
request_idr6   )rn   rv   ro   r   stdin_send_frameuuiduuid4hexwait_SHUTDOWN_TIMEOUT_SECSr<   _terminate_process_remove_registryr   s     r'   r   zHostSupervisor.shutdown   s    ZZ 	 DM::D	 <	$yy{"tzz'=  *iPTPZPZP\P`P`OaDb!cd		"8	9 !!#	 	  	*##D)	* !!#s/   B#A%B/ #B,/C	C CC C!c                   	 t        j                  | j                  j                  d            }	 t        |j                  d      xs d      }|dk  st        |      s| j                          y| j                  |      s| j                          y| j                  |t        	       | j                          y
# t        $ r Y yt
        $ r | j                          Y yw xY w# t
        $ r d}Y w xY w)z>Terminate a stale registered host, guarding against PID reuse.r   encodingnonezinvalid-registryhost_pidr   znot-runningzpid-reuse-ignoredr   
terminated)jsonloadsrZ   	read_textFileNotFoundErrorr<   r   rk   getrG   _pid_matches_compute_host_terminate_pidr   )r~   r%   rF   s      r'   r   z'HostSupervisor.reconcile_startup_orphan   s    	&::d00::G:LMD	dhhz*/a0C !8:c?!!# --c2!!#&C)?@) ! 	 	&!!#%	&  	C	s(   /B; C$ ;	C!C! C!$C21C2)on_completec               ^   | j                          t        |j                  d      xs t        j                         j
                        }t        |j                  d      xs d      }t        |      }d|d<   ||d<   | j                  5  ||f| j                  |<   d d d        	 | j                  |       |S # 1 sw Y   xY w# t        $ r_}| j                  5  | j                  j                  |d        d d d        n# 1 sw Y   nxY wd||dt        |      d}| ||        d }~ww xY w)	Nr   sidrI   z
turn.startr   
turn.errorsend_failedr   r   r   r   message)r}   r   r   r   r   r   r   rn   ry   r   r<   pop)r~   framer   r   r   payloadexcerrs           r'   submit_turnzHostSupervisor.submit_turn   s'    	

<0DDJJL4D4DE
%))E"(b)u+& *ZZ 	A/2K.@D
+	A	W% #	A 	A  	 :##''
D9: : : %('s8C &C 	s<   
B8%C 8C	D,D'C?6	D'?D	#D''D,)r   c                   | j                          | j                  d||xs t        j                         j                  d       y )N	interruptr   r   r   )r}   r   r   r   r   r~   r   r   s      r'   r   zHostSupervisor.interrupt  s5    

+cIgW[WaWaWcWgWghir.   c               t    | j                  |dd||xs t        j                         j                  dd      S )Nr	   
reload_mcpr   T)
route_namer   r   )controlr   r   r   r   s      r'   r   zHostSupervisor.reload_mcp  s@    ||#)#ZEcSWS]S]S_ScScd	  
 	
r.   g      >@)r   r   r6   c                  |t         vrt        d|       | j                          t        |xs i j	                  d      xs t        j                         j                        }t        |xs i       }|j                  dd       ||d<   ||d<   ||d<   d }|r:t        j                  d      }| j                  5  || j                  |<   d d d        | j                  |       |r|d	|d
S 	 |j	                  |      | j                  5  | j                  j                  |d        d d d        S # 1 sw Y   gxY w# 1 sw Y   S xY w# | j                  5  | j                  j                  |d        d d d        w # 1 sw Y   w xY wxY w)Nz!unclassified host mutator route: r   r   r   r   r   r*   )maxsizesent)statusr   r   )r
   
ValueErrorr}   r   r   r   r   r   r   
setdefaultqueueQueuern   rz   r   r   )	r~   r   r   r   r   r6   r   r   qs	            r'   r   zHostSupervisor.control  sk    00@MNN

'-R,,\:Ndjjl>N>NO
W]#+e(l(l&*A&A 756&&z27qy$J??	=555) =&&**:t<= =7 7= = =&&**:t<= = =s<   ;D1.E	 D=1D:=E		FE<3	F<FFc                  | j                   rt        d      | j                  j                          i | _        t        d      }|j                  t        j                         | j                  r|j                  | j                         t        | j                        |d<   |j                  dt        t                            t        t                     |d   j                  t        j                        vr-t        t                     t        j                  z   |d   z   |d<   t!        j"                  | j$                  t        | j&                        |t         j(                  t         j(                  t         j(                  ddd	      }|| _        t-        | j.                  |fdd	      | _        t-        | j2                  |fd
d	      | _        t-        | j6                  |fdd	      | _        | j0                  j;                          | j4                  j;                          | j8                  j;                          | j                  j=                  d      s,| j?                  |       t        d| j@                  dd         | jC                          | jE                          tF        jI                  d|jJ                  |       y )Nz.compute host respawn disabled after crash loopT)inherit_credentials"HERMES_COMPUTE_HOST_HEARTBEAT_SECS
PYTHONPATHr*   )r4   r\   r   stdoutr5   r$   bufsizestart_new_sessionzcompute-host-stdout)targetargsnamedaemonzcompute-host-stderrzcompute-host-waitr   r   z(compute host did not send hello; stderr=z%compute host started pid=%s reason=%s)&rw   RuntimeErrorrt   clearru   r   updater   environr\   r   r_   r   r/   splitpathsepr8   Popenr[   r4   PIPEro   _Thread_drain_stdoutrp   _drain_stderrrq   _wait_for_exitrr   r}   r   r   r{   _validate_hello_persist_registryloggerinforF   )r~   r   r\   r   s       r'   r   zHostSupervisor._spawn_locked4  s!   ##OPP!#=

2::88JJtxx 478K8K4L01|S%67z|C$5$;$;BJJ$GG #JL 1BJJ >\AR RCIIDHH//????"

 
%T-?-?tgTirvw%T-?-?tgTirvw#4+>+>dWSfost!!#!!#!  %%d%3##D)!I$J[J[\^\_J`Iabcc ;TXXvNr.   c                ~   | j                   }|st        d      t        |j                  d      xs d      }|r*|| j                  k7  rt        d| d| j                         t        |j                  d      xs d      }| j
                  dk7  r-|dd| j
                  hvrt        d| d| j
                         y y )	Nzcompute host missing hellohermes_homerI   z#compute host HERMES_HOME mismatch: z != 	build_shar7   zcompute host build mismatch: )ru   r   r   r   ra   r`   )r~   r   got_homegot_shas       r'   r   zHostSupervisor._validate_helloZ  s    ;<<uyy/526D$=$==!DXJdSWSlSlRmnooeii,23""i/GB	SWSjSjCk4k!>witDLcLcKdeff 5l/r.   c                   | j                   j                  j                  dd       | j                   j                  | j                   j                  dz         }| j
                  | j                  j                  d      xs d| j                  j                  d      xs dt        j                         | j                  d}|j                  t        j                  |d      d	
       |j                  | j                          y )NTr   z.tmpboot_idrI   r   )r   r   r   
started_atr[   )	sort_keysr   r   )rZ   r   r   with_suffixsuffixrF   ru   r   timer[   
write_textr   dumpsr   )r~   tmpr   s      r'   r   z HostSupervisor._persist_registrye  s    !!''t'D  ,,T-?-?-F-F-OP{{y17R5;))+II
 	tzz'T:WMD&&'r.   c                    	 | j                   j                          y # t        $ r Y y t        $ r t        j                  dd       Y y w xY w)Nz&failed to remove compute host registryTexc_info)rZ   unlinkr   r<   r   debugr   s    r'   r   zHostSupervisor._remove_registryr  sG    	R%%'  	 	RLLADLQ	Rs    	A
A
	A
c                L   | j                   5  | j                  }||j                         |j                  t	        d      |j                  j                  t        j                  |dd      dz          |j                  j                          d d d        y # 1 sw Y   y xY w)Nzcompute host is not running),:F)
separatorsensure_asciir   )	rn   ro   r   r   r   r   r   r   flush)r~   r   r   s      r'   r   zHostSupervisor._send_framez  s    ZZ 	::D|tyy{6$**:L"#@AAJJTZZ*SXY\``aJJ	 	 	s   BBB#c                   |j                   J |j                   D ]:  }	 t        j                  |      }t        |t              s*| j                  |       < y # t        j                  $ r t        j                  d|d d        Y lw xY w)Nz%compute host emitted invalid json: %r   )	r   r   r   JSONDecodeErrorr   warning
isinstancer   _handle_host_frame)r~   r   rawr   s       r'   r   zHostSupervisor._drain_stdout  s    {{&&&;; 	/C

3 %&''.	/ '' FDS	Rs   A,BBc                    |j                   J |j                   D ]D  }|j                  d      }|s| j                  |gz   dd  | _        t        j	                  d|       F y )Nr   izcompute host stderr: %s)r5   rstripr{   r   r  )r~   r   r  r$   s       r'   r   zHostSupervisor._drain_stderr  sc    {{&&&;; 	@C::d#D%)%6%6$%?$F!8$?		@r.   c                (   t        |j                  d      xs d      }|dk(  r+t        |      | _        | j                  j                          y |dk(  rDt        |j                  d      xs | j                        | _        t        j                  d|       y |dk(  r3|j                  d      }t        |t              r| j                  |       y |d	v r| j                  |       y |d
v rdt        |j                  d      xs d      }| j                  5  | j                  j                  |      }d d d        	 |j                  |       y y |dk(  rv|j                  d      rdt        |j                  d      xs d      }| j                  5  | j                  j                  |      }d d d        	 |j                  |       y y y y # 1 sw Y   xY w# t         j"                  $ r Y y w xY w# 1 sw Y   FxY w# t         j"                  $ r Y y w xY w)Nr   rI   r   hbprogress_counterzcompute host heartbeat: %srpcr   >   turn.endr   >   control.ackshutdown.ackcontrol.errorinterrupt.ackreload_mcp.ackr   error)r   r   r   ru   rt   setrk   r|   r   r   r  r]   _complete_turnrn   rz   
put_nowaitr   Full)r~   r   ftyper   r   r   s         r'   r  z!HostSupervisor._handle_host_frame  s   EIIf%+,Gu+DK!!#D=*-eii8J.K.jtOjOj*kD'LL5u=E>ii	*G'4(g&..&ggUYY|4:;J ;**..z:;}LL' G		, 7UYY|4:;J ;**..z:;}LL' 	 !8; ;
 zz 
; ;
 zz s<   G
;G G/5G; 
GG,+G,/G8;HHc                *   t        |j                  d      xs d      }| j                  5  | j                  j	                  |d       }d d d        y |\  }}|
	  ||       y y # 1 sw Y   xY w# t
        $ r t        j                  d       Y y w xY w)Nr   rI   z,compute host turn completion callback failed)r   r   rn   ry   r   r<   r   	exception)r~   r   r   pending_sidcbs         r'   r  zHostSupervisor._complete_turn  s    <06B7
ZZ 	@))--j$?G	@?b>Q5	 	@ 	@  Q  !OPQs   A%A1 %A.1BBc                *   |j                         }| j                  ry | j                  5  | j                  |ur
	 d d d        y d | _        d d d        | j	                          | j                  dd|        | j                          y # 1 sw Y   @xY w)Ncrashzcompute host exited with code )r   r   )r   rv   rn   ro   r   _fail_pending_turns_maybe_respawn_after_crash)r~   r   codes      r'   r   zHostSupervisor._wait_for_exit  s    yy{==ZZ 	zz%	 	 DJ	 	  ;YZ^Y_9` a'')	 	s   B	B		Bc          
     J   | j                   5  | j                  }i | _        d d d        j                         D ]8  \  }\  }}d||||d}| j                  ddd|||ddd       |0	  ||       : y # 1 sw Y   UxY w# t        $ r t
        j                  d	       Y hw xY w)
Nr   r   z2.0eventr  )r   r   )r   
session_idr   )jsonrpcmethodparamsz"compute host error callback failed)rn   ry   itemsr]   r<   r   r  )r~   r   r   r  r   r   r  r   s           r'   r  z"HostSupervisor._fail_pending_turns  s    ZZ 	%))G"$D	% &-]]_ 	K!J	b$( "E MM$% '&)/6&#I
 ~KuI+	K	% 	%2 ! K$$%IJKs   A5*B5A>B"!B"c                    t        j                         } j                  D cg c]  }||z
  t        k  s| c} _        t	         j                         j
                  k\  r(d _        t        j                  d j
                         y  j                  j                  |       t        dddt        dt	         j                        dz
        z  z        d fd}t        |d	d
      j                          y c c}w )NTzIcompute host crash loop: max %s restarts per 5min reached; not respawningg      @g      ?r3   r   r*   c                 <   t        j                          j                  5  j                  sj                  sj
                  
	 d d d        y 	 j                  d       d d d        y # t        $ r t        j                  d       Y )w xY w# 1 sw Y   y xY w)Nr  r   zcompute host respawn failed)
r   sleeprn   rv   rw   ro   r   r<   r   r  )delayr~   s   r'   _respawnz;HostSupervisor._maybe_respawn_after_crash.<locals>._respawn  s    JJu D==D$<$<

@VD DD&&g&6	D D
 ! D$$%BCDD Ds)   &BA..BBBBBzcompute-host-respawn)r   r   r   returnNone)r   	monotonicrx   _RESPAWN_WINDOW_SECSlenr^   rw   r   r  appendminrj   r   r}   )r~   nowtr-  r,  s   `   @r'   r   z)HostSupervisor._maybe_respawn_after_crash  s    nn*.*=*=aQqL`A`qat""#t'7'77'+D$LLdfjfvfvw""3'Cc!S1D1D-E-I&J!JKL	D 	x&<TJPPR' bs
   C>C>c                    t        |      S re   )rU   )r~   rF   s     r'   r   z(HostSupervisor._pid_matches_compute_host  s    ',,r.   r   c               *   	 t        j                  |t        j                         t        j                         |z   }t        j                         |k  r9t        |      sy t        j                  d       t        j                         |k  r9	 t        j                  |t        j                         y # t        $ r Y y t
        $ r t        j                  d|d       Y y w xY w# t        $ r Y y t
        $ r t        j                  d|d       Y y w xY w)Nz%failed to SIGTERM compute host pid=%sTr   g?z%failed to SIGKILL compute host pid=%s)r   rC   signalSIGTERMrD   r<   r   r   r   r1  rG   r+  SIGKILL)r~   rF   r6   deadlines       r'   r   zHostSupervisor._terminate_pid  s    	GGC( >>#g-nn)c?JJt nn)	VGGC( " 	 	LL@#PTLU	 " 	 	VLL@#PTLU	Vs/   $B3 $C$ 3	C!> C! C!$	D/ DDc                   |j                         y 	 |j                          |j                  t               y # t        $ r Y nw xY w	 |j                          n# t        $ r Y nw xY w	 |j                  d       y # t        $ r Y y w xY w)Nr   r3   )r   	terminater   r   r<   rC   r   s     r'   r   z!HostSupervisor._terminate_process  s    99;"	NNII4I5 			IIK 			IIaI  		s3   &: 	AA
A 	A'&A'+A> >	B
	B
)rZ   str | Path | Noner[   zlist[str] | Noner4   r@  r\   zdict[str, str] | Noner]   Callable[[dict], None] | Noner^   rk   r_   rk   r`   
str | Nonera   rB  rb   boolr/  r0  )r/  rk   )r/  dict[str, Any])r/  rC  r.  r/  r   )r   rD  r   rA  r/  r   )r   r   r   rB  r/  r0  )r   r   r   rB  r/  r   )r   r   r   r   r   zdict[str, Any] | Noner   rC  r6   floatr/  r   )r   r   r/  r0  )r   rD  r/  r0  )r   zsubprocess.Popen[str]r/  r0  )r   r   r   r   r/  r0  rF   rk   r/  rC  )rF   rk   r6   rF  r/  r0  ) __name__
__module____qualname____doc__r   propertyrF   r   r   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r   r  r   r   r   r   r   r-   r.   r'   rW   rW   ~   s   E
 ,0!%!%%)26 )-+/( )( 	(
 ( #( 0( ( ( '( )( ( 
(T = = ! !Y1$< 6:	 3	
 
> ?C j @D 
 *.== 	=
 '= = = 
=@$OL	g(R	/@$LQ
*K:S.- <R V(r.   rW   )r
   rW   r(   rU   )r!   z
str | Pathr"   r   r/  r0  )r/  r   rE  rG  )rF   rk   r/  r   )*rK  
__future__r   r   loggingr   r   r:  r8   rh   rl   r   r   collections.abcr   pathlibr   typingr   hermes_constantsr   tools.environments.localr   	getLoggerrH  r   Threadr   r
   __annotations__r@   r2  r   r(   r/   r=   rA   rG   rQ   rU   rW   __all__r-   r.   r'   <module>rX     s    #   	    
    $   , :			8	$


 !$"$*% "!*' ^  /  
/
8,-
q qhr.   