
    *"hjZ9                    ~   d Z ddlmZ ddlZddlZddlZddlZddlmZm	Z	 ddl
mZ  ej                  e      ZdZ ej                   dej"                        Zd	Zd
dddd	Zd!dZd"dZd#dZd$d%dZd&dZd'dZd'dZd'dZd'dZd(dZdd	 	 	 	 	 d)dZdd	 	 	 	 	 d)dZ d*dZ!d+dZ"	 	 	 	 	 	 d,dZ#	 	 	 	 	 	 d-d Z$y).a"  
Session-level LLM judge for classic evolve-server sessions.

This stage runs after summarization so it can reuse the generated
``_trajectory`` and ``_summary`` fields. It only backfills sessions that
do not already have a reliable session-level score from benchmark /
aggregate pipelines.
    )annotationsN)AnyOptional   )AsyncLLMClienta  You are a session-level evaluator for SkillClaw trajectories.

You will receive one session with:
- a lossless trajectory
- an LLM-generated analysis summary
- extracted source artifacts that the agent read
- lightweight metadata such as prior PRM scores and tool-error flags
- extracted final output artifacts when the agent wrote files

Score the session on a 0.0-1.0 scale for:
- task_completion: whether the user's goal was completed
- response_quality: correctness, completeness, and clarity of the final outcome
- efficiency: whether the path avoided unnecessary retries / detours
- tool_usage: whether tool usage was appropriate and effective

Use this weighting for the overall score:
- task_completion: 0.55
- response_quality: 0.30
- efficiency: 0.05
- tool_usage: 0.10

Guidelines:
- 1.0 means clearly excellent on that dimension.
- 0.5 means mixed / uncertain / partially successful.
- 0.0 means clearly failed on that dimension.
- Prefer the trajectory as ground truth; use the summary as supporting analysis.
- Distinguish "missing evidence" from "clear failure". If evidence is weak, be conservative rather than extreme.
- Do not assume benchmark labels exist.
- Prioritize factual correctness and goal completion over polish.
- Do not heavily penalize framework/runtime startup noise (for example benign prologue reads,
  environment initialization, or short non-blocking detours) unless it materially interferes
  with solving the task.
- Use low efficiency scores only for severe wasted effort: repeated failed retries, long
  thrashing loops, or large amounts of irrelevant work.
- Judge tool_usage mainly by whether the core tools chosen were appropriate for reaching a
  correct result; do not over-penalize incidental startup/tooling noise.
- If the session includes concrete output artifacts (for example file contents written by the
  agent), treat those artifacts as strong evidence for task_completion and response_quality.
- If the session includes concrete source artifacts that the agent read from the task workspace,
  use those source artifacts as the primary factual basis for judging whether the final outputs
  are accurate.
- When written outputs match the requested schema/format and are consistent with the available
  evidence, score completion/quality based primarily on correctness of those outputs even if
  earlier exploration was noisy.
- Only lower completion/quality sharply when the final outputs are missing, malformed, clearly
  contradicted by evidence, or unsupported by the available facts.

Return EXACTLY one JSON object with:
{
  "task_completion": <float 0..1>,
  "response_quality": <float 0..1>,
  "efficiency": <float 0..1>,
  "tool_usage": <float 0..1>,
  "overall_score": <float 0..1>,
  "rationale": "<brief explanation>"
}

No markdown fences. No extra text.
z\{.*\})task_completionresponse_quality
efficiency
tool_usageg?g333333?g?皙?c                T    t        | t        t        f      xr t        | t               S )N)
isinstanceintfloatbool)values    ?/root/.hermes/SkillClaw/evolve_server/pipeline/session_judge.py
_is_numberr   b   s"    ec5\*J:eT3J/JJ    c                   t        j                  ddt        | xs d            j                         j	                  d      }|sy 	 t        j                  |      }t        |t              r|S 	 t        j                  |      }|sy 	 t        j                  |j                  d            }t        |t              r|S d S # t
        j                  t        f$ r Y lw xY w# t
        j                  t        f$ r Y y w xY w)Nz```(?:json)?\s* `r   )resubstrstriprstripjsonloadsr   dictJSONDecodeError
ValueError_JSON_BLOCK_REsearchgroup)textcleanobjmatchs       r   _extract_json_objectr*   f   s    FF%r3tzr?;AACJJ3OEjjc4 J !
 !!%(EjjQ( S$'31T1   *-    *- s$   &B? $C ?CCC:9C:c           	     p    t        |       sy t        dt        dt        |                   }t	        |d      S )N              ?   )r   maxminr   round)r   scores     r   _normalize_scorer3   {   s0    eSeEl+,E?r   c                l    t        | xs d      j                         }t        |      |k  r|S |d | dz   S )Nr   z...)r   r   len)r   	max_charsr&   s      r   
_clip_textr7      s<    u{!!#D
4yI
e##r   c                X    d}t         D ]  }|| |   t        |   z  z  } t        |d      S )Nr,   r.   )_DIMENSION_KEYS_WEIGHTSr1   )scorestotalkeys      r   _compute_weighted_overallr>      s9    E -x},,-?r   c                |    | j                  d      }t        |t              rt        |j                  d            ryy)N	benchmarkoverall_scoreTFgetr   r    r   )sessionr@   s     r   _has_benchmark_overall_scorerE      s0    K(I)T"z)--2P'Qr   c                |    | j                  d      }t        |t              rt        |j                  d            ryy)N	aggregate
mean_scoreTFrB   )rD   rG   s     r   _has_aggregate_mean_scorerI      s0    K(I)T"z)--2M'Nr   c                   | j                  d      }t        |t              r|syt        |d   t              r|d   ni }|j                  d      }t	        |      sydt        |      cxk  rdk  sy yg }|d d D ]8  }t        |t              s|j                  d      }|(|j                  |       : t        | j                  d      xs$ | j                  d      xs | j                  d	            }|xr | S )
NturnsF	prm_scorer,   r-   task_idrG   phase)rC   r   listr    r   r   appendr   )rD   rK   	last_turn
last_scoreearlier_scoresturnprmhas_benchmarkish_contexts           r   -_looks_like_existing_session_level_turn_scorerX      s    KK EeT"%'b	48b	bI{+Jj!5$++ ,Ncr
 '$%hh{#?!!#&'  $GKK	$:$ngkk+>V$nZaZeZefmZno#:N(::r   c                   | j                  d      }t        |t              r|sy| j                  d      }t        |t              rt	        |j                  d            ryt        |       ryt        |       ryt        |       ryy)NrK   T_judge_scoresrA   F)rC   r   rP   r    r   rE   rI   rX   )rD   rK   existing_judges      r   _should_skip_judgingr\      sp    KK EeT"%[[1N.$'J~7I7I/7Z,[#G, )4W=r   c                   | j                  d      | j                  d      t        | j                  d      xs g       t        | j                  d            t        | j                  d      xs g       | j                  d      t	        |       t        |       | j                  d      xs d| j                  d	      xs dd

S )N
session_id	num_turns_skills_referenced_has_tool_errors_prm_scores_avg_prm_trajectoryr   _summary)
r^   r_   skills_referencedhas_tool_errorsprior_prm_scoresavg_prm_before_judgesource_artifactsoutput_artifacts
trajectorysummary)rC   sortedr   rP   _extract_source_artifacts_extract_output_artifacts)rD   s    r   _build_judge_payloadrq      s    kk,/[[-#GKK0D$E$KL,> ?@ ]!;!ArB 'J 75g>5g>kk-06B;;z*0b r      )max_artifactsc                  g }| j                  d      xs g D ]>  }t        |t              s|j                  d      xs g D ]  }t        |t              s|j                  d      }t        |t              s7t        |j                  d      xs d      j	                         dk7  rg|j                  d      }t        |t              s	 t        j                  |      }t        |j                  d      xs d      j	                         }|j                  d	      }	|r|	|j                  |t        |	      d
       t        |      |k\  s|c c S  A |S # t
        j                  $ r Y 0w xY w)NrK   
tool_callsfunctionnamer   write	argumentspathcontentrz   r{   )rC   r   r    r   r   r   r   r!   rQ   r7   r5   )
rD   rs   	artifactsrU   	tool_callrv   raw_argsargsrz   r{   s
             r   rp   rp      s\   
 ')IG$* !$%,/52 	!Ii. }}Z0Hh-8<<'-2.446'A||K0Hh,zz(+ txx'-2.446Dhhy)G7? )'2 9~.  5	!!<  '' s   EE32E3   c                  g }t               }| j                  d      xs g D ]  }t        |t              si }|j                  d      xs g D ]  }t        |t              s|j                  d      }t        |t              s6t	        |j                  d      xs d      j                         dk7  rf|j                  d      }t        |t              s	 t        j                  |      }	t	        |j                  d      xs d      j                  d	d      }
|
s|	||
<    |j                  d
      xs g D ]k  }t        |t              st	        |j                  d      xs d      j                         dk7  rEt        |j                  d            r`t	        |j                  d      xs d      j                  d	d      }|j                  |      }t        |t              st	        |j                  d      xs d      j                         }|r||v r|j                  d      rt	        |j                  d      xs d      j                         }|r|dk(  r*|j                  |t        |      d       |j                  |       t        |      |k\  sh|c c S   |S # t        j                  $ r Y uw xY w)NrK   ru   rv   rw   r   readry   id_tool_results	tool_name	has_errortool_call_idrz   z/root/r{   z(see attached image)r|   )setrC   r   r    r   r   r   r   r!   replacer   
startswithrQ   r7   addr5   )rD   rs   r}   
seen_pathsrU   call_args_by_idr~   rv   r   parsed_argscall_idtool_resultresult_call_idr   rz   r{   s                   r   ro   ro     s   
 ')I5JG$* 3!$%57,/52 	7Ii. }}Z0Hh-8<<'-2.446&@||K0Hh,"jj2 )---34<<S"EG+6(#	7&  88N39r 	!Kk40;??;/526<<>&HKOOK01 !@!FBGOOPSUWXN"&&~6DdD)txx'-2.446D4:-x(+//)4:;AACGg)?? )'2 NN4 9~.  7	!13!h E '' s   J--KKc                4   | j                  d      xs g }t        | j                  d      xs g       }d }|r2t        |d   t              r|d   j                  d      }|d   |d   d<   t        |      }|r||d<   |||d<   || d<   |d   g| d<   |d   | d	<   y )
NrK   rb   rL   rM   rA   original_prm_scoresprevious_last_prm_scorerZ   rc   )rC   rP   r   r    )rD   r;   rK   previous_prm_scoresprevious_last_prmjudge_scoress         r   _apply_judge_scoresr   ?  s    KK &BEw{{=9?R@E"It,!"IMM+6!'!8b	+<L.A*+$2C./+GO$_56GM 1GJr   c                @   t        |       }|sy i }t        D ]%  }t        |j                  |            }| y |||<   ' t	        |      }i ||t        |j                  d      xs d      j                         d}t        |j                  d            }|||d<   |S )N	rationaler   )rA   r   rA   model_overall_score)r*   r9   r3   rC   r>   r   r   )rawpayloadr;   r=   
normalizedoverallresultraw_overalls           r   _parse_scoresr   R  s    "3'G!F !%gkk#&67
 s	! (/G
 [17R8>>@F
 #7;;#?@K(3$%Mr   c                  K   t        |      ryt        |      }dt        ddt        j                  |d      dg}	 | j                  |dd	       d{   }t        |      }|s&t        j                  d|j                  d             yt        ||       |S 7 E# t        $ r0}t        j                  d
|j                  d      |       Y d}~yd}~ww xY ww)z<Judge one session and backfill session-level score metadata.Nsystem)roler{   userF)ensure_ascii  r   )
max_tokenstemperaturez1[SessionJudge] LLM call failed for session %s: %sr^   z:[SessionJudge] could not parse judge output for session %s)r\   rq   _JUDGE_SYSTEMr   dumpschat	ExceptionloggerwarningrC   r   r   )llmrD   r   messagesr   excr;   s          r   judge_sessionr   j  s     
 G$"7+Gm4DJJwU$KLHHHX$CHHH 3FHKK%	
 (M% I ?KK%	

 sA   ;CB BB ACB 	C&&CCCCc           	       K   |sy|D cg c]  }t        |      r| }}|syt        j                  |D cg c]  }t        | |       c}ddi d{   }d}t	        ||      D ]D  \  }}t        |t              r't        j                  d|j                  d      |       =|@|dz  }F t        j                  d|t        |      t        |             |S c c}w c c}w 7 w)	z<Judge all sessions that lack a reliable session-level score.r   return_exceptionsTNz5[SessionJudge] exception while judging session %s: %sr^      z9[SessionJudge] judged %d/%d candidate sessions (%d total))r\   asynciogatherr   zipr   BaseExceptionr   r   rC   infor5   )r   sessionsrD   
candidatesresultsjudgedr   s          r   judge_sessions_parallelr     s     
 )1Wg9Mg9V'WJWNN5?	@'-W
%	@ G
 Fz73 	fm,NNGL)
 aKF	 KKCJH	 M7 X
 
As2   C&CCC&C

C&C$AC&'>C&)r   r   returnr   )r&   r   r   Optional[dict[str, Any]])r   r   r   zOptional[float])r   )r   r   r6   r   r   r   )r;   zdict[str, float]r   r   )rD   dict[str, Any]r   r   )rD   r   r   r   )rD   r   rs   r   r   zlist[dict[str, str]])rD   r   r;   r   r   None)r   r   r   r   )r   r   rD   r   r   r   )r   r   r   zlist[dict[str, Any]]r   r   )%__doc__
__future__r   r   r   loggingr   typingr   r   core.llm_clientr   	getLogger__name__r   r   compileDOTALLr#   r9   r:   r   r*   r3   r7   r>   rE   rI   rX   r\   rq   rp   ro   r   r   r   r    r   r   <module>r      s9   #    	   ,			8	$;z Iryy1 	K2*$;6$$ $$ $ 	$T ;; ; 	;|2&0 	    F#	#"# 	#r   