
    `gj              /       >   d Z 	 ddlZddlZddlZddlZddlZddlmZ ddl	m
Z
mZmZmZmZ ddlmZ ddlmZmZ ddlZddlmZmZmZmZmZmZ ddlmZ  ej:                  e      Zddl Z dd	l!m"Z" dd
l#m$Z$m%Z%m&Z& ddl'm(Z( i Z) e* e(jV                               Z,ddddZ-dee.ee.e/f   f   dee.ee.e/f   f   fdZ0dee.e/f   dee.e/f   fdZ1de
ee.ef      dee.ee.e/f   f   fdZ2de
ee.ef      dee.e/f   fdZ3de/dee.ef   de/dee.ef   dee.ef   f
dZ4dedee.ef   fdZ5 G d d      Z6	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d7de.de/d e.d!e.d"e.d#e.d$e.d%e/d&e/d'e7d(e7d)e7d*e.d+e/d,e.d-e.d.e.d/e.d0e/d1e.d2e7d3e.d4e/f.d5Z8ed6k(  r e jr                  e8       yy# e$ r Y w xY w)8a  
Batch Agent Runner

This module provides parallel batch processing capabilities for running the agent
across multiple prompts from a dataset. It includes:
- Dataset loading and batching
- Parallel batch processing with multiprocessing
- Checkpointing for fault tolerance and resumption
- Trajectory saving in the proper format (from/value pairs)
- Tool usage statistics aggregation across all batches

Usage:
    python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run
    
    # Resume an interrupted run
    python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run --resume
    
    # Use a specific toolset distribution
    python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run --distribution=image_gen
    N)Path)ListDictAnyOptionalTuple)datetime)PoolLock)ProgressSpinnerColumn	BarColumn
TextColumnTimeRemainingColumnMofNCompleteColumn)Console)AIAgent)list_distributions!sample_toolsets_from_distributionvalidate_distribution)TOOL_TO_TOOLSET_MAPcountsuccessfailure
tool_statsreturnc                     i }t         D ]4  }|| v r| |   j                         ||<   t        j                         ||<   6 | j                         D ]  \  }}||vs|j                         ||<    |S )a  
    Normalize tool_stats to include all possible tools with consistent schema.
    
    This ensures HuggingFace datasets can load the JSONL without schema mismatch errors.
    Tools that weren't used get zero counts.
    
    Args:
        tool_stats (Dict): Raw tool statistics from extraction
        
    Returns:
        Dict: Normalized tool statistics with all tools present
    )ALL_POSSIBLE_TOOLScopyDEFAULT_TOOL_STATSitems)r   
normalizedtoolstatss       ?/root/.hermes/venv/lib/python3.12/site-packages/batch_runner.py_normalize_tool_statsr'   G   s     J # 9:)$/446Jt1668Jt	9 "'') ,ez!$zz|Jt,     tool_error_countsc                     i }t         D ]  }| j                  |d      ||<    | j                         D ]  \  }}||vs|||<    |S )z
    Normalize tool_error_counts to include all possible tools.
    
    Args:
        tool_error_counts (Dict): Raw error counts mapping
        
    Returns:
        Dict: Normalized error counts with all tools present
    r   )r   getr"   )r)   r#   r$   r   s       r&   _normalize_tool_error_countsr,   e   si     J # :,00q9
4: )..0 %ez!$Jt% r(   messagesc                 V   i }i }| D ]F  }|d   dk(  rVd|v rR|d   rM|d   D ]D  }|rt        |t              s|d   d   }|d   }||vr	dddd||<   ||   d	xx   d
z  cc<   |||<   F b|d   dk(  sk|j                  dd      }|j                  dd      }d}	 t        |t              rt	        j
                  |      n|}	t        |	t              rOd|	v r|	d   d}d|	v r+t        |	d   t              r|	d   }
|
j                  d      d}|	j                  d      du rd}||v s||   }|r||   dxx   d
z  cc<   7||   dxx   d
z  cc<   I |S # t        j                  t        t        f$ r7 |sd}n/|j                         j                         j                  d      rd}Y w xY w)z
    Extract tool usage statistics from message history.
    
    Args:
        messages (List[Dict]): Message history
        
    Returns:
        Dict: Tool statistics with counts and success/failure rates
    role	assistant
tool_callsfunctionnameidr   r   r      r$   tool_call_id contentTerrorFr   zerror:r   )
isinstancedictr+   strjsonloadsJSONDecodeError
ValueError	TypeErrorstriplower
startswith)r-   r   tool_calls_mapmsg	tool_call	tool_namer6   r8   
is_successcontent_jsoninner_contents              r&   _extract_tool_statsrL   }   s    J N ?:v;+%,#*=#lBS . 9	 
9d(CX%j1&9	( J.!"#$#$-Jy) 9%g.!3./8|,9" [F"77>26Lggi,G J'6@#6Ntzz'2T[lD1,.<3H3T%*
 !L0ZY@WY]5^(4Y(? ),,W5A).J $''	2e;%*
 ~-*<8	y))494y))494?:B # ((*i@ ' !&J]]_**,77A!&J's   BEAF('F(c                 &   d}d}| D ]|  }|j                  d      dk7  r|dz  }|j                  dd      xs d}d|v }|j                  d      r)t        |j                  dd      j                               nd	}|s|sx|dz  }~ ||||z
  |dkD  d
S )a  
    Count how many assistant turns have reasoning vs no reasoning.
    
    Checks for <REASONING_SCRATCHPAD> in content or a non-empty 'reasoning' field
    (native thinking tokens). Returns counts for tracking reasoning coverage.
    
    Args:
        messages: Message history
        
    Returns:
        Dict with 'total_assistant_turns', 'turns_with_reasoning', 'turns_without_reasoning'
    r   r/   r0   r5   r8   r7   z<REASONING_SCRATCHPAD>	reasoningF)total_assistant_turnsturns_with_reasoningturns_without_reasoninghas_any_reasoning)r+   boolrB   )r-   totalwith_reasoningrF   r8   has_scratchpadhas_native_reasonings          r&   _extract_reasoning_statsrX      s     EN 
 776?k)
'')R(.B1W<ILQ\I]tCGGK$<$B$B$DEch1aN
  "' .#(>#9+a/	 r(   prompt_indexprompt_data	batch_numconfigc                    |d   }d|  }|j                  d      xs |j                  d      }|r't        j                  dd      }|dk(  rdd	l}	 |j	                  ddd
|gdd      }	|	j
                  dk7  r|j                  d      rt        d|  d| dd       |j	                  dd|gddd      }
|
j
                  dk7  r@d| d| d|
j                  d	d  d	i g |t        j                         j                         ddS ddlm} ||d| |d}|j                  d       r|d    |d <    |||       |j                  d      rt        d|  d!|        	 t        |d"         }|j                  d      rt        d|  d#|        d$| d%|  d&}t!        dEi d'|j                  d'      d(|j                  d(      d)|d)   d*|d*   d+|d,dd-|j                  dd      d.|j                  d.      d/|j                  d/d0      d1|d2|j                  d2      d3|j                  d3      d4|j                  d4      d5|j                  d5      d6|j                  d6      d7|j                  d7      d8|j                  d8      d9|j                  d9      d:dd;d}|j#                  ||<      }t%        |d=         }t'        |d=         }|j)                  |d=   ||d>         }d| ||||d>   |j                  d?d      |d@   ||t        j                         j                         |d)   dAdB
S # t        $ r Y <t        $ r/}|j                  d      rt        d|  d| d       Y d	}~nd	}~ww xY w# t        $ rv}t        dC|  dD|        |j                  d      rt+        j,                          d| t/        |      d	i g |t        j                         j                         ddcY d	}~S d	}~ww xY w)Fa  
    Process a single prompt with the agent.
    
    Args:
        prompt_index (int): Index of prompt in dataset
        prompt_data (Dict): Prompt data containing 'prompt' field and optional 'image' field
        batch_num (int): Batch number
        config (Dict): Configuration dict with agent parameters
        
    Returns:
        Dict: Result containing trajectory, stats, and metadata
    prompttask_imagedocker_imageTERMINAL_ENVlocaldockerr   NinspectT
   )capture_outputtimeoutverbosez
   Prompt z: Pulling docker image ...flushpulliX  )rg   textrh   FzDocker image not available: 
i  )r[   	timestamp)r   rY   r9   
trajectoryr   toolsets_usedmetadataz: Docker image check failed: )register_task_env_overridesz	docker://)ra   modal_imagesingularity_imagedaytona_imagecwdz: Using container image distributionz: Using toolsets z[Bz:P]base_urlapi_keymodelmax_iterationsenabled_toolsetssave_trajectoriesverbose_loggingephemeral_system_promptlog_prefix_charsd   
log_prefixproviders_allowedproviders_ignoredproviders_orderprovider_sortopenrouter_min_coding_score
max_tokensreasoning_configprefill_messagesskip_context_filesskip_memory)task_idr-   	completedpartial	api_calls)r[   rp   r}   )
r   rY   rq   r   reasoning_statsr   r   r   rr   rs   u   ❌ Error processing prompt :  )r+   osgetenv
subprocessrun
returncodeprintstderrr	   now	isoformatFileNotFoundError	Exceptiontools.terminal_toolrt   r   r   run_conversationrL   rX   _convert_to_trajectory_format	traceback	print_excr<   )rY   rZ   r[   r\   r^   r   container_imageenv_type_spproberm   img_errrt   	overridesselected_toolsetsr   agentresultr   r   rq   es                         r&   _process_single_promptr      s   $ "Fl^$G "oog.Q+//.2QO 99^W5x$iw	?C#'    ##q(zz),
<.8OP_O``cdlpq77!6?;'+$ # D !+',,8'COCTTVW[WbWbcgdgWhVi%j*.*,-/6?hllnNfNfNh(i   	D+*#,_,=!>,	
	 ??5!*51Ie#GY7::i J|n,D_DUVWQ
=f^>TU::i J|n,=>O=PQR )B|nA6
 
ZZ
+
JJy)
 /
 ""23	

 /
 $
 #JJy%8
 %+JJ/H$I
 $ZZ(:C@
 "
 %jj)<=
 %jj)<=
 #JJ'89
 !**_5
 )/

3P(Q
  zz,/!
" $ZZ(:;#
$ $ZZ(:;%
&  $'
( )
0 '''@ )
);<
 36*3EF 88:;

 ($$.,zz)U3,.&%\\^557
 	
G %  i::i(J|n4QRYQZ[cghid  
,\N"QC@A::i ! (V&%\\^557
 	

s?   B2L5 GM; 5	M8M8	$M33M8;	O:A+O5/O:5O:argsc                 n   | \  }}}}}t        |      }t        d| dt        |       d       |d| dz  }|D cg c]  \  }}||vr||f }	}}|	s t        d| d       |dt        |      i g d	S t        d
t        |	       dt        |      t        |	      z
   d       i }
dddd}g }d}|	D ]%  \  }}t        ||||      }|d   r|d   r|j	                  di       }|j	                  dd      s&t        d| d       |dz  }|j                  |       k|j	                  di       }t        |      }|j                         D ci c]  \  }}||j	                  dd       }}}t        |      }||d   |d   |d   |j	                  dd      |d   |d   ||d	}t        |dd !      5 }|j                  t        j                  |d"      d#z          d$d$d$       |j	                  di       j                         D ]K  \  }}||
vr	dddd%|
|<   |
|   d&xx   |d&   z  cc<   |
|   dxx   |d   z  cc<   |
|   dxx   |d   z  cc<   M |D ]/  }||xx   |j	                  di       j	                  |d      z  cc<   1 |d   r?|d   r:|j                  |       |j	                  d      rd'nd(}t        d)| d*| d+       t        d,| d-       ( t        d| d.t        |	       d/       |t        |	      t        |      t        |	      z
  |
|||d0S c c}}w c c}}w # 1 sw Y   QxY w)1z
    Worker function to process a single batch of prompts.
    
    Args:
        args (Tuple): (batch_num, batch_data, output_dir, completed_prompts, config)
        
    Returns:
        Dict: Batch results with statistics
    u   
🔄 Batch z: Starting (	 prompts)batch_z.jsonlu
   ✅ Batch z: Already completed (skipping)r   )r[   	processedskippedr   completed_promptsz   Processing z prompts (skipping z already completed)rO   rP   rQ   r   rq   r   rR   Tu      🚫 Prompt z% discarded (no reasoning in any turn)r5   r   r   rs   r   r   Fr   rr   )	rY   conversationsrs   r   r   r   rr   r   r)   autf-8encoding)ensure_asciiro   Nr   r   u   ⚠️  partialu   ✅z   z Prompt z
 completedu      ❌ Prompt z failed (will retry on resume)z: Completed (z prompts processed))r[   r   r   r   r   discarded_no_reasoningr   )r   r   lenr   r+   appendr'   r"   r,   openwriter=   dumps)r   r[   
batch_data
output_dircompleted_prompts_setr\   batch_output_fileidxdataprompts_to_processbatch_tool_statsbatch_reasoning_statscompleted_in_batchr   rY   rZ   r   rN   raw_tool_statsr   rH   r%   raw_error_countsr)   trajectory_entryfkeystatuss                               r&   _process_batch_workerr     s6    HLDIz:'<fj!J	M)LZ0A
KL #vi[%?? &0!T++ 
d 
 
9+%CDE":!#
 	
 
N31233Fs:Y\]oYpGpFq  rE  F  G 67QRopq &8 EQ!k'	
 )!5

#4b9I==!4d;~5Z[\&!+&")),7 $ZZb9N.~>J
 )7(<(<(> $Iu 599Y22    !==M N !-!'!5":.#K0!::i7#K0!'!8(%6
  'w? Q1

#3%H4OPQ !'

< < B B D 
	GIu 00  / + Y'0E'NB0Y'	2eI6FF2Y'	2eI6FF2
	G ) 	XC!#&&**5F*K*O*OPSUV*WW&	X )!5%%l3*0**Y*?&UFCxx~Z@AN<.0NOPKEQN 
Jyks3E/F.GGZ
[\ +,z?S);%<<&0"8/ C` &Q Qs   LL$1*L**L4	c            ,          e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d%dedededededed	ed
edededededee   dee   dee   dedee	   dede
eef   dee
eef      def*dZdee
eef      fdZdeeeee
eef   f         fdZde
eef   fdZd&de
eef   dee   fdZdefd Zd!edeee
   ee   f   fd"Zd'd#efd$Zy)(BatchRunnerzV
    Manages batch processing of agent prompts with checkpointing and statistics.
    Ndataset_file
batch_sizerun_namery   r~   r{   r|   r}   num_workersri   r   r   r   r   r   r   r   r   r   r   max_samplesc           	         t        |      | _        || _        || _        || _        || _        || _        || _        || _        |	| _	        |
| _
        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        t-        |      s0t/        d| dt1        t3               j5                                      t        d      |z  | _        | j6                  j9                  dd       | j6                  dz  | _        | j6                  dz  | _        | j?                         | _         | j*                  rq| j*                  tC        | j@                        k  rOtC        | j@                        }| j@                  d| j*                   | _         tE        d	| d
| j*                   d       | jG                         | _$        tE        d       tE        d| j                   dtC        | j@                         d       tE        d| j                          tE        dtC        | jH                                tE        d| j                          tE        d| j                          tE        d| j6                          tE        d| j                          | j                  rFtC        | j                        dkD  r| j                  dd dz   n| j                  }tE        d| d       yy)a  
        Initialize the batch runner.

        Args:
            dataset_file (str): Path to the dataset JSONL file with 'prompt' field
            batch_size (int): Number of prompts per batch
            run_name (str): Name for this run (used for checkpointing and output)
            distribution (str): Toolset distribution to use (default: "default")
            max_iterations (int): Max iterations per agent run
            base_url (str): Base URL for model API
            api_key (str): API key for model
            model (str): Model name to use
            num_workers (int): Number of parallel workers
            verbose (bool): Enable verbose logging
            ephemeral_system_prompt (str): System prompt used during agent execution but NOT saved to trajectories (optional)
            log_prefix_chars (int): Number of characters to show in log previews for tool calls/responses (default: 20)
            providers_allowed (List[str]): OpenRouter providers to allow (optional)
            providers_ignored (List[str]): OpenRouter providers to ignore (optional)
            providers_order (List[str]): OpenRouter providers to try in order (optional)
            provider_sort (str): Sort providers by price/throughput/latency (optional)
            max_tokens (int): Maximum tokens for model responses (optional, uses model default if not set)
            reasoning_config (Dict): OpenRouter reasoning config override (e.g. {"effort": "none"} to disable thinking)
            prefill_messages (List[Dict]): Messages to prepend as prefilled conversation context (few-shot priming).
                NOTE: Anthropic Sonnet 4.6+ and Opus 4.6+ reject a trailing assistant-role prefill
                (400 error).  For those models use output_config.format or structured-output
                schemas instead.  Safe here for user-role priming and for older Claude / non-Claude models.
            max_samples (int): Only process the first N samples from the dataset (optional, processes all if not set)
        zUnknown distribution: z. Available: r   T)parentsexist_okzcheckpoint.jsonzstatistics.jsonNu   ✂️  Truncated dataset from z to z samples (--max_samples)u   📊 Batch Runner Initializedz   Dataset:  (r   z   Batch size: z   Total batches: z   Run name: z   Distribution: z   Output directory: z   Workers: <   rj   u"      🔒 Ephemeral system prompt: '')%r   r   r   r   ry   r~   r{   r|   r}   r   ri   r   r   r   r   r   r   r   r   r   r   r   r   r@   listr   keysr   mkdircheckpoint_file
stats_file_load_datasetdatasetr   r   _create_batchesbatches)selfr   r   r   ry   r~   r{   r|   r}   r   ri   r   r   r   r   r   r   r   r   r   r   r   
full_countprompt_previews                           r&   __init__zBatchRunner.__init__  s   h !.$ (, 
&'>$ 0!2!2.*+F($ 0 0& %\25l^=QUVhVjVoVoVqQrPstuu v,1dT:  $1BB //,== ))+ 0 03t||3D DT\\*J<<(9)9)9:DL3J<tDDTDTCUUmno ++--.T../r#dll2C1DINO012"3t||#4"567dmm_-.!$"3"3!456%doo%678T--./0''JMdNjNjJknpJpT99#2>Fvz  wS  wSN6~6FaHI (r(   r   c           	      $   | j                   j                         st        d| j                          g }t        | j                   dd      5 }t	        |d      D ]S  \  }}|j                         }|s	 t        j                  |      }d|vrt        d| d       C|j                  |       U 	 ddd       |st        d| j                          |S # t        j                  $ r}t        d	| d
|        Y d}~d}~ww xY w# 1 sw Y   VxY w)zy
        Load dataset from JSONL file.
        
        Returns:
            List[Dict]: List of dataset entries
        zDataset file not found: rr   r   r5   r^   u   ⚠️  Warning: Line z! missing 'prompt' field, skippingu&   ⚠️  Warning: Invalid JSON on line r   Nz(No valid entries found in dataset file: )r   existsr   r   	enumeraterB   r=   r>   r   r   r?   r@   )r   r   r   line_numlineentryr   s          r&   r   zBatchRunner._load_dataset  s      '')#&>t?P?P>Q$RSS$##S7; 	q"+Aq/ $zz| JJt,Eu, 6xj@abc NN5)	  GHYHYGZ[\\ ++ B8*BqcRS	 	sB   &D4(CDC.DD(C>9D>DDDc                    g }t        dt        | j                        | j                        D ]N  }t	        | j                  ||| j                  z    |      D cg c]	  \  }}||f }}}|j                  |       P |S c c}}w )z
        Split dataset into batches with indices.
        
        Returns:
            List of batches, where each batch is a list of (index, entry) tuples
        r   )start)ranger   r   r   r   r   )r   r   ir   r   batchs         r&   r   zBatchRunner._create_batches  s     q#dll+T__= 	"A4=dll1QQUQ`Q`M`>aij4kljc5c5\lElNN5!	"  ms   B c                 J   | j                   j                         s| j                  g i ddS 	 t        | j                   dd      5 }t	        j
                  |      cddd       S # 1 sw Y   yxY w# t        $ r)}t        d|        | j                  g i ddcY d}~S d}~ww xY w)z
        Load checkpoint data if it exists.
        
        Returns:
            Dict: Checkpoint data with completed prompt indices
        Nr   r   batch_statslast_updatedr   r   r   u,   ⚠️  Warning: Failed to load checkpoint: )r   r   r   r   r=   loadr   r   )r   r   r   s      r&   _load_checkpointzBatchRunner._load_checkpoint  s     ##**, MM%'! $	 
	d**C'B $ayy|$ $ $ 	@DE MM%'! $	 	s;   A0 A$	A0 $A-)A0 -A0 0	B"9BB"B"checkpoint_datalockc                     t        j                         j                         |d<   ddlm} |r|5   || j
                  |       ddd       y || j
                  |       y# 1 sw Y   yxY w)z
        Save checkpoint data.
        
        Args:
            checkpoint_data (Dict): Checkpoint data to save
            lock (Lock): Optional lock for thread-safe access
        r   r   )atomic_json_writeN)r	   r   r   utilsr  r   )r   r  r  r  s       r&   _save_checkpointzBatchRunner._save_checkpoint  sg     +3,,.*B*B*D'+ I!$"6"6HI I d22ODI Is   A!!A*c                    t               }t        | j                  j                  d            }|s|S t	        dt        |       d       |D ]  }	 t        |dd      5 }|D ]  }	 t        j                  |j                               }|j                  dd      r:|j                  d	g       }|D ]K  }|j                  d
      dk(  s|j                  dd      j                         }	|	r|j                  |	        n  	 ddd        |S # t        j                  $ r Y w xY w# 1 sw Y   &xY w# t        $ r&}
t	        d|j                   d|
        Y d}
~
d}
~
ww xY w)ah  
        Scan all batch files and extract completed prompts by their actual content.
        
        This provides a more robust resume mechanism that matches on prompt text
        rather than indices, allowing recovery even if indices don't match.
        
        Returns:
            set: Set of prompt texts that have been successfully processed
        batch_*.jsonlu   📂 Scanning z% batch files for completed prompts...r   r   r   failedFr   fromhumanvaluer7   Nu!     ⚠️  Warning: Error reading r   )setsortedr   globr   r   r   r=   r>   rB   r+   addr?   r   r3   )r   r   batch_files
batch_filer   r   r   r   rF   prompt_textr   s              r&   "_scan_completed_prompts_by_contentz.BatchRunner._scan_completed_prompts_by_content  sn     ET__11/BC$$s;/00UVW% 	RJR*cG< % ! %%$(JJtzz|$<E  %yy59 ( -2IIor,JM'4 *#&776?g#=25'''22F2L2L2NK'2(9(=(=k(J$)*%%	R0 !   $33 %$%#% %&  R9*//9J"QCPQQRsf   D2D&&5DD&+D6D>D&D2D#	 D&"D#	#D&&D/	+D22	E!;EE!r   c                    g }g }t        | j                        D ]  \  }}|j                  dd      j                         }|sv|j                  dg       }|D ]_  }|j                  d      xs |j                  d      }	|	dv s,|j                  d      xs |j                  dd      j                         } n ||v r|j	                  |       |j	                  ||f        ||fS )	a  
        Filter the dataset to exclude prompts that have already been completed.
        
        Args:
            completed_prompts: Set of prompt texts that have been completed
            
        Returns:
            Tuple of (filtered_dataset, skipped_indices)
        r^   r7   r   r/   r  >   userr  r8   r  )r   r   r+   rB   r   )
r   r   filtered_datasetskipped_indicesr   r   r  r   rF   r/   s
             r&   _filter_dataset_by_completedz(BatchRunner._filter_dataset_by_completed  s     #DLL1 	6JC))Hb1779K  %		/2 >( C776?=cggfoD00'*wwy'9'QSWWWb=Q&X&X&Z	 //&&s+ !''e5#	6&  00r(   resumec                 V   t        d       t        d       t        d       t               }|r*| j                         }|rt        dt        |       d       |r|r| j	                  |      \  }}|st        d       yg }t        dt        |      | j                        D ]%  }|||| j                  z    }|j                  |       ' || _        t        d       t        d	       t        d       t        d
t        | j                        dd       t        dt        |      dd       t        d       t        dt        |      dd       t        dt        |              t        d       | j                         }|j                  d      | j                  k7  r| j                  g i dd}t        | j                        r*t        | j                  t               sd}	t        dd       n| j                  }	i d| j"                  d| j$                  d| j&                  d| j(                  d|	d| j*                  d| j,                  d| j.                  d| j0                  d | j2                  d!| j4                  d"| j6                  d#| j8                  d$| j:                  d%| j<                  d&| j>                  }
t        |j                  d'g             }i }tA        j@                         }t        d(| jB                   d)       tE               }tG        | jB                  *      5 }tI        | j                        D cg c]  \  }}||t!        | jJ                        ||
f! }}}t        d+t        |       d,       t        d-       g }tM        d.      }tO        tQ               tS        d/      tU        d01      tW               tS        d2      tY               |d3d4d4d45      5 }|j[                  d6t        |      7      }t]        j^                         }|j`                  }|jc                  t\        jd                         	 |jg                  th        |      D ]  }|j                  |       |jk                  |d89       	 |j                  d:      }|j                  d'g       xs g }|jk                  |       t        |tl              rR|j                  d;d      |j                  d<d      |j                  d=d      d>|jo                  d?i       t!        |      <   tq        |      |d'<   | js                  ||@        	 |jc                  |       	 ddd       ddd       ddddD}D ]  }|j                  dEi       j{                         D ]K  \  }} ||vr	ddddF||<   ||   dGxx   | dG   z  cc<   ||   dHxx   | dH   z  cc<   ||   dIxx   | dI   z  cc<   M |D ]/  }!||!xx   |j                  dJi       j                  |!d      z  cc<   1  	 tq        |      |d'<   | js                  ||@       |D ]R  }||   } | dH   | dI   z   }"|"dkD  r1t}        | dH   |"z  dLz  d3      | dM<   t}        | dI   |"z  dLz  d3      | dN<   IdO| dM<   dO| dN<   T | jJ                  dPz  }#t        dQ|#j~                   dR       t        }$d}%d}d}&tq        | jJ                  j                  dS            }'t        |#dTdUV      5 }(|'D ]  })|&d8z  }&|)j                  j                  dW      d8   }t        |)dXdUV      5 }*|*D ]  }+|%d8z  }%	 t        j                  |+      },|,j                  dEi       }-|-D .cg c]	  }.|.|$vs|. }/}.|/r9|d8z  }t        |/d         dYkD  r|/d   ddY dRz   n|/d   }0t        dZ| d[|0 d\       |(j                  |+        	 ddd        	 ddd       |dkD  rt        d_| d`|% da       t        db|& dc|%|z
   dd       | j                  | j"                  t        | j                        t        | j                        | j                  | j$                  t        j                         j                         t}        tA        j@                         |z
  d3      ||de
}1t        | j                  dTdUV      5 }2t        j                  |1|2d3d4f       ddd       t        d       t        dg       t        d       t        dht        di |D                      t        dj|%|z
          t        dk|&        t        dlt}        tA        j@                         |z
  d3       dm       t        dn       t        do       |r|tq        |j{                         dp dq      }3t        drdsdtdudvdtdwdvdtdxdvdtdydz	       t        do       |3D ]0  \  }} t        |dsdt| dG   dvdt| dH   dvdt| dI   dvdt| dM   d{d|
       2 nt        d}       t        d~ |D              }4t        d       t        do       |d   }5|d   }6|d   }7|5dkD  rZt}        |6|5z  dLz  d8      }8t}        |7|5z  dLz  d8      }9t        d|5d       t        d|6dd|8 d       t        d|7dd|9 d       nt        d       |4dkD  rt        d|4d       t        d| jJ                          t        d       t        d       t        d| j                  j~                          t        d| j                  j~                          yc c}}w # tt        $ r}t        dA|        Y d}~d}~ww xY w# tt        $ r}tv        jy                  dB|dC        d}~ww xY w# |jc                  |       w xY w# 1 sw Y   >xY w# 1 sw Y   CxY w# tt        $ r}t        dK|        Y d}~d}~ww xY wc c}.w # t        j                  $ r |d8z  }t        d]| d^       Y w xY w# 1 sw Y   xY w# 1 sw Y   GxY w# 1 sw Y   TxY w)z
        Run the batch processing pipeline.
        
        Args:
            resume (bool): Whether to resume from checkpoint
        zG
======================================================================u   🚀 Starting Batch ProcessingF======================================================================z	   Found z. already-completed prompts by content matchingu-   
✅ All prompts have already been processed!Nr   u   📊 RESUME SUMMARYz   Original dataset size:     ,z promptsz   Already completed:         u~      ─────────────────────────────────────────u       🎯 RESUMING WITH:          z   New batches created:       zG======================================================================
r   r   up   ℹ️  Detected Entra ID bearer provider — workers will rebuild credentials from config.yaml in each process.Trk   ry   r}   r~   r{   r|   ri   r   r   r   r   r   r   r   r   r   r   r   u   
🔧 Initializing z worker processes...)	processesu   ✅ Created z batch tasksu+   🚀 Starting parallel batch processing...
)force_terminalu   [bold blue]📦 Batches(   )	bar_widthu   •   F)consolerefresh_per_second	transientredirect_stdoutredirect_stderr
Processing)rT   r5   )advancer[   r   r   r   )r   r   r   r   )r  u8   ⚠️  Warning: Failed to save incremental checkpoint: zBatch worker failed: %s)exc_infor   r   r   r   r   r   r   u8   âš ï¸  Warning: Failed to save final checkpoint: r   success_ratefailure_rateg        ztrajectories.jsonlu%   
📦 Combining ALL batch files into rj   r
  wr   r   _r   2   u,      ⚠️  Filtering corrupted entry (batch z): invalid tool 'r   u/      ⚠️  Filtering invalid JSON entry (batch )u   ⚠️  Filtered z corrupted entries out of z totalu   ✅ Combined z& batch files into trajectories.jsonl (z	 entries))
r   ry   total_promptstotal_batchesr   r}   completed_atduration_secondstool_statisticsreasoning_statistics)indentr   u   📊 BATCH PROCESSING COMPLETEu    ✅ Prompts processed this run: c              3   @   K   | ]  }|j                  d d        yw)r   r   Nr+   .0r   s     r&   	<genexpr>z"BatchRunner.run.<locals>.<genexpr>E  s     4\qQUU;5J4\   u'   ✅ Total trajectories in merged file: u   ✅ Total batch files merged: u   ⏱️  Total duration: su   
📈 Tool Usage Statistics:zF----------------------------------------------------------------------c                     | d   d   S )Nr5   r   r   )xs    r&   <lambda>z!BatchRunner.run.<locals>.<lambda>P  s    ad7m r(   )r   reversez	Tool Namez<25 Countz<10SuccessFailurezSuccess Ratez<12z.1f%z(No tool calls were made during this run.c              3   @   K   | ]  }|j                  d d        yw)r   r   Nr;  r<  s     r&   r>  z"BatchRunner.run.<locals>.<genexpr>b  s     RQaee$<a@Rr?  u   
🧠 Reasoning Coverage:rO   rP   rQ   z   Total assistant turns:    z   With reasoning:           r   z%)z   Without reasoning:        z   No assistant turns recorded.u,      🚫 Samples discarded (zero reasoning): u   
💾 Results saved to: z0   - Trajectories: trajectories.jsonl (combined)z6   - Individual batches: batch_*.jsonl (for debugging)z   - Statistics: z   - Checkpoint: )Pr   r  r  r   r  r   r   r   r   r   r  r+   r   callabler|   r:   r<   ry   r}   r~   r{   ri   r   r   r   r   r   r   r   r   r   r   timer   r   r
   r   r   r   r   r   r   r   r   r   add_tasklogging	getLoggerlevelsetLevelWARNINGimap_unorderedr   updateint
setdefaultr  r  r   loggerr9   r"   roundr3   r   r  r   stemsplitr=   r>   r   r?   r	   r   r   r   dumpsumr   ):r   r  completed_prompt_textsfiltered_entriesr  batches_to_processr   r   r  worker_api_keyr\   r   total_tool_stats
start_timecheckpoint_lockpoolr[   r   tasksresultsr%  progresstaskroot_loggeroriginal_levelr   r   ckpt_errr   total_reasoning_statsbatch_resultrH   r%   r   total_callscombined_fileVALID_TOOLStotal_entriesbatch_files_foundall_batch_filesoutfiler  infiler   r   r   kinvalid_toolsinvalid_previewfinal_statsr   sorted_toolstotal_discardedtotal_turnsrU   without_reasoningpct_withpct_withouts:                                                             r&   r   zBatchRunner.run*  s    	o./h "%%)%L%L%N"%	#&<"=!>>lmn ,040Q0QRh0i-o#FG "$1c"23T__E 1(1t+>?"))%01 .DL /"'((O23t||3DQ2GxPQ233G2J(ST  S  T4S9I5J14MXVW237I3J2KLM/" //1z*dmm; MM%'! $	O" DLL!*T\\3*G!N@ "\\N
D--
TZZ
 d11
 	

 ~
 t||
 &t'C'C
  5 5
  !7!7
  !7!7
 t33
 T//
 *4+K+K
 $//
  5 5
   5 5!
( !$O$7$78KR$P Q YY[
$T%5%5$66JKL & D,,- C	9 .7t||-D	 *Iz ()	E 	 LUL9:@A GT2G45B'"$5!#%#$ % % /9 ((SZ(H &//1!,!2!2$$W__59"&"5"56KU"S iv. a8i(.

;(?I(.

3F(K(QrI188C))S917K1K/5zz)Q/G>DjjIacd>ea" : :=" McR[n ] DJJ_C`O,?@ 11/1X%i4  ((8_/9)C	9L ;<UVst u# 	bL$0$4$4\2$F$L$L$N 
K 	5$44!"#$#$3$Y/ !+G4gF4 +I6%	:JJ6 +I6%	:JJ6
K - b%c*l.>.>?PRT.U.Y.YZ]_`.aa*b	b&	Y39:O3PO/0!!/!H
 * 	,I$Y/E	*U9-==KQ(-eI.>.Ls.RTU(Vn%(-eI.>.Ls.RTU(Vn%(+n%(+n%	, *>>6}7I7I6J#NO ) !!5!5o!FG-w7 	b7- b
!Q&!&OO11#6q9	*cG< b & b%*b#'::d#3D)-,)CJ 9C,[1a{FZQ,[M,[, 0A 5 0SVWdefWgShkmSm-2B3B2G%2O  tA  BC  tD %(TU^T__p  rA  qB  BC  'D  !E (#MM$/bb b	b	b4 a%&6%77QR_Q``fgh/00VWdgwWwVx  yB  C  	D  -- . .//ZZ$LLN446 %diikJ&> B/$9
 $//39 	DQIIk1QUC	D 	o./h04\T[4\1\0]^_7HX8X7YZ[./@.ABC(tyy{Z/G)K(LANO-.h! &&(+L [%Qwsm1YsO1YsOSTUcdgThij(O$0  	5 oQW~c*!Y',AY',A^,S1	4 <= R'RR*+h+,CD./EF12KL?^k9C?CH 1K ?# EqIK1+aAB1.1C2hZrRS12CA1FbUWXY34Q@QR@STU)$//):;<@AFG!$//"6"6!789!$"6"6";";!<=>A	t  ) i!$\]e\f"ghhi ! LL!:ALM  ((8_/9 /9)C	9 C	9|  	YLXJWXX	YV -\  $33 b,1,!$ST]S^^_"`ab#b b	b 	bX	D 	Ds#  0n	$l'-A>n+An=mB;l-mnn!n, 47p+p7+o"	o,o0<o,p-o>p
p2p'n-	m	6m
	m
m	m	m9m44m99m<<nnn	nn),	o5o		oo'p=p ppp	ppp()defaultrf   NNzclaude-opus-4-20250514   FNr   NNNNNNNNN)N)F)__name__
__module____qualname____doc__r<   rU  rS   r   r   floatr   r   r   r   r   r   r  r   r  r  r  r  r   r   r(   r&   r   r     s#    & -'+ #'+'+%)!7;+/15-lJlJ lJ 	lJ
 lJ lJ lJ lJ lJ lJ lJ "%lJ lJ  9lJ  9lJ  c!lJ" #lJ$ &.e_%lJ& 'lJ( sCx.)lJ* tCH~.+lJ, -lJ\tDcN3 @d5d38n1D+E&F!G $sCx. 6ES#X Ehtn E"*!C *!X 1c  1eDQUJX\]`XaLaFb  1DN?$ N?r(   r   r   r   r   ry   r}   r|   r{   	max_turnsr   r  ri   r   r   r   r   r   r   r   r   reasoning_effortreasoning_disabledprefill_messages_filer   c                 p   |rkddl m} t        d       t        d        |       }t        |j	                               D ]
  } ||        t        d       t        d       t        d       y| st        d	       y|r|d
k  rt        d       y|st        d       y|r-|j                  d      D cg c]  }|j                          c}nd}|r-|j                  d      D cg c]  }|j                          c}nd}|r-|j                  d      D cg c]  }|j                          c}nd}d}|rddi}t        d       n;|r9g d}||vrt        ddj                  |              yd|d}t        d|        d} |rc	 t        |dd      5 }!t        j                  |!      } ddd       t        | t              st        d       yt        dt        |        d|        	 t        d4i d| d|d |d!|d"|d#|d$|d%|d&|d'|
d(|d)|d*|d+|d,|d-|d.|d/|d0| d1|}#|#j!                  |	2       yc c}w c c}w c c}w # 1 sw Y   xY w# t        $ r}"t        d|"        Y d}"~"yd}"~"ww xY w# t        $ r.}"t        d3|"        |
rt#        j$                          Y d}"~"y
d}"~"ww xY w)5ax  
    Run batch processing of agent prompts from a dataset.

    Args:
        dataset_file (str): Path to JSONL file with 'prompt' field in each entry
        batch_size (int): Number of prompts per batch
        run_name (str): Name for this run (used for output and checkpointing)
        distribution (str): Toolset distribution to use (default: "default")
        model (str): Model name to use (default: "claude-opus-4-20250514")
        api_key (str): API key for model authentication
        base_url (str): Base URL for model API
        max_turns (int): Maximum number of tool calling iterations per prompt (default: 10)
        num_workers (int): Number of parallel worker processes (default: 4)
        resume (bool): Resume from checkpoint if run was interrupted (default: False)
        verbose (bool): Enable verbose logging (default: False)
        list_distributions (bool): List available toolset distributions and exit
        ephemeral_system_prompt (str): System prompt used during agent execution but NOT saved to trajectories (optional)
        log_prefix_chars (int): Number of characters to show in log previews for tool calls/responses (default: 20)
        providers_allowed (str): Comma-separated list of OpenRouter providers to allow (e.g. "anthropic,openai")
        providers_ignored (str): Comma-separated list of OpenRouter providers to ignore (e.g. "together,deepinfra")
        providers_order (str): Comma-separated list of OpenRouter providers to try in order (e.g. "anthropic,openai,google")
        provider_sort (str): Sort providers by "price", "throughput", or "latency" (OpenRouter only)
        max_tokens (int): Maximum tokens for model responses (optional, uses model default if not set)
        reasoning_effort (str): Reasoning effort: "none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra" (default: "medium")
        reasoning_disabled (bool): Completely disable reasoning/thinking tokens (default: False)
        prefill_messages_file (str): Path to JSON file containing prefill messages (list of {role, content} dicts)
        max_samples (int): Only process the first N samples from the dataset (optional, processes all if not set)
        
    Examples:
        # Basic usage
        python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run
        
        # Resume interrupted run
        python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run --resume
        
        # Use specific distribution
        python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=image_test --distribution=image_gen
        
        # With disabled reasoning and max tokens
        python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run \
                               --reasoning_disabled --max_tokens=128000
        
        # With prefill messages from file
        python batch_runner.py --dataset_file=data.jsonl --batch_size=10 --run_name=my_run \
                               --prefill_messages_file=configs/prefill_opus.json
        
        # List available distributions
        python batch_runner.py --list_distributions
    r   )print_distribution_infou$   📊 Available Toolset Distributionsr  u   
💡 Usage:zD  python batch_runner.py --dataset_file=data.jsonl --batch_size=10 \z@                         --run_name=my_run --distribution=<name>Nu%   ❌ Error: --dataset_file is requiredr5   u2   ❌ Error: --batch_size must be a positive integeru!   ❌ Error: --run_name is requiredr  effortnoneu&   🧠 Reasoning: DISABLED (effort=none))r  minimallowmediumhighxhighmaxultrau.   ❌ Error: --reasoning_effort must be one of: z, T)enabledr  u   🧠 Reasoning effort: r   r   r   uF   ❌ Error: prefill_messages_file must contain a JSON array of messagesu   💬 Loaded z prefill messages from u$   ❌ Error loading prefill messages: r   r   r   ry   r~   r{   r|   r}   r   ri   r   r   r   r   r   r   r   r   r   r   )r  u   
❌ Fatal error: r   )toolset_distributionsr  r   r  r   rZ  rB   joinr   r=   r  r:   r   r   r   r   r   r   r   )$r   r   r   ry   r}   r|   r{   r  r   r  ri   r   r   r   r   r   r   r   r   r  r  r  r   r  	all_dists	dist_namepproviders_allowed_listproviders_ignored_listproviders_order_listr   valid_effortsr   r   r   runners$                                       r&   mainr  {  s>   V A45h&(		 01 	/I#I.	/ 	oUVPQ 56aBC12 Sd1B1H1H1MNAaggiNimRc1B1H1H1MNAaggiNimN]/D/DS/IJ!AGGIJcg $f-67	]=0B499]C[B\]^'+7GH'(8'9:; 		+S7C 0q#'99Q< 0.5^_L%5!6 77NOdNefg 
%
!
 
 &	

 %
 
 
 
 $
 
 %<
 .
 5
 5
 1
  (!
" "#
$ .%
& .'
( $)
. 	

&
!w ONJ,0 0  	8<=	>  #A3'(!	sa   2H?#II	I I(#I I 'AI> II 	I;#I66I;>	J5$J00J5__main__)NNNr  zanthropic/claude-sonnet-4.6Nzhttps://openrouter.ai/api/v1rf   r  FFFNr   NNNNNNFNN):r  hermes_bootstrapModuleNotFoundErrorr=   rN  r   rL  pathlibr   typingr   r   r   r   r   r	   multiprocessingr
   r   r   rich.progressr   r   r   r   r   r   rich.consoler   rO  r  rW  fire	run_agentr   r  r   r   r   model_toolsr   _WORKER_CONFIGr  r   r   r!   r<   rU  r'   r,   rL   rX   r   r   r   rS   r  Firer   r(   r&   <module>r     sG  .	   	   3 3  &  q q  			8	$   
 ,  1,1134   !Q1= d3S#X+>&? DdSVX[S[nI\D] <DcN tCQTH~ 0P$tCH~"6 P4T#s(^@S;T Pf!tDcN'; !S#X !HY
Y
c3hY
 Y
 cN	Y

 
#s(^Y
x| |$sCx. |~i	? i	?Z !.2$#'!! $!%/iii i 	i
 i i i i i i i i !i i i  !i" #i$ %i& 'i( )i* +i, -i. /iX zDIIdO Y(  	
 		s   F FF