
    \j;S                        d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZd dl	Z	d dl
mZ d dlmZmZ eeeee   z  f   Z ej(                  e      Z G d d      Z G d	 d
      Z G d d      Z G d de      Z G d d      Zy)    N)OrderedDict)Mapping)Any)TensorProto)InferenceSession
RunOptionsc                   n   e Zd Zedededefd       Zededefd       Zedefd       Zedefd       Z	edede
eef   fd	       Zedefd
       Zedej                  fd       Zedej                  fd       Zedede
eej                  f   fd       Zedede
eej                  f   fd       Zy)
TypeHelperort_sessionnamereturnc                     t        | j                               D ]"  \  }}|j                  |k(  s|j                  c S  t	        d| d      )Nzinput name 
 not found)	enumerate
get_inputsr   type
ValueError)r   r   _iinputs       ]/root/.hermes/venv/lib/python3.12/site-packages/onnxruntime/transformers/io_binding_helper.pyget_input_typezTypeHelper.get_input_type   sO    ";#9#9#;< 	"IBzzT!zz!	" ;tfJ788    c                     t        | j                               D ]"  \  }}|j                  |k(  s|j                  c S  t	        d| d      )Nzoutput name r   )r   get_outputsr   r   r   )r   r   r   outputs       r   get_output_typezTypeHelper.get_output_type   sO    #K$;$;$=> 	#JB{{d"{{"	# <vZ899r   ort_typec                    t         j                  t         j                  t         j                  t         j                  t
        t         j                  t         j                  t         j                  t         j                  t         j                  t         j                  t         j                  t         j                  t         j                  d}| |vrt        |  d      ||    S )N)tensor(int64)tensor(int32)tensor(float)tensor(float16)tensor(bool)tensor(uint8)tensor(int8)tensor(double)tensor(int16)tensor(uint16)tensor(uint32)tensor(uint64)tensor(complex64)tensor(complex128) not found in map)numpyint64int32float32float16booluint8int8float64int16uint16uint32uint64	complex64
complex128r   )r   ort_type_to_numpy_type_maps     r   ort_type_to_numpy_typez!TypeHelper.ort_type_to_numpy_type#   s     #[["[["]]$}} "[[!JJ#mm"[[#ll#ll#ll!&"'"2"2&
"  55z):;<<)(33r   c                    i dt         j                  dt         j                  dt         j                  dt         j                  dt         j
                  dt         j                  dt         j                  dt         j                  d	t         j                  d
t         j                  dt         j                  dt         j                  dt         j                  dt         j                  dt         j                  dt         j                   dt         j"                  t         j$                  t         j&                  t         j(                  t         j*                  d}| |vrt-        |  d      ||    S )Nr   r    r!   r"   tensor(bfloat16)r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   tensor(float8e4m3fn)tensor(float8e4m3fnuz))tensor(float8e5m2)tensor(float8e5m2fnuz)tensor(int4)tensor(uint4)r-   )torchr/   r0   r1   r2   bfloat16r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   float8_e4m3fnfloat8_e4m3fnuzfloat8_e5m2float8_e5m2fnuzint4uint4r   )r   ort_type_to_torch_type_maps     r   ort_type_to_torch_typez!TypeHelper.ort_type_to_torch_type:   sl   &
U[[&
U[[&
 U]]&
 u}}	&

 &
 EJJ&
 U[[&
 EJJ&
 emm&
 U[[&
 ell&
 ell&
 ell&
  &
 !%"2"2&
  #E$7$7!&
" %e&;&;#&
$ #("3"3&+&;&;!JJ"[[+&
". 55z):;<<)(33r   c                    i }| j                         D ].  }t        j                  |j                        ||j                  <   0 | j                         D ].  }t        j                  |j                        ||j                  <   0 |S )z9Create a mapping from input/output name to onnx data type)r   r
   ort_type_to_onnx_typer   r   r   )r   name_to_onnx_typer   r   s       r   get_io_onnx_type_mapzTypeHelper.get_io_onnx_type_mapX   s      ++- 	YE,6,L,LUZZ,Xejj)	Y "--/ 	[F-7-M-Mfkk-Zfkk*	[  r   c                 0   i dt         j                  dt         j                  dt         j                  dt         j                  dt         j
                  dt         j                  dt         j                  dt         j                  d	t         j                  d
t         j                  dt         j                  dt         j                  dt         j                  dt         j                  dt         j                  dt         j                   dt         j"                  t         j$                  t         j&                  t         j(                  t         j*                  t         j,                  t         j.                  d}| |vrt1        |  d      ||    S )Nr   r    r!   r"   r@   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   rA   rB   )rC   rD   ztensor(float4e2m1)rE   rF   ztensor(string)r-   )r   INT64INT32FLOATFLOAT16BFLOAT16BOOLUINT8INT8DOUBLEINT16UINT16UINT32UINT64	COMPLEX64
COMPLEX128FLOAT8E4M3FNFLOAT8E4M3FNUZ
FLOAT8E5M2FLOAT8E5M2FNUZ
FLOAT4E2M1INT4UINT4STRINGr   )r   ort_type_to_onnx_type_maps     r   rR   z TypeHelper.ort_type_to_onnx_typec   s   %
[..%
[..%
 [..%
 {22	%

  4 4%
 K,,%
 [..%
 K,,%
 k00%
 [..%
 k00%
 k00%
 k00%
  !6!6%
 !+"8"8%
  #K$<$<!%
" %k&@&@#%
$ #."8"8&1&@&@"-"8"8',,(..)00/%
!2 44z):;<<(22r   
numpy_typec                 h   t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j
                  t        j
                  t        t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  i}| |vrt!        |  d      ||    S Nr-   )r.   r/   rG   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r   )rn   numpy_type_to_torch_type_maps     r   numpy_type_to_torch_typez#TypeHelper.numpy_type_to_torch_type   s     KKKKMM5==MM5==%**KKJJ

MM5==KKLL%,,LL%,,LL%,,OOU__e..(
$" 99
|+<=>>+J77r   
torch_typec                 h   t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j
                  t        j
                  t         j                  t        t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  t         j                  t        j                  i}| |vrt!        |  d      ||    S rp   )rG   r/   r.   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r   )rs   torch_type_to_numpy_type_maps     r   torch_type_to_numpy_typez#TypeHelper.torch_type_to_numpy_type   s     KKKKMM5==MM5==JJKKJJ

MM5==KKLL%,,LL%,,LL%,,OOU__e..(
$" 99
|+<=>>+J77r   c                    i }| j                         D ].  }t        j                  |j                        ||j                  <   0 | j                         D ].  }t        j                  |j                        ||j                  <   0 |S )z:Create a mapping from input/output name to numpy data type)r   r
   r>   r   r   r   )r   name_to_numpy_typer   r   s       r   get_io_numpy_type_mapz TypeHelper.get_io_numpy_type_map          ++- 	[E-7-N-Nuzz-Zuzz*	[ "--/ 	]F.8.O.OPVP[P[.\v{{+	]!!r   c                    i }| j                         D ].  }t        j                  |j                        ||j                  <   0 | j                         D ].  }t        j                  |j                        ||j                  <   0 |S )z:Create a mapping from input/output name to torch data type)r   r
   rP   r   r   r   )r   name_to_torch_typer   r   s       r   get_io_torch_type_mapz TypeHelper.get_io_torch_type_map   rz   r   N)__name__
__module____qualname__staticmethodr   strr   r   r>   rP   dictintrT   rR   r.   dtyperr   rG   rv   ry   r}    r   r   r
   r
      sh   9$4 9C 9C 9 9 :3 :3 : : 4 4 4, 4 4 4: !*: !tCH~ ! ! 3 3 3> 8U[[ 8 8. 8U[[ 8 8. "+; "S%++EU@V " " "+; "S%++EU@V " "r   r
   c            
           e Zd Zedefd       Zedej                  dej                  dej                  deej                     fd       Z	ed
d       Z
y	)IOBindingHelperr   c                     i }|j                         D ]^  \  }}t        j                  | |      }t        j                  |      }t	        j
                  t        j                  |      ||      ||<   ` |S )zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.)r   device)itemsr
   r   rP   rG   emptyr.   prod)r   output_shapesr   output_buffersr   shaper   rs   s           r   get_output_buffersz"IOBindingHelper.get_output_buffers   sr     (..0 	cKD%!11+tDH#::8DJ#(;;uzz%/@
[a#bN4 	c r   	input_idsposition_idsattention_maskpastc                 4   t         j                  |       }| j                         }|j                         sJ |j	                  d|j
                  j                  d|d   t        |j                               |j                                |t        |      D ]  \  }	}
|
j                         sJ |
j                         }|dk(  r|j                         }|j	                  d|	 |
j
                  j                  d|d|	    t        |
j                               |        |d|j                         sJ |j	                  d|j
                  j                  d|d   t        |j                               |j                                |d|j                         sJ |j	                  d|j
                  j                  d|d   t        |j                               |j                                | j                         D ]  }|j                  }||   }t        j                  | d|j
                  j                   dt        |j                                       |j                  ||j
                  j                  d||   ||   |j                                 |S )zdIO binding for a session: bind inputs (input_ids, position_ids, attention_mask, past_*) and outputs.r   r   past_r   r   z device type=z shape=)r
   rT   
io_bindingis_contiguous
bind_inputr   r   listsizedata_ptrr   r   r   loggerdebugbind_output)r   r   r   r   r   r   r   rS   r   ipast_ir   r   output_nameoutput_buffers                  r   prepare_io_bindingz"IOBindingHelper.prepare_io_binding   s    ';;KH !++-
 &&(((!!k*!" 	
 &t_ 	6++---!??,q=  )113H%%A3KMM&&%aSk2'$ %!//111!! %%**!"23^((*+'') #--///!!##((!.1\&&()%%' "--/ 	F ++K*;7MLLK=m6J6J6O6O5PPWX\]j]o]o]qXrWstu""$$))!+.k*&&(		 r   c                 h   g }| j                         D ]  }|j                  }||   }||   }|dt        j                  |       j	                  |      j                         j                         }	|r.|j                  |	j                         j                                |j                  |	        |S )z3Copy results to cpu. Returns a list of numpy array.r   )	r   r   r.   r   reshapeclonedetachappendcpu)
r   r   r   return_numpyort_outputsr   r   bufferr   copy_tensors
             r   "get_outputs_from_io_binding_bufferz2IOBindingHelper.get_outputs_from_io_binding_buffer*  s     !--/ 	0F ++K#K0F!+.E UZZ%67??FLLNUUWK"";??#4#:#:#<="";/	0 r   N)T)r~   r   r   r   r   r   rG   Tensorr   r   r   r   r   r   r   r      s    (8   R<<R llR 	R
 5<< R Rh  r   r   c                       e Zd ZdZddedej                  fdZdedefdZ	d Z
d	ed
ej                  fdZdefdZddeeej                  f   dedefdZeddedededeeef   fd       Zy)CudaSessionzLInference Session with IO Binding for ONNX Runtime CUDA or TensorRT providerr   r   c                 6   || _         | j                   j                         D cg c]  }|j                   c}| _        | j                   j	                         D cg c]  }|j                   c}| _        t        j                  | j                         | _        t        j                  | j                         | _
        | j                   j                         | _        || _        t               | _        t               | _        || _        i | _        y c c}w c c}w N)r   r   r   input_namesr   output_namesr
   rT   io_name_to_onnx_typer}   io_name_to_torch_typer   enable_cuda_graphr   input_tensorsoutput_tensorsr   buffer_sharing)selfr   r   r   r   r   s         r   __init__zCudaSession.__init__=  s    &484D4D4O4O4QR5EJJR7;7G7G7S7S7UVVV[[V$.$C$CDDTDT$U!%/%E%EdFVFV%W"**557!2(])m /1 SVs   DD
input_namer   c                     || j                   v sJ || j                  v sJ || j                  |<   || j                  |<   y r   )r   r   r   )r   r   r   s      r   set_buffer_sharingzCudaSession.set_buffer_sharingM  sJ    T-----d/////*5J'+5K(r   c                     | ` | `| `y r   )r   r   r   )r   s    r   __del__zCudaSession.__del__S  s    Or   r   tensorc           	      l   |j                   j                  |j                   j                  nd}t        |j                        dk(  rdgnt	        |j                        }| j
                  j                  ||j                   j                  || j                  |   ||j                                || j                  v ry| j
                  j                  | j                  |   |j                   j                  || j                  |   ||j                                || j                  | j                  |   <   y y )Nr      )r   indexlenr   r   r   r   r   r   r   r   r   r   )r   r   r   	device_idtensor_shapes        r   bind_input_and_buffer_sharingz)CudaSession.bind_input_and_buffer_sharingX  s   +1==+>+>+JFMM''PQ	!&,,/14s$v||:L""MM%%d+OO	
 4&&&OO''##D)""))$/! >DD 3 3D 9: 'r   
shape_dictc                    | j                   r|j                         D ]  \  }}|| j                  v s|| j                  v r:t	        | j                  |   j
                        t	        |      k(  rRt        d      | j                  |   }t        j                  t	        |      |      j                  | j                        }|| j                  |<   | j                  ||        |j                         D ]K  \  }}|| j                  v s|| j                  v r/t	        | j                  |   j
                        t	        |      k(  rS|| j                  v rb| j                  |   }t        j                  t	        |      |      j                  | j                        }|| j                  |<   | j                   j#                  ||j                  j$                  |j                  j&                  |j                  j&                  nd| j(                  |   t+        |j-                               |j/                                N y)z Allocate tensors for I/O Bindingz(Expect static input shape for cuda graph)r   )r   Nr   )r   r   r   r   tupler   RuntimeErrorr   rG   r   tor   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   torch_dtyper   s         r   allocate_bufferszCudaSession.allocate_buffersp  s   !!)//1 Ee4+++t111 !3!3D!9!?!?@E%LP$*+UVV"&"<"<T"BK"[[u[ILLTXT_T_L`F/5D&&t,66tVDE &++- 	KD%t(((4...59L9LT9R9X9X3Y]bch]i3i4..."88>U5\EHHPTP[P[H\,2##D)++MM&&+1==+>+>+JFMM''PQ--d3'OO%	r   N	feed_dictrun_optionssynchronizec                 <   |j                         D ]  \  }}t        |t        j                        r|j	                         sJ || j
                  v sA| j                  r| j                  |   j                         |j                         k(  sJ | j                  |   j                  |j                  k(  sJ |j                  j                  dk(  sJ | j                  |   j                  |       | j                  ||        |rf| j                  j                          | j                   j#                  | j                  |       | j                  j%                          | j&                  S | j                   j#                  | j                  |       | j&                  S )z$Bind input tensors and run inferencecuda)r   
isinstancerG   r   r   r   r   r   nelementr   r   r   copy_r   r   synchronize_inputsr   run_with_iobindingsynchronize_outputsr   )r   r   r   r   r   r   s         r   inferzCudaSession.infer  sV   %OO- 		ELD&fell38L8L8NNNt'''))--d3<<>&//BSSSS--d399V\\III!==--777&&t,226:66tVD		E OO..0//MOO//1 """ //M"""r   r   r   streamr   c                 8    | d|d}|dk7  rt        |      |d<   |S )NkSameAsRequested)r   arena_extend_strategyr   r   user_compute_stream)r   )r   r   r   optionss       r   get_cuda_provider_optionsz%CudaSession.get_cuda_provider_options  s1     #%7!2
 Q;-0[G)*r   F)NT)r   )r~   r   r   __doc__r   rG   r   r   r   r   r   r   r   	ShapeDictr   r   r   r3   r   r   r   r   r   r   r   r   r   r   :  s    V1$4 1ell 1 6S 6s 6
D# Du|| D0$9 $L#tC$56 #Z #ei #, S T SV _cdgildl_m  r   r   c                        e Zd Z	 	 	 	 ddedej
                  dedededede	e
e
f   dz  f fd	Zdd
edefdZdde	e
ej                  f   d
ef fdZ xZS )
GpuBindingNr   r   r   enable_gpu_graphgpu_graph_idr   r   c                    t         
|   |||       |r*|j                         D ]  \  }}	| j                  ||	        | j	                  |       || _        |rt        j                  |      nd | _        || _	        d | _
        y r   )superr   r   r   r   r   copydeepcopyr   r   last_run_gpu_graph_id)r   r   r   r   r   r   r   r   r   r   	__class__s             r   r   zGpuBinding.__init__  s     	f.>?+9+?+?+A A'
K''
K@A 	j)(7G$--
3T%)"r   disable_cuda_graph_in_runr   c                 ~    t               }|rdn| j                  }|j                  dt        |             || _        |S )Nr   )r   r   add_run_config_entryr   r   )r   r   r   r   s       r   get_run_optionszGpuBinding.get_run_options  s:    ,6rD<M<M$$^S5FG%1"r   r   c                     | j                  |      }| j                  r|j                  dd       t        |   ||      S )N'disable_synchronize_execution_providers1)r   r   r   r   r   )r   r   r   r   r   s       r   r   zGpuBinding.infer  s>    **+DE;;,,-VX[\w}Y44r   )Fr   r   Nr   )r~   r   r   r   rG   r   r   r3   r   r   r   r   r   r   r   r   __classcell__)r   s   @r   r   r     s     "'04*%* * 	*
 * * * S#X-*.	 	* 	5tC$56 5SW 5 5r   r   c            	       l    e Zd ZdZddedej                  dedefdZ	 	 dde	d	e
d
eeef   dz  defdZy)GpuBindingManagerzA manager for I/O bindings that support multiple CUDA Graphs.
    One cuda graph is reused for same input shape. Automatically add a new cuda graph for new input shape.
    r   r   r   max_cuda_graphsc                 X    || _         || _        g | _        d | _        || _        || _        y r   )r   r   graph_bindingsno_graph_bindingr   r   )r   r   r   r   r   s        r   r   zGpuBindingManager.__init__  s4    & ! !%.r   Nr   use_cuda_graphr   r   c           	      6   | j                   D ]  }|j                  |k(  s|c S  t        | j                         | j                  k\  s|sr| j                  ?t        | j                  | j                  || j                  |      | _        | j                  S | j                  j                  |       | j                  S t        | j                  | j                  |dt        | j                         | j                  |      }| j                   j                  |       |S )N)r   r   T)r   r   r   r   )r  r   r   r   r  r   r   r   r   r   r   )r   r   r  r   gpu_graph_bindings        r   get_bindingzGpuBindingManager.get_binding  s    "&!4!4 	) ++z9((	) t""#t';';;N$$,(2$$dkk:dkkbp)%
 ((( %%66zB((( 'KK!T001;;)
 	""#45  r   )r   r   )FN)r~   r   r   r   r   rG   r   r   r   r   r3   r   r   r   r  r   r   r   r   r     so    /$4 /ell /TW /nq /"  %04	 ! !  ! S#X-	 !
 
 !r   r   )r   loggingcollectionsr   collections.abcr   typingr   r.   rG   onnxr   onnxruntimer   r   r   r   r   r   r   	getLoggerr~   r   r
   r   r   r   r   r   r   r   <module>r     s      # #     4 Cc**+				8	$t" t"nm m`~ ~B)5 )5X3! 3!r   