
    \j|[                     v    d dl Z d dlZd dlmZmZ d dlmZ d dlm	Z	  e j                  e      Z G d de      Zy)    N)AttentionMaskFusionAttention)helper)	OnnxModelc                   :     e Zd ZdZdedededef fdZd Z xZ	S )FusionBartAttentionz?
    Fuse Bart Attention subgraph into one Attention node.
    modelhidden_size	num_headsattention_maskc                 *    t         |   ||||       y )N)super__init__)selfr	   r
   r   r   	__class__s        a/root/.hermes/venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_bart_attention.pyr   zFusionBartAttention.__init__   s     	YG    c                    | j                   j                  |g dg d      }d }|-| j                   j                  |g dg d      }|
|d   }|dd  }|	|\  }}}	}
}nt        j                  d       y |?|j                  d   |j
                  d   k(  r|j                  d   }nc|j                  d   }nSg }|j                  D ].  }||vr||d   j
                  d   k(  r|j                  |       0 t        |      dk7  ry |d   }	 ||   }|j                  dv r| j                   j                  |      d   }|j
                  D ];  }|s||   }|D cg c]  }|j                   }}|j                  d	      dk\  s9|} n | j                   j                         j                  D ch c]  }|j                   }}| j                   j                         j
                  D ch c]  }|j                   }}| j                   j                  |g d
g d      }| j                   j                  |g dg d      }| j                   j                  |g dg d      }d\  }}g d d }}}||}|\  }} }}||j
                  d      }!|!D ]Z  }"|"j                  dk(  s| j                   j                  |"ddgddg      }#|#|#d   j                  d   }||"j
                  d      }! n |!D ]D  }"||"j
                  d      }$|$D ]$  }%|%j
                  d   |v s|%j
                  d   } n |dk7  sD n{ ny|)|}|\  }&}} }}|&j                  d   }|&j
                  d   }nN|j                  d   |v r|j                  d   }n-||}|d   j                  d   }nt        j                  d       y ||v r|nd}||v r|nd}| j                   j                  |dd	gddg      }'| j                   j                  |g dg d      }(| j                   j                  |g dg d      })| j                   j                  |g dg d      }*g d },}+|'|'\  }-}.|'}+n8|(	|(\  }-},}.|(}+n-|)	|)\  }-}-}.|)}+n"|*
|*\  }-}-},}.|*}+nt        j                  d       y | j                   j                  |.g dg d       }/| j                   j                  |.g d!g d       }0| j                   j                  |.g d!g d"      }1g }2|/|/}2|2\  }3}4}5}6}7n0|0|0}2|2\  }5}3}4}6}7n#|1|1}2|2\  }5}3}4}6}7nt        j                  d#       y | j                   j                  |.g d$g d%      }8| j                   j                  |.g d&g d'      }9| j                   j                  |.g d(g d      }:| j                   j                  |.g d)g d*      };| j                   j                  |.g d+g d,      }<d\  }=}>g d d }A}@}?|8F|8}?|?\  }B}C}A||Cj
                  d      }D|DD ]%  }E|Ej
                  d   |v sEj
                  d   }> nz nw|9*|9}?|?\  }-}F}B}C}A|Fj                  d   }=|Fj
                  d   }>nK||.j                  d      j                  d   |v r'||.j                  d      g}?|?d   j                  d   }=n|<|<}?|?\  }-}-}-}-}B}C}@}An|:|:}?|?\  }-}B}C}A||Aj
                  d      }!|!D ]Z  }"|"j                  dk(  s| j                   j                  |"ddgddg      }G|GGd   j                  d   }=||"j
                  d      }! n |!D ]D  }"||"j
                  d      }$|$D ]$  }%|%j
                  d   |v s|%j
                  d   }> n |>dk7  sD n/ n-|;|;}?|?d   j                  d   }=nt        j                  d-       y |=|v r|=nd}=|>|v r|>nd}>A@| j                   j                  |j                  d         }H|Hj                  d   }I|Hj                  }Jd.}K| j                   j                  |K      }L|LB| j!                  KJIgt#        j$                  d/g|Iz  t'        j(                  |J      0      1       | j                   j+                  d2      }Mt'        j,                  d2KAj
                  d   gCj                  g|M      }@t/        |=      xr t/        |      xr
 Ad u xr |d u }N|N xr: |7j                  d   |k(  xr& Aj                  d   |k(  xr |j                  d   |k(  }ON xrT |7j                  d   |k(  xr@ Aj                  d   |j                  d   k(  xr Aj                  d   |7j                  d   k7  }P|+|(|*fv }Q|Q }ROxr R}SOxr Q}TPxr R}UTxr t/        |=      xr t/        |      }VNxr R}WQ}Xg }Y|Xr| j                   j                  |,d3gdg      }Z| j                   j                  |,g d4g d5      }[| j                   j                  |,g d6g d7      }\| j                   j                  |,d8d8gddg      }]|[[}Yn%\\}Yn ]]}YnZZ}Ynt        j                  d9       y t        Y      dkD  sJ Ss	TsUsVsWrq|	}^| j1                  |4      \  }_}`|_dk  s`dk  rc| j2                  dkD  rT| j4                  dkD  rEt        j                  d:_`| j2                  | j4                         | j2                  }_| j4                  }`_dk  s`dk  s`_z  dk7  rt        j                  d;       y d }aVsUsWr[| j6                  rL| j9                  |7UsVrAn|=UsVr|n||6UsVr@nd UsVr|nd _`^j
                  d   XVr|=ndVr|nd|>|<      nd }anH| j6                  }bd=| _        | j;                  d |7A||6@|_`|^j
                  d   X|=||>|>      }a|b| _        at        j                  d?       y | j<                  j                  a       | j>                  | j@                  |aj                  <   | jB                  jE                  ^|
|g       | jB                  jE                  |+       VsUsWr,t        |2      dkD  r"|2d   j                  d	k(  r|2jG                          t        |?      dkD  r"|?d   j                  d	k(  r|?jG                          t        |      dkD  r"|d   j                  d	k(  r|jG                          | jH                  rt        |2      dkD  r"|2d   j                  d2k(  r|2jG                          t        |?      dkD  r"|?d   j                  d2k(  r|?jG                          t        |      dkD  r"|d   j                  d2k(  r|jG                          | jB                  jE                  |2       | jB                  jE                  |?       | jB                  jE                  |       d@| _%        y y c c}w c c}w c c}w )AN)AddMatMulReshape	Transposer   )   r   r   r   r   )r   r   r   r   r   r   )r   Nr   r   r   r   r   r   z(fuse_attention: failed to match qkv path>   r   Clipr   )r   r   r   r   )r   r   r   N)Concatr   r   r   r   )r   r   r   r   N)r   r   r   r   )r   r   r   r   ) r   r   r   r   r   z&fuse_attention: failed to match v pathSoftmax)r   r   r   )r   r   r   )Wherer   r   )r      r   )r   r   r   r   )r   r    r   r   z'fuse_attention: failed to match qk path)r   r   Mulr   r   )r   r   r   r   r   )r!   r   r   r   r   )r   r   r   r   Nz&fuse_attention: failed to match q path)r   r   r   )r   r   r   )r   r   r   r   r   )r   r   r   r   r   )r!   r   r   r   )r!   r   r   r   r   )r   r   r   r   r   )r!   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   Nz&fuse_attention: failed to match k path
empty_biasg        )dtype)dimsvalsr   r   )SliceExpandr   )r   r   r   )r&   	UnsqueezeGatherShaper   )r   r    r   r   r   r&   z*fuse_attention: failed to match mask nodeszfuse_attention: reshape dims invalid (num_heads=%d, hidden_size=%d), falling back to user-specified num_heads=%d, hidden_size=%dz9fuse_attention: failed to detect num_heads or hidden_size)q_matmulk_matmulv_matmulq_addk_addv_addr   r
   outputunidirectionalpast_kpast_v	present_k	present_vF)
mask_indexr+   r,   r-   r.   r/   r0   r   r
   first_inputr1   causalr3   r4   r5   r6   z+fuse_attention: failed to create fused nodeT)&r	   match_parent_pathloggerdebuginputr1   appendlenop_typeget_childrencountgraphnameget_initializerr$   	data_typeadd_initializernparrayr   tensor_dtype_to_np_dtypecreate_node_name	make_nodeboolget_num_heads_and_hidden_sizer   r
   use_multi_head_attentioncreate_multihead_attention_nodecreate_attention_nodenodes_to_addthis_graph_namenode_name_to_graph_namenodes_to_removeextendpop!disable_multi_head_attention_biasprune_graph)cr   normalize_nodeinput_name_to_nodesoutput_name_to_node	qkv_nodesadd_before_layernormqkv_nodes_with_residualadd_out
matmul_outreshape_qkvtranspose_qkv
matmul_qkv
root_inputother_inputsinput_skip_layernormr1   childrenchildchildren_typesnodegraph_input_namesgraph_output_namesv_nodes_past_or_presentv_nodes_with_pastv_nodes_past_only_oair4   r6   v_nodesadd_vmatmul_vtranspose_v	reshape_vstart_child_nodesstart_child_nodeconcat_v_nodesstart_grandchild_nodesstart_grandchild_nodeconcat_vqk_nodes_no_maskqk_nodes_with_maskqk_nodes_sdpa_no_maskqk_nodes_sdpa_with_maskqk_nodesadd_qk_	matmul_qk
q_nodes_hfq_nodes_oaiq_nodes_sdpaq_nodestranspose_q	reshape_qmul_qadd_qmatmul_qk_nodes_no_past_hfk_nodes_with_past_hfk_nodes_past_or_present_oaik_nodes_past_only_oaik_nodes_sdpar3   r5   k_nodesadd_kmatmul_ktranspose_k	reshape_ktranspose_k_nodestranspose_k_nodeconcat_kconcat_k_nodesadd_v_tensorbias_dimr#   empty_bias_nameempty_tensoradd_namethree_root_inputsone_root_inputtwo_root_inputshas_maskno_maskencoder_attentiondecoder_self_attentiondecoder_cross_attention decoder_self_attention_with_past!decoder_cross_attention_with_pastcausal_mask
mask_nodesmask_nodes_bartmask_nodes_whisper_hfmask_nodes_whisper_oai mask_nodes_whisper_oai_unit_testattention_last_noder   r
   new_node%use_multi_head_attention_ground_truthsc                                                                                                      r   fusezFusionBartAttention.fuse   s   JJ00?
	  $&*jj&B&BJ%'#
 '2'>q'A$3AB7	   LLCD+#))!,q0AA177:
177:
L(.. ,!44Yq\0033##F+, < A%%aJ	 -Z8 !!_4!ZZ44^DQGN$++ 	F*62H9ABemmBNB##H-2#
	 48::3C3C3E3K3KL4TYYLL48JJ4D4D4F4M4MNDdiiNN"&**">">5#

 !JJ88?

 !%

 < <<!

 #	#%tT".-G8?5[)UH !4ELLO D$5 
 #++x7%)ZZ%A%A("K0A&N
 &1!/!3!9!9!!<(;<L<S<STU<V(W%
 %6  )<=M=T=TUV=W)X&-C ),33A6:LL$9$@$@$C	 ? *'GBI?X{Iuh^^A&F *Ia $55%%a(F".+GR[&&q)FLLAB!%66B!*.@!@Ib	::77
YPXDY\]_`[ab!ZZ99*Fbdmn !%

 < <ZIgir s"&**">">=|#
 t&'+LAy'H+#5 Avy)H".3OAq),H$0&=#Aq&).HLLBCZZ11<


 jj22<
 zz33<

 ! G?F<[)UE8$!G?F<UKE8%"G?F<UKE8LLAB!ZZ99.

  $zz;;E 

 '+jj&B&B5'
#
 !%

 < <C!
 zz33_'

 #	#%tT)(G18.[)X !4I4D4DQ4G H$5  #**1-1CC 0 7 7 :I "-*G>E;Q+y(^^A&F *I !34::1=ARR*9??1+=>?GQZ%%a(F%"GDKAQ1ai(41G4;1QY !4HOOA4F G$5 
 #++x7%)ZZ%A%A("K0A&N
 &1!/!3!9!9!!<(;<L<S<STU<V(W%
 %6  )<=M=T=TUV=W)X&-C ),33A6:LL$9$@$@$C	 ? #.+GR[&&q)FLLAB!%66B!*.@!@Ib	EM::55ekk!nEL#((+H **E*O::55oFL#$$#"3%("2&:Y:YZ_:`a	 %  zz2259H$$U_hooa>P,QT]TbTbScemnE LcT&\ch$>NcS[_cSc!! 0q!Z/0q!Z/0 q!Z/	 	 "! 7q!Z/7q!X^^A%667 q!X^^A%66	 	   24KLL,*6w!/!<H"1"=g+A+cd6l+cW[\bWc(,=,I') 
"jj::	O
 %)JJ$@$@,%!
 &*ZZ%A%A@&"
 04zz/K/K'"A0,
 %02
'33
1==
 ,,
IJz?Q&&& %&/0"-%)%G%G	%R"I{ Q+"28JtO_O_bcOcRNN$$ !NN	"..A~!1kI6MRS5SXYH/3JNo* 44! 88!)-DHhnt-DHhnt#'>Bbehl'>Bbehl"+$/299!<'2)Ivr)Ivr"+"+ 9 " % , 9=8U8U505-55#%%%' + *.55a8&!!''! 6 $ 1V-JK$$X.:>:N:ND((7  '')<mZ(XY  ''1 03JNow<!#(;(;x(GKKMw<!#(;(;x(GKKMw<!#(;(;x(GKKM997|a'GBK,?,?5,H7|a'GBK,?,?5,H7|a'GBK,?,?5,H  ''0  ''0  ''0  $DM 1_	 C
 MNs   w*w%w)
__name__
__module____qualname____doc__r   intr   r   r   __classcell__)r   s   @r   r   r      s>    HH H 	H
 &H]$r   r   )loggingnumpyrH   fusion_attentionr   r   onnxr   
onnx_modelr   	getLoggerr   r;   r    r   r   <module>r      s6   
   ;   			8	$k$/ k$r   