
    \jB^                     v    d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZ d dlmZ  ee      Z G d de      Zy)	    )	getLoggerN)Fusion)FusionUtils)	NodeProtohelpernumpy_helper)	OnnxModelc                       e Zd ZdZdef fdZdededz  fdZdede	dz  fdZ
d	ed
ede	fdZdede	fdZd	ede	fdZdedede	fdZdededefdZdedefdZdedefdZ	 ddedededede	dedz  defdZd Z xZS ) FusionMultiHeadAttentionDiTu  
    Fuse MultiHeadAttention for Diffusion Transformer (DiT) models like F5-TTS.

    Recognizes attention patterns where Q, K, V are pre-computed (e.g., after RoPE)
    and K is pre-transposed, with optional Cast nodes for mixed-precision (FP16) inference
    and a custom scalar scale factor before Softmax.

    Supported patterns (anchored at Softmax):

        MatMul(Q, K^T) → [Cast(FP16→FP32)] → Mul(scale) → Softmax → [Cast(FP32→FP16)] → MatMul(attn, V)
            → Transpose(perm=0,2,1,3) → Reshape → output

    Where:
        - Q is in BNSH format (post-RoPE or post-projection)
        - K is pre-transposed to BNHS format (via Transpose(perm=0,1,3,2) or natively)
        - V is in BNSH format
        - Scale is an arbitrary scalar constant (e.g., 100.0 for DiT, or 1/sqrt(d_k))
        - Cast nodes are optional (present in FP16 models for FP32 Softmax stability)
    modelc                 ,    t         |   |ddg       y )NMultiHeadAttentionSoftmax)fused_op_typesearch_op_types)super__init__)selfr   	__class__s     Z/root/.hermes/venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_mha_dit.pyr   z$FusionMultiHeadAttentionDiT.__init__%   s    .BU^T_`    mul_nodereturnNc                 J   t        d      D ]  }| j                  j                  |j                  |         }|.t	        |t
        j                        r+|j                  dk(  sXt        |j                               c S t	        |t        t        f      st        |      c S  y)zExtract the scalar scale constant from a Mul node.

        The scale can be in either input[0] or input[1].

        Returns:
            float: the scale value, or None if not found.
           N   )ranger   get_constant_valueinput
isinstancenpndarraysizefloatitemint)r   r   ivalues       r   get_scale_from_mulz.FusionMultiHeadAttentionDiT.get_scale_from_mul(   s     q 	(AJJ11(..2CDE eRZZ0zzQ$UZZ\22U|4 <'	( r   c                 @   ddg}t        d      D ]v  }| j                  j                  |j                  |         }|.t	        |t
        j                        r|j                  dk(  ||<   [t	        |t        t        f      srd||<   x |d   r|d   sy|d   r|d   syy)zDetermine which input of Mul is the data (non-constant) input.

        Returns:
            int: the input index (0 or 1) of the data input, or None if ambiguous.
        Fr   Nr   Tr   )
r   r   r   r   r    r!   r"   r#   r&   r$   )r   r   is_scalar_constantr'   r(   s        r   get_data_input_of_mulz1FusionMultiHeadAttentionDiT.get_data_input_of_mul:   s     $U^q 	1AJJ11(..2CDE eRZZ0,1JJ!O&q)U|4,0&q)	1 a );A)>a );A)>r   tensor_nameoutput_name_to_nodec                 f   ||vry||   }d}d}|||k  r|j                   dk(  rZt        j                  |d      }|g dk(  r=|j                  d   |v r*||j                  d      }| j	                  |      }|dkD  r|S 	 y|j                  d   |v r||j                  d      }n	 y|dz  }|||k  ry)u  Detect num_heads by walking upstream from a BNSH tensor looking for a Reshape node.

        Typical upstream patterns:
            Reshape(shape=[B, S, N, H]) → Transpose(perm=0,2,1,3) → ... → tensor_BNSH
            Reshape(shape=Concat(..., N, H)) → Transpose(perm=0,2,1,3) → ... → tensor_BNSH

        Returns:
            int: number of heads, or 0 if not detected.
        r   
   	Transposepermr   r   r      r   )op_typer	   get_node_attributer   _get_num_heads_from_reshape)	r   r-   r.   currentdepth	max_depthr2   parent	num_headss	            r   detect_num_headsz,FusionMultiHeadAttentionDiT.detect_num_headsO   s     11 &k2	!ei&7+- 33GVD<'}}Q'+>>!4W]]15E!F$($D$DV$L	$q=#,,  }}Q#66-gmmA.>?  QJE% !ei&7( r   nodec                    |j                   dk7  ryt        |j                        dk\  ra| j                  j	                  |j                  d         }|7t        |t        j                        r|j                  dk(  rt        |d         S t        |j                        dk\  r|j                  d   | j                  j                  d      D ch c]  }|j                  d    c}v r| j                  j                  d      D cg c]$  }|j                  d   |j                  d   k(  s#|& }}|rt        |d   j                        dk(  ro| j                  j	                  |d   j                  d         }|Bt        |t        j                        r(|j                  dk(  rt        |j                               S yc c}w c c}w )zExtract num_heads from a Reshape node's shape parameter.

        Handles:
            - Static shape constant: [B, S, num_heads, head_dim]
            - Concat-based shape: Concat([B_dim], [S_dim], [num_heads], [head_dim])
        Reshaper   r   r      Concat)r5   lenr   r   r   r    r!   r"   r#   r&   get_nodes_by_op_typeoutputr%   )r   r>   shape_valuenconcat_nodesr(   s         r   r7   z7FusionMultiHeadAttentionDiT._get_num_heads_from_reshapew   sp    <<9$ tzz?a**77

1FK&:k2::+NS^ScScghSh;q>** tzz?aDJJqM4::KjKjksKt5uaahhqk5u$u'+zz'F'Fx'Pq!TUT\T\]^T_cgcmcmnocpTpAqLqLO$9$9 :a ?

55l1o6K6KA6NO$!%4q"5::<00 6vqs   F=$G*Gc                    | j                   j                   j                  j                  D ]z  }|j                  |k(  s|j                  j
                  j                  }|s6t        |j                        dk(  sO|j                  d   }|j                  dkD  sn|j                  c S  | j                   j                   j                  j                  D ]z  }|j                  |k(  s|j                  j
                  j                  }|s6t        |j                        dk(  sO|j                  d   }|j                  dkD  sn|j                  c S  y)zTry to detect num_heads from a BNSH tensor's shape in graph inputs or value_info.

        For BNSH tensors, the N dimension (index 1) is num_heads.
        rA   r   r   )r   graphr   nametypetensor_typeshaperC   dim	dim_value
value_info)r   r-   inprN   dim_nvis         r   !detect_num_heads_from_input_shapez=FusionMultiHeadAttentionDiT.detect_num_heads_from_input_shape   s    ::##))// 	/Cxx;&,,22S^q0!IIaLE*$.	/ **""((33 	/Bww+%++11S^q0!IIaLE*$.	/ r   reshape_outtranspose_outc                 >    | j                  |j                  d         S )zTry to detect num_heads from the output Transpose's input shape.

        The Transpose converts BNSH -> BSNH. The N dimension gives us num_heads.
        r   )rU   r   )r   rV   rW   s      r   detect_num_heads_from_outputz8FusionMultiHeadAttentionDiT.detect_num_heads_from_output   s     
 55m6I6I!6LMMr   
input_nameoutput_namec                    d}| j                   j                  |      }|Tt        j                  t	        j
                  g dd      |      }| j                   j                  || j                         t        j                  d||g|g| j                   j                  d            }| j                  j                  |       | j                  | j                  |j                  <   |S )a  Add a Reshape node to convert 4D BxSxNxH to 3D BxSxD.

        Args:
            input_name: input name for the 4D tensor of shape BxSxNxH.
            output_name: output name for the 3D tensor of shape BxSxD.

        Returns:
            str: the output name.
        bsnh_to_bsd_reshape_dims)r   r   int64)dtype)rK   r@   inputsoutputsrK   )r   get_initializerr   
from_arrayr!   arrayadd_initializerthis_graph_namer   	make_nodecreate_node_namenodes_to_addappendnode_name_to_graph_namerK   )r   rZ   r[   new_dims_namenew_dimsreshape_nodes         r   reshape_to_3dz)FusionMultiHeadAttentionDiT.reshape_to_3d   s     3::--m<#..rxx
'/RYfgHJJ&&x1E1EF''. M,,Y7	
 	  .:>:N:N$$\%6%67r   c           	          |dz   }t        j                  d|g|g| j                  j                  dd      g d      }| j                  j                  |       | j                  | j                  |j                  <   |S )z4Add a Transpose node to convert BNSH to BSNH format._BSNHr1   Transpose_BNSH_to_BSNHname_prefixr3   rK   r2   	r   ri   r   rj   rk   rl   rh   rm   rK   r   rZ   r[   transpose_nodes       r   transpose_bnsh_to_bsnhz2FusionMultiHeadAttentionDiT.transpose_bnsh_to_bsnh   |     7*))LM,,[F^,_
 	  0<@<P<P$$^%8%89r   c           	          |dz   }t        j                  d|g|g| j                  j                  dd      g d      }| j                  j                  |       | j                  | j                  |j                  <   |S )z4Add a Transpose node to convert BNHS to BNSH format._BNSHr1   Transpose_BNHS_to_BNSHru   r   r   r4   r   rw   rx   ry   s       r   transpose_bnhs_to_bnshz2FusionMultiHeadAttentionDiT.transpose_bnhs_to_bnsh   r|   r   qkvrE   r<   scalec                 Z   |dkD  sJ |||g}|g}t        j                  d||| j                  j                  d            }	d|	_        |	j
                  j                  t        j                  d|      g       |0|	j
                  j                  t        j                  d|      g       |	S )a  Create a MultiHeadAttention node.

        Args:
            q: name of query input (BSD format, 3D).
            k: name of key input (BNSH format, 4D).
            v: name of value input (BNSH format, 4D).
            output: output name of MHA.
            num_heads: number of attention heads.
            scale: optional custom scale factor for attention logits.

        Returns:
            NodeProto: the node created.
        r   r   ra   zcom.microsoftr<   r   )r   ri   r   rj   domain	attributeextendmake_attribute)
r   r   r   r   rE   r<   r   
mha_inputsmha_outputsmha_nodes
             r   create_multihead_attention_nodez;FusionMultiHeadAttentionDiT.create_multihead_attention_node   s    , 1}}AY
h## ,,-AB	
 *!!6#8#8i#P"QR%%v'<'<We'L&MNr   c                    |j                   dk(  sJ |}| j                  j                  |j                  d         ry t	        j
                  |d      }||dvry || j                  j                         dk  ry d }| j                  j                  |g dg d|      }||\  }}}	}
n+| j                  j                  |g dg d	|      }|y |\  }}	}
t        j                  |	d
g d      sy d }| j                  j                  |g dg d      }||\  }}}n)| j                  j                  |ddgdd g      }|y |\  }}| j                  |      }|t        j                  d       y | j                  |      }|y |r|j                  d   n|j                  d   }|j                  |   |k7  r]|r/| j                  j                  |g ddd|z
  dg      }|y |\  }}}n,| j                  j                  |ddgdd|z
  g      }|y |\  }}|j                  d   }|j                  d   }|j                  d   }d}|)||v r%||   }|j                   dk(  r|j                  d   }d}| j                  j!                  |dd|      }|)t        j                  |d
g d      r|j                  d   }n| j#                  |      }d}|}||v r%||   }|j                   dk(  r|j                  d   }d}| j                  j%                  |      }| j                  j%                  |      }| j                  j%                  |      }||||k7  rt        j                  d||       y ||||k7  rt        j                  d||       y |||s||t        j                  d||       y |r!||||k7  rt        j                  d||       y | j'                  ||      } | dk  r| j'                  ||      } | dk  r| j)                  |      } | dk  r| j)                  |      } | dk  r| j+                  |
|	      } | dk  rt        j                  d       y | j-                  |      }!| j/                  |!|!dz         }"|j                  d   |j                  d   |j                  d   g}#||#j1                  |j                  d          ||#j1                  |j                  d          |#D ]0  }$|$|v st3        ||$         dkD  st        j                  d|$        y  | j5                  |"|||
j                  d   | |      }%| j6                  j1                  |%       | j8                  | j:                  |%j<                  <   ||	|
g}&||&j1                  |       | j                  j?                  |&|
j                  d   g||      st        j                  d        y | j@                  jC                  |&       d| _"        y )!Nr   r   axis)r^   r4      )CastMatMulr1   r@   )r   r   r   r   r   )r   r1   r@   )r   r   r   r2   r3   )Mulr   r   )r   Nr   r   r   z9fuse_dit_attention: failed to extract scale from Mul noder   Fr   Tr1   )input_indexr.   r   zBfuse_dit_attention: Q/V element type mismatch (%s vs %s), skippingzBfuse_dit_attention: Q/K element type mismatch (%s vs %s), skippingzlfuse_dit_attention: cast nodes present, V not traced through Cast, types unverifiable (q=%s, v=%s), skippingzLfuse_dit_attention: K Cast source dtype mismatch with Q (%s vs %s), skippingz.fuse_dit_attention: failed to detect num_heads_BSDzDfuse_dit_attention: intermediate %s has multiple consumers, skipping)r   r   r   rE   r<   r   zFfuse_dit_attention: downstream nodes have external consumers, skipping)#r5   r   find_graph_outputrE   r	   r6   get_opset_versionmatch_child_pathr   check_node_attributematch_parent_pathr)   loggerdebugr,   r   match_parentr   	get_dtyper=   rU   rY   r{   rq   rl   rC   r   rk   rh   rm   rK   is_safe_to_fuse_nodesnodes_to_remover   prune_graph)'r   r>   input_name_to_nodesr.   softmaxr   cast_after_softmaxchild_nodes	matmul_svrW   rV   cast_before_softmaxparent_nodes	mul_scale	matmul_qkr   data_input_idxexpected_data_sourceparent_nodes_altq_bnshk_transposed_inputv_bnshv_traced_through_cast
v_producerk_transpose_nodek_bnshk_traced_through_castk_bnsh_for_dtype
k_producerq_dtypek_dtypev_dtyper<   q_bsnhq_bsdintermediate_outputsr-   r   r   s'                                          r   fusez FusionMultiHeadAttentionDiT.fuse  s   ||y((( ::''q(9: ++GV<G 3<DJJ88:R?
 " jj116,	
 "HSE	=+ **552(#	K "4?1I}k //v|T
 # zz33%

 #8D5I*I  ::77!D	L
 ##/ Iy
 ''	2=LLTU 33I>! AT299!<YbYiYijkYl??>*.BB"#'::#?#?-N*A.$ 
 $+<L9	.	#'::#?#?H%N*+$ 
 $+'7$	9
 #&__Q/# !&)f8K.K,V4J!!V+#))!,(,%  ::22{GZ 3 
 'K,L,LM]_egs,t%++A.F 001CDF !&!22,-=>J!!V+#-#3#3A#6 (,% **&&v.**&&'78**&&v.7#67g;MLL]_fhop7#67g;MLL]_fhop  +/A/MWl'/@	  !W%8W=PU\`gUgLL^
 
 ))&2EF	>--f6IJI>>>vFI>>>vFI>99+}UI>LLIJ
 ,,V4""66F?; !* 0 0 3Y5E5Ea5H'..YZJ[\* ''(;(B(B1(EF) ''(:(A(A!(DE/ 	K11c:Mk:Z6[^_6_cepq	 77%%a( 8 
 	  *6:6J6J$$X]]3 %m[A)""#56
 zz//k00346IK^
 LLab##O4  r   )N)__name__
__module____qualname____doc__r	   r   r   r$   r)   r&   r,   strdictr=   r7   rU   rY   rq   r{   r   r   r   __classcell__)r   s   @r   r   r      s>   (ai a9  $i C$J *&C &d &s &P	 c 6S S 0N	 NR[ N`c N # # 2    * #(( ( 	(
 ( ( t|( 
(TE r   r   )loggingr   numpyr!   fusion_baser   fusion_utilsr   onnxr   r   r   
onnx_modelr	   r   r   r    r   r   <module>r      s4   
    $ 0 0  	8	E & E r   