
    \jd                     x    d dl Z d dlZd dlmZmZ  G d de j                        Z G d dej                        Z	y)    N)common_spec
model_specc                       e Zd ZdZdZdZdZy)RotaryScalingTypezRoPE scaling type.r         N)__name__
__module____qualname____doc__LinearSuLlama3     S/root/.hermes/venv/lib/python3.12/site-packages/ctranslate2/specs/attention_spec.pyr   r   	   s    F	
BFr   r   c                   <    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZy)MultiHeadAttentionSpecNc                    t         j                  | _        |rt        j                  |      | _        t        |rdnd      D cg c]  }t        j                          c}| _        |r6t        j                  |      | _	        t        j                  |      | _
        |rt        j                  d      | _        |rd | _        d | _        |rd | _        d | _        |rd | _        d | _        d | _        |dk7  r)t'        j(                  d      j+                  |      | _        |dk7  r)t'        j(                  d      j+                  |      | _        |t'        j(                  d      j+                  |      | _        || _        t'        j(                  d      j+                  |
      | _        |)t'        j(                  d      j+                  |      | _        |t8        j:                  u r*t'        j(                  d      j+                  |	      | _        nA|t8        j>                  u rd | _         d | _!        n |t8        jD                  u rd | _#        d | _$        |)t'        j(                  d      j+                  |      | _%        |)t'        j(                  d      j+                  |      | _&        |)t'        j(                  d      j+                  |      | _'        |r!d| _(        t        j                         | _)        y y c c}w )	N)rms_normr      Tr   int32float32int8)*r   OPTIONALqueries_scaler   LayerNormSpec
layer_normrange
LinearSpeclinearq_normk_normv_normrelative_position_keysrelative_position_valuesrelative_attention_biasrelative_attention_max_distance!relative_asymmetric_position_keysrelative_left_max_positionrelative_right_max_positionnpdtypetype original_max_position_embeddingsmax_position_embeddings
rotary_dimrotary_interleaverotary_baserotary_scaling_typer   r   rotary_scaling_factorr   rotary_scaling_long_factorrotary_scaling_short_factorr   rotary_low_freq_factorrotary_high_freq_factornum_heads_kvhead_dimsliding_windowmerged_encoder_attention	memory_kv)selfself_attentionrelative_positionrelative_asymmetric_positionr'   r   r1   r2   r4   r5   r3   r/   r0   r:   r;   r<   qk_normqk_norm_rmsr$   has_normr=   _s                         r   __init__zMultiHeadAttentionSpec.__init__   s   . (00)77JDO.3AQ.O
)*K""$
 %33[IDK%33[IDK%33TBDK*.D',0D)"+/D(37D0'59D2.2D+/3D,+q046HHW4E4J4J05D1 #a'+-88G+<+A+A',D( ! hhw/44Z@DO%6D"!xx	277DD".+-88F+;+@+@AT+U("&7&>&>>-/XXi-@-E-E).* %(9(<(<<26/370$(9(@(@@.2+/3,# " 1 6 6| DDHHW-228<DM%"$((7"3"8"8"HD#,0D)(335DN $y
s   K8)FFFFFNTNr   i'  r   r   NNNFTFTF)r	   r
   r   rG   r   r   r   r   r      sF     %* % )* !!&+Y6r   r   )
enumnumpyr,   ctranslate2.specsr   r   IntEnumr   	LayerSpecr   r   r   r   <module>rM      s2      5 Z6Z11 Z6r   