
    Bvhr                     z    d dl Z d dlZd dlmZmZ d dlmZmZ d dl	m
Z
  e j                  e      Z G d de      Zy)    N)AttentionMaskFusionAttention)TensorProtohelper)	OnnxModelc                   F     e Zd ZdZdedededef fdZd Zd Z	d	 Z
 xZS )
FusionBartAttentionz?
    Fuse Bart Attention subgraph into one Attention node.
    modelhidden_size	num_headsattention_maskc                 *    t         |   ||||       y )N)super__init__)selfr
   r   r   r   	__class__s        X/RAG/venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_bart_attention.pyr   zFusionBartAttention.__init__   s     	YG    c                    | j                   j                  |dgdg      }|y|d   }| j                   j                  |g dg d      }	| j                   j                  |g dg d      }
|	|
y|	\  }}}|
\  }}}|j                  d   |k7  s|j                  d   |k7  ry| j                   j                  |g dg d      }| j                   j                  |g dg d	      }||y|d
   j                  |j                  k7  s|d
   j                  |j                  k7  ry| j                   j                  |g dg d      }| j                   j                  |g dg d      }| j                   j                  |g dg d      }|||y|d
   }|d
   }|d
   }|j                  d   }|j                  d   |k7  s$|j                  d   |k7  s|j                  d   |k7  ryy)NConcat   Fr   	UnsqueezeGatherShaper   r   r   )r   r   r   )r   r   r   )r      r   )r   r   MulT)r
   match_parent_pathinputnameoutput)r   reshape_qkv_2reshape_qkv_1reshape_q_2reshape_k_2reshape_v_2
root_inputconcat_qkv_2_pathconcat_qkv_2reshape_qkv_2_path_1reshape_qkv_2_path_2_gather_1shape_1gather_2shape_2reshape_qkv_1_path_1reshape_qkv_1_path_2reshape_q_2_pathreshape_k_2_pathreshape_v_2_pathmul_qmul_kmul_vgather_1_outs                            r   check_runtime_shape_pathz,FusionBartAttention.check_runtime_shape_path   s    !JJ88
UVTWX$(+#zz;;LJjluv#zz;;LJjluv'+?+G38W38W==z)W]]1-=-K#zz;;MKlnwx#zz;;MKlnwx'+?+G#((HMM9=QRT=U=Z=Z^f^k^k=k::77Eceno::77Eceno::77Eceno#'7'?CSC[ $ $ $q);;q>\)U[[^|-Ku{{[\~amOmr   c                    | j                   j                  |g dg d      }|"|d   j                  d   |j                  d   k7  ry| j                   j                  |g dg d      }| j                   j                  |g dg d      }||y|d   }	|d   }
|	j                  d	   |
j                  d	   k7  ry|d   j                  d   |j                  d   k7  r#|d   j                  d   |j                  d   k7  ry|| j                   j                  |d
gd	g      }|y| j                   j                  |d   g dg d      }| j                   j                  |d   g dg d      }||y|\  }}}}|\  }}}|j                  d   |j                  d   k7  ry|d   j                  d   |j                  d   k7  r#|d   j                  d   |j                  d   k7  ryy)N)r   Slicer   	Transposer   r   r   r   r   r   F)r   PowCastDivr   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r>   )r>   r   r   r   )r   r   r   r   r   )r   r   r   T)r
   r    r!   r#   )r   r$   
matmul_qkvadd_qk	matmul_qkadd_qreshape_qkv_pathmatmul_qk_path_1matmul_qk_path_2mul_1mul_2add_qk_pathslice_q_path_1slice_q_path_2r.   unsqueeze_1unsqueeze_2s                    r   check_runtime_shape_path_openaiz3FusionBartAttention.check_runtime_shape_path_openaiM   s     ::77Dl
 #'7';'A'A!'D
HYHYZ[H\'\::77GI[
  ::77GI[
 #'7'? # #;;q>U[[^+B%%a(ELLO;@PQS@T@Z@Z[\@]afamamnoap@p **66vy1#NK"!ZZ99A I<N "ZZ99+a.JjluvN%.*@#1 A{Aq .KA  #{'8'8';;b!''*ell1o=.QSBTBZBZ[\B]afamamnoapBpr   c                    d}| j                   j                  |g dg d      }| j                   j                  |g dg d      }|
|\  }}}	}
}}n||}|\  }}}	}
}d}ny g }|j                  D ].  }||vr||d   j                  d   k(  r|j	                  |       0 t        |      dk7  ry |d   }	 ||   }|j                  d	k(  r| j                   j                  |      d   }|j                  D ];  }|s||   }|D cg c]  }|j                   }}|j                  d
      dk\  s9|} n | j                   j                         j                  D ch c]  }|j                   }}| j                   j                         j                  D ch c]  }|j                   }}| j                   j                  |g dg d      }| j                   j                  |g dg d      }| j                   j                  |g dg d      }| j                   j                  |dgdg      }| j                   j                  |g dg d      }d\  }}d\  }} ||\  }}!}"} }#|!j                  d   }n<||}|\  }!}"} }#| j                   j                  | ddg|"g      }$| j                   j                  | g d      }%|$%|$\  }&}'|'j                  d   |v r|'j                  d   }|%|%\  }(}&})|)j                  d   |v r|)j                  d   }| j                   j                  |(ddgddg      }*|*\  }&}+|+j                  d   }nk|+|\  }}(}!}"} }#|}|(j                  d   }|(j                  d   }n>||d   j                  d   |v r~|}|d   j                  d   }|d   j                  d   }||vrt        t        d | j                   j                         |               },t        |,      dk(  r|,d   j                  d   nd}n||d   j                  d   |v r~|}|d   j                  d   }|d   j                  d   }||vrjt        t        d | j                   j                         |               },t        |,      dk(  r|,d   j                  d   nd}nt        j!                  d       y ||v r|nd}||v r|nd}| j                   j                  |dd
gddg      }-| j                   j                  |g dg d       }.| j                   j                  |g d!g d"      }/d }0|-|-\  }&}1|-}2n|.|.\  }&}&}0}&}1|.}2n|/	|/\  }&}0}1|/}2ny | j                   j                  |1g d#g d$      }3| j                   j                  |1g d%g d&      }4d }5|3
|3\  }5}6}7}8}9}:n|4|4}3|3\  }8}6}7}9}:ny | j                   j                  |1g d'g d(      };| j                   j                  |1g d)g d      }<| j                   j                  |1g d*g d+      }=| j                   j                  |1g d,g d-      }>| j                   j                  |1ddgddg      }?| j                   j                  |1g d.g d+      }@d\  }A}Bd/\  }C}D}E|;|;\  }&}C}F}D}G}E|;}HnS|<|<\  }I}F}D}E|<}H|Ej                  d   }B| j                   j                  |Eddg|Dg      }$| j                   j                  |Eg d      }%|$%|$\  }&}J|Jj                  d   |v rJj                  d   }B|%|%\  }K}&}L|Lj                  d   |v rLj                  d   }B| j                   j                  Kddgddg      }*|*\  }&}M|Mj                  d   }Ans|=|=\  }&}C}F}D}E|=}H|Fj                  d   }BnV|>+|>\  }&}C}K}&}D}E|>}H|Kj                  d   }A|Kj                  d   }Bn)|?|?d   j                  d   |v r~|?}H|Hd   j                  d   }A|Hd   j                  d   }B|B|vrt        t        d0 | j                   j                         A               }Nt        |N      dk(  rNd   j                  d   nd}Bn@@d   j                  d   |v r~@}H|Hd   j                  d   }A|Hd   j                  d   }B|B|vrUt        t        d1 | j                   j                         A               }Nt        |N      dk(  rNd   j                  d   nd}Bny A|v rAnd}AB|v rBnd}BH|<|=|>fv r| j                   j#                  | j                  d         j$                  d   }Od2}P| j                   j#                  |P      }Q|QK| j'                  Pt(        j*                  Ogt-        j.                  d3g|Oz  t,        j0                  4      5       | j                   j3                  d	      }Rt5        j6                  d	PEj                  d   gDj                  g|R      }G|r!t9        A      s| j;                  |	||0|1|9      sy |s"t9        A      s| j=                  |	|5C||      sy t9        A      xr t9        |      xr Ed u xr d6t?               v}S|S xr: Ej                  d   |k(  xr& |:j                  d   |k(  xr #j                  d   |k(  }TS xrT |:j                  d   |k(  xr@ Ej                  d   #j                  d   k(  xr Ej                  d   |:j                  d   k7  }UTxr |2|-k(  }VTxr |2|.|/fv }W|sVnWxr t9        A      xr t9        |      }XUxr |2|-k(  }YSxr |2|-k(  }Zd g }\}[Wrm| j                   j                  |0d7gdg      }]| j                   j                  |0g d8g d      }^|^^d   j                  d   }[|^}\n]]d   j                  d   }[|]}\Vs	WsXsYsZr|	}_| jA                  |7      \  }`}a|`dk  sadk  sa`z  dk7  rt        j!                  d9       y d }bXsYsZr\| jB                  rM| jE                  |:YsXrEnAYsXr#n||9YsXrGnd YsXr| nd `a_j                  d   X|XrAndXr|ndB|X:      nd }bna| jB                  }cd| _!        Wr[r[nd}d| jG                  d |:E#|9G| `a|_j                  d   t        \      dkD  rd ndA|B|W;      }bc| _!        by | jH                  j	                  b       | jJ                  | jL                  |bj                  <   | jN                  jQ                  _|
|g       | jN                  jQ                  |2       XsYsZr|3d   j                  d
k(  r|3jS                          Hd   j                  d
k(  rHjS                          |d   j                  d
k(  r|jS                          | jT                  rjYsZrf|3d   j                  d	k(  r|3jS                          Hd   j                  d	k(  rHjS                          |d   j                  d	k(  r|jS                          | jN                  jQ                  |3       | jN                  jQ                  H       | jN                  jQ                  |       d| _+        y y c c}w c c}w c c}w )<NF)AddMatMulReshaper?   rW   rV   rD   )rU   rV   rW   r?   rV   )r   r   r   r   r   Tr   r   rU   rV   )rW   r?   rW   rU   rV   )r   r   r   r   N)r?   rW   rU   rV   )r   r   r   N)rW   r   r?   rW   rU   rV   )r   r   r   r   r   NrW   )r?   rW   rW   r?   r@   ) rX   )NNr?   )exclude)r   rW   r?   r   c                      | j                   dk(  S NIdentityop_typenodes    r   <lambda>z*FusionBartAttention.fuse.<locals>.<lambda>      
(Br   rX   c                      | j                   dk(  S r[   r]   r_   s    r   ra   z*FusionBartAttention.fuse.<locals>.<lambda>)  rb   r   z&fuse_attention: failed to match v pathSoftmax)rd   rW   rU   rW   rV   )r   r   r   r   r   )rd   rU   rV   r   )rW   r?   rW   r   rU   rV   )r   r   r   r   r   r   )r   r?   rW   rU   rV   )r   r   r   r   r   )r?   rW   r?   rW   rU   rV   )r   r   r   r   r   r   )r   r?   rW   rV   )r?   rW   r?   rW   rV   )r   r   r   r   r   )r?   rW   r   r?   rW   rV   )r   r   r   r   r   r   )r   r?   rW   rW   r?   )NNNc                      | j                   dk(  S r[   r]   r_   s    r   ra   z*FusionBartAttention.fuse.<locals>.<lambda>  rb   r   c                      | j                   dk(  S r[   r]   r_   s    r   ra   z*FusionBartAttention.fuse.<locals>.<lambda>  rb   r   
empty_biasg        )dtype)dimsvalsmatmul_vWhere)Expandr   r   rl   z9fuse_attention: failed to detect num_heads or hidden_size)q_matmulk_matmulv_matmulq_addk_addv_addr   r   r#   unidirectionalpast_kpast_v	present_k	present_v
packed_qkv)
mask_indexrn   ro   rp   rq   rr   rs   r   r   first_inputr#   
add_qk_strru   rv   rw   rx   causal),r
   r    r!   r#   appendlenr^   get_childrencountgraphr"   match_child_pathlistfilterinput_name_to_nodesloggerdebugget_initializerri   add_initializerr   FLOATnparrayfloat32create_node_namer   	make_nodeboolrS   r<   localsget_num_heads_and_hidden_sizeuse_multi_head_attentioncreate_multihead_attention_nodecreate_attention_nodenodes_to_addthis_graph_namenode_name_to_graph_namenodes_to_removeextendpop!disable_multi_head_attention_biasprune_graph)er   normalize_noder   output_name_to_nodemodel_impl_openai	qkv_nodesqkv_nodes_openaiadd_out
matmul_outr$   transpose_qkvr%   rE   other_inputsr!   r)   skip_layernormr#   childrenchildchildren_typesr`   graph_input_namesgraph_output_namesv_nodesv_nodes_openaiv_nodes_with_past_self_attnv_nodes_with_past_cross_attn#v_nodes_with_past_cross_attn_openairv   rx   r(   add_vtranspose_vreshape_v_1rk   reshape_pathconcat_pathr.   transpose_add_vconcat_vtranspose_concat_vconcat_nodestranspose_concat_v_inidentity_node_v
qk_nodes_1
qk_nodes_2qk_nodes_2_openairF   rG   qk_nodesq_nodesq_nodes_openair&   transpose_qreshape_q_1r8   rH   matmul_qk_nodes_with_biask_nodes_no_bias_openaik_nodes_no_bias#k_nodes_no_bias_with_past_self_attn$k_nodes_no_bias_with_past_cross_attn+k_nodes_no_bias_with_past_cross_attn_openairu   rw   r'   reshape_k_1matmul_ktranspose_k_1add_kk_nodesr9   transpose_matmul_kconcat_ktranspose_concat_ktranspose_concat_k_inidentity_node_kbias_dimempty_bias_nameempty_tensoradd_namethree_root_inputsone_root_inputtwo_root_inputsencoder_attentiondecoder_attentiondecoder_attention_with_pastdecoder_cross_attention!decoder_cross_attention_with_pastrz   
mask_nodesmask_nodes_bartmask_nodes_whisperattention_last_noder   r   new_node%use_multi_head_attention_ground_truthr|   se                                                                                                        r   fusezFusionBartAttention.fuse   s   ! JJ00J
	
  ::77?

   )(I  !%#))E//	!++A..& * |!!!_
	 -Z8 !!U*!ZZ44^DQGN$++F*62H9ABemmNB##H-2#
 , 48::3C3C3E3K3KL3K4TYY3KL48JJ4D4D4F4M4MN4MDdii4MN**..@

 555

 '+jj&B&BJ!	'
# (,zz'C'CKC	(
$ /3jj.J.J</
+
 #	'UGND[+{E8#**1-I'$G:A7[+uh  ::66K($ 7 L **55e=_`K''3$O"))!,0BB / 6 6q 9I&4?110%,,Q/3EE 2 9 9! <I#zz;;HyR]F^abde`fg+7((.44Q7(4QlN[(KeX1G^^A&F *I(49UVX9Y9_9_`a9bfw9w2GR[&&q)F**1-I 22"&BDJJDbDbDdekDlm# =@<PTU<UOA.55a8[]	/;3B7==a@DUU9GR[&&q)F**1-I 22"&BDJJDbDbDdekDlm# =@<PTU<UOA.55a8[]	LLAB!%66B!*.@!@Ib	ZZ11*y(>SVWYZU[\
ZZ11JO

 !JJ88Eaclm!%LAy!H#)3&Aq&!Y!H*#4 Avy(H**..G

 55<

 LSIKk5%'$G?F<E;UH JJ88M

 "&!=!=5"

 **66F

 /3jj.J.JP	/
+ 04zz/K/K)$F	0
, 7;jj6R6RC	7
3 #	-=*[((J[GA{M;x'G#/:P7E=+x,G *I  ::66K($ 7 L **55h@bcK'*6'&%,,Q/3EE 2 9 9! <I&4?110%,,Q/3EE 2 9 9! <I#zz;;HyR]F^abde`fg+7((.44Q7(CR@A{M;%G%,,Q/I0<Ad>A{Hah9G^^A&F *I0<4R8>>qAEVV:GR[&&q)F**1-I 22"&BDJJDbDbDdekDlm# =@<PTU<UOA.55a8[]	7C;B?EEaHL]]AGR[&&q)F**1-I 22"&BDJJDbDbDdekDlm# =@<PTU<UOA.55a8[]	!%66B!*.@!@Ib	-@cddzz11%++a.AFFqIH*O::55oFL#$$#%%"3%("2"**E	 %  zz2259H$$U_hooa>P,QT_TdTdSegopE L88 !L11  LmT&\mh$>NmS]ekemSm!! 0q!Z/0q!Z/0 q!Z/	 	 "! 7q!Z/7q!X^^A%667 q!X^^A%66	 	 +Ex:/E*ZxJHY;Z/Z&7=NqTXY_T`qeijpeq 	$ #2"Lh*6L,=,X(jBX) "&rJ
"jj::	O
 "&!=!=="
 "-/299"=
/
 ,,Q/66r:
,
  *&0"/%)%G%G%T"I{A~!1kI6MRS5SXYH*.EIj, 44# 88!)-DHcio-DHcio#'>B]ecg'>B]ecg"+$/299!<'B)Dv")Dv""+"+#> 9 $ ' . 9=8U8U505-+<ZQS
55#%%%' + *.55a8 #J! 3!!'',' 6 * 1V-$$X.:>:N:ND((7  '')<mZ(XY  ''1 +.EIj2;&&(2KKM2;&&(2KKM2;&&(2KKM99+/Pr{**e3r{**e3r{**e3  ''0  ''0  ''0  $D} 1E C
 MNs   yy#y()__name__
__module____qualname____doc__r   intr   r   r<   rS   r   __classcell__)r   s   @r   r	   r	      sJ    HH H 	H
 &H.`0dE$r   r	   )loggingnumpyr   fusion_attentionr   r   onnxr   r   
onnx_modelr   	getLoggerr   r   r	    r   r   <module>r      s6   
   ; $  			8	$u	$/ u	$r   