BvhSddlZddlZddlZddlmZddlmZddlZddl Z ddl Z ddl m Z ddl mZddl mZmZddlmZddlmZdd lmZdd lmZmZmZmZdd lmZej<eZ Gd d e jBjDZ#y)N)chain)Path)convert_float_to_float16)RepeatedCompositeFieldContainer) ModelProtoValueInfoProto) OnnxModel)PastKeyValuesHelper) WhisperConfig)convert_inputs_for_ortget_model_dynamic_axesget_sample_decoder_inputsgroup_past_key_values)InferenceSessioncbeZdZdZd%dedej jdede ffd Z d&dejd ejdzd e e ejdzfd Z d&dejd ejdzd e e ejdzfd Z d&dejd ejdzd e e ejdzfd ZdZdZdZd%de de de fdZd'dede de fdZd%dede fdZdefdZdede fdZ d(dedede d e de de d!e d"e fd#Zdedede de fd$ZxZS))WhisperDecoderz/A Whisper decoder with optional past key valuesconfigmodel model_implno_beam_search_opct|||_|j|_||_||_|dk(rdn|j j|_|dk(rdn |j|_|dk(r|nd|_|jj|_ |jj|_ |jj|jz|_ yNopenai)super__init__rdevicerrrdecoderproj_outmax_source_positionsdecoder_attention_heads num_headsd_model head_size)selfrrrr __class__s a/RAG/venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_decoder.pyrzWhisperDecoder.__init__%s  ll $!2)X5t5;;;N;N *h 6ENN (H4U$ $(KK$D$D!<<,,>Ndecoder_input_idsencoder_hidden_statespast_key_valuesc|j|||d}|j|j}|j}|||fSt j |\}}||fS)NT)r) input_idsr* use_cache)rrlast_hidden_stater*r group_by_self_and_cross) r$r(r)r*outputslogitspresent_key_values present_self present_crosss r& hf_forwardzWhisperDecoder.hf_forward4s| ,,"7'+   w889$44  "-- - ':&Q&QRd&e# m|##r'ci}|[t|\}}|Dcgc]}|jdd}}|Dcgc]#}|j|jdddz%}}|Dcgc]}|jdd}}|Dcgc]#}|j|jdddz%}}t |j j jD]\}} |d|z|| jj<|d|zdz|| jj<|d|z|| jj<|d|zdz|| jj<|j j\} } |j j |||} ||j j jD]} tj|| jj| | jjgdj| | jj<tj|| jj| | jjgdj| | jj<gg}} |j j jD]} | j!| | jj| j!| | jj|V|j!| | jj|j!| | jj| Dcgc]?}|j|jddd|j"fzjddA} }|Dcgc]?}|j|jddd|j"fzjddA}}| D]}|j%|)t'j(| |zt+| dz}| |fS| | fScc}wcc}wcc}wcc}wcc}wcc}w)N))xxakv_cache)dimr9)r transposereshapeshape enumeraterrblocksattnkeyvalue cross_attninstall_kv_cache_hookstorchcatdetachappendr#remover group_by_layerlen)r$r(r)r* past_kv_cacheself_attn_kv_cachescross_attn_kv_cachespast_kvidxblockr<hooksr1r3r4 present_kvhookr2s r& oai_forwardzWhisperDecoder.oai_forwardPs,   &8Mo8^ 5 !5J]"^J]w7#4#4Q#:J] "^]p"q]pRY7??7==!3Du3L#M]p "qK_#`K_G$5$5a$;K_ #`^r#s^rSZGOOGMM"14E4M$N^r #s' (:(:(A(AB U0CAG0L ejjnn-2Ea#gPQk2R ejj../6J1s76S e..2238LQQTWWX[8Y e..445 C**;;=%##&7>2HUZZ^^4LMST,&((.3YY"5::#3#34huzz?O?O6PQWX.&())* 3')"m ZZ''..E    8 9   )9)9 : ;&$$Xe.>.>.B.B%CD$$Xe.>.>.D.D%EF/+ *    z//3r4>>6JJ K U UVWYZ [*  , +    z//3r4>>6JJ K U UVWYZ [+  D KKM  "!4!C!C},c,.?1.D" -- -|##]#_"q#`#sl  s$Q(Q%Q (Q+AQ5AQcl|jdk(r|j|||S|j|||Sr)rrXr5)r$r(r)r*s r&forwardzWhisperDecoder.forwards< ??h &##$57Lo^ ^02GYYr'c |jrddg}|Sddgttjdt |j j D}|S)Nr,r)c3@K|]}d|d|d|d|fyw)past_key_self_past_value_self_past_key_cross_past_value_cross_N.0is r& z-WhisperDecoder.input_names..sC(!EA*!-1A!/EYZX[G\`qrsqt^uv!E first_passlistr from_iterablerangernum_hidden_layers)r$ input_namess r&rmzWhisperDecoder.input_namessl ??&(?@K' ''(!&t{{'D'D!E( Kr'c 6|jrGdgttjdt |j j D}|Sdgttjdt |j j D}|S)Nr1c3@K|]}d|d|d|d|fyw)present_key_self_present_value_self_present_key_cross_present_value_cross_Nrarbs r&rez.WhisperDecoder.output_names..sH("FA 0s31!50421#6  "Frfc30K|]}d|d|fyw)rprqNrarbs r&rez.WhisperDecoder.output_names..s,(!EA-QC04Gs2KL!Esrg)r$ output_namess r&ruzWhisperDecoder.output_namess ?? ''("'t{{'D'D!E(  L0''(!&t{{'D'D!E(Lr'c`t|j||}d|vr|js|dd=|S)Nr,r7)r rr)r$rmru dynamic_axess r&rwzWhisperDecoder.dynamic_axess7-dkk; U , &t/E/E[)!,r'use_fp16_inputsuse_int32_inputs return_dictc t|j|jd|jrdnd|jrdnd||}|r|jr|d=|S|jr |d|dfS|d|d|dfS) Nr8rr7) batch_sizepast_sequence_lengthsequence_lengthuse_fp16 use_int32r*r(r))rrrrh)r$rxryrzinputss r&rzWhisperDecoder.inputss* KK KK'+!A"&//Qq$&  ,-M ??*+./  & ' * + $ %  r'iois_cross is_outputcJ|jjjjd}d|jvr!|j |j |_|jjjjd}d|jvr/|j |r|j|_n |rdnd|_|jjjjd}d|jvr!|j |j|_|S)Nr7_dim_r8total_sequence_lengthr~) type tensor_typer@r= dim_paramClearr! dim_valuerr#)r$rrrr!rr#s r&fix_key_value_cache_dimsz'WhisperDecoder.fix_key_value_cache_dimssGG''--11!4 i)) ) OO "&..I ''--3377: o// /  ! ! #,0,E,E)GP,CVl)GG''--11!4 i)) ) OO "&..I  r'io_listcg}g}g}|D]}d|jvr d|jvr|j|1d|jvrD|j|d|}|jr|j|q|j||j|d|}|jr|j||j||js|||zz }|S)Npastpresentr$F)rrT)namerKrr)r$rr reordered_iorPrQrnew_ios r&fix_iozWhisperDecoder.fix_ios  !BRWW$"'')A##B'277"66rEU^6_)) ''/'..v666rDT]6^)) ''/(//7!$%% /2FF FLr'c|j|jjd}t|jjdkDrG|jjj t|jjdkDrG|jjj ||j|jj d}t|jj dkDrG|jj j t|jj dkDrG|jj j ||S)NF)rrT)rgraphinputrNpopextendoutput)r$rreordered_inputsreordered_outputss r&fix_inputs_and_outputsz%WhisperDecoder.fix_inputs_and_outputs1s ;;u{{'8'8E;J%++##$q( KK   ! ! #%++##$q(   !12 KK (:(:dKK%++$$%) KK   " " $%++$$%) !!"34 r'c>|jdk(r |r t|}|Sr)rr)r$rrxs r&fix_layernorm_weightsz$WhisperDecoder.fix_layernorm_weights?s! ??h &?-U3E r'onnx_model_pathproviderverboseuse_external_data_formatuse_encoder_hidden_statesuse_kv_cache_inputsc R|xr| |_| xr||_|js|jsJd|j||} |j} |j } |j | | } t |jjddtj5} tjj| d}t |jjdd|r|n|}tjj!|| |d| | | dd| tj"||}|j%|}|j'||}t)j*|||d d d d |j-||||y #1swYxYw) alExport decoder to ONNX Args: onnx_model_path (str): path to save ONNX model provider (str): provider to use for verifying parity on ONNX model verbose (bool, optional): print verbose information. Defaults to True. use_external_data_format (bool, optional): use external data format or not. Defaults to False. use_fp16_inputs (bool, optional): use float16 inputs for the KV caches. Defaults to False. use_int32_inputs (bool, optional): use int32 inputs for the decoder_input_ids. Defaults to True. use_encoder_hidden_states (bool, optional): use encoder_hidden_states as model input for decoder-init/decoder-without-past models. Defaults to False. use_kv_cache_inputs (bool, optional): use KV caches as model inputs for decoder-with-past models. Defaults to True. zVOnly one of `use_encoder_hidden_states` and `use_kv_cache_inputs` can be true at once.)rxryT)parentsexist_okz decoder.onnx) argsf export_paramsrmrurw opset_versiondo_constant_foldingr)load_external_data)save_as_external_dataall_tensors_to_one_fileN)rh later_passrrmrurwrparentmkdirtempfileTemporaryDirectoryospathjoinrHonnxexport load_modelrrr save verify_onnx)r$rrrrrxryrrrrmrurw tmp_dir_nametemp_onnx_model_pathout_pathrs r& export_onnxzWhisperDecoder.export_onnxKsH4O(,  ++8 (OEUV9+ *s 7C FF&c|j||d}g}|jr|jd i|}|j|dj j j |dD]D}|D]=} |j| j j j ?Fn|jd i|}|j|dj j j |dD]=} |j| j j j ?t||g} | jdt|| } t|jD]g\} }tj|| | | z }tjd|dtjd tj |iy#YyxYw) awVerify ONNX model outputs and PyTorch model outputs match Args: onnx_model_path (str): path to save ONNX model provider (str): execution provider for ONNX model use_fp16_inputs (bool, optional): use float16 inputs for the KV caches use_int32_inputs (bool, optional): use int32 inputs for the decoder_input_ids T)rxryrzrr7) providersNz Comparing z...z Max diff: ra)rrhrZrKrJcpunumpyrrunr rArunpabsloggerwarningmax)r$rrrxryr pt_outputsoutpresent_key_value_layerpresent_key_valuepresent_self_key_valuesess ort_outputsrd output_namediffs r&rzWhisperDecoder.verify_onnxs8_O_mqr ??$,,((C   c!fmmo11399; <+.q6')@%%%&7&>&>&@&D&D&F&L&L&NO*A,2$,,((C   c!fmmo11399; <*-a&&!!"8"?"?"A"E"E"G"M"M"OP+1 H:Fhht%;FD%IJ  "+D,=,=,?"@;vvjmk!n<=K=<=BFF4L>:;#A  s BHH)F)NN)FF)TFFTFT)__name__ __module__ __qualname____doc__r rHnnModulestrboolrTensorritupler5rXrZrmrurwrrrrrrrrrr __classcell__)r%s@r&rr"s29 ?} ?UXX__ ?RU ?jn ?$6:<@ $ <<$ %||d2$eELL12T9 $>6:<@ X$ <<X$ %||d2X$eELL12T9 X$z6:<@ Z <<Z %||d2ZeELL12T9 Z 8 d d QU 4>T^b(=$8 J  :   ). %!%*/$(OWOWOW OW #' OW  OWOW$(OW"OWb555 5  5r'r)$loggingrr itertoolsrpathlibrrrrrHfloat16r#google.protobuf.internal.containersrrr onnx_modelr past_helperr transformersr whisper_inputsr r rr onnxruntimer getLoggerrrrrrrar'r&rsh  ,O+ +&)   8 $oUXX__or'