BvhlddlZddlZddlZddlZddlZddlZddlZddlmZddl Z ddl Z ddl Z ddl m Z ddlmZddlmZddlmZddlmZddlmZdd lmZdd lmZmZmZmZej>e Z!gd Z"e jFd e jHd e jJdiZ&GddeZ'GddeZ(GddeZ)GddeZ*GddeZ+e*ddfe+ddfe)ddfdZ,GddZ-Gd d!Z.y)"N)Path) Precision)float_to_float16_max_diff) FusionOptions)IOBindingHelper) OnnxModel)optimize_model)torch_onnx_export) GPT2ConfigGPT2LMHeadModel GPT2Model TFGPT2Model) distilgpt2gpt2z gpt2-mediumz gpt2-largezgpt2-xlMb@?g?g@c,eZdZdZfdZfdZxZS)GPT2ModelNoPastState2Here we wrap a class to disable past state output.c$t||yNsuper__init__selfconfig __class__s Z/RAG/venv/lib/python3.12/site-packages/onnxruntime/transformers/models/gpt2/gpt2_helper.pyrzGPT2ModelNoPastState.__init__*  c(t||ddS)NF) use_cache return_dict)rforwardr input_idsrs rr$zGPT2ModelNoPastState.forward-swyEuMMr __name__ __module__ __qualname____doc__rr$ __classcell__rs@rrr's<!NNr rc,eZdZdZfdZfdZxZS)TFGPT2ModelNoPastStaterc2d|_t| |y)NF)r"rrrs rrzTFGPT2ModelNoPastState.__init__4s   r c&t||dS)NF)r")rcallr%s rr$zTFGPT2ModelNoPastState.forward8sw|I|77r r'r-s@rr/r/1s<!88r r/c<eZdZdZfdZedZfdZxZS) MyGPT2ModelzMHere we wrap a class for Onnx model conversion for GPT2Model with past state.c$t||yrrrs rrzMyGPT2Model.__init__?rr c t|ddttfrt|d|k(rt|dddk(sJg}t |D]Z}|j t j|d|djd|d|djdfd\|dt|fS|S)Nr)dim) isinstancetuplelistlenrangeappendtorchcat unsqueeze)result num_layerpresentis r post_processzMyGPT2Model.post_processBs fQilUDM 2vay>Y.3vay|3D3I IIG9%II1a2215vay|A7P7PQR7ST&1IuW~. . r ct|||||d}tj||jj SNF) position_idsattention_maskpast_key_valuesr#rr$r4rGrn_layerrr&rJrKpastrCrs rr$zMyGPT2Model.forwardUsD %)  ! '' 0C0CDDr ) r(r)r*r+r staticmethodrGr$r,r-s@rr4r4<s+W!$EEr r4c,eZdZdZfdZfdZxZS)MyGPT2LMHeadModelzSHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state.c$t||yrrrs rrzMyGPT2LMHeadModel.__init__crr ct|||||d}tj||jj SrIrMrOs rr$zMyGPT2LMHeadModel.forwardfsD %)  ! '' 0C0CDDr r'r-s@rrSrS`s]! E Er rSc,eZdZdZfdZfdZxZS)MyGPT2LMHeadModel_NoPaddingaHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state and no padding. When you always use batch_size=1 in inference, there is no padding in inputs. In such case, position_ids and attention_mask need no be in inputs. c$t||yrrrs rrz$MyGPT2LMHeadModel_NoPadding.__init__xrr c|t|||d}tj||jj S)NF)rLr#rM)rr&rPrCrs rr$z#MyGPT2LMHeadModel_NoPadding.forward{s4DeT'' 0C0CDDr r'r-s@rrWrWrs !EEr rWlogitsTF last_state)r GPT2LMHeadModel_NoPaddingr c0eZdZdZdefdZdefdZdZy) Gpt2Inputsc<||_||_||_||_yr)r&rJrKrP)rr&rJrKrPs rrzGpt2Inputs.__init__s +4.:WeFJ r returnc|j|j|jfDcgc]}|| }}|jr|j |j|Scc}wr)r&rJrKrPextend)rv input_lists rto_listzGpt2Inputs.to_listsZ"&..$2C2CTEXEX!Yk!YA]^]ja!Y k 99   dii ( ls AAc~td|j|j|j|jfDS)Nc3&K|] }|| ywr).0rcs r z&Gpt2Inputs.to_tuple..su c1ghgtQ cs)r;r&rJrKrP)rs rto_tuplezGpt2Inputs.to_tuples3u1B1BDDWDWY]YbYb cuuur cd}|j]|jjtjk(r*|jj tj n |j}|j Dcgc]"}|j tj $}}t|j|j||Scc}w)N)dtype) rKrmr@float16tofloat32rPr^r&rJ)rrKprPs rto_fp32zGpt2Inputs.to_fp32s    *''-->##&&U]]&;((  4899=9a5==)9=$..$*;*;^TRR>s:'CN) r(r)r*rr<rer;rkrrrhr rr^r^s(K v%v Sr r^c"peZdZdZedddej ej ej dfdededededed ed ed ejd e d e de dejdejdejde de f dZ e dAdededede dedeeeeff dZedZedBdZedBdZedCdZedDdZeddddej ej ej fdede d e d e de dejdejdejfd!Ze dEd#Zegd$fd%ed&eefd'ZedFd(e d)efd*ZedFd(e d)efd+Zed,ZedGd-Ze dHd(e d.eeej>fd/eeeefd)ed0e d1e f d2Z ed3Z!ed4Z"edd5d5d6d7ddddej ej ej d"ddfd8Z#edd9ddddej ej ej d:d7d;f d<Z$edId=Z%edddgd>fdefd?Z&y@)J Gpt2HelperzEA helper class for Gpt2 model conversion, inference and verification.FT batch_sizepast_sequence_lengthsequence_lengthnum_attention_heads hidden_sizerD vocab_sizedevicernhas_position_idshas_attention_maskinput_ids_dtypeposition_ids_dtypeattention_mask_dtypeleft_side_paddingr`c|rtjntj}d|||t||z g}t |Dcgc] }tj |||dzdz "}}tj d|dz ||f| |}d}| re||z}tj||g| |}|dk\rAt |D]3}tj d|dz }|r d||d|f<(d||||z df<5d}| rQ|jjd dz }|j|dkd|dd|dfj| }t||||Scc}w) zCreate random inputs for GPT2 model. Returns torch tensors of input_ids, position_ids, attention_mask and a list of past state tensors. r8rmr{g@?rr7)lowhighsizermr{N)r@rnrpintr>randrandintonesrandomlongcumsum masked_fill_ror^)rurvrwrxryrDrzr{rnr|r}r~rrr float_type past_shape_rPr&rKtotal_sequence_lengthrFpadding_lengthrJs rget_dummy_inputszGpt2Helper.get_dummy_inputss*'.U]]5==    11 2  `een_op_oZ[JjH3NQTT_opMMao.!   $8?$J !"ZZ23*N %)z*A%+^^A7Lq7P%QN(=>q/>/'9:VWq*?.*P*R'RS + )..077;a?L  % %lQ&6 :'+?+@(@ADDEWXL)\>4HHCqs%Er r model_classc |j}|j}|j}|j}t|d} ||| dk(r|n|g} d||||zt ||z g} | | i} t |D]} | | dt| z<| S)zAReturns a dictionary with output name as key, and shape as value.r7rZr8present_)rxrynum_hidden_layersrz MODEL_CLASSESrr>str)rurvrwrrrxryrDrz output_namelast_state_shapepresent_state_shape output_shapesrFs rget_output_shapeszGpt2Helper.get_output_shapess%88(( ,, && #K03   %1J{     ? 2  11 2  %&67 y!A1DM*s1v- ."r c|D]|}||vsJ||}tj|||jkDs8tjtj|||j |j ||<~y)Nr)numpyprodnelementr@emptyrmr{)output_buffersrkeybuffers rauto_increase_buffer_sizez$Gpt2Helper.auto_increase_buffer_sizess C.( ((#C(Fzz-,-0AA&+kkJJ}S12 ,,!=='s# !r c|rtjntj}i}|jD]3\}}tjt j |||||<5|S)zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.r)r@rnrpitemsrrr)rr{ is_float16 data_typernameshapes rget_output_bufferszGpt2Helper.get_output_bufferssX&0EMMU]] (..0KD%#(;;uzz%/@ Z`#aN4 1r c|djj}tj||dz }|r.tj|tj|dzz Stj|S)zGReturns the maximum difference between PyTorch and OnnxRuntime outputs.rư>)cpurabsamax) torch_outputs ort_outputsrelativeexpected_outputsdiffs r diff_outputszGpt2Helper.diff_outputs%sn)+//1779yy)KN:; ::deii0@&AD&HIJ J::d# #r c tj|d|djj||}tj d||}t |dz }t |D]g}tj|d|z|d|jj||}tj d|d|d||xr|}i|s/tj||} tjd| d |S) zReturns True if torch and ORT outputs are close for given thresholds, and False otherwise. Note: need kwargs since Gpt2BeamSearchHelper.compare_outputs has an extra parameter model_class r)rtolatolz9PyTorch and OnnxRuntime output 0 (last_state) are close: r7zPyTorch and OnnxRuntime layer z state (present_z ) are close:z@PyTorch and OnnxRuntime results are not all close: max_abs_diff=.5f) rallcloserloggerdebugr=r>rtrinfo) rrrrkwargsis_close is_all_close num_layerslayer max_abs_diffs rcompare_outputszGpt2Helper.compare_outputs/s >>+a.-2B2F2F2H2N2N2PW[bfg PQYPZ[\ %) :&E~~AI&a '++-335 H LL9%@PQVPWWcdlcmn o'4HL'%22=+NL KKZ[ghkZlm nr cd}d}g}g}tt|D]}||}|dk(r|dn |d|dz jj} tj|| |d} |j tj tj| |z |xr| }tj| jrtjd|dtj| jrtjd|dtj|jrtjd |dtj|jrtjd |dtj|| z } tj| j| j } |j d | | d d | d || d dt#| | d |dk(stjtj|d|j } tjtj| d| j }tj$| |}|j't)|}|t)||||fS)aCompare outputs from PyTorch and OnnxRuntime Args: torch_outputs (Tuple[Torch.Tensor]): PyTorch model output ort_outputs (List[numpy.ndarray]): OnnxRuntime output atol (float, optional): Absolute tollerance. Defaults to 1e-06. Returns: is_all_close(bool): whether all elements are close. max_abs_diff(float): maximum absolute difference. messages(str): a list of debug message for each output TFrr7)rrzPyTorch output z has nanz has infz ORT output zdiff=z.9fz index=z ort=z torch=N)axis)r>r=rrrr?rrisnananyrrisinffabs unravel_indexargmaxrfloat array_equalindexmax)rrrris_top1_matched max_diffsmessagesrF ort_output torch_outputrridx ort_max_indextorch_max_indexmax_diff_output_indexs rcompare_outputs_v2zGpt2Helper.compare_outputs_v2Js`  s;'(A$QJ01QM!,M!rr?rbr=rrr&rPrparentmkdirtempfileTemporaryDirectoryospathjoinr r;onnx load_modelrsave)modelr{rrrr|r}r~rrrrD dummy_inputsrdoutputsrF past_names present_namesrrrr tmp_dir_nametemp_onnx_model_paths r export_onnxzGpt2Helper.export_onnxs#\\NN !22!" & : :**((-1+1!53 "))+ ]]_Z(G,1+;<+;aaSk+; <16y1AB1AA8A31A Bqz"f&7&7771:;K;KA;NRXRdRd;ddd$+AJ$4$4Q$76;L;L$LR^oano  \i8 O)< D%1n!EL !D%1o!FL "#m /; +JL (   ~ . 1=/-RL) *   / 0:&7|q S_ %AAA  !7!7!=!= >f\EVEVWXEYE_E_D``hipqrisiyiyhz{DELMNEOPQEREXEXDY Z  _$$**4$*G #,,.,')ww||L+'N$)*1177t7T!z**"& +!-!-"$(,-1# (L$ L14 L6%BL;$L.;Mrc td} t|d||d| d} |r5|rtj| nd|vrd|d<| jdddi|| j ||| S) zHOptimize ONNX model with an option to convert it to use mixed precision.rrF) model_type num_headsry opt_leveloptimization_optionsuse_gpu keep_io_typesuse_symbolic_shape_inferTrh)rr rtauto_mixed_precisionconvert_float_to_float16save_model_to_file) roptimized_model_pathrrxryrrstagerrms r optimize_onnxzGpt2Helper.optimize_onnxs -V4  )#!5  #//2"&0.3F?+***SDSFS 13KLr )AddLayerNormalizationSkipLayerNormalizationFastGeluEmbedLayerNormalization onnx_model op_block_listcn|jDchc]}|j}}t|}|j|}tj d|d||j jdj}d}|j}||vsJ||}d} |jdk(r|} tj d|jd} |jD]} |j| } | nt| } tjd|jd | | d k}n/tjd |jd |jg} g}|s| |g} | jg}| |||d }tj d||jdddi||Scc}w)a?Convert GPT-2 model to mixed precision. It detects whether original model has fp16 weights, and set parameters for float16 conversion automatically. Args: onnx_model (OnnxModel): optimized ONNX model op_block_list (List[str], optional): operators to compute in fp32. Defaults to ["Add", "LayerNormalization", "SkipLayerNormalization", "FastGelu", "EmbedLayerNormalization"] Returns: parameters(dict): a dictionary of parameters used in float16 conversion z fp32 op: z fp16 op: rFNMatMulz#Found last MatMul node for logits: z3max diff of converting weights in last MatMul node : rz-Failed to find MatMul node for logits. Found z of node )rr!node_block_listforce_fp16_initializersz!auto_mixed_precision parameters: rTrh)nodesop_typeset differencerrgraphoutputroutput_name_to_nodeinputget_initializerrrwarningr)r r!node op_full_set fp32_op_set fp16_op_setlogits_output_nameis_weight_fp16_precisionr-last_matmul_node initializerr.max_diffrr% parameterss rrzGpt2Helper.auto_mixed_precisions(1;0@0@0BC0Bt||0B C-( !,,[9  i }J{mDE(--/66q9>>$) (<<>!%8888"#56 <<8 ##  KK=dii[I JK(88? *$1=H LLNtyykY[\d[ef g'/$ $ NNJ4<<.Xabfbkbkalm n (/?/K/0M/445O+*.'?    7 |DE+ ++XTXZXaDsF2inputs total_runsc`tjd|jj}t j 5||}ddd|dk(rSg}t j 5t |D]A}tj}||}|jtj|z C dddt|dzt|z }tjdjt|d|fS#1swYxYw#1swYaxYw)zfRun inference of PyTorch model, and returns average latency in ms when total_runs > 0 besides outputs.zstart pytorch_inferenceNrzPyTorch inference time = {} ms.2f) rrrrrer@rr>timer?sumr=format) rr;r<rdrlatencyrstartaverage_latencys rpytorch_inferencezGpt2Helper.pytorch_inferencebs  ./^^%--/ ]]_Z(G ?N ]]_:& ,tyy{U23' g,-G < 5<|j>tj|jj j|d<|j>tj|jj j|d<|jd|}|dk(r|Sg}t|D]N}tj} |jd|}|jtj| z Pt|dzt|z } tjd j!t!| d || fS) zcRun inference of ONNX model, and returns average latency in ms when total_runs > 0 besides outputs.zstart onnxruntime_inferencer&NrrKrJrr>z"OnnxRuntime Inference time = {} msr?)rrrascontiguousarrayr&rrP enumeraterKrJrunr>r@r?rAr=rB) ort_sessionr;r< ort_inputsrFpast_irrCrrDrEs ronnxruntime_inferencez Gpt2Helper.onnxruntime_inference|s  23!5#:#:6;K;K;O;O;Q;W;W;Y#Z[ ;; "&v{{3 6*/*A*A&**,BTBTBV*W U1#;'4  ,+0+B+B6CXCXC\C\C^CdCdCf+gJ' (    *).)@)@ATATAXAXAZA`A`Ab)cJ~ &!oodJ7 ? z"AIIKE%//$ ;K NN499;. /# g,-G < 9@@Y^A_`aO++r c 8tj|||||||S)z)Returnas IO binding object for a session.)rprepare_io_binding)rKr&rJrKrPrrs rrPzGpt2Helper.prepare_io_bindings,11         r c2tj||||S)z3Copy results to cpu. Returns a list of numpy array.)r"get_outputs_from_io_binding_buffer)rKrr return_numpys rrRz-Gpt2Helper.get_outputs_from_io_binding_buffers AA    r rrrSinclude_copy_output_latencyc htjdtj||j|j |j |j||}|j|tj||||}|dk(r|Sg} t|D]g} tj} |j||rtj||||} | jtj| z it| dzt| z } tjd| || fS)zUInference with IO binding. Returns outputs, and optional latency when total_runs > 0.z*start onnxruntime_inference_with_binded_iorr>z4OnnxRuntime with IO binding inference time = %.2f ms)rrrtrPr&rJrKrPrun_with_iobindingrRr>r@r?rAr=) rKr;rrr<rSrT io_bindingrrCrrDrEs r$onnxruntime_inference_with_binded_ioz/Gpt2Helper.onnxruntime_inference_with_binded_ios  AB 22         ! ! KK    &&z2!CC    ? z"AIIKE  * *: 6*AA  NN499;. /#g,-G < K_]O++r cTtd|dd5}tj||dddtj d|dtd|dd5}tj||dddtj d|dy#1swYkxYw#1swY/xYw)N ort_outputs_.picklewbz$ORT output are saved to ort_outputs_torch_outputs_z(Torch output are saved to torch_outputs_openpickledumprr)rFrrrs r save_outputszGpt2Helper.save_outputss L7+T 2a KK Q '3 :1#WEF N1#W-t 4 KK q )5 >qcIJ 3 25 4sBBBB'ctd|dd5}tj||dddtj d|dy#1swY#xYw)N dummy_inputs_r[r\z!inputs are saved to dummy_inputs_r^)rFrrrrs r save_inputszGpt2Helper.save_inputssI M!G,d 3q KK a (4 7s'BC4 3s A  Ari'r7c|j}tjd|d|d|d|d| d|d d}d }d }d }|r0tj ||||| }tj |||}d }d }g}d g|z}||z}t |D]E}t||z }tjd |}|d k(rd ntjd |} tjd |}!tjd|!d| dtj|!| ||j|j|j|j||| | | | |d}"tj!||"}#|rtj#||"}$n1tj |!| ||| }%tj%||"||%}$tj'|#|$|\}&}'}(})}*t)j*|'s|j-|'|&r|d z }|*r|d z }||xxd z cc<|rr|&sptjd|d|!d| d|d|' t/|)D]>\}}+tjd|d|j1|j2d|+@|st)j*|'s |'d|zkDstj5||"tj7||$|#H|r*dD,cic]},d|,t)j8||,d }-},ndD,cic]},d|,d }-},|d z|z |-d!<|D.cgc] }.|.d z|z  c}.|-d"<|d z|z |-d#<|t;|z d z|z |-d$<tjd%|d&|d'|t;|z d(||d)|zkDr)tjd*t|dz|z d+d,|-Scc},wcc},wcc}.w)-zKGenerate random inputs and compare the results of PyTorch and Onnx Runtime.zRunning parity test (atol=z , test_cases=z, runs=z, use_io_binding=z, model_class=z , is_float16=z) ...r8Nrr7z#Running parity test for batch_size=z past_sequence_length=z...T)r~rrr)rz test_case=z batch_size=z sequence_length=z MaxDiff= z: Name=z, d)2Z_cmax_diff_percentile_rnanrtop1_match_ratetop1_match_rate_per_rundiff_pass_ratenan_ratezParity Test Cases=z ; Passed=z; Nan=z; Top1_Matched=gffffff?zParity is good: passed rate=z.0f%)rrrrtrrr>rrrrrrxryrNrzrFrNrXrrrr?rI get_outputsrrerb percentiler=)/rKrr{rrrtest_cases_per_runr<use_io_bindingrr|r}r~rrrrenable_pickle_outputrmax_batch_sizemax_past_seq_len max_seq_lenrmax_output_shapespassed_test_casestop1_matched_casesmax_abs_diff_listtop1_matched_cases_per_runtotal_test_casesrFrun_idrwrvrurrrrrrrrrmessagerqrCxs/ r test_parityzGpt2Helper.test_paritys.#\\ (m:J LL5j\AWXlWmmpq &66$**""!! " /#5%9"&7L"!225,GG(>>{LY * < <(# ! )MM~} --g{-N  %;;|,!((6!Q&!"a'"*62a72|  < |;QRfQggxzIyJJST`Sab#,H"5JAwKK"QCw{/F/F/H/K/P/P.QQST[S\ ]^#6$\)BlUX[_U_F_&&q,7'';@G)J eueu`a&qc*u/?/?@QST/UVY.Z[eu BRRAQA,QC0%7AQFR$6$"Orjrg c|j}d}|r0tj| || ||}tj|||}tj | || |j |j |j|j||||| | | }|rtj|||\}}|Stj||||\}}|S)zCGenerate random inputs and measure average latency of Onnx Runtime.N)r~rr) rrtrrrrxryrNrzrNrX)rKrr{rr<ryrr|r}r~rrrurwrvrrrrrrCs rtest_performancezGpt2Helper.test_performances(#\\ &880/6;M(::=&R\]N!22    & &    NN       +1!53 " #99+|U_`JAw  $HH\>=*JAwr c tjddd|j|j|j|j |d|| j }tjj||S)zJIT trace for TorchScript.r7F) rurvrwrxryrDrzr{rnr|r}) rtrrxryrNrzrer@jittrace)rrr{r|r}rds r torchscriptzGpt2Helper.torchscriptss 00!" & : :**nn((-11   ') yyuj11r rawfp32fp16int8c |}tjj|rt|jd}n|j dd|dk7r|d|zz }|r|dz }|rdddd d }d D]}tjj ||||z} tjj| sI||vr/ tj| tjd | |tjd|d| tjj tjj |||dztjj tjj ||dz|dztjj tjj ||dz|dztjj tjj ||d z|dzd Stjj ||dztjj ||dztjj ||dztjj ||dzd S#t$r0} tjd | d | jYd} ~ dd} ~ wwxYw)z=Build a path name for given model based on given attributes.r/r r_past_fp32_fp16_int8rzRemoved the existed directory: zFailed to remove the directory r$NzDirectory for z existed: z.onnxz _fp32.onnxz _fp16.onnxz _int8.onnx)rrisdirrpartssplitrexistsshutilrmtreerrOSErrorstrerror) output_dirmodel_name_or_pathrhas_past new_folderremove_existing model_namesuffixr new_dires rget_onnx_pathszGpt2Helper.get_onnx_pathss`( 77==+ ,0177;J   S !" % + + # + +J  ' !J '7SF= '',,z:z@R3RS77>>'*!_4c"MM'2"KK*I'(ST nZL 7)$TU>ww||BGGLLZ$H*W^J^_ GGLLZ'-AB- GGLLZ'-AB- GGLLZ'-AB- "77<< J,@AGGLLZ,-FGGGLLZ,-FGGGLLZ,-FG   - 'c"KK*I'RTUVU_U_T`(abbcs=-J K(%KKN)r )F)MbP?r)r)FFr)r)T)rTF)TT)'r(r)r*r+rQr@int32rr{boolrmr^rr rdictr<rrrrrrr rrrrFrNrPrRTensorrXrbrerrrrrhr rrtrtsO!%#'',{{*/++,1KK"&>I>I!>I>I! >I  >I  >I>I >I>I>I!>I>I"KK>I$kk>I >I !>I>I@ -  !      c49n   D  $$43 3 j ).!%#'',{{*/++,1KKuu u #' u  u!uu"KKu$kkuun"'"!!F$ CCCyCCJ((((2,:,3,,>  (   !,10,0,S%,,./0,CcN+ 0,  0,  0,&*0,0,dKKDD     %  ;;"[["%EEN %  ;;"[[44l22"-7 : : : r rt)/loggingrr`rrrr@pathlibrrrr@benchmark_helperrrnrfusion_optionsrio_binding_helperrr r optimizerr torch_onnx_export_helperr transformersr r r r getLoggerr(rPRETRAINED_GPT2_MODELSFLOAT32FLOAT16INT8DEFAULT_TOLERANCErr/r4rSrWrr^rtrhr rrs   &-(- $6LL   8 $Wv s NNCN9N8[8!E)!EHEE$ E/ E"*8T:"=x!O|T2 SS>_  _  r