U T?hY @sddlZddlZddlmZddlmZmZmZddlZ ddl Z ddl m Z ddl mZddlmZddlmZddlmZmZmZdd lmZdd lmZmZmZdd lmZmZdd l m!Z!m"Z"dd l#m$Z$e%e&Z'dddddddddddg Z(GdddZ)dS)N)Path)DictTupleUnion)float_to_float16_max_diff) OnnxModel)optimize_model)version) WhisperConfigWhisperForConditionalGenerationWhisperProcessor) __version__)WhisperDecoderWhisperDecoderHelperWhisperDecoderInit)WhisperEncoderWhisperEncoderHelper)WhisperEncoderDecoderInitWhisperEncoderDecoderInitHelper)InferenceSessionz whisper-tinyzwhisper-tiny.enz whisper-basezwhisper-base.enz whisper-smallzwhisper-small.enzwhisper-mediumzwhisper-medium.enz whisper-largezwhisper-large-v2zwhisper-large-v3c @s6eZdZed#eeeeedddZeeeejej j dddZ ed$eeeejeee eej j fd d d Z ed%eeeeefejeeeeed ddZed&eeedddZed'eeeeeeeeed ddZed(eej j ejeedddZeeedddZed)eeeejeedd d!Zd"S)* WhisperHelperF) output_dirmodel_name_or_pathsuffix new_folderreturncCs^|}tj|r t|jd}n|dd}||7}|rHtj||n|}tj||dS)aBuild onnx path Args: output_dir (str): output directory model_name_or_path (str): pretrained model name, or path to the model checkpoint suffix (str, optional): suffix like "_encoder" or "_decoder_fp16" will be appended to file name. Defaults to None. new_folder (bool, optional): create a new directory for the model. Defaults to False. Returns: str: path of onnx model /z.onnx)ospathisdirrpartssplitjoin)rrrr model_name directoryr'h/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/models/whisper/whisper_helper.py get_onnx_path,s zWhisperHelper.get_onnx_path)r cache_dirdevicerc Csddlm}m}m}ddlm}m}d}|dddd} d \} } | |krf||| ||} || } t| d } t j | |d } W5QRX~ |f| d }||}| | d | dk r| | | |S)Load model given a pretrained name or path, then build models for ONNX conversion. Args: model_name_or_path (str): pretrained model name or path cache_dir (str): cache directory device (torch.device): device to run the model merge_encoder_and_decoder_init (bool, optional): Whether merge encoder and decoder initialization into one ONNX model. Defaults to True. Returns: Dict[str, torch.nn.Module]: mapping from name to modules for ONNX conversion. r)_ALIGNMENT_HEADS_MODELS _download)ModelDimensionsWhisperFrrN)NNrb)Z map_locationdimsZmodel_state_dict)Zwhisperr-r.r/Z whisper.modelr0r1r#opentorchloadload_state_dictZset_alignment_headsto)rr*r+r-r.r/r0r1Z in_memoryr%Zcheckpoint_fileZalignment_headsfp checkpointr4modelr'r'r(load_model_openaiIs"  zWhisperHelper.load_model_openaiT)r model_implr*r+merge_encoder_and_decoder_initstate_dict_pathrcCsi}tttdkr d|d<tj|fd|i|}|dkr`t|||}|j|j} } |} n||} } d} |r|j t |ddt | |j || d } | ||rt| | |j d|| d } | | d St|jj|j }||t|j|j }|||| |d SdS) r,4.36.0eagerattn_implementationr*ZopenaiNF)strict)r>r<)decoder_start_token_idr>r<)encoder_decoder_initdecoder)encoderrG decoder_init)r parsetransformers_versionr from_pretrainedrr=rHrGr8r6r7rconfigevalr9rrr<r)rr>r*r+r?r@ extra_kwargsr<Z openai_modelZ model_encoderZ model_decoderZ passed_modelrGrFrHrIr'r'r( load_modelps@  zWhisperHelper.load_modelr<r+onnx_model_pathverboseuse_external_data_formatZuse_decoder_input_idsZuse_int32_inputsc CsXt|trt|||||n6t|tr@t|||||||nt||||||dSN) isinstancerr export_onnxrrrrQr'r'r(rWs4   zWhisperHelper.export_onnxZSimplifiedLayerNormalizationZ SkipSimplifiedLayerNormalizationZReluAdd) onnx_model op_block_listcCs\tdd|D}t|}||}td|d||jdj}d}|}||ksft ||}d} |j dkr|} td |jd} |j D]} | | } | dk rqqt | } td |jd | | d k}ntd |j d|jg} g}|s"| dk r"|g} | jg}| t|||d}td||jfddi||S)aConvert model to mixed precision. It detects whether original model has fp16 precision weights, and set parameters for float16 conversion automatically. Args: onnx_model (OnnxModel): optimized ONNX model op_block_list (List[str], optional): . Defaults to ["SimplifiedLayerNormalization", "SkipSimplifiedLayerNormalization", "Relu", "Add"] Returns: parameters(dict): a dictionary of parameters used in float16 conversion cSsg|] }|jqSr')op_type).0noder'r'r( sz6WhisperHelper.auto_mixed_precision..z fp32 op: z fp16 op: rFNZMatMulz#Found last MatMul node for logits: z3max diff of converting weights in last MatMul node z: gư>z-Failed to find MatMul node for logits. Found z of node ) keep_io_typesr[node_block_listZforce_fp16_initializersz!auto_mixed_precision parameters: Zuse_symbolic_shape_inferT)setZnodes differenceloggerinfographoutputnameoutput_name_to_nodeAssertionErrorr\inputZget_initializerrdebugwarninglistZconvert_float_to_float16)rZr[Z op_full_setZ fp32_op_setZ fp16_op_setZlogits_output_nameZis_weight_fp16_precisionrir^Zlast_matmul_nodeZ initializerrkmax_diffr`ra parametersr'r'r(auto_mixed_precisionsF      z"WhisperHelper.auto_mixed_precisioncpu) rRoptimized_model_path is_float16num_attention_heads hidden_sizerTrquse_gpuproviderc Csxddlm} | d} d| _|dk| _t|d|||s6dnd| |dd } |rd|rXt| n | jdd | j||dd dS) zHOptimize ONNX model with an option to convert it to use mixed precision.r) FusionOptionsZbartTZrocmNF)Z model_typeZ num_headsrv opt_leveloptimization_optionsrwZonly_onnxruntime)Zcast_input_output)Zall_tensors_to_one_file) Zfusion_optionsryZuse_multi_head_attentionZ!disable_multi_head_attention_biasrrrqZ convert_model_float32_to_float16Zsave_model_to_file) rRrsrtrurvrTrqrwrxryr|mr'r'r( optimize_onnxs&      zWhisperHelper.optimize_onnx) processorpt_modelr+ batch_size prompt_modec s2zddlm}WnVtk rf}z8tjd|ddd}td|dt|W5d}~XYnXddlm}|d d d d }g} |d kr|dddgddj} n^|dddgddj|dddgddjg} t | |kst t | d| d f} d\} } } }d\}}| || | | |||ddd }|rddg}fdd|D}g}g}t|D]d}t |||d<| | ||d<|jf|}|||j|dddql| |d<|d=n8g}|jf|}j|dddg}t|}|d=|d=||||fS)Nr) load_datasetz.An error occurred while importing `datasets`: T)exc_infozpip install datasetszCCould not import `datasets`. Attempting to install `datasets` via `z`.z)hf-internal-testing/librispeech_asr_dummycleanZ validation)r#rZaudioarraypt)Zreturn_tensors)rrr)?r) input_features max_length min_length num_beamsnum_return_sequenceslength_penaltyrepetition_penaltyearly_stopping use_cachezJohn has doubtszMaria has grave doubtscsg|]}|qSr')Zget_prompt_idsr]prr'r(r_kszBWhisperHelper.pt_transcription_for_verify_onnx.. prompt_idsrZskip_special_tokensrr)Zdatasetsr Exceptionrderrorrmrsystemrlenrjr6catr9rangeZ from_numpygeneratedetachrrnumpyappend batch_decodern)rrr+rrreZ install_cmdZdsZinput_features_rrrrrrrinputsZpromptsrpt_transcription pt_outputsiZ pt_outputr'rr( pt_transcription_for_verify_onnx<sb      z.WhisperHelper.pt_transcription_for_verify_onnxrrc CsD|dkr*|r*d}d}d}d}||||h}nd}d}d} ||| h}|S) Nrz{ John has doubts whether Sir Frederick Layton's work is really Greek after all and can discover in it but little of Rocky Izy John has doubts whether Sir Frederick Latins work is really Greek after all and can discover in it but little of Rocky Iz Maria has grave doubts whether Sir Frederick Layton's work is really Greek after all and can discover in it but little of Rockyz Maria has grave doubts whether Sir Frederick Latins work is really Greek after all and can discover in it but little of Rocky IzX Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.zY Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.zZ "Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.r') rrZ'expected_transcription_no_comma_prompt1Z)expected_transcription_misspelled_prompt1Z'expected_transcription_no_comma_prompt2Z)expected_transcription_misspelled_prompt2expected_transcription_optionsZexpected_transcription_no_commaZ!expected_transcription_with_commaZ+expected_transcription_with_quote_and_commar'r'r(select_transcription_optionss* z*WhisperHelper.select_transcription_options)rr* ort_sessionr+rrc%Cs$i}tttdkr d|d<tj|fd|i||}tj||d}tj||d} tj |||||d\} } } } | j g}|j ddd }t t d d |}||}t t d d |}t t d d |}tjtjtjtjtjtjd}d|k}t||D]\}}|dkr0| || |<q|dkrVtj| j||d| |<q|dkrtj|| jf||d| |<q|dkr@|s|r|gn|g}tj|||d| |<ng}t|D]}|| |qt dd|D}g}|D],}|| j!g|t"|}|||qtj|||d| |<n|dkrdtjdg||d| |<n|dkrtjddgg||d| |<nn|dkrt#tj|g||d|d| |<n@|dkrtjdg||d| |<ntj| |g||d| |<q|$d| ddddddf}|j%|dd}t&||} d}!t|D]$}|!| || kod||| k9}!qFd}"|!st|D]v}| |j'||j'kr| |||dddt"| |f}#n| |||}#t |#(|# t)d }$t |"|$}"q~|"dkr t*+d!| t*+d"||"S)#zRCompare the result from PyTorch and ONNX Runtime to verify the ONNX model is good.rArBrCr*)r*renglishZ transcribe)languagetaskcSs|dS)Nrr')tokenr'r'r(z+WhisperHelper.verify_onnx..cSs|jSrU)rhentryr'r'r(rrcSs|jSrU)typerr'r'r(rr)z tensor(float)ztensor(float16)z tensor(int64)z tensor(int32)z tensor(int8)z tensor(uint8)Zextra_decoding_idsrZ vocab_mask)dtypeZprefix_vocab_maskZdecoder_input_idscss|]}t|VqdSrU)rrr'r'r( sz,WhisperHelper.verify_onnx..Zlogits_processorrZcross_qk_layer_headrZ temperaturerNTr)keyzPyTorch outputs: zONNX Runtime outputs: ),r rJrKr rLr9r r rrrEZget_decoder_prompt_idsrnmap get_inputsnpZfloat32float16Zint64Zint32Zint8Zuint8ziprrrrZonesZ vocab_sizerrrtolistmaxZ pad_token_idrrepeatrunrrshapeminabsrdrm)%rr*rr+rrrOrrrMrrrZdecoder_prompt_idsZstart_idrZforced_decoder_idsZ ort_namesZ ort_dtypesZ ort_to_npZuse_extra_decoding_idsrhrZ raw_input_idsZ ort_promptsrmax_lenZpadded_promptsrZ padded_promptZ ort_outputsZort_transcriptionrZparityrodiffZ max_diff_ir'r'r( verify_onnxs           $  "    * zWhisperHelper.verify_onnxN)rF)Tr)TFTF)rX)FTFrr)rF)rF)__name__ __module__ __qualname__ staticmethodstrboolr)r6r+nnModuler=rrPrrrrrrWrrrqintr~r rrrrr'r'r'r(r+s&:%C&E r)*loggingrpathlibrtypingrrrrrr6rrrZrZ optimizerr packagingr Z transformersr r r r rKZwhisper_decoderrrrZwhisper_encoderrrZwhisper_encoder_decoder_initrrZ onnxruntimer getLoggerrrdZPRETRAINED_WHISPER_MODELSrr'r'r'r(s: