U T?h_ @sddlZddlZddlmZddlZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZddlmZddlmZmZmZmZmZdd lmZmZmZd ejd <eeZd ej iZ!d:d dZ"ddZ#ddZ$ej%fddZ&ddZ'ddZ(ddZ)ddZ*d;ddZ+e,e,e-e.e.e e.e.dd d!Z/e,e,e,d"d#d$Z0d%d&Z1dN)Path)AffinitySetting) OptimizerInfo Precisioncreate_onnxruntime_session) MODEL_CLASSES)QuantizeHelper)torch_onnx_export) AutoConfigAutoFeatureExtractor AutoTokenizer LxmertConfigTransfoXLConfig)PRETRAINED_GPT2_MODELSGPT2ModelNoPastStateTFGPT2ModelNoPastState2ZTF_CPP_MIN_LOG_LEVELtriucCs|dks tt|jdkr.|d|dks2ttd}|tjdtjd|}|d|dd|df}t| |t |S)Nrr)rdtype) AssertionErrorlenshapesize torch_functorchonesuint8whereboolZ zeros_like)xZdiagonaloutZ torch_triutemplatemaskr'X/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/onnx_exporter.py triu_onnx#s  & r)cCs tt_dSN)r)rrr'r'r'r(replace_torch_functions-sr+cCstdt_dS)Nr)rrrr'r'r'r(restore_torch_functions1sr,c Cs|jdkr4tj|d|j|jtj}d|i}|Stjjd|d||f|d}d|i}d|krztj||g|d }||d<d |krtj ||g|d } | |d <|j r||d <t |t rtj dd|jtj|d <tj dd|jtj|d <t |trtj |jgtjd |d<|S)NZvitswin pixel_valuesrrlowhighrr input_idsattention_maskrZtoken_type_idsdecoder_input_ids visual_feats visual_posz@tf_transfo_xl_model/transformer/pos_emb/einsum/Einsum/inputs_1:0) model_typenumpyrandomZrand image_sizeZastypeZfloat32randintrZzerosis_encoder_decoder isinstancer Zrandnvisual_feat_dimvisual_pos_dimr hidden_size) vocab_size batch_sizesequence_length input_namesconfigZ data_typer4inputsr5Z segment_idsr'r'r(create_onnxruntime_input5s.    rIcCs&i}|D]}||kr||||<q|Sr*r')rHrFZremaining_model_inputsZ input_namer'r'r( filter_inputsSs rJcCs"t|ttfrdd|Dn|gS)NcSsg|] }t|qSr')flatten.0ir'r'r( \szflatten..)r?listtuple)rHr'r'r(rK[srKcCs0|D]&}t|ttfs ||nt||q|Sr*)r?rPrQappendupdate_flatten_list)rHZres_listrNr'r'r(rS_s$rSc Cs|djd}dd|D}ddtt|D}t|D]J\}}ddi||<||j}t|D]"\}} | |kr`|||d iq`q:||fS) Nr4cSsi|]}|dddqS)rDseq_len)rrr'rMkeyr'r'r( hsz&build_dynamic_axes..cSsg|]}dt|dqS)Zoutput_r)strrLr'r'r(rOjsz&build_dynamic_axes..rrDrU)rranger enumerateupdate) example_inputsZoutputs_flattenrE dynamic_axes output_namesrNZ output_namedimsjdimr'r'r(build_dynamic_axeses  rcc CsNt||dd}|dkr*t|ddSt|ddd|D}|||}t|t|krtdt|dt|dStt|D]} t t || ||  } | d krtd | d | |rd nd } |rd nd } tj || ||  | | dstd| d| d| dSqtd|dS)NF)Zenable_all_optimizationz is an invalid ONNX modelz is a valid ONNX modelcSsi|]\}}||qSr')r:)rMktr'r'r(rXsz'validate_onnx_model..z"Number of output tensors expected z, got g-C6?zMax absolute diff=z for output tensor g?g?)rtolatolzOutput tensor z is not close: rtol=z, atol=z0inference result of onnxruntime is validated on T) rloggererrorinfoitemsrunrrZr:ZamaxabscpuZallclose) onnx_model_pathr]example_outputs_flattenuse_gpuZfp16r_Z test_sessionZexample_ort_inputsZexample_ort_outputsrNZabs_diffrfrgr'r'r(validate_onnx_modelts8 $  rr)onnx_dir model_name input_countoptimized_by_scriptrq precisionoptimized_by_onnxruntimeuse_external_datac Csddlm}|dd|} |s,| d|} n&|r4dnd} | d|d|d| } |r^| d7} |} |r|stj|| } tj| st| tj| | dS) Nr)subz [^a-zA-Z0-9_]_Zgpurn_ortz.onnx)rerzospathjoinexistsmakedirs) rsrtrurvrqrwrxryrzZnormalized_model_namefilenameZdevice directoryr'r'r(get_onnx_file_paths     r) file_pathsuffixreturncCs&t|}t|j|j||jS)a Append a suffix at the filename (before the extension). Args: path: pathlib.Path The actual path object we would like to add a suffix suffix: The suffix to add Returns: path with suffix appended at the end of the filename and before extension )rrYparentjoinpathstem with_suffixr)rrrr'r'r(add_filename_suffixsrcCsf|stj|sRt|jjdddddlm}m}||||dd}||||<nt d|dS)NTparentsexist_okr)get_fusion_statisticsoptimize_by_onnxruntimec)rqoptimized_model_path opt_level'Skip optimization since model existed: ) r~rrrrmkdir optimizerrrrhrj)roort_model_pathrq overwritemodel_fusion_statisticsrrr{r'r'r(optimize_onnx_model_by_ortsrc  Cs|stj|st|jjdddddlm} ddlm } | dkrL| |} | ||t j krfd| _ |t jkrvd| _|dkrd}d}| ||||d| |dd}|d ks|d kr||| |<|t j kr|jdd ||| ntd |dS) NTrr) FusionOptions)optimize_modelFr.)Z num_headsrBroptimization_optionsrqZonly_onnxruntimeZ bert_kerasZbert_tf)Z keep_io_typesr)r~rrrrrfusion_optionsrrruse_raw_attention_maskrFLOAT16Zenable_gelu_approximationINT8Zenable_embed_layer_normZuse_dynamic_axesZget_fused_operator_statisticsZconvert_float_to_float16Zsave_model_to_filerhrj)rorr9num_attention_headsrBrqrwrrruse_external_data_formatrrrZ opt_modelr'r'r(optimize_onnx_models>         rcCsz|dk r&|tkr|Stddt|tkr2dSddl}|d|dk rNdS|d|dk rbdS|d |dk rvd Sd S) NzValid model class:  rrz-squad$ZAutoModelForQuestionAnsweringz-mprc$Z"AutoModelForSequenceClassificationZgpt2ZAutoModelWithLMHeadZ AutoModel)r Exceptionrrr}search)rtcustom_model_classr}r'r'r(modelclass_dispatchersrFcCszt||}|dkr6|r&tj|||dStj|||dS|rBd|}td|gd}td|t||}|j|||dS)Nr)rG cache_dirZTF transformers)fromlistzModel class name: )rrfrom_pretrainedr __import__rhrjgetattr)rtrGrr is_tf_modelZmodel_class_nameZtransformers_module model_classr'r'r(load_pretrained_model/s  rcCs@tj||d}t|drd|_||t||||d}||fS)Nr return_dictF)rGrr)r rhasattrrmodifyr)rtrrconfig_modifierrGmodelr'r'r( load_pt_modelBs   rcCsHtj||d}||t}|t||||dd}|||fS)NrT)rGrrr)r rrrZ get_affinityrZ set_affinity)rtrrrrGZaffinity_settingrr'r'r( load_tf_modelNs rcCs ddlm}||\}}||fS)Nr)tf2pt_pipeline)Zconvert_tf_models_to_pytorchr)rtrrGrr'r'r(load_pt_model_from_tfcs  rc Cs*d}|rt| |||d|}|tjkr0| || jfS|tjksN|tjksN|tjkrt||t |d||d|}t | ||| j | j ||| | | || |} |rt| ||||tjk|}|tjkrt d| t| | |t d| |tjkr|rt| d}t| ||| | | ||dkr"| jn| jfS)NTFzQuantizing model: zFinished quantizing model: r|r-)rrrZNOOPTrCZBYSCRIPTrrrrrrrrBrhrjrZquantize_onnx_modelZBYORTrrZ num_labels)rtrr9rsrFrqrwZ optimize_info validate_onnxrrrGrror]rpr_ris_valid_onnx_modelrrr'r'r(validate_and_optimize_onnxms       rc Cst||||\}}|d}d}|dkrvtj||d}tjjdd|j|jdtjd |j|jd}||dd}n*t j||d}|j |d }|j d dd}t||}|f|}t|ttfstd t|t|}t|g}t||t|d | | d |}|stj|std |t|jjdddd}d}|dkrbdd|Ddg}}nt ||\}}t!t"|t|#|t|$||d||d t%ntd|t&|||||| | | | | ||||||d|\}}}||||fS)Nr-rrr/r1pt)return_tensorsrThis is a sample inputz%type of output is not list or tuple: FExporting ONNX model to TrcSsi|]}|ddiqS)rr0r'rVr'r'r(rXsz-export_onnx_model_from_pt..Zlogits) rargsfrFr_r^Zdo_constant_folding opset_versionr!Skip export since model existed: )'rrnr rr:r;r=r<r Zreshaper max_model_input_sizesget encode_plusrJr?rPrQrtyperKrSrrr~rrrhrjrrrrcr+r valueskeysr,r) rtrrr9rrrrsrFrqrwoptimizer_inforrrrrrGrr]max_input_sizeZimage_processordata tokenizerexample_outputsrpror^r_Zonnx_model_filerrCr'r'r(export_onnx_model_from_pts          rc(Csddl}|jgdtj||d}|jdkr<|ddi|j|d}t ||||\}}| t ||j dd|d d d }t ||}|jr|j dd|d d d j|d <|d kr|jdd|jg|d<|jdd|jg|d<z|jrd|_Wntk rYnX||dd}d}|dks,|dkr:dg}|d}ddlm}||}t||t |d| | d|}|r||ddn|}|stj|std||st|j j!d d dddl"}ddl#}|j$%|j$j&g}|'D]6\} }!dgt |!j(}"|)|j*t+|"|!j,| dq|j-j.|t+||||d\}#}#|r|/|d}$|$0tj1|W5QRXtj2tj1|d}tj|rt3|t4||ntd||d}t5|||||| | | | | ||||||||\}%}&}'|%|&|'|fS) NrZGPUr pad_tokenz[PAD]rrtf max_lengthT)rrpaddingZ truncationr6zunc-nlp/lxmert-base-uncasedrr7r8F)Ztrainingzxlnet-base-casedzxlnet-large-casedZlast_hidden_state)nestrr)name)Zinput_signatureZopsetZ large_modelZ output_pathrz__MODEL_PROTO.onnxrZ_tf)6Z tensorflowrGZset_visible_devicesr rrZadd_special_tokensrrrZresize_token_embeddingsrrrJr>r4r;normalr@rAZ use_cacherZtensorflow.python.utilrrKrr~rrrhrjrrrzipfiletf2onnxloggingZ set_levelERRORrkrrRZ TensorSpecrQrconvertZ from_kerasZipFile extractalldirnamerremoverenamer)(rtrrr9rrrrsrFrqrwrrrrrrrrrrGrr]rr_rrproZtf_internal_model_pathrrspecsrvaluer`r{zZoptimized_onnx_pathrrCr'r'r(export_onnx_model_from_tf;s            r)rN)N)N)F);rr~pathlibrr:rZaffinity_helperrZbenchmark_helperrrrZhuggingface_modelsrZquantize_helperrZtorch_onnx_export_helperr rr r r r rZ0onnxruntime.transformers.models.gpt2.gpt2_helperrrrenviron getLogger__name__rhrrr)r+r,Zint64rIrJrKrSrcrrrYintr"rrrrrrrrrrrrr'r'r'r(sZ          - !  ;   _o