U
    T?h/                     @   s~  d dl Z d dlZd dlZd dlZzd dlmZmZ W n$ ek
rX   d dlmZmZ Y nX d dl	m
Z
 d dlmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ eeedk rede ddddgZe dddZ!d$e ee  e"dddZ#eee ef  e dddZ$eee ef  e e dddZ%dd  Z&d%d!d"Z'e(d#krze&  dS )&    N)PackageNotFoundErrorversion)Path)AnyDictListOptional)load_dataset)	evaluator)ORTModelForQuestionAnswering)__version__)r   )AutoTokenizerpipelinez1.13.1z1Please install optimum>=1.13.1. Current version: .z5bert-large-uncased-whole-word-masking-finetuned-squadzdeepset/roberta-base-squad2z%distilbert-base-cased-distilled-squadpackage_namec                 C   s&   z
t | W S  tk
r    Y d S X d S )N)r   r   r    r   a/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/models/bert/eval_squad.pyget_package_version7   s    
r   CUDAExecutionProviderF)model_id	onnx_pathuse_io_bindingc                 C   st   |dkrLt j| d||d}tjd| }|| tj|d}td| n t jtj|t|j	||d}||fS )a  Load onnx model given pretrained model name and optional ONNX model path. If onnx_path is None,
    the default onnx model from optimum will be used.

    Args:
        model_id (str): pretrained model name or checkpoint path
        onnx_path (Optional[str], optional): path of onnx model to evaluate. Defaults to None.

    Returns:
        model: ORTModel for the onnx model
        onnx_path: the path of onnx model
    NT)Zexportproviderr   r   z
model.onnxzModel is exported to onnx file:)	file_namer   r   )
r   from_pretrainedospathjoinZsave_pretrainedprintdirnamer   name)r   r   r   r   modelZsave_onnx_dirr   r   r   load_onnx_model>   s$       

r#   )resultscsv_filenamec                 C   s   t |ddddd}ddddd	d
ddddddddddddddg}tj||d}|  | D ]}|| qX|  W 5 Q R X td|  dS )zOutput a CSV file with detail of each test results.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
    a asciimodenewlineencodingpretrained_model_namer   r   disable_fused_attention
batch_sizesequence_lengthr   exactf1totalZHasAns_exactZ	HasAns_f1ZHasAns_totalZ
best_exactZbest_exact_threshZbest_f1Zbest_f1_threshZtotal_time_in_secondssamples_per_secondZlatency_in_seconds
fieldnamesz&Detail results are saved to csv file: N)opencsv
DictWriterwriteheaderwriterowflushr   )r$   r%   csv_fileZcolumn_names
csv_writerresultr   r   r   output_detailsb   s8    r@   )r$   r%   metric_namec              	      s  t |ddddz}ddddd	g td
d | D }|  tdd | D }|  tdd | D }|  g }|D ]$}|D ]}	|d|	 d|  qqxtj| | d}
|
  |D ]}i }i }|dd |D  | D ]r}|d |kr|| r fdd| D }|s|| |d }	|d }d|	 d| }||kr|| ||< q|r|D ]}|	|d||< q\|

| q|  W 5 Q R X td| d|  dS )a  Output a CSV file with summary of a metric on combinations of batch_size and sequence_length.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
        metric_name (str): the metric to summarize
    r&   r'   r(   r)   r-   r   r   r.   r   c                 S   s   h | ]}|d  qS )r   r   .0r?   r   r   r   	<setcomp>   s     z!output_summary.<locals>.<setcomp>c                 S   s   h | ]}|d  qS )r/   r   rB   r   r   r   rD      s     c                 S   s   h | ]}|d  qS )r0   r   rB   r   r   r   rD      s     bZ_sr5   c                 S   s   i | ]
}|d qS )r'   r   )rC   kr   r   r   
<dictcomp>   s      z"output_summary.<locals>.<dictcomp>c                    s   i | ]\}}| kr||qS r   r   )rC   rF   vheader_namesr   r   rG      s       r/   r0   zSummary results for z are saved to csv file: N)r7   listsortappendr8   r9   r:   updateitemsgetr;   r<   r   )r$   r%   rA   r=   Z
model_listZbatch_sizessequence_lengthsZ	key_namesr0   r/   r>   r"   rowvaluesr?   headerskeyr   rI   r   output_summary   sN    
rV   c                  C   s>  t  } t|  dD ] }t|}|rt| d| q| j}| jr^tj| js^td| j tj	
dddk}g }t|}| jD ]}||_t|d d|_| jd krtd	 t }t|| j| j| j\}	}
t | }td
|dd t|	j ||	jjkrtdtd|	|d| jd}td}td t }td| jdkrZd| j dndd}t | }td|dd td t }|j||ddd}t | }td|dd | j|d< ||d< ||d< |
|d < | j|d!< ||d"< | j|d#< t| || qt|d$ d%D ]}t|| d&| q d S )'N)zonnxruntime-gpuZonnxruntimeonnxZtorchtransformersZoptimumdatasetsevaluatez versionz Onnx model path does not exist: ZORT_DISABLE_FUSED_ATTENTION01      z4Exporting onnx model. It might take a few minutes...z!Onnx model exported or loaded in z.1fz secondszTsequence length should not be larger than {ort_model.config.max_position_embeddings}zquestion-answeringT)r"   	tokenizerZquestion_firstr/   zLoading dataset...Zsquadr   zvalidation[:]Z
validation)splitzDataset loaded in z<Evaluating squad_v2 with ORT. It might take a few minutes...Zsquad_v2)Zmodel_or_pipelinedataZmetricZsquad_v2_formatzEvaluation done in r   r.   r-   r   r/   r0   r   z
detail.csv)r2   r1   r4   z.csv) parse_argumentsr   r   Z
model_namerW   r   r   existsRuntimeErrorenvironrP   r   r   rQ   Zmodel_max_lengthminZ
doc_stridetimer#   r   r   configZmax_position_embeddingsr   r/   r
   r	   r3   ZcomputerM   r@   rV   )argsr!   Zpackage_versionr-   r.   Zall_resultsr_   r0   
start_timeZ	ort_modelr   ZlatencyZqa_pipelineZtask_evaluatorZsquad_datasetr?   rA   r   r   r   main   sv    


    &



rl   c                 C   s   t  }|jdddttd dt d |jddd	td
gdd |jddtddd |jddtddd |jddtd dd |jddddd |jddddd |jdd || }|S ) Nz-mz--model_nameFr   z=Checkpoint directory or pre-trained model names in the list: )requiredtypedefaulthelpz-sz--sequence_lengths+i  zFSequence lengths for onnx model inputs. It could have multiple values.)nargsrn   ro   rp   z-bz--batch_size   zbatch size for inference.)rn   ro   rp   z-tz--totalz+Total samples to test. 0 means all samples.z--onnxzbOptional onnx model path. If not specified, optimum will be used to export onnx model for testing.z
--providerr   zRSelect which Execution Provider to use for runs. Default is CUDAExecutionProvider.)rm   ro   rp   z--use_io_binding
store_truezUse IO Binding for GPU.)rm   actionrp   )r   )argparseArgumentParseradd_argumentstrPRETRAINED_SQUAD_MODELSintset_defaults
parse_args)argvparserrj   r   r   r   rc     sT    		
rc   __main__)Nr   F)N))rv   r8   r   rh   importlib.metadatar   r   ImportErrorZimportlib_metadatapathlibr   typingr   r   r   r   rY   r	   rZ   r
   Zoptimum.onnxruntimer   Zoptimum.versionr   Zoptimum_version	packagingZversion_checkrX   r   r   parserz   ry   r   boolr#   r@   rV   rl   rc   __name__r   r   r   r   <module>   sH          $)AF
6
