U
    T?hI                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZ eeZdd Zdd Zd	d
 Zdd Zdd Zedkre  dS )    N)setup_logger)BenchmarkRecord)WhisperConfigWhisperProcessorc                  C   s  t  } | jddtddd | jddd dd	 | jd
dd ddgdd | jddtdd | jddtdd | jddddd | jddddd | jdtdd  | jd!td"d  | jd#tdd$d | jd%tdd&d'd(gd)d* | jd+tdd,d-d.gd/d* | jd0td1d2d3 | jd4ddd5d | jd6tdd7d3 | jd8td d9d3 | jd:ddd; |  }t|d<|jd=d> 	d?d@ dA|j
 d@|j }|js||_tj|jddB | jdC9  _|S )DNz-a--audio-pathTz0Path to folder of audio files for E2E evaluation)typerequiredhelpz-lz
--languagezLanguage of audio file)defaultr	   z-tz--taskZ
transcribe	translatezTask to complete)r
   choicesr	   z-w--warmup-runs   )r   r
   z-n
--num-runs
   z--hf-pt-eagerF
store_truez,Benchmark in PyTorch without `torch.compile`)r
   actionr	   z--hf-pt-compilez)Benchmark in PyTorch with `torch.compile`--hf-ort-dir-pathzDPath to folder containing ONNX models for Optimum + ORT benchmarking)r   r	   --ort-model-pathz'Path to ONNX model for ORT benchmarking--model-namez9Model name in Hugging Face (e.g. openai/whisper-large-v2)--precisionZint8Zfp16Zfp32zPrecision to run model)r   r   r   r	   --devicecpucudaZrocmzDevice to benchmark models--device-idr   zGPU device ID)r   r
   r	   z	--verbosezPrint detailed logsz	--timeoutz8Number of mins to attempt the benchmark before moving on--log-folderz'Path to folder to save logs and results--tune)r
   r   
model_size/.-z./)exist_ok<   )argparseArgumentParseradd_argumentstrint
parse_argssetattr
model_namesplitreplacer   	precision
log_folderosmakedirstimeout)parserargsZlog_folder_name r5   g/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/models/whisper/benchmark_all.pyget_args   s     r7   c                  C   s  g }d }d}d}d}d}d\}	}
d\}}d\}}}}d\}}d}d}d	}d
}t |(}|D ]}|dd}||kr|d}n"||krd}n||ks||krd}||krt|t||d }q\||krt|t||d }|dk r|| }	}
d }|dkrx|| }}d }q\||krNt|t||d }|| }|d }q\||kr\d|krt||dd |d d }n@|||t| d  dd}t||  d }t|d }|	r|	nd|r|nd |r|nd }|d }|r|| nd}t	
d| d t	
d| d t	
d|  |||	|
|rP|nd|r\|nd|||||g
 }|| q\W 5 Q R X |S )NzLoad audio: zFeature extraction: zEvaluating PyTorch...zEvaluating ONNX Runtime...)NN)NNNNz	Latency: zThroughput: zGenerated token length: zpeak=
 z
load-audiozfeature-extractionprocess   ZCPU=   z MB'"Zmax_used_MBr   r   zTotal latency: z szAudio duration: zReal-time factor: )openr-   floatlenrfindr(   findjsonloadsloggerinfoappend) 	device_idlog_filebase_resultsentriesstepZload_audio_patternZfeat_ext_patternZpytorch_patternZonnxruntime_patternZload_audio_latency_sZload_audio_throughput_sZfeat_ext_latency_sZfeat_ext_throughput_stoken_lengthZ	latency_sZper_token_latency_sper_token_latency_msZ
throughputZmemoryZlatency_patternZthroughput_patternZtoken_length_patternZmemory_patternfZ
input_linelineZpeakusageZtotal_latencyZaudio_durationrtfentryr5   r5   r6   process_log_file   s    








&"rW   c                 C   s  dd l }|j| dddddddd	d
dddddddddgd}|d d|d< |d d|d< |d	 d|d	< |d
 d|d
< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< dd l}|j}tdd |D }d}d}|rv|d dd }|d dd }g }	| D ]<\}
}|d dkrt|d |d |d |d ||}n&t|d |d |d |d t	j
t	j}|d |jjd< |d |j_|d |j_|d	 |jjd< |d
 |jjd< |d |jjd < |d |jjd!< |d |jjd"< |d |jjd#< |d |jjd$< |d |jjd%< |d d& |j_|d |j_|d |j_|	| qt||	 t|d'd(|	 td)| d* d S )+Nr   zWarmup RunszMeasured Runsz
Model NameZEngineZ	PrecisionZDevicez
Audio FilezDuration (s)zToken LengthzLoad Audio Latency (s)zLoad Audio Throughput (qps)zFeature Extractor Latency (s)z"Feature Extractor Throughput (qps)zLatency (s)zPer Token Latency (ms/token)zThroughput (qps)zMemory (GB)zReal Time Factor (RTF))columnsr(   rB   c                 S   s(   g | ] }|j d kr|j  d|j qS ))zort-nightly-gpuzort-nightlyonnxruntimezonnxruntime-gpu==)keyversion).0ir5   r5   r6   
<listcomp>&  s   
z save_results.<locals>.<listcomp>r9   rZ   r>   rY   
audio_filedurationrP   Zload_audio_latencyZload_audio_throughputZfeature_extractor_latency_sZ feature_extractor_throughput_qpsrQ   rU   r<   .csvz.jsonzResults saved in !)ZpandasZ	DataFrameZastypepkg_resourcesworking_setsortedr,   Ziterrowsr   torch__name____version__configZ
customizedwarmup_runsZmeasured_runsmetricsZlatency_ms_meanZthroughput_qpsZmax_memory_usage_GBrJ   Zsave_as_csvZsave_as_jsonr-   rH   rI   )resultsfilenamepddfrd   Zinstalled_packagesZinstalled_packages_listZort_pkg_nameZort_pkg_versionrecords_rowrecordr5   r5   r6   save_results   s              ru   c              	   C   s   | dt j  dd}tj| j|}t|dF}tj|||d}z|	| j
 W n tjk
rr   |  Y nX W 5 Q R X td | j| j| j|| j| j||g}	t| j||	}
|
S )Nrr   %Y-%m-%d_%H:%M:%Sz.logw)stdoutstderrz Gathering data from log files...)datetimenowr0   pathjoinr/   rA   
subprocessPopenwaitr2   TimeoutExpiredkillrH   rI   rk   num_runsr+   r.   devicerW   rK   )r4   benchmark_cmdZenginer`   ra   Zlog_filenamelog_pathrL   r:   rM   rm   r5   r5   r6   	benchmarkU  s(    

r   c                  C   sv  t  } t| j t| j dtjj_	t
| j}t| j}|j| j| jd}|jgttdd | }| jr| jrdt|gng }| jr| jrdt|gng }| jrdgng }g }t| jD ]t}	tj| j|	}
ztj|
d}W nN tk
r6 } z.d}tjd	| dd
 td|	 d W 5 d }~X Y nX td|
 d | jrdddd|
ddd| jd| j d| j!dt| j"dt| j#dt| j$d| j%g| }td t	| |d|	|}|&| | j'r:dddd|
ddd| jd| j d| j!dt| j"dt| j#dt| j$d| j%g| }td t	| |d |	|}|&| | j(rdddd|
dd!d"| j(d| jd| j d| j!dt| j"dt| j#dt| j$d| j%g| }td# t	| |d$|	|}|&| | j)rdddd|
dd%d&| j)d| jd| j d| j!dt| j"dt| j#dt| j$d| j%g| | }td' t	| |d(|	|}|&| q| j* d)| j  d*t+j+, d+d,}t-|tj| j%| d S )-NT)languagetaskc                 S   s   | d S )Nr>   r5   )Ztoken_idr5   r5   r6   <lambda>|      zmain.<locals>.<lambda>z--decoder-input-idsr   )r|   r   z@An error occurred while trying to calculate the audio duration: )exc_infozJIf you get an error that says:
	soundfile.LibsndfileError: Error opening 'zt': File contains data in an unknown format.
you may not have installed `ffmpeg` in addition to installing `librosa`.zTesting z...pythonz-mzmodels.whisper.benchmarkr   z--benchmark-typezhf-pt-eagerr   r   r   r   r   r   r   z'Benchmark PyTorch without torch.compilezpytorch-eagerzhf-pt-compilez$Benchmark PyTorch with torch.compilezpytorch-compilezhf-ortr   z Benchmark Optimum + ONNX Runtimezoptimum-ortZortr   zBenchmark ONNX RuntimerY   r!   rr   rv   rb   ).r7   r   verboserH   rI   __dict__rg   backendsZcudnnr   r   Zfrom_pretrainedr+   r   Zget_decoder_prompt_idsr   r   Zdecoder_start_token_idlistmapr'   Ztuner0   listdir
audio_pathr|   r}   librosaZget_duration	ExceptionwarningZhf_pt_eagerr.   r   rK   rk   r   r/   extendZhf_pt_compileZhf_ort_dir_pathZort_model_pathr   rz   r{   ru   )r4   rj   	processorZhf_forced_decoder_idsZort_forced_decoder_idsZhf_decoder_input_ids_cmdZort_decoder_input_ids_cmdZort_tune_cmdZall_resultsr`   r   ra   er   rm   Zcsv_filer5   r5   r6   mainp  s(   









"r   __main__)r$   rz   rF   loggingr0   r~   r   rg   Zbenchmark_helperr   rl   r   Ztransformersr   r   	getLoggerrh   rH   r7   rW   ru   r   r   r5   r5   r5   r6   <module>   s(   
 [^ $