U T?ha@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl ZddlZddlmZddlmZmZddlmZmZmZmZddlZeeZdddd Zd d Z d d Z!ddZ"ddZ#ddZ$ddZ%edkre%dS)) annotationsN) setup_logger)add_io_bindings_as_tensorsget_initial_inputs_and_outputs) AutoConfigAutoModelForCausalLM AutoTokenizerBitsAndBytesConfigzargparse.Namespace)argsc Cs|jdkrJd}|jdkrt|jdkrttdddtjd}tj|jdkrJ|jn|j |j |j |j |j dd||jd id }nzLtj|jdkr|jn|j |j |j |j |j d|jdkrdnd d |j}Wnhtk r(}zHtd |tj|jdkr|jn|j |j |j |j |j ddd |j}W5d}~XYnX||jdkrt|}n:t}|jdkrldd|jifnd}tj|j||gd}|S)N pt-compilept-eagerint4cudaTZnf4)Z load_in_4bitZbnb_4bit_use_double_quantZbnb_4bit_quant_typeZbnb_4bit_compute_dtypeZflash_attention_2Z80GB) cache_dir torch_dtypeuse_auth_tokentrust_remote_code use_cacheattn_implementationZquantization_configZ max_memoryZsdpa)rrrrrrz&Try to load a model using eager mode: eagerr CUDAExecutionProvider device_idZCPUExecutionProvider) sess_options providers)benchmark_typeonnx_precisiondevicer torchfloat16rfrom_pretrained hf_dir_path model_namerrauthtrustrto target_device ExceptionprintevalcompileortZSessionOptionsZInferenceSessiononnx_model_path)r modelZ bnb_configerepr1e/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/models/llama/benchmark_e2e.py get_model8sn       r3c Cs|jdkr(t|f|}W5QRXd}|jdkrP|jdkrltj|jnt||||j|j }| t }t |D]X}|jdkrt(|f|}|jdkrtj|jW5QRXq||||q|t }|||} | |fS)Nr r cpu)rrZno_gradrrZ synchronizer'ruse_fp16use_buffer_shareZsynchronize_inputstime perf_counterrangeZrun_with_iobindingZsynchronize_outputs) r r.runsinputsoutputsZ io_bindingstart_endavgr1r1r2 run_inferencexs*            rAc CsFtt|||||j|j|j|j\}}t|||j||\}}||fSN) clear_cacherr'r5r6enginerAZ warmup_runs) r r.config tokenizer prompt_lengthpromptr;r<r>r1r1r2prepare_model_for_inferencesrIcCsttjdSrB)gcZcollectrrZ empty_cacher1r1r1r2rCsrCcCsvtj|ddddddddd |d d d |d d d |d d |d ddgd}|j|ddtd|ddS)Nz Batch Sizez Prompt LengthzPrompt Processing Latency (ms)z"Prompt Processing Throughput (tps)zSampling Latency (ms)zSampling Throughput (tps)z"First Token Generated Latency (ms)z&First Token Generated Throughput (tps)Average Latency of First z Tokens Generated (ms)Average Throughput of First z Tokens Generated (tps)zWall-Clock Latency (s)zWall-Clock Throughput (tps))columnsF)indexzResults saved in !)pdZ DataFrameZto_csvloggerinfo)resultsfilenameZ gen_lengthdfr1r1r2 save_resultss(  rWc Cst}|jddtddddgd|jdd td d d |jd dd ddd|jddd ddd|jddttjdddd|jdtddd|jddd d d!|jd"d#dtjdd$d%d&d'd(|jd)d dd*d|jd+d dd,df|jd-d.d/d0|jd1d2d3d0|jd4d5dtd6d7d8d9d6gd:d;|jdd?d|jd@dAttj rVdBndCdCdBgdD|jdEdFtdGdH|jdIdJtdKdH|jdLdMtdNdH|jdOtdPdH| }t j |jt|jd|jkrt|dQ|jdR|jdSkr|jdT|jif|_|jdkr*|js*tdU|jdV|_|jdV|_t|dW|j|jdXksx|jd7kr||jdCkr|d6nd9|_|jdCkrdY|jn|j}|jd9krtjntj}|jdkrdndZ}t|d[|t|d\|t|d]|t|d^|jd9k|jo|dk|_|S)_Nz-btz--benchmark-typeTr r r,)typerequiredchoicesz-mz --model-nameFzrbrczcuda:ptr'rrDr5)argparseArgumentParser add_argumentstrospathjoinintrrZ is_available parse_argsnprandomseedZ manual_seedrsetattrrupperrgrr-AssertionError batch_sizessplitprompt_lengths precisionr Zfloat32r6)parserr r'rrDr1r1r2get_argss        *   r~c9st}tdt|jd}t|j}tj|ddd}W5QRXt j |j dkrZ|j n|j |j |j|jd}tj |j dkr|j n|j |j |j|jd}t|}g}t|j|jD]\}}t|t|}}td|d|t||j} ||krTttd |d |jd |jd |d |d|d|d|d|jd|d||g|} ||g} ztdt|||||| \} } t|||j| | \}} |d}|||}td|dtd|||d| ||gtdtt|||||| \} } | d }|j!d}|j"}t#|drF|j$n |j%|j&}t'j(||j)t'j*d}g}g}t+,}|| krt||d| | \}} |-|t+,}| d j!ddkr| d!.dd}|j/dd"0d|j12|d|j1}t'3| d d|4}n| d dddddf}t'j5|dd"}||B|j6k}|7||j68|dg}t+,} |-| |t'j9||gdd"}|d7}|| d<t'9| d!|:t'j;8|dgd| d!<d#| krt'j<| d#dd"d$8|dd| d#<| d j!ddkr | d ddddddf=| d <| d >|j?d%krF| d&| d&<n|j@svtA|jBD]>}!| d'|!d(| d)|!d(<| d'|!d*| d)|!d*<qX| d!j!d}"tA|jBD]d}!t'j(|||"||j)|jCd}#t'j(|||"||j)|jCd}$| Dd'|!d(|#=d'|!d*|$=iqqvt+,}%|Ed$|jFrd+tG|tH|}&tItJfd,d|}tH|}'td-|&|'d.d/dd0t.|tH|}(|(d})|d|(}*td1|)dtd2|*d|d$}+|+d},|d|+}-td3|,dtd4|-d|jd5}.t.|d|.tH|d|.}/|/d}0|d|/}1td6|.d7|0dtd8|.d7|1dt.|tH|}2|2d}3|d|2}4td6|jd7|3dtd8|jd7|4d|%|}5|||j|5}6td9|5d:td;|||j|5dtd<| |)|*|,|-|0|1|3|4|5|6g |-| WqtKk r}7z td=|d|d>|7W5d}7~7XYqXqd?|j?d@tLjLMdAdB}8tN||8|jdS)CNFcSsdd|DS)NcSsi|]\}}t||qSr1)rq).0kvr1r1r2 hsz*main....)items)dr1r1r2hzmain..) object_hookr)rrrzRunning batch size = z, prompt length = z2 A prompt of size z was not found in 'zv'. There are a couple of solutions to fix this. 1) You can change one of the keys in 'z' to be z). If za < actual prompt's length, the benchmark E2E tool will repeat the first word in the prompt until zB = actual prompt's length. If zm > actual prompt's length, the benchmark E2E tool will automatically trim the actual prompt's length so that zd = actual prompt's length. 2) You can add a new key-value entry in 'z' of the form 'z,': 'your prompt goes here'. zMeasuring prompt processing...iz&Average Latency of Prompt Processing: z msz)Average Throughput of Prompt Processing: z tpszMeasuring token generation...Z input_idshead_dim)rZdtypeZlogitsZattention_mask)dimZ position_idsrriZpast_key_valueszpresent.z.keyzpast_key_values.z.value cs |kSrBr1)Zacc_timeZanomaly_threshold_factorZ min_time_sr1r2r rz Filtered out z$ anomaly accelerator times that are zx greater than z ms...zAverage Latency of Sampling: z Average Throughput of Sampling: z"Latency of First Token Generated: z%Throughput of First Token Generated: rLrKz Tokens Generated: rMzWall-Clock Latency: z szWall-Clock Throughput: zAdding results to CSVz$Could not benchmark at batch size = z - Z benchmark_Z_e2e_z%Y-%m-%d_%H:%M:%Sz.csv)Or~rrRrS__dict__openZ prompts_filejsonloadrr!r"r#rr$r%rr3 itertoolsproductryr{rqrCZgeneration_lengthNotImplementedErrortextwrapdedentrIrAZnum_runsextendcloneshapeZnum_key_value_headshasattrrZ hidden_sizeZnum_attention_headsrZzerosr'boolr7r8appendsumZ unsqueezerepeatZ vocab_sizeviewZgatherZsqueezeZargmaxZ eos_token_idZ masked_fillZreshapecatr&Zint64max contiguousZzero_rDr6r9Znum_hidden_layersrupdatepopZanomaly_filteringminlenlistfilterr(datetimenowrW)9r Zsize_to_promptfrErFr.Zall_csv_metricsZ batch_sizerG max_lengthrHZ csv_metricsr;r<Zaccelerator_prompt_latency_sZaccelerator_prompt_latency_msZaccelerator_prompt_thrptZ all_token_idsZcurrent_lengthZ num_headsZ head_sizeZhas_eosZaccelerator_timesZsampling_timesZwall_clock_start_timeZaccelerator_time_latency_sZsampling_start_timeZprompt_end_indicesZidxsZnext_token_logitsZ next_tokensZ tokens_to_addZsampling_end_timeiZnew_sequence_lengthZ present_keyZ present_valueZwall_clock_end_timeZ orig_sizenew_sizeZavg_sampling_latency_sZavg_sampling_latency_msZavg_sampling_thrptZfirst_token_latency_sZfirst_token_latency_msZfirst_token_thrptZhalfwayZhalfway_token_latency_sZhalfway_token_latency_msZhalfway_token_thrptZall_token_latency_sZall_token_latency_msZall_token_thrptZwall_clock_latency_sZwall_clock_thrptr/rUr1rr2main`s              &&               0r__main__)& __future__rrjrrJrrloggingrnrr7numpyrsZpandasrQrZbenchmark_helperrZ llama_inputsrrZ transformersrrrr Z onnxruntimer, getLogger__name__rRr3rArIrCrWr~rr1r1r1r2 s8   @ #~