U T?hN @spddlmZddlZddlZddlmZmZddlm Z m Z ddddd Z d;d d dddddddZ dd dddddddZd d!d"d#Zd?d%dddd&d'd(Zd%ddd)d*d+Zd,d%d-d.d/Zd,d%dddd%d0d1d2Zd,d%d%ddd3d4d5Zd d6dd7d dddd8d9d:ZdS)@) annotationsN) AutoConfig AutoTokenizer)InferenceSessionOrtValuez torch.Tensorbool)attention_mask use_past_kvcCs@|dd}||dkd|r<|dddfd}|S)Nr)longZcumsumZ masked_fill_ unsqueeze)rr position_idsrd/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/models/llama/llama_inputs.pyget_position_idss rptFrz torch.deviceintstr)configdevice batch_sizeseq_lenengine return_dictc Cstjd|j||ftjd}tj||tjd}t|dd}|dkrH|n||}|dkrb|n||}|dkr||n||}|s|||fS|||d} | S)NrlowhighsizedtyperFr ort input_idsrr)torchrandint vocab_sizeint64onesrnumpyto) rrrrrrr$rrinputsrrrget_sample_inputss  r-r )rrr past_seq_lenuse_fp16rr world_sizec s.tjd|j|dftjd}tj||dtjd} t| dd} t|||||d} |dkr^|n|}|dkrx| n| } |dkr| n| } |dkrt | nt t fd d | } |st | t st || | | fS|| | d } |dkrt | tst | | nt | t s"t | | d <| S) Nrr rr Tr!r0r"cs|d|dfSNrr r+kvrrrYz0get_sample_with_past_kv_inputs..r#past_key_values)r%r&r'r(r)rget_past_kv_inputsr*r+flatten_past_kv_inputslistmap isinstanceAssertionErrordictupdate) rrrr.r/rrr0r$rrpast_kvr,rr6rget_sample_with_past_kv_inputsBs0      rC) rrrrr. max_seq_lenr/use_buffer_sharerrr0c sDtjd|j||ftjd} tj|||tjd} t| |dkd} t||||| d}|dkrb| n| } |dkr|| n| } |dkr| n| } |dkrt |nt t fdd|}| st |t st | | | |fS| | | d }|dkr(t |ts t |||r@t|||}nt |t s8t ||d <|S) Nrrr r!r1r"cs|d|dfSr2r3r4r6rrr7r8z7get_merged_sample_with_past_kv_inputs..r#r9)r%r&r'r(r)rr:r*r+r;r<r=r>r?r@rA enable_past_present_share_buffer)rrrrr.rDr/rErrr0r$rrrBr,rr6r%get_merged_sample_with_past_kv_inputszs4     rG)rrr.rrDr/rEsplit_kvc Csp|r tjntj}|j|j} |stj|||j|dtjt |||fdd|tj||j ||j| |tj||j ||j| |tj |tj dd} ntj|||j|tjtj |||ftj ddddtj tj |tj dd} t|j D]T} | d| dtj||j|| |d | dtj||j|| |iq|rlt| ||} | S) Ngr )kr )x attn_maskZk_cacheZv_cachepos)rJrKrLZk__cacheZv_)npfloat16float32 hidden_sizenum_attention_headsrandomrandZastypeZtriur)num_hidden_layersarrayr(int32rangerArF) rrr.rrDr/rErHZnp_dtype head_size ort_inputsirrrget_msft_sample_inputssp  " $    r\)rrr.r/r0csZ|j|t|dr|jn |j|j|r0tjntjfddt|j D}|S)Nhead_dimc s4g|],}tjdtjdfqS)r )r%rT).0_rrY num_headsr. torch_dtyperr sz&get_past_kv_inputs..) num_key_value_headshasattrr]rQrRr%rOrPrXrU)rrr.r/r0rBrr`rr:s r:z'list[tuple[torch.Tensor, torch.Tensor]])r9cCsVi}t|D]D\}\}}||d|d<||d|d<q |S)Npast_key_values..key.value) enumeratedetachcpur*)r9rBr[Zpast_kZpast_vrrrr;s r;r@) pt_inputsrEr.rDcCsli}|D]J\}}t|tjr*|||<q |dkrB|t|q |||<q |rht |||}|S)Nr9) itemsr>rNZndarrayrAr;rjrkr*rF)rmrEr.rDrZrIvrrrconvert_inputs_for_orts   rp)rZr.rDc Cst|D]f\}}d|ks d|kr|j\}}}}tj||||f|jd} || d|d|d|d|f<| ||<q|S)Ncacher9r )rnshaperNzerosr) rZr.rDrIrorrar_rYnew_vrrrrFs  rFr)modelrZcCslttdd|}t|}||}t|rHtd|td||}t|rh|D] }||=q\|S)NcSs|jSNname)Z model_inputrrrr7$r8z#verify_ort_inputs..z(The following model inputs are missing: zEThere are missing inputs to the model. Please add them and try again.)setr= get_inputskeyslenprint Exception)rurZ model_inputsZ user_inputsZmissing_inputsZunnecessary_inputsZunnecessary_inputrrrverify_ort_inputs"s r)rurZr device_idrEkv_cache_ortvaluescCs"|}ttdd|}|D]\}} ||kr8q&|rd|ksLd|kr||krztj| ||d} ||| | ||<q||| ||||q&tj| ||d} ||| q&| D]X} | j } |rd| ksd| kr| dd dd} | | || q|j | ||dq||fS)NcSs|jSrvrw)r[rrrr7@r8z.add_io_bindings_as_ortvalues..rqr9) device_typeroutpresent) io_bindingryr=rzrnrZortvalue_from_numpyZbind_ortvalue_inputZupdate_inplace get_outputsrxreplaceZbind_ortvalue_output bind_output)rurZrrrErrrrIroZv_deviceoutputrxZ input_namerrradd_io_bindings_as_ortvalues6s*   r)rur,outputsr/rEc Cst||}d}tjtjtjtjd}|}|D]P\}} |j|| j j | j j dkrXdn| j j |t | j t| j| d| j }q4|D]l} | j} |rd| kr|| ddn|| } |j| |j |j dkrdn|j |rtjntjt| j| dq|S)N)z torch.int32z torch.int64z torch.float16z torch.float32rkr)rxrrZ element_typerrZ buffer_ptrrr9)rrNrWr(rOrPrrnZ bind_inputrtypeindexreprrtuplerrZdata_ptrrrxrr) rur,rr/rErZpt_to_nprrIrorrxrrradd_io_bindings_as_tensorscsF    rrz list[str])r tokenizerrequested_lengthpromptrr/rErc Cs|j|_|j|dd}|r tjntj} tj|d|tjd} tj|d|tjd} t| dd} | j d} | |kr| ddd|f} | ddd|f} t| dd} nt| |kr| ddd f d j }| ddd f d j }t || D] }t || f} t || f} qt| dd} | j d} | |ks2t|d krD| n| |d krX| n| |d krl| n| d }|d krg|d <| j \}}|j}|j}t|d r|jn |j|j}t |jD]}tj|||r|nd ||| d}tj|||r|nd ||| d}|d kr>|d|d|d|d|in|d ||fqd}|d krtj|||j|| d}d|i}|st |jD]\}tj|||||| d}tj|||||| d}|d|d|d|d|iq||fS)NT)paddingr$)rrrFr!r rr"r#r9r]rfrgrhlogitszpresent.)Z eos_tokenZ pad_tokenZbatch_encode_plusr%rOrPZtensorr(rrrr TrXZhstackr? contiguousZmax_position_embeddingsrdrer]rQrRrUrsrAappendr')rrrrrr/rErZencodings_dictrbr$rrZtokenized_lengthZinput_ids_first_colZattention_mask_first_colr_r,rZsequence_lengthZmax_sequence_lengthrarYr[Zpast_keyZ past_valuerrZ present_keyZ present_valuerrrget_initial_inputs_and_outputss               "r)rF)FrFr )FFrFr )r )Frrl) __future__rr*rNr%Z transformersrrZ onnxruntimerrrr-rCrGr\r:r;rprFrrrrrrrrs> (?$65  -0