U
    T?h(                     @   sh   d dl mZ d dlmZ d dlmZmZ d dlmZ d dl	m
Z
 d dlmZ eeZG dd deZd	S )
    )	getLogger)Tuple)AttentionMaskFusionAttention)AttentionMaskFormat)	NodeProto)	OnnxModelc                       sJ   e Zd ZdZeeed fddZeeeef dddZ	dd	 Z
  ZS )
FusionAttentionClipzB
    Fuse Attention subgraph of Clip into one Attention node.
    )modelhidden_size	num_headsc                    s.   t |}tj|_t j||||ddgd d S )NFSkipLayerNormalization)Zuse_multi_head_attentionZsearch_op_types)r   r   ZNoMaskZmask_formatsuper__init__)selfr
   r   r   Zattention_mask	__class__ `/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/fusion_attention_clip.pyr      s    zFusionAttentionClip.__init__)	reshape_qreturnc                 C   sf  | j |dd}|dks&t|jdkr2| j| jfS | j |jd }|dkrX| j| jfS t|dksp|d dkr|| j| jfS |d }| j |jd }|dkr| j| jfS t|dks|d dkr| j| jfS |d }|| }| jdkr|| jkr| jrt	d| j d	| d
 d| _| jdkr^|| jkr^| j
r^t	d| j d	| d
 d| _
||fS )zDetect num_heads and hidden_size for ONNX model from MiDaS
        Args:
            reshape_q (NodeProto): reshape node for q
        Returns:
            Tuple[int, int]: num_heads and hidden_size
        Concat   N      r      z--num_heads is z. Detected value is z. Using detected value.Fz--hidden_size is )r
   match_parentleninputr   r   Zget_constant_valueZnum_heads_warningloggerwarningZhidden_size_warning)r   r   concatZnum_head_valuer   Zhead_size_valueZ	head_sizer   r   r   r   get_num_heads_and_hidden_size'   s6    z1FusionAttentionClip.get_num_heads_and_hidden_sizec           -      C   s  d }d }dD ]$}| j |d|}|d k	r|}|}qd }|d k	rJ|jd }ndD ]z}d }| j |d|}	| j |d|}
|	d k	r|	}n|
d k	r|
}|d krqN| j |d|d}|d krqN|jd }|} qqN|d krd S | j |ddd	d
d	dgd| d d dddg}|d krd S |\}}}}}}| j |d	d
d	ddgddddd g}|d kr\td d S |\}}}}}d }g }d }| j j|dd	dd	dgdddd dg|d}| j |ddgddg}|d k	r|}t|dkstd|d  }|\}}}}}n&|d k	r
|}|\}}ntd d S | j |d	d
d	dddgddddd d g}|d krXtd d S |\}}}} }!}"| j |d
d	d
d	ddgdddddd g}#|#d krtd d S |#\}$}%}}}&}'|"j	d |ks|'j	d |ks|j	d |krtd d S | 
|\}(})|(dks|)dkr&td d S |}*|d k	r| j |ddddddg|dddddg}+|+d kr| j |dddddg|ddddg}+|+d krtd d S | jd |"|'||!|&||(|)||*jd d d |d k	d},|,d krd S | j|, | j| j|,j< | j|*|g d| _d S )N)r   r   r   r   )r   r   AddZLayerNormalizationFZMatMulZReshapeZ	Transposer   z&fuse_attention: failed to match v pathZSoftmax)Zreturn_indicez'fuse_attention: failed to match qk pathZMulz&fuse_attention: failed to match q pathz&fuse_attention: failed to match k pathz>fuse_attention: expect to have same input to q, k and v matmulz9fuse_attention: failed to detect num_heads or hidden_sizer   ZExpandZ	UnsqueezeZWhereZLessz4fuse_attention: failed to match causal mask subgraph)Z
mask_indexZq_matmulZk_matmulZv_matmulZq_addZk_addZv_addr   r   r   outputZ
add_qk_strscaleZcausalT)r
   r   r$   Zfind_first_child_by_typeZmatch_parent_pathr   debugr   AssertionErrorr   r"   Zcreate_attention_nodeZnodes_to_addappendZthis_graph_nameZnode_name_to_graph_namenameZnodes_to_removeextendZprune_graph)-r   Znormalize_nodeZinput_name_to_nodesZoutput_name_to_nodeZskip_input_indexZnode_before_layer_normiparentZ
root_inputZnode_before_layer_norm_1Znode_before_layer_norm_2childZ	qkv_nodes_Zreshape_qkvZtranspose_qkvZ
matmul_qkvZv_nodesZ	reshape_vZadd_vZmatmul_vZadd_maskZadd_mask_indicesZqk_nodesZ
qk_nodes_1Z
qk_nodes_2Zcausal_mask_input_indexZ_softmax_qkZ	matmul_qkZq_nodesZ_transpose_qr   Zmul_qZadd_qZmatmul_qZk_nodesZ_transpose_kZ
_reshape_kZadd_kZmatmul_kr   r   Zattention_last_nodeZcausal_mask_nodesnew_noder   r   r   fuseU   s   

   

  





  

  

0






zFusionAttentionClip.fuse)__name__
__module____qualname____doc__r   intr   r   r   r"   r0   __classcell__r   r   r   r   r	      s   .r	   N)loggingr   typingr   Zfusion_attentionr   r   Zfusion_optionsr   Zonnxr   Z
onnx_modelr   r1   r   r	   r   r   r   r   <module>   s   