U
    T?h1                     @   sh   d dl mZ d dlmZ d dlmZ d dlmZ d dlm	Z	 ee
ZG dd deZG dd	 d	eZd
S )    )	getLogger)Dict)Fusion)helper)	OnnxModelc                       s2   e Zd Zed fddZeedddZ  ZS )FusionLayerNormalizationmodelc                    s   t  |dd d S )NLayerNormalization
ReduceMeansuper__init__selfr	   	__class__ [/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/fusion_layernorm.pyr      s    z!FusionLayerNormalization.__init__input_name_to_nodesoutput_name_to_nodec                 C   s  g }| j ||}t|dks*t|dkr.dS |jd }|d jdksX|d jd |kr\dS t|dkr|d jdks|d jd |krdS d}|D ]L}| j j|d|dd}	| j j|d	dgg d
}
|	dk	r|	}q|
dk	r|
d }q|dkrdS | j |dddddgdddddgfddddd	dgddddddgfg|\}}}|dk rJdS |d }||kr`dS |d }| j |\}}|dks|dks|dkrt	
d|  dS |d }| j |ddkrdS ||jd  d }|jd	kr|| ||jd  d }n|}|jdkrdS ||jd  d }|jdkr<dS || || ||dd  ||||g | j ||j||st	
d dS |jd	kr|n|}|jd| j |jd |  }| j |ddsdS |jd| j |jd |  }| j |ddsdS | j| tjd|jd ||g|jd g| j jdddd}|jtdt|g | j| | j| j|j< dS )a  
        Fuse Layer Normalization subgraph into one node LayerNormalization:
              +----------------------+
              |                      |
              |                      v
          [Root] --> ReduceMean -->  Sub  --> Pow --> ReduceMean --> Add --> Sqrt --> Div --> Mul --> Add
                     (axis=2 or -1)  |      (Y=2)   (axis=2 or -1)  (E-6 or E-12 or 0)    ^
                                     |                                               |
                                     +-----------------------------------------------+

         It also handles cases of duplicated sub nodes exported from older version of PyTorch:
              +----------------------+
              |                      v
              |           +-------> Sub-----------------------------------------------+
              |           |                                                           |
              |           |                                                           v
          [Root] --> ReduceMean -->  Sub  --> Pow --> ReduceMean --> Add --> Sqrt --> Div  --> Mul --> Add
              |                      ^
              |                      |
              +----------------------+
        r      NSub   DivF)	recursiveCast)excludeSqrtAddr   Powg-C6?zHskip SkipLayerNormalization fusion since epsilon value is not expected:    g       @Mulz4It is not safe to fuse LayerNormalization node. Skipzlayernorm weightzlayernorm biasr
   	LayerNormZname_prefixinputsoutputsnameepsilon)r	   Zget_childrenleninputop_typeZfind_first_child_by_typeZmatch_child_pathmatch_parent_pathsget_constant_inputloggerdebugZfind_constant_inputoutputappendextendis_safe_to_fuse_nodesZinput_indexZ$is_constant_with_specified_dimensionnodes_to_remover   	make_nodecreate_node_name	attributemake_attributefloatnodes_to_addthis_graph_namenode_name_to_graph_namer*   )r   noder   r   subgraph_nodeschildrenZ
root_inputZdiv_nodechildZ
div_node_1Z
div_node_2Zpath_idparent_nodes_Zsub_nodeZsecond_add_nodeiZ
add_weightZpow_nodeZ	temp_nodeZmul_nodeZlast_add_nodeZnode_before_weightweight_input
bias_inputZnormalize_noder   r   r   fuse   s    
  








zFusionLayerNormalization.fuse__name__
__module____qualname__r   r   r   rI   __classcell__r   r   r   r   r      s   r   c                       s2   e Zd Zed fddZeedddZ  ZS )FusionLayerNormalizationTFr   c                    s   t  |ddd d S )Nr
   r!   ZTFr   r   r   r   r   r      s    z#FusionLayerNormalizationTF.__init__r   c                 C   s  g }| j |ddddddddddg
dddd	d	d	dd	d	dg
fddddddd
ddddgdddd	d	d	d	dd	d	dgfg|\}}}|dkrdS t|dkst|d	 dkr|d dkr|d dkstd dS |dd \}}}	}
}}|dd \}}}}d}t|dkr|d }|jd
kst| j |dd	|}|dkrHtd dS | j |d	|}|dkrf|n| j |d	|}|dkrtd dS | j 	|\}}|dks|d	ks|dkr|dkrtd dS |dkr|j
d	 |j
ks|j
d	 |j
krtd dS |dk	rL|j
d	 |j
ks>|j
d	 |j
krLtd dS |j
d	 |j
d krptd dS ||||	|
|||||||g}|dk	r| j |d
d	|}|dkrtd dS ||||g | j ||j| j  | j  s td dS | j| |	j
d }|j
d	 }tjd|j
d	 ||g|jd	 g| j jdddd}|jtdt|g | j| | j| j|j< dS )aU  
         Layer Norm from Tensorflow model(using keras2onnx or tf2onnx):
          +------------------------------------+
          |                                    |
          |                                    |
        (Cast_1)                               |
          |                                    |
          |                                    v                                           (B)                             (B)             (A)
         Add --> (Cast_1) --> ReduceMean -->  Sub  --> Mul --> ReduceMean --> (Cast_3) --> Add --> Sqrt --> Reciprocol --> Mul --> Mul --> Sub --> Add
          |                       |                                                                                         |       ^              ^
          |                       |                                                                                         |       |              |
          |                       +--------------------------------------------------(Cast_2)-------------------------------|-------+              |
          |                                                                                                                 v                      |
          +---------------------------------------------------------------------------------------------------------------> Mul--------------------+
        r   r$   Z
Reciprocalr    r!   r   r   Nr   r   r#   )r   r   r   z=return indice is exepected in [0, 1], but got {return_indice}      zmul_node_3 not foundzroot node is nonegh㈵>zepsilon is not matchedz;reduce_mean_node_1 and mul_node_3 shall link from root nodez%mul_node_2 shall have two same inputszcast_node_2 not foundz$not safe to fuse layer normalizationr
   r%   r&   r'   r+   )r	   r/   r,   AssertionErrorr1   r2   r.   Zmatch_parentZ
get_parentr0   r-   r5   r6   r3   r   r   r7   r   r8   r9   r:   r;   r<   r=   r4   r>   r?   r*   )r   r@   r   r   Zreturn_indicerE   rD   Z
sub_node_0Z
mul_node_0Z
mul_node_1Zreciprocol_nodeZ	sqrt_nodeZ
add_node_0Zreduce_mean_node_0Z
mul_node_2Z
sub_node_1Zreduce_mean_node_1Zcast_node_3Z
mul_node_3Znode_before_reduceZ	root_noderF   r+   rA   Zcast_node_2rG   rH   Z
fused_noder   r   r   rI      s    !
&$
	




(












zFusionLayerNormalizationTF.fuserJ   r   r   r   r   rO      s   rO   N)loggingr   typingr   Zfusion_baser   Zonnxr   Z
onnx_modelr   rK   r1   r   rO   r   r   r   r   <module>   s    