U T?hc@sddlmZddlmZmZmZmZmZddlm Z ddl m Z ddl m Z mZmZddlmZeeZGddde ZGd d d eZd S) ) getLogger)DictListOptionalTupleUnion)Fusion) FusionUtils) NodeProto TensorProtohelper) OnnxModelcs eZdZdZd*eedfdd Zeede eeffddd Z ee ee efe e d d d Zd dZddZddZddZddZee eedeffdddZd+eeeeedefeedddZddZd d!Zd,d"d#Zd$d%Zd&d'Zd(d)ZZS)-FusionEmbedLayerNoMaskz Fuse embedding layer into one node (EmbedLayerNormalization). It supports the following model types: BERT, DistilBert, ALBert. no mask)model descriptioncs<t|dddg|t||_d|_d|_d|_d|_dS)NEmbedLayerNormalizationLayerNormalizationSkipLayerNormalizationF)super__init__r utils shape_infershape_infer_done attention embed_node)selfrr __class__\/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/fusion_embedlayer.pyrs zFusionEmbedLayerNoMask.__init__N)addreturncCsP|j|dgdg}|dkr dS|j|dgdg}|dkr@dS|d|dfS)NGatherr)rmatch_parent_path)rr! gather_0_path gather_1_pathrrr match_two_gather&sz'FusionEmbedLayerNoMask.match_two_gather) layernorminput_name_to_nodesis_distil_bertr"c Cs|jj|d|dd|_|jdk r$dS|jd|kr6dS||jd}tdd|D}|d d d d gkr|D]^}|jd krj|j|d d d d gddddg}|dk rj|d jd|jdkrj|d|_dSqjt |dkr|djd kr|djd|kr||djd}t |dkr|djd kr|djd|kr||djd} | D]}|jdkrV||_dSqVtdd| D}|r|d d d dd gkr|d d d d ddgkr|d d d d dgkrt ddSn2|d d d d gkr|d d d d gkrt ddSdS)aCheck that LayerNormalization has a child of Attention node or subgraph like Attention. Args: layernorm (NodeProto): LayerNormalization node input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes is_distil_bert (bool): whether it is DistilBert or not Returns: bool: whether there is Attention node or subgraph like Attention AttentionF) recursiveNTrcSsg|] }|jqSrop_type.0childrrr KszCFusionEmbedLayerNoMask.check_attention_subgraph..ZMatMulrAddMultiHeadAttentionr$cSsg|] }|jqSrr.r0rrr r3hsShapez Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Mul -- Cast(to=7) -- Add(B=1) -- Cast(to=7)* --> Gather | ^ V | +------------------------------+ Roberta new pattern from transformers v4.9: (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Add(B=0) -- Mul -- Cast(to=7) -- Add(B=1) --> Gather | ^ V | +-------------------------------------------+ start_node = position_embedding_gather start_index = 1 # match optional Cast node. parent = self.model.get_parent(start_node, start_index, output_name_to_node) if parent is None: return if parent.op_type == "Cast": if OnnxModel.get_node_attribute(parent, "to") != 7: return start_node = parent start_index = 0 i, path, return_indices = self.model.match_parent_paths( start_node, [ (['Add', 'Cast', 'Mul', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0]), (['Add', 'Cast', 'Mul', 'Add', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0, 0])], output_name_to_node) if path is not None: # constant input of Add shall be 1. i, value = self.model.get_constant_input(path[0]) if value != 1: return False _, self.padding_word_id = self.model.get_constant_input(path[-1]) return input_ids == path[-1].input[0] FrrrIrJrKrrr match_position_embedding_robertas-z7FusionEmbedLayerNoMask.match_position_embedding_robertac Cs|j|ddgddg|}|dkr&dS|\}}|j|jd}|dk rt|jdkr|jddkr|j|ddgr|j|ddgrt|jd ks|j|d dgsdS|j}|d krt |d dgsdSn|j|ddgsdS|j |d|} | dkrdS| j d kr<|j| dds*dS|j | d|} n| } | dksV| j d krZdS|j| ddspdS|j | d|} | dks| j dkrdS|| jdkS)a  Match position embedding path from input_ids to Gather for BERT. BERT Embedding Layer Pattern: (input_ids) / / Shape / | / Gather (indices=1) / | / Add (optional, B=0) / | Gather (segment_ids) Unsqueeze (axes=0) \ | | \ Gather Slice (data[1,512], starts=0, ends=*, axes=1, steps=1) \ / | Add Gather \ / Add | LayerNormalization SlicerDr$r7NFr Zaxesr4r#r8) rr%get_constant_valuer;r=rMrrHZget_opset_versionr Zcheck_node_attributeZ get_parentr/) rrIrJrKpathsliceZ unsqueezeZ slice_weightZ opset_versionrAZgatherrMrrr match_position_embedding_berts^      z4FusionEmbedLayerNoMask.match_position_embedding_bertcCs(||||rdS||||r$dSdS)NTF)rYrOrPrrr match_position_embedding:s z/FusionEmbedLayerNoMask.match_position_embeddingc Cs|jd}|r|jdnd}|jd}|jsB|jjdd|_d|_|jdk r|j|}|j|}|rl|sptt|dkrt|dkr|d|dkst d|d|dS|r|j ||st d |d |j|dS|j |jd } | dkst| j dkr t d dS|j |jd } | dksbt| j dksb| j d| j dkrpt d dS|r|j |jd } | dkst| j dks| j d| j dkrt ddS| j d | j d krt d|jd d| j d d|jd d| j d |r| j d | j d krlt d|jd d| j d d|jd d| j d | j d | j d krt d|jd d| j d d|jd d| j d dS)zXSanity check of embedding weights, and match hidden_size of weights and shape of inputs.r$NT)updater7z^Cannot fuse EmbedLayerNormalization: input_ids and position_ids not matched in 2nd dimension: z vs FzYCannot fuse EmbedLayerNormalization: input_ids and segment_ids does not have same shape: z != rzICannot fuse EmbedLayerNormalization: word embedding table is not expectedzMCannot fuse EmbedLayerNormalization: position embedding table is not expectedzLCannot fuse EmbedLayerNormalization: segment embedding table is not expectedzword_embedding_table (z) size z <= position_embedding_table (z <= segment_embedding_table (zposition_embedding_table ()r;rrZinfer_runtime_shaperZget_edge_shapeAssertionErrorr=r>infoZ compare_shaperVrMwarning) rword_embedding_gathersegment_embedding_gatherrIrJ segment_ids position_idsZinput_ids_shapeZposition_ids_shapeZword_embedding_tableZposition_embedding_tableZsegment_embedding_tablerrr check_embeddingHs|             222z&FusionEmbedLayerNoMask.check_embedding) input_namer"cCsXd}|j|}|dk r@|jjjtjkr:|j|\}}qP|}n|j|\}}||fS)aCast a graph input or node input to int32. Args: input_name (str): name of graph input or node input Returns: A tuple of casted input name and the cast node. int32_output (str): If input is int32, it is the input name, Otherwise it is output name of Cast node. input_cast_node (Union[None, NodeProto]): Cast node. It could be None if input is int32. N) rfind_graph_inputtypeZ tensor_typeZ elem_typer ZINT32rZcast_input_to_int32)rrdZinput_cast_nodeZ graph_inputZ int32_outputrrr cast_to_int32s  z$FusionEmbedLayerNoMask.cast_to_int32F)rJr)r_rIr`rbc Csg} ||\}} |jd} |jdkr>|jd} |jd} n|jd} |jd} d}|dk r||jd\}} |||jd|jd|jd| | g}n|d|jd|jdd| | g}|dk r|d||\}} ||| d | d g}|r|dk r|n| d }||tjd||| d }d |_|j D] }|j dkr4|j |gq4t |j dkr||j t ddg| || D]}|j|j|j <q|j | ||_|S)agCreate an EmbedLayerNormalization node. Note that segment embedding is optional. Args: input_ids (str): input_ids for word embeddings layernorm (NodeProto): LayerNormalization or SkipLayerNormalization node. word_embedding_gather (NodeProto): the Gather node for word embedding position_embedding_gather (NodeProto): the Gather node for position embedding segment_embedding_gather (Union[None, NodeProto]): the Gather node for segment embedding, or None. Returns: NodeProto: the EmbedLayerNormalization node created. rrr$r7rSNr_outputZ_dummy_mask_indexZ_embedding_sum)outputsnamez com.microsoftepsilong-q=)rgrZcreate_node_namer/r;appendr Z make_nodedomain attributerkextendr=Zmake_attributeZthis_graph_nameZnode_name_to_graph_name nodes_to_addr)rrJr)r_rIr`rbembedding_sum_outputembedding_sum_namerqrNZ node_namegammabetaZembed_node_inputsraZembed_node_outputsrkrZattrArrr create_fused_nodesj               z(FusionEmbedLayerNoMask.create_fused_nodecCs$|j|jd|jdd|_dS)NrT)rreplace_input_of_all_nodesr9Z prune_graph)rr)rrrr finish_fusion sz$FusionEmbedLayerNoMask.finish_fusioncCs*|jdko(t|jdko(t|jddkS)NrrSr)r/r=r9)rrArrr "is_skip_layer_norm_with_sum_outputsz9FusionEmbedLayerNoMask.is_skip_layer_norm_with_sum_outputc Cs||}|dkrdS|\}}|jd} |jd} |j||ddsFdS||d|sXdS|jdkr||} d} |} | r|jdnd}|dk o|j|dk }n|} |jdkrdnd} t |j| kr|j| nd}|dk o|j|dk }|o||kot ||dk}|dk o(|jdkp(|p(|} |j | ||||| | |rF|ndd}| rxd | j| <|sx|j ||jd | ||d S) NFr$r+rrSr4r)rrrsZ_no_use__to_be_removed_r7T) r(r;rCrcr/ryr9rZfind_graph_outputr=rvrwrx)rr)add_before_layernormr*rKoptional_segment_gather two_gatherr_rIrJrbZneed_embedding_sum_outputZsum_output_indexZnode_with_sum_outputZ sum_outputZis_sum_graph_outputZis_sum_used_by_multiple_nodesrrrr fuse_gpt2sX           z FusionEmbedLayerNoMask.fuse_gpt2c Cs||}|dkrdS|\}}|jd}|j||dds   z FusionEmbedLayerNoMask.fuse_bertc Cs|j|dgdg}|jdkr8|dkr*dS|d}d}n|j|dgdg}|j|dgdg}|dkr|dk r|dkr|dS|d}|d}nJ|dk r|dkr|j|dgdg}|dkrdS|d}|d}n|}d}||||||rdS|||||rdS|||||rdSdS)Nr4rrr#r$)rr%r/r~rr) rrAr*rKZfirst_add_pathr{r|r&r'rrr fusesB   zFusionEmbedLayerNoMask.fuse)r)NFN)N)__name__ __module__ __qualname____doc__r strrr rrr(rrboolrCrOrQrYrZrcrgrrvrxryr~rrr __classcell__rrrr rs@  T>/HJ   b Q)2rcs8eZdZd edfdd ZddZfddZZS) FusionEmbedLayerNormalizationF)rcst|d||_dS)Nz with mask)rruse_mask_index)rrrrrr rsz&FusionEmbedLayerNormalization.__init__cCs|j}t|jdkr0|j|td|jnDt|jdkrb|jdsb||jd<td|jntd|jdS|D]H}td|j|jdkr|jd|jd<qx|jd krx|jd|jd <qxdS) Nzappend mask to %szreplace mask in %szskip mask in %szupdate mask_index in %sr,r$rSr5rT) rr=r;rmr>r?rkr/r9)r mask_int32attention_nodesrZattention_noderrr replace_masks    z*FusionEmbedLayerNormalization.replace_maskcshd|_d|_d|_t||||jdkr0dS|jsNtd|ddS|jdkrz|jdkrztd|ddS|jr|jj d}n |jj d}||}|j |rdd|D}| |||ddS||krtd ||ddS||}|j d krdd d|D}|j d krN|j d }t|t|krN|j|| |||ddS)NzG--use_mask_index is not set: EmbedLayerNormalization will not have maskz EmbedLayerNormalization(no mask)zLEmbedLayerNormalization will not have mask since attention node is not foundrSrTcSsg|]}|jdkr|qS)r,r5r.r1rArrr r3s z6FusionEmbedLayerNormalization.fuse..z"EmbedLayerNormalization(with mask)zHEmbedLayerNormalization will not have mask since %s is not a node output) ReduceSumrFcSsg|]}|jdkr|qSrr.rrrr r3%s rr)rr<rrrrr>r?Zincrease_counterr;rrerr/r=Znodes_to_removerm)rrAr*rKrZchildren_nodesrrrr rsH                z"FusionEmbedLayerNormalization.fuse)F)rrrr rrrrrrrr rsrN)loggingrtypingrrrrrZ fusion_baserZ fusion_utilsr Zonnxr r r Z onnx_modelr rr>rrrrrr s    X