U
    zh	                    @  s  U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*Z*ddl+Z*ddl,Z*ddl-m.  m/Z0 dd	l1m2Z2 dd
l3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9 ddl:m;Z; ddl<m=Z=m>Z> ddl?m@ZA ddlBmCZCmDZD ddlEmFZF ddlGmHZH ddlImJZJ ddlKm@Z@ ddlLmMZM ddlNmOZO e	PeQZRe*jSjTZTe*jSjUZUeZVe&eVe*jIjWf ZXG dd de ZYG dd de ZZG dd de Z[e%dZ\e&e*jIj]j^e_f Z`G dd  d Zaea ZbG d!d" d"ZcG d#d$ d$edZee&eceef Zfd%d&d'd(d)ZgG d*d+ d+ZhG d,d- d-eZiG d.d/ d/eiZjG d0d1 d1eiZkG d2d3 d3eiZlG d4d5 d5eiZmG d6d7 d7eiZne#ed8f ZoG d9d: d:enZpG d;d< d<epZqG d=d> d>epZrG d?d@ d@epZsG dAdB dBenZtG dCdD dDetZuG dEdF dFetZvG dGdH dHetZwG dIdJ dJeiZxG dKdL dLeiZyG dMdN dNeiZzG dOdP dPZ{G dQdR dRe Z|ej}G dSdT dTZ~ej}G dUdV dVe~Zej}G dWdX dXe~Zej}G dYdZ dZe~Zd"d[d\d]d^Zd_d`dadbdcddZeddedfdddfddgdhdidjdkd[dl
dmdnZe Zdoedp< dqddfddid-drdsdtZeejdu dv Zg Zdwedx< eddedyfdqdddfddgdhdidjd[dadzd{d|ZeAjdyd}ddd~ddidjd-dddZefdydd-dhdRd[ddddZefdydd-dhdRd[ddddZddd[dddZedZdd[dddZddddddZdd[dddZddadddZG dd dZddddddZddddjdidjd-dddZe* ddd_d~d[ddddZe* d_d~ddddZdddddZddadddZd_ddddZdqddddZdddddZe Zdoed< ddddddddĄZddddǜddɄZddddd̄ZdS )a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)Path)AnyCallableDefaultDictDict	GeneratorIterableListMappingNoReturnOptionalProtocolSequenceSetTupleTypeTypeVarUnion)Self	TypeGuard)enable_python_dispatcher)counters)is_integer_dtype)make_fxmaybe_disable_fake_tensor_modeguard_size_oblivious)immutable_dictimmutable_list   config)aot_functionmake_boxed_func)default_partition)FakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_typec                   @  s(   e Zd ZU ded< ddddddZdS )SearchFnstr__name__r   argskwargsreturnc                 O  s   d S N selfr2   r3   r6   r6   Q/var/www/html/venv/lib/python3.8/site-packages/torch/_inductor/pattern_matcher.py__call__l   s    zSearchFn.__call__N)r0   
__module____qualname____annotations__r:   r6   r6   r6   r9   r.   i   s   
r.   c                   @  s   e Zd ZddddddZdS )	ReplaceFnr   r1   c                 O  s   d S r5   r6   r7   r6   r6   r9   r:   q   s    zReplaceFn.__call__Nr0   r;   r<   r:   r6   r6   r6   r9   r>   p   s   r>   c                   @  s    e Zd ZdddddddZdS )TraceFnzUnion[SearchFn, ReplaceFn]r   torch.fx.GraphModule)fnr2   r3   r4   c                 O  s   d S r5   r6   )r8   rB   r2   r3   r6   r6   r9   r:   v   s    zTraceFn.__call__Nr?   r6   r6   r6   r9   r@   u   s   r@   Tc                   @  s   e Zd ZddddZdS )MultipleNoner4   c                 C  s   dt  ks| tkstd S )NMULTIPLE)globalsrG   AssertionErrorr8   r6   r6   r9   __init__   s    zMultiple.__init__N)r0   r;   r<   rK   r6   r6   r6   r9   rD      s   rD   c                      s   e Zd ZU dZded< ded< ded< ded	< d
ed< ded< ded< d7dddddd fddZeddddZd ddddZd dddZ	d dd!d"Z
ddd#d$d%Zd&dd'd(Zd)dd*d+Zdd,dd-d.d/Zd8d1d,d2d3dd4d5d6Z  ZS )9Matcha  
    Represents a successfully matched pattern.

    The `Match` object is returned to represent a successfully matched
    pattern. Included in the Match are the pattern that was matched, the graph
    nodes matched, and any args that were used during the matching.

    The args and kwargs are specific to the type of pattern that was matched and
    provide hints about what was matched.
    PatternExprpattern	List[Any]r2   zDict[str, Any]r3   List[torch.fx.Node]nodesz'Dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.Graph]replacement_graphNzOptional[Sequence[Any]]zOptional[Dict[str, Any]]rE   )rT   rN   r2   r3   r4   c                   sD   t    || _t|pg | _|p$i | _g | _i | _|| _d | _	d S r5   )
superrK   rN   listr2   r3   rQ   rR   rT   rU   )r8   rT   rN   r2   r3   	__class__r6   r9   rK      s    

zMatch.__init__torch.fx.GraphrF   c                 C  s   | j jS r5   )rT   graphrJ   r6   r6   r9   r[      s    zMatch.graphotherr4   c                 C  s   | j rFt| j  t|j  @ D ]"}| j | |j | kr"td|q"| j|j | j|j | j |j  | j|j d S )Nzkwarg mismatch: {})	r3   setkeysFailedMatchr2   extendrQ   updaterR   )r8   r]   keyr6   r6   r9   ra      s     zMatch.extendc                 C  s   | j rt| j gng | _ | S r5   )r2   tuplerJ   r6   r6   r9   bundle   s    zMatch.bundler/   c                 C  s   d| j  d| j dS )NzMatch(..., , )r2   r3   rJ   r6   r6   r9   __repr__   s    zMatch.__repr__r[   r4   c                 C  s$   t | jD ]}|js
|| q
d S r5   )reversedrQ   _erased
erase_node)r8   r[   nr6   r6   r9   erase_nodes   s    zMatch.erase_nodeszList[Optional[torch.fx.Node]]c                   s    fdd j jD S )Nc                   s$   g | ]}|d k	r j j| nd qS r5   )rT   pattern_to_node.0prJ   r6   r9   
<listcomp>   s   z&Match.output_nodes.<locals>.<listcomp>)rT   outputsrJ   r6   rJ   r9   output_nodes   s    
zMatch.output_nodestorch.fx.Nodec                 C  s   t dd |  D S )Nc                 s  s   | ]}|r|V  qd S r5   r6   rq   r6   r6   r9   	<genexpr>   s      z$Match.output_node.<locals>.<genexpr>)nextrv   rJ   r6   r6   r9   output_node   s    zMatch.output_nodeSequence[Any])rU   r2   r4   c                 C  s   t | | jj|| d S r5   )ReplacementPatternEntryreplace_with_graphrT   r[   )r8   rU   r2   r6   r6   r9   r}      s       zMatch.replace_with_graphTr>   zOptional[TraceFn]bool)replacement_fnr2   trace_fnrun_dcer4   c              	   C  sx   ddl m} |jd k	r|jntj}|H |d kr>tjt|d}||tj	
|dd }t| | jj|| W 5 Q R X d S )Nr   )Vr   c                 S  s
   | j d S Nvalmeta)argr6   r6   r9   <lambda>       z*Match.replace_by_example.<locals>.<lambda>)Ztorch._inductor.virtualizedr   Z	fake_mode
contextlibnullcontext	functoolspartialfwd_onlytorchfxmap_argr|   r}   rT   r[   )r8   r   r2   r   r   r   contextreplacementr6   r6   r9   replace_by_example   s     zMatch.replace_by_example)NN)NT)r0   r;   r<   __doc__r=   rK   propertyr[   ra   re   ri   ro   rv   rz   r}   r   __classcell__r6   r6   rX   r9   rL      s.   
  
  rL   c                   @  sJ   e Zd ZU dZded< dddddddZdd	d
dZdd	ddZdS )r`   z
    Represents a unsuccessful match.

    The `FailedMatch` object is returned to represent a failure to match a
    pattern.
    r/   format_stringr   rE   )r   r2   r3   r4   c                 O  s0   || _ t|dkr td| || _|| _d S )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r   lenRuntimeErrorr2   r3   )r8   r   r2   r3   r6   r6   r9   rK     s    zFailedMatch.__init__rF   c                 C  s   | j j| j| jS r5   )r   formatr2   r3   rJ   r6   r6   r9   __str__  s    zFailedMatch.__str__r~   c                 C  s   dS NFr6   rJ   r6   r6   r9   __bool__  s    zFailedMatch.__bool__N)r0   r;   r<   r   r=   rK   r   r   r6   r6   r6   r9   r`      s
   
r`   MatchResultzTypeGuard[Match])mr4   c                 C  s   t | S )z
    TypeGuards cannot act on `self`. Thus this function exists to let mypy
    recognize FailedMatch.__bool__ as a TypeGuard.
    )r~   )r   r6   r6   r9   is_match  s    r   c                   @  sh   e Zd ZU dZded< ded< ded< ded	< ddddddddZddddddZddddZd
S )rS   zC
    Internal state needed while running PatternExpr._match().
    List[Optional[PatternExpr]]ru   z*Dict[PatternExpr, Optional[torch.fx.Node]]rp   rZ   r[   zList[NodeOrConstant]exclusive_node_setNz*Optional[Dict[PatternExpr, torch.fx.Node]]rE   )ru   rp   r[   r4   c                C  s,   || _ |d kri nt|| _|| _g | _d S r5   )ru   dictrp   r[   r   )r8   ru   rp   r[   r6   r6   r9   rK   (  s    zMatchContext.__init__rM   NodeOrConstantr   )rN   noder4   c                 C  sZ   || j kr*| j | |kr"t| |S tdS ||| }|| j ksDt|rL|nd| j |< |S )z)wrapper to check reused nodes in patternszrepeated pattern differsN)rp   rL   r`   _matchrI   )r8   rN   r   r   r6   r6   r9   match4  s    

zMatchContext.matchz Dict[PatternExpr, torch.fx.Node]rF   c                 C  s   dd | j  D S )Nc                 S  s&   i | ]\}}|  r|d k	r||qS r5   )has_multiple_users)rr   rN   r   r6   r6   r9   
<dictcomp>A  s
     z;MatchContext.filter_multi_user_patterns.<locals>.<dictcomp>)rp   itemsrJ   r6   r6   r9   filter_multi_user_patterns@  s    z'MatchContext.filter_multi_user_patterns)N)r0   r;   r<   r   r=   rK   r   r   r6   r6   r6   r9   rS     s   
 rS   c                   @  st   e Zd ZdZeddddddZdddd	d
ZddddZddddZddddddZ	dddddZ
dS )rM   z+
    Base class for types of patterns.
    rw   rS   r   r   rT   r4   c                 C  s   d S r5   r6   r8   r   rT   r6   r6   r9   r   M  s    zPatternExpr._matchr   r4   c              
   C  sJ   zt | g|jd| |W S  tk
rD } z| W Y S d }~X Y nX d S Nr[   )rS   r[   r   r`   r8   r   er6   r6   r9   r   Q  s    zPatternExpr.matchr~   rF   c                 C  s   dS r   r6   rJ   r6   r6   r9   r   W  s    zPatternExpr.has_multiple_usersr/   c                 C  s   | j jd S )Nz())rY   r0   rJ   r6   r6   r9   ri   Z  s    zPatternExpr.__repr__Set[torch.fx.Node].Generator[Optional[torch.fx.Node], None, None]rT   searchedr4   c                 c  s   | |j kr|j |  V  d S r5   )rp   r8   rT   r   r6   r6   r9   find_anchor_nodes]  s    
zPatternExpr.find_anchor_nodesr   r\   c                 C  s   t || jS )z
        Compare two `PatternExpr`s and return true if they are the
        same. Note this is NOT matching a pattern - it is comparing the pattern
        structures (for debugging).
        )
isinstancerY   r8   r]   r6   r6   r9   
pattern_eqc  s    zPatternExpr.pattern_eqN)r0   r;   r<   r   r   r   r   r   ri   r   r   r6   r6   r6   r9   rM   H  s   rM   c                   @  s"   e Zd ZdZddddddZdS )	Argzn
    Capture an arg which will become an input to the handler.  Args are
    passed in depth first order.
    r   rS   r   r   c                 C  s   t || |gdS )N)r2   rL   r   r6   r6   r9   r   r  s    z
Arg._matchNr0   r;   r<   r   r   r6   r6   r6   r9   r   l  s   r   c                   @  s@   e Zd ZdZddddddZdd	d
dZdddddZdS )Ignoredz4
    Match an arg, but don't pass it to handler
    r   rS   r   r   c                 C  s
   t || S r5   r   r   r6   r6   r9   r   {  s    zIgnored._matchr/   rF   c                 C  s   dS )N*r6   rJ   r6   r6   r9   ri   ~  s    zIgnored.__repr__PatternPrettyPrinterppr4   c                 C  s   dS )Nz	Ignored()r6   )r8   r   r6   r6   r9   pretty_print  s    zIgnored.pretty_printN)r0   r;   r<   r   r   ri   r   r6   r6   r6   r9   r   v  s   r   c                      s\   e Zd ZdZddd fddZdddd	Zd
dddddZddd fddZ  ZS )
KeywordArgD
    Capture a kwarg which will become an input to the handler.
    r/   rE   namer4   c                   s   t    || _d S r5   rV   rK   r   r8   r   rX   r6   r9   rK     s    
zKeywordArg.__init__rF   c                 C  s   d| j dS )NzKeywordArg(rg   r   rJ   r6   r6   r9   ri     s    zKeywordArg.__repr__r   rS   r   r   c                 C  s   t || | j|idS )Nr3   )rL   r   r   r6   r6   r9   r     s    zKeywordArg._matchr   r~   r\   c                   s$   t t|}t |o"| j|jkS r5   typingcastr   rV   r   r   r   rX   r6   r9   r     s    zKeywordArg.pattern_eq	r0   r;   r<   r   rK   ri   r   r   r   r6   r6   rX   r9   r     s
   r   c                      sf   e Zd ZU dZded< ddd fddZddd	d
ZddddddZddd fddZ  Z	S )ExclusiveKeywordArgr   r/   r   rE   r   c                   s   t    || _d S r5   r   r   rX   r6   r9   rK     s    
zExclusiveKeywordArg.__init__rF   c                 C  s   d| j dS )NzExclusiveKeywordArg(rg   r   rJ   r6   r6   r9   ri     s    zExclusiveKeywordArg.__repr__r   rS   r   r   c                 C  s2   ||j krtdS |j | t|| | j|idS )Nzexclusive arg appears twicer   )r   r`   appendrL   r   r   r6   r6   r9   r     s    
zExclusiveKeywordArg._matchr   r~   r\   c                   s$   t t|}t |o"| j|jkS r5   r   r   rX   r6   r9   r     s    zExclusiveKeywordArg.pattern_eq)
r0   r;   r<   r   r=   rK   ri   r   r   r   r6   r6   rX   r9   r     s   
r   c                      s   e Zd ZU dZded< ded< d)ddd	d
 fddZeeddddZddddZ	ddddZ
ddddZddddddZdddd d!Zdddd"d#d$Zd%dd& fd'd(Z  ZS )*_TargetExprz7
    Base class for filtering match by node.target
    zList[FnsType]fnszSet[FnsType]fns_setr+   z!Union[FnsType, Sequence[FnsType]]zUnion[Multiple, int]rE   )r   usersr4   c                   sx   t    t|st|tr"|gnt|}|D ]. t tjjr.|	 fdd 
 D  q.|| _t|| _|| _d S )Nc                 3  s   | ]}t  |V  qd S r5   getattrrr   overloadrB   r6   r9   rx     s     z'_TargetExpr.__init__.<locals>.<genexpr>)rV   rK   callabler   r/   rW   r   _opsOpOverloadPacketra   	overloadsr   r^   r   r   )r8   r   r   rX   r   r9   rK     s    
 
z_TargetExpr.__init__r/   rF   c                 C  s   d S r5   r6   rJ   r6   r6   r9   op  s    z_TargetExpr.opc                 C  s~   | j d }t|ts|j}t| j dkr4d| dS | j d tt|d krTd| S t| j d tjjrvt| j d S |S d S )Nr   r+   [z, ...]torch.)	r   r   r/   r0   r   r   r   r   Z
OpOverload)r8   Z
first_reprr6   r6   r9   fns_repr  s    


z_TargetExpr.fns_reprc                 C  sJ   | j tkrd}n| j dkr*d| j  d}nd}| jj d|   | dS )Nz
, MULTIPLEr+   rf   rg    ()r   rG   rY   r0   r   )r8   Zcomma_usersr6   r6   r9   ri     s    

z_TargetExpr.__repr__r~   c                 C  s   t | jtp| jdkS )Nr+   )r   r   rD   rJ   r6   r6   r9   r     s    z_TargetExpr.has_multiple_usersrS   r   r   r   c                 C  s   t d S r5   NotImplementedErrorr   r6   r6   r9   r     s    z_TargetExpr.find_anchor_nodesrw   r   c                 C  s(   t |tjjo&|j| jko&t|| jkS r5   )r   r   r   Noder   extract_targetr   )r8   r   r6   r6   r9   
_match_fns  s
    
z_TargetExpr._match_fnsr   c                 C  s$   | |j kp"| jtkp"t|j| jkS r5   )ru   r   rG   r   r   r6   r6   r9   _match_users  s
    
z_TargetExpr._match_usersr   r\   c                   s<   t t|}t |o:| j|jko:| j|jko:| j|jkS r5   )r   r   r   rV   r   r   r   r   r   rX   r6   r9   r     s    


z_TargetExpr.pattern_eq)r+   )r0   r;   r<   r   r=   rK   r   r   r   r   ri   r   r   r   r   r   r   r6   r6   rX   r9   r     s   
 	r   .c                      s   e Zd ZdZdddddddd fd	d
ZeddddddZeddddddZddddZdddddZ	dddddd Z
dd!d"d#d$d%Zdd&d' fd(d)Z  ZS )*_TargetArgsExprzE
    Base class for filtering match by node.{target,args,kwargs}
    r+   )_usersz/Union[torch.fx.node.Target, str, Sequence[Any]]r   zUnion[int, Multiple]rE   )r   r2   r   r3   r4   c                  sh   t  || t|| _t|| _tdd t||	 D rJ| j
| _n| j| _| | j| j| _d S )Nc                 s  s   | ]}t |tttfV  qd S r5   )r   r   rW   rd   rr   xr6   r6   r9   rx     s   z+_TargetArgsExpr.__init__.<locals>.<genexpr>)rV   rK   rd   r2   r   r3   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r8   r   r   r2   r3   rX   r6   r9   rK     s    


z_TargetArgsExpr.__init__r{   zMapping[Any, Any]z9Tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]r1   c                 C  s&   | |  }t| f| }||fS r5   )r   r   r_   )r2   r3   r   specr6   r6   r9   r     s    z_TargetArgsExpr.simple_flattenc                   s6   ddd fdd t | |g\}} |}||fS )Nzpytree.TreeSpec)sr4   c                   sF   | j d kr| S tttttti}t|| j | j | j	tt
 | jS r5   )typer"   rW   rd   r!   r   pytreeZTreeSpecgetr   mapZchildren_specs)r   mapping	norm_specr6   r9   r  +  s    
z1_TargetArgsExpr.pytree_flatten.<locals>.norm_spec)r   Ztree_flatten)r2   r3   Zflatr   r6   r  r9   r   '  s    
z_TargetArgsExpr.pytree_flattenr/   rF   c                 C  st   |   ftt| jdd | j D }| jtkr>|d n| jdkrZ|d| j  | j	j
 dd| dS )	Nc                 S  s   g | ]\}}| d | qS =r6   rr   kvr6   r6   r9   rt   =  s     z,_TargetArgsExpr.__repr__.<locals>.<listcomp>_users=MULTIPLEr+   _users=r   rf   rg   )r   r   reprr2   r3   r   r   rG   r   rY   r0   join)r8   r2   r6   r6   r9   ri   9  s    


z_TargetArgsExpr.__repr__r   r   c                   s   |   f fdd| jD  fdd| j D }| jtkrJ|d n| jdkrf|d| j  d}| jj d	|	| d
S )Nc                 3  s   | ]}  |V  qd S r5   r   r   r   r6   r9   rx   H  s     z/_TargetArgsExpr.pretty_print.<locals>.<genexpr>c                   s$   g | ]\}}| d   | qS r  r  r  r  r6   r9   rt   I  s     z0_TargetArgsExpr.pretty_print.<locals>.<listcomp>r	  r+   r
  rf   r   rg   )
r   r2   r3   r   r   rG   r   rY   r0   r  )r8   r   r2   
joiner_strr6   r  r9   r   E  s    

z_TargetArgsExpr.pretty_printrw   rS   r   r   c                   s   |rt|jtjkr*td|S ||s@tdS |j}|j t tjk rddlm} ||j|j|j}|d krtd|S |\} t|tjkrt tjkr҇ fdd D  qtd|S n fdd D  	| \}}j
\}}	||	kr$td||	S t|t|ks:tt|}
tt ||D ]l\}}}t|tr|||}t|s|  S |
| n*t|tjjs||krTtd	||  S qT|
j| |j|
j< |
S )
Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionc                   s    i | ]}|j kr| | qS r6   r   rr   i_kwargsr8   r6   r9   r   h  s     
  z*_TargetArgsExpr._match.<locals>.<dictcomp>c                   s    i | ]}|j kr| | qS r6   r   r  r  r6   r9   r   n  s     
  zargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})r   r   r2   r`   r   r3   Ztorch.fx.operator_schemasr  targetr   r   rI   rL   zipr   countr   rM   r   r   ra   r   r   r   rQ   r   rR   )r8   r   rT   _argsr  Znormalized_args_and_kwargsZ
node_itemsZ	node_specZ
self_itemsZ	self_specr   r  rN   
child_nodechild_matchr6   r  r9   r   S  sZ    
  $  



  z_TargetArgsExpr._matchr   r   r   c                 c  s   | |j kr|j |  V  dS | jd D ]^}t|tr$|||D ]B}t|tjjsRq>|jD ]&}||krX| 	|rX|V  |
| qXq>q$dS )a  
        This is used when we are matching a pattern with multiple outputs.
        There is a partial match (stored in ctx) and we want to walk
        this pattern to find a connection to an already-matched node.

        Yields candidate nodes that `self._match` might like.
        Nr   )rp   r   r   rM   r   r   r   r   r   r   add)r8   rT   r   rN   Z
other_noder   r6   r6   r9   r     s    




z!_TargetArgsExpr.find_anchor_nodesr~   r\   c                   sP   t t|}t |oN| jd |jd koNtdd t| jd |jd D S )Nr+   c                 s  s.   | ]&\}}t |tr||n||kV  qd S r5   r   rM   r   rr   abr6   r6   r9   rx     s   z-_TargetArgsExpr.pattern_eq.<locals>.<genexpr>r   )r   r   r   rV   r   r   allr  r   rX   r6   r9   r     s    z_TargetArgsExpr.pattern_eq)r0   r;   r<   r   rK   staticmethodr   r   ri   r   r   r   r   r   r6   r6   rX   r9   r     s   2r   c                   @  s   e Zd ZdZdZdS )CallFunctionzR
    Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
    call_functionNr0   r;   r<   r   r   r6   r6   r6   r9   r"    s   r"  c                   @  s   e Zd ZdZdZdS )
CallMethodzW
    Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
    call_methodNr$  r6   r6   r6   r9   r%    s   r%  c                   @  s   e Zd ZdZdZdS )
CallModulezP
    Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
    call_moduleNr$  r6   r6   r6   r9   r'    s   r'  c                   @  s"   e Zd ZdZddddddZdS )	_TargetExprVarArgsz[
    Matches a call_function node with any arguments which are passed into the pattern
    rw   rS   r   r   c                 C  sh   |  |stdS | ||s&tdS t|| }|j| |j|j| < |j	|j |j
|j
 |S )NZfunction_mismatchZmultiple_users)r   r`   r   rL   rQ   r   r  rR   r2   ra   r3   rb   )r8   r   rT   r   r6   r6   r9   r     s    

z_TargetExprVarArgs._matchNr   r6   r6   r6   r9   r)    s   r)  c                   @  s   e Zd ZdZdS )CallFunctionVarArgsr#  Nr0   r;   r<   r   r6   r6   r6   r9   r*    s   r*  c                   @  s   e Zd ZdZdS )CallMethodVarArgsr&  Nr+  r6   r6   r6   r9   r,    s   r,  c                   @  s   e Zd ZdZdS )CallModuleVarArgsr(  Nr+  r6   r6   r6   r9   r-    s   r-  c                      s`   e Zd ZdZddddd fddZd	d
ddZddddddZddd fddZ  ZS )ListOfz$
    Matches a repeated pattern
    FrM   r~   rE   )rN   r   r4   c                   s(   t    t|tst|| _|| _d S r5   )rV   rK   r   rM   rI   rN   r   )r8   rN   r   rX   r6   r9   rK     s    
zListOf.__init__r/   rF   c                 C  s   | j j d| j dS Nr   rg   )rY   r0   rN   rJ   r6   r6   r9   ri     s    zListOf.__repr__rP   rS   r   r   c           
      C  s   t |ttfrt|dkr"tdS t|| }| }d}t|D ]b\}}t|j	||j
d}|| j|}	| }t|	s| js@td||	  S q@d}||	  q@|stdS | S )Nr   Znon_listFr   zlist[{}]: {}Tzlist: no_match)r   rW   rd   r   r`   rL   r   	enumeraterS   ru   r[   r   rN   r   r   ra   re   )
r8   r   rT   r   rp   Zmatchedr  r  Z	child_ctxr  r6   r6   r9   r     s,    
  zListOf._matchr   r\   c                   s2   t t|}t |o0| j|jo0| j|jkS r5   )r   r   r   rV   r   rN   r   r   rX   r6   r9   r     s    
zListOf.pattern_eq)Fr   r6   r6   rX   r9   r.    s
   r.  c                      s   e Zd ZU ded< ddd fddZedd	d
dZdd	ddZdddddZddddddZ	ddddddZ
dddddZd d!d" fd#d$Z  ZS )%MultiOutputPatternr   ru   zSequence[Optional[PatternExpr]]rE   )ru   r4   c                   sP   t    t|d tsttdd |D s6t|t|| _|d j| _d S )Nr   c                 s  s    | ]}|d kpt |tV  qd S r5   )r   rM   r   r6   r6   r9   rx     s     z.MultiOutputPattern.__init__.<locals>.<genexpr>)	rV   rK   r   r   rI   r   rW   ru   r   )r8   ru   rX   r6   r9   rK     s
    

zMultiOutputPattern.__init__z-Union[Callable[..., Any], str, Sequence[Any]]rF   c                 C  s   t t| jd }|jS )Nr   )r   r   r   ru   r   )r8   outputr6   r6   r9   r     s    zMultiOutputPattern.fnsr/   c                 C  s   | j j d| j dS r/  )rY   r0   ru   rJ   r6   r6   r9   ri   !  s    zMultiOutputPattern.__repr__r   r   c                   sD    fdd| j D }dd }| jj d|| }| d}|S )Nc                   s   g | ]}  |qS r6   r  r   r  r6   r9   rt   %  s     z3MultiOutputPattern.pretty_print.<locals>.<listcomp>z,
z  z([z
]))ru   rY   r0   r  )r8   r   r2   r  Zstr_outr6   r  r9   r   $  s
    

zMultiOutputPattern.pretty_printrw   rS   r   r   c                 C  sr   t t| jd }|||}t|s*|S | jdd  D ]4}|d krFq8| ||}t|sb|  S || q8|S )Nr   r+   )r   r   r   ru   r   r   _match_from_anchorsra   )r8   r   rT   r2  r   rN   r  r6   r6   r9   r   +  s    zMultiOutputPattern._matchrM   )rN   rT   r4   c                 C  sP   t |j}td}||t D ]*}|||}t|r@|  S t ||_q |S )Nzno anchor found)r   rp   r`   r   r^   r   r   )r8   rN   rT   Zpriorr   r   r6   r6   r9   r3  ;  s    
z&MultiOutputPattern._match_from_anchorsr   c              
   C  sJ   zt | j|jd| |W S  tk
rD } z| W Y S d }~X Y nX d S r   )rS   ru   r[   r   r`   r   r6   r6   r9   r   H  s    zMultiOutputPattern.matchr   r~   r\   c                   sH   t t|}t |oFt| jt|jkoFtdd t| j|jD S )Nc                 s  s.   | ]&\}}t |tr||n||kV  qd S r5   r  r  r6   r6   r9   rx   S  s   z0MultiOutputPattern.pattern_eq.<locals>.<genexpr>)	r   r   r   rV   r   r   ru   r   r  r   rX   r6   r9   r   N  s    zMultiOutputPattern.pattern_eq)r0   r;   r<   r=   rK   r   r   ri   r   r   r3  r   r   r   r6   r6   rX   r9   r1    s   
r1  c                      s`   e Zd ZdZddd fddZeddd	d
ZddddddZddd fddZ  Z	S )RepeatedExprzp
    Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
    r   rE   )inner_patternr4   c                   s   t    || _|j| _d S r5   )rV   rK   r5  r   )r8   r5  rX   r6   r9   rK   _  s    
zRepeatedExpr.__init__zSequence[FnsType]rF   c                 C  s   | j jS r5   )r5  r   rJ   r6   r6   r9   r   d  s    zRepeatedExpr.fnsrw   rS   r   r   c                 C  sv   | | j|}t|s|S |j| j | j|t D ]8}t| g|jd | j|}t|sf|  S |	| q8|S r   )
r   r5  r   rp   popr   r^   rS   r[   ra   )r8   r   rT   r   Zanchor_nodeZanchor_mr6   r6   r9   r   h  s     zRepeatedExpr._matchr   r~   r\   c                   s&   t t|}t |o$| j|jS r5   )r   r   r   rV   r   r5  r   rX   r6   r9   r   y  s    zRepeatedExpr.pattern_eq)
r0   r;   r<   r   rK   r   r   r   r   r   r6   r6   rX   r9   r4  Z  s   r4  c                   @  s`   e Zd ZdZddddZeedddd	d	d
ddZdd	dddZ	dd	dddZ
dS )r   z
    Serializes Patterns to executable python.
    XXX: currently only used and tested for fuse attention patterns. May not cover
    all patterns.
    rE   rF   c                 C  s   t jj | _i | _i | _d S r5   )r   r   r[   Z
_Namespace	namespacememoized_objs_namesmemoized_objs_pprJ   r6   r6   r9   rK     s    zPatternPrettyPrinter.__init__Nr2  rM   r/   )objoutput_namer4   c                 C  sn   t  }t| dst| j|d}g }|jD ]$}||j|  d|j|   q*|| d|  d|S )zU
        Serializes obj to python code with obj written out to `output_name`
        r   r  z = 
)r   hasattrrI   r   r8  r   r9  r  )r:  r;  r   Zout_strr2  rc   r6   r6   r9   run  s    
"zPatternPrettyPrinter.runr   )r:  r4   c                 C  sD   t |tr(| j| }r|S | |S t|dr<|| S t|S )Nr   )r   r   r8  r   memoizer=  r   r  )r8   r:  Zmemoized_namer6   r6   r9   r     s    



z!PatternPrettyPrinter.pretty_printr   c                 C  sN   | | }| }dD ]}||d}q| j|d }|| j|< || j|< |S )N)zaten.r   zprims.r   )r   r   replacer7  create_namer8  r9  )r8   r:  Zobj_strobj_nameprefixtmp_namer6   r6   r9   r?    s    


zPatternPrettyPrinter.memoize)r2  )r0   r;   r<   r   rK   r!  r   	lru_cacher>  r   r?  r6   r6   r6   r9   r     s   r   c                   @  s   e Zd ZdddddZdS )_PassDictsType Tuple[str, torch.fx.node.Target]List[PatternEntry])r  r4   c                 C  s   d S r5   r6   )r8   r  r6   r6   r9   __getitem__  s    z_PassDictsType.__getitem__N)r0   r;   r<   rI  r6   r6   r6   r9   rF    s   rF  c                   @  sH   e Zd ZU ded< ded< ddddd	d
dZddddddddZdS )PatternEntryrM   rN   Callable[[Match], bool]extra_checkrL   rZ   rw   rE   r   r[   r   r4   c                 C  s   t d S r5   r   r8   r   r[   r   r6   r6   r9   apply  s    zPatternEntry.applyNF/Union[_PassDictsType, Sequence[_PassDictsType]]z!Union[torch.fx.node.Target, None]r~   )
pass_dictsr  prependr4   c                 C  s   |d kr8t | jdst| jjD ]}| j|||d q n~t|ttfrt | jdsVt|rt|| jj|f 	d|  q|| jj|f 
|  n*ttt |}|D ]}| j|||d qd S )Nr   rR  r   r   )r=  rN   rI   r   registerr   r   PatternMatcherPassr   insertr   r   r   r   rF  )r8   rQ  r  rR  rB   r   r6   r6   r9   rT    s    zPatternEntry.register)NF)r0   r;   r<   r=   rO  rT  r6   r6   r6   r9   rJ    s   
  rJ  c                   @  s*   e Zd ZU ded< ddddddd	Zd
S )LoweringPatternEntryCallable[..., Any]handlerrL   rZ   rw   rE   rM  c              	   C  s~   t | jt | j|}||4 ||t|j|j}|j	
|j	 || W 5 Q R X |jd |kspt|| d S )N)r   wrapsrY  r   inserting_beforer#  rd   r2   r3   r   rb   replace_all_uses_withrQ   rI   ro   )r8   r   r[   r   rY  r   r6   r6   r9   rO    s    zLoweringPatternEntry.applyN)r0   r;   r<   r=   rO  r6   r6   r6   r9   rW    s   
rW  c                   @  s.   e Zd ZU dZded< dddddd	d
ZdS )GraphPatternEntryz8
    A pattern that runs a function on the FX graph
    rX  rY  rL   rZ   rw   rE   rM  c              	   C  s0   | | | j|f|j|j W 5 Q R X d S r5   )r\  rY  r2   r3   rN  r6   r6   r9   rO    s    zGraphPatternEntry.applyN)r0   r;   r<   r   r=   rO  r6   r6   r6   r9   r^    s   
r^  c                   @  sD   e Zd ZU ded< eddddddd	d
ZdddddddZdS )r|   zCallable[..., List[Any]]normalize_argsrL   rZ   z+Union[torch.fx.Graph, torch.fx.GraphModule]zSequence[torch.fx.Node]rE   )r   r[   rU   r2   r4   c              	     sf  |   }|d }G fdddtjj}|   }t|dkrF|d }nD|d sRtt|d jjfdd|D }t	|t
ddd }dd	d
dddd| ||j  }	t|	tjjr|	g}	ddddddddd fddt|t|	kr.t||	D ]\}
}|
| qn t|dks@t|d |	 W 5 Q R X |  d S )Nr   c                      s2   e Zd ZdZdZdZddd fddZ  ZS )z<ReplacementPatternEntry.replace_with_graph.<locals>.ReplacerNrw   r   r   c                   s   |j dkrt |S |j dkr|j}| |\}}|||}d|jkrd|jkr|jd |jd< t|jd tj	rd|jkst
|jd |jd< |S td| d S )N)placeholderr2  r#  r   Ztensor_metaz
unhandled )r   rV   run_noder  Zfetch_args_kwargs_from_envr#  r   r   r   TensorrI   r   )r8   r   r  r2   r3   result)rY   r[   r6   r9   ra    s    

zEReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_node)r0   r;   r<   r&  r(  get_attrra  r   r6   r   rX   r9   Replacer  s   re  r+   c                   s(   g | ] }t |tjjr ||fqS r6   )r   r   r   r   index)rr   rn   )rQ   r6   r9   rt     s   z>ReplacementPatternEntry.replace_with_graph.<locals>.<listcomp>)rc   rw   r/   r   rE   )r   recompute_taginput_stopsr4   c                 S  sX   | g}t  }|rT| }||kr||krt|dr|| ||jd< ||j qd S )Nr   	recompute)r^   r6  r=  r  r   ra   Zall_input_nodes)r   rg  rh  queuevisitedr   r6   r6   r9   percolate_tags$  s    

zBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tagsr   r   c                 S  s:   | j dkrd S | jtjkrd S t| jdks0t| jd S )Nr#  r#   r+   )r   r  operatorgetitemr   r2   rI   r   r6   r6   r9   maybe_getitem:  s    
zAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitemzUnion[torch.fx.Node, None]z3Union[torch.fx.Node, Sequence[torch.fx.Node], None])oldnewr4   c                   s  | d kr|d kst d S t| tjjs*t |d krJ| d  |  d S t|tjjrd|jkrp|j| j d| jkr|| jd t	  | | |  d S t
ttjj |}t| j }|D ]*}|}|d krt d|||  qΈ|  d S )Nr   ri  zcan't handle)rI   r   r   r   r   r]  rm   r   rb   r^   r   r   r   rW   r   r_   )rq  rr  Zold_usesuseridx)r2   r[   rp  rl  r@  r6   r9   r@  B  s0    






z;ReplacementPatternEntry.replace_with_graph.<locals>.replace)rv   r   r   Interpreterr   rI   rW   r[   rQ   minrm  
itemgetterr\  r>  r   r   r  ro   )r   r[   rU   r2   rv   Z
first_nodere  Z	last_nodeindicesr   rq  rr  r6   )r2   r[   rp  rQ   rl  r@  r9   r}     s2    

Cz*ReplacementPatternEntry.replace_with_graphrw   rM  c                 C  s0   |j d k	st| |||j | j|j|j d S r5   )rU   rI   r}   r_  r2   r3   rN  r6   r6   r9   rO    s    zReplacementPatternEntry.applyN)r0   r;   r<   r=   r!  r}   rO  r6   r6   r6   r9   r|     s
   
 r|   r~   r   r4   c                 C  s   dS )NTr6   r   r6   r6   r9   _return_true  s    r{  rX  r   rE   )	search_fnr   r4   c                 C  s   t d| j| d S )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor0   )r|  r   r6   r6   r9   log_trace_failure  s
    r  r6   zIterable[Any]rP  rK  z)Union[Dict[str, Union[float, int]], None]zSequence[str]zUnion[PatternExpr, None])
r|  
replace_fnexample_inputsr   rQ  rL  scalar_workaroundexclusive_arg_namessearch_fn_patternr4   c	              
     s   t j  ddd f	dd}	ddd fd	d
}
tkr^t r^dS tjddx dd |D dkrt	|}n}t
|}|tkstt| t||	|
d}|| |jW  5 Q R  S Q R X dS )a  
    Create a replacement rule based on example functions that get traced
    to create patterns.  This supports both training and inference when
    run on a joint forward+backward graph.

    Args:
        search_fn: traced to give original pattern
        replace_fn: traced to give replacement graph
        example_inputs: example inputs for initial trace
        trace_fn: fwd_only or joint_fwd_bwd
        pass_dict: dict of passes to register to
        extra_check: additional check to run on match(using real shapes)
    rL   r~   ry  c                   s  t }|D ]$}|jkrtd| dj qt tjfdd|D dd  g }tjj  t	D ]\}}t
 | tjrt|rt | jr W 5 Q R  dS tj |   |   | j | j|d |< t | j |  D ]0t
tjrtfd	d
|D r| qqt
}|s|rLddd 	fdd}z||  }W n> tk
r }	 zt	|	 W Y W 5 Q R  dS d}	~	X Y nX g }
ttt|t  |jjD ]v\}}|t|k r|
|j q|j|< |j ||t|  }|j!|_|"| |j#| W 5 Q R X q|
| }nNz	 }W n> tk
r }	 zt	|	 W Y W 5 Q R  dS d}	~	X Y nX t$||d}% d }|dk	st&|'|}t(|r|r _)W 5 Q R  dS W 5 Q R  dS Q R X dS )z
        Often shapes get burned into the pattern, so our initial match ran with
        `ignore_types=(int, ...)`.

        Recheck the match with the correct shapes.
        z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                   s   g | ]} j | qS r6   r   )rr   r   rz  r6   r9   rt     s     z:register_replacement.<locals>.check_fn.<locals>.<listcomp>c                 S  s
   | j d S r   r   )rn   r6   r6   r9   r     r   z8register_replacement.<locals>.check_fn.<locals>.<lambda>F)dtypedevicerequires_gradc                 3  s   | ]}t  |kV  qd S r5   r   rr   r  )r  r6   r9   rx     s    z9register_replacement.<locals>.check_fn.<locals>.<genexpr>r   )args_newr4   c                    s   | t | t   d   S r5   )r   )r  )r2   r|  r6   r9   search_fn_new  s    z=register_replacement.<locals>.check_fn.<locals>.search_fn_newN)argnamesr  r  r   T)*rW   r3   r   r   r   r   Z_dynamoutilsZdetect_fake_moder0  r   rb  r   r  Zempty_stridedsizeZstrider  r   r   shapeZSymIntr   r   r  r  ranger   r[   rQ   r  Zinserting_afterr`  r   r]  rm   fx_to_patternrv   rI   r   r   rU   )r   r  r   Zsym_argsr  ZgradZspecific_patternr  Zspecific_graphr   Zsym_arg_namesr`  new_noder   Zspecific_pattern_match	argnames_staticr  rL  r  r  r  r|  r  r   )r2   r   r  r9   check_fn  s    
 



"


"
z&register_replacement.<locals>.check_fnr   rO   )r3   r4   c                    st   g } D ]}| | | qtdt| d D ],}d| | krF q^| | d|  q0| rptd| |S )Nr+   Z	tangents_zleftover kwargs: )r   r6  r  r   rI   )r3   r2   r   r  )r  r6   r9   r_  (  s    z,register_replacement.<locals>.normalize_argsFZfunctionalize_rng_opsc                 S  s   g | ]}t |tjo|jqS r6   )r   r   rb  r  r   r6   r6   r9   rt   ;  s    z(register_replacement.<locals>.<listcomp>N)rN   rL  r_  )inspect	signature
parametersr_   joint_fwd_bwdr   Zis_inference_mode_enabledfunctorch_configpatchgen_patternr   r>  _seen_patternsrI   r  r|   rT  rN   )r|  r  r  r   rQ  rL  r  r  r  r  r_  rN   Zpattern_reprr6   r  r9   register_replacement  s:    $j


r  zSet[str]_serialized_patternsr/   )unique_namer|  r  r   r  r4   c              	   C  s   dddd}t  s$tdt  |j}ddlm} |jdd	 t||||}W 5 Q R X tj	|| d
}	|t
krd}
t
| nd}
| }tt | d |
8}|
dkr|| n
|d ||	 |d W 5 Q R X |S )Nr/   rF   c                  S  s   t d} t dj| d}g }ttjjD ]4}ttjj|}t|t	r,t
|ttfr,|| q,d|}d| d}| | S )Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            z            # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr   dirr   Z	_inductorZpattern_matcherr   r   r   
issubclassrM   r   r   r  )Zauto_generated_msgfile_templateZpattern_matcher_importsr   attrZformatted_importsr6   r6   r9   get_file_template_  s     
z-_serialize_pattern.<locals>.get_file_templatez0Could not find serialized patterns directory at r   r$   Fr  )r;  wr  z.pyz

r<  )SERIALIZED_PATTERN_PATHis_dirr   r0   Ztorch._functorchr%   r  r  r   r>  r  r  openwrite)r  r|  r  r   r  r  pattern_namer  rN   Zserialized_patternZ
write_moder  fr6   r6   r9   _serialize_patternX  s,    "

r  Z	fx_passesZserialized_patternszvList[Tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsF)r  r|  r  r  r   rQ  rL  r  r  skip_duplicatesr4   c
                 C  s   t |}dtjkr$t| ||||}
n:|j}td| }|rHt|| sTt	d|  t
|| }
t|D ]"}tjj|rh|jd k	rhd |_qht|
tkr|	rd S t|||||
f t|||||||||
d	 d S )NZPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r  )rd   osenvironr  r0   	importlibimport_moduler=  r}  warningr   r   Z	tree_iterr   _subclassesZfake_tensorZis_fakeZconstantr   r>  r  r  r   r  )r  r|  r  r  r   rQ  rL  r  r  r  patr  r   r   r6   r6   r9   gen_register_replacement  sL    
    
r  r  r{   )r|  r  r   r  r  r4   c           
      C  s   t | j }|d kri }g }d}|D ]2}||krF|||  q*|||  |d7 }q*|| |}	t|	tttt	j
t	jf|||dS )Nr   r+   )ignore_typesr  r  r  )r  r  r  r_   r   r  intfloatrW   r   r  r  )
r|  r  r   r  r  r  Zflat_inputsZ	input_idxargnameZ	search_gmr6   r6   r9   r    s$    

r  rS  z2Callable[[Callable[..., Any]], Callable[..., Any]])rN   rL  	pass_dictrR  r4   c                  s   ddd fdd}|S )z
    Register an aten to inductor IR replacement pattern.  The decorated
    function is saved and then called a lowering time allowing direct
    pattern to inductor IR conversion.
    rX  rY  r4   c                   s.   t | stt | djd d| _| S )NrN   rL  rY  rS  T)r   rI   rW  rT  Z_inductor_lowering_functionrY  rL  r  rN   rR  r6   r9   	decorator  s       z,register_lowering_pattern.<locals>.decoratorr6   rN   rL  r  rR  r  r6   r  r9   register_lowering_pattern
  s    r  c                  s   ddd fdd}|S )zk
    Register a pattern that runs a function on the FX graph, allowing
    custom transformation code.
    rX  r  c                   s(   t | stt | djd | S )Nr  rS  )r   rI   r^  rT  r  r  r6   r9   r  .  s       z)register_graph_pattern.<locals>.decoratorr6   r  r6   r  r9   register_graph_pattern"  s    r  rZ   rw   )r[   r   r4   c                 C  s   |t t| jkS r5   )ry   iterrQ   )r[   r   r6   r6   r9   is_start_of_fx_graph8  s    r  z)_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_)r   c                 C  sH   | j dkrt| jjr8dS n| j dkr8t| jr8dS | jdd k	S )Nr#  Tr&  out)r   _mutation_op_researchr  r0   r3   r   ro  r6   r6   r9   is_mutation_opA  s    

r  r  c                 C  s\   |}d|j kr t| |s |j}q|j dd}||k	rX|j}t|rL|d7 }||j d< q.|S )Nmutation_region_idr   r+   )r   r  prevr   ry   r  )r[   r   rn   r  r6   r6   r9   get_mutation_region_idK  s    r  rA   rj   c                 C  s   dt t| jjkS )Nr  )ry   r  rQ   r   r   r6   r6   r9   "should_compute_mutation_region_idsX  s    r  c                 C  s.   d}| j D ]}t|r|d7 }||jd< q
d S )Nr   r+   r  )rQ   r  r   )r[   r  ndr6   r6   r9   compute_mutation_region_ids\  s
    
r  c                      sV   e Zd Zddddd fddZd	d
dddZdddddZddddZ  ZS )rU  FNr~   Optional[str]rE   )prevent_match_across_mutations	pass_namer4   c                   s$   t    tt| _|| _|| _d S r5   )rV   rK   r   rW   patternsr  r  )r8   r  r  rX   r6   r9   rK   e  s
    
zPatternMatcherPass.__init__rG  rH  )itemr4   c                 C  s
   | j | S r5   )r  )r8   r  r6   r6   r9   rI  q  s    zPatternMatcherPass.__getitem__rA   r  rj   c              	   C  s  | j s
dS t|tjjr|j}| jr@t|r4t| t	
t|}d}g }d}| j D ],\}}|dkrhd}qR||j||dd qR|r||jddd ttj|ddD ]}t|}|jdkr|j|f| j krqt|ddrq| j |j|f D ]}	|jr q|	j|}
| jr>t|
r>ttt||
jd	kr>qtjd
|j krht!"d||j#|
|	j t|
r|	$|
r|d	7 }|	%|
|| t&d d  d	7  < t&d d  t|
j7  < qq|S )Nr   Fr(  T)r   r  sort)r   r  )reverse)Zallow_cpu_inputsr+   Z!TORCHINDUCTOR_PATTERN_MATCH_DEBUGz
%s%s %s %sinductorZpattern_matcher_countZpattern_matcher_nodes)'r  r   r   r   ZGraphModuler[   r  r  r  r   r   r  r   Z
find_nodessortedr   r   from_iterabler   r   r-   rl   rN   r   r   r   r^   r   rQ   r  r  r   r   r}  r  r2   rL  rO  r   )r8   r[   Zget_mutation_region_id_partialr  rQ   Zhas_call_moduler   r  r   entryr   r6   r6   r9   rO  t  s\     
zPatternMatcherPass.applyrF   c                 C  s   | j   d S r5   )r  clearrJ   r6   r6   r9   r    s    zPatternMatcherPass.clear)FN)r0   r;   r<   rK   rI  rO  r  r   r6   r6   rX   r9   rU  d  s     5rU  r   r   r1   c                  O  s   t d S r5   r   rh   r6   r6   r9   _not_implemented  s    r  z+Union[torch.fx.GraphModule, torch.fx.Graph]zSequence[Type[Any]])gmr  r  r  r  r4   c                   s   |pi }dd |  D tt|ks.tdddfddt G  fdd	d	tjj}||  }t	|t
stt|S |S )
z
    Convert an FX graph into a PatternExpr.  This is useful for simple
    patterns that can only match single functions and fixed-length lists.
    c                 S  s   i | ]\}}||qS r6   r6   r  r6   r6   r9   r     s      z!fx_to_pattern.<locals>.<dictcomp>rC   zUnion[T, KeywordArg, Ignored])r   r4   c                   s^   t | ttfr"| kr"t|  S t|  kr4t S t | trZtdd | D rZ| rZt S | S )Nc                 s  s   | ]}t |tV  qd S r5   )r   r   )rr   yr6   r6   r9   rx     s     z5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>)r   r  r  r   r   r   rW   r   )r   )r  inv_scalar_workaroundr6   r9   process_arg  s     z"fx_to_pattern.<locals>.process_argc                      sf   e Zd ZeZeZeZdddddfddZdddddfd	d
Zddd fddZ	  Z
S )z fx_to_pattern.<locals>.Converterr/   r{   zMapping[str, Any]z&Union[ExclusiveKeywordArg, KeywordArg])r  r2   r3   r4   c                   sd   t }|t k r | }n* r6|ds0t|}ntdd|}|}|krXt|S t|S d S )NZtangentz_\d+$r   )ry   r   
startswithrI   resubr   r   )r8   r  r2   r3   rn   r   )r  argnumr  r6   r9   r`    s    
z,fx_to_pattern.<locals>.Converter.placeholderrM   c                   sT   t ||f\}}t krDfdd|D }fdd| D }t|f||S )Nc                   s   g | ]} |qS r6   r6   r  r  r6   r9   rt     s     zBfx_to_pattern.<locals>.Converter.call_function.<locals>.<listcomp>c                   s   i | ]\}}| |qS r6   r6   )rr   r  r  r  r6   r9   r     s      zBfx_to_pattern.<locals>.Converter.call_function.<locals>.<dictcomp>)r   Ztree_maprW   r   r"  )r8   r  r2   r3   )r  r  r6   r9   r#    s
    z.fx_to_pattern.<locals>.Converter.call_functionrw   r   rn   r4   c                   sr   t  |}|jdkrbt|trbt|t|jd ks:tt||jd D ]\}}t|j	|_	qJnt|j	|_	|S )Nr2  r   )
rV   ra  r   r   rd   r   r2   rI   r  r   )r8   rn   rvrr   rX   r6   r9   ra    s    z)fx_to_pattern.<locals>.Converter.run_node)r0   r;   r<   r  r&  r(  rd  r`  r#  ra  r   r6   )r  r  r  r  r  rX   r9   	Converter  s   
r  )r   r   rI   r   r  r   r   ru  r>  r   rM   r1  r   Ztree_leaves)r  r  r  r  r  r  rN   r6   )r  r  r  r  r  r  r9   r    s    	 *
r  Tr   )rB   r2   r   r4   c             	   C  sV   t   t| t dd| }W 5 Q R X ddlm} ||j |rJ|j  |  |S )z>Build a normalized inference graph, for use with fx_to_patternreal)Ztracing_moder+   remove_noop_ops)r   r   r,   fx_passes.post_gradr  r[   eliminate_dead_code	recompile)rB   r2   r   r  r  r6   r6   r9   r     s    

r   )rB   r2   r4   c              	     s   d ddddd fdd}t jd$ t| d	d
 |t ddd|  W 5 Q R X  sZtddlm} | j ddl	m
} t }tt jjjjtdtd}t||td|j | j t jj  j_ j      S )z=Build a normalized training graph, for use with fx_to_patternNrA   r{   r   z1Tuple[torch.fx.GraphModule, torch.fx.GraphModule])joint_graphinputsr3   r4   c                   s    rt t|  t| |f|S r5   )rI   clone_graphr(   )r  r  r3   r  r6   r9   record_joint_graph  s    z)joint_fwd_bwd.<locals>.record_joint_graphc                 S  s   t | S r5   )r'   )gr  r6   r6   r9   r      r   zjoint_fwd_bwd.<locals>.<lambda>TF)Zpartition_fnZdecompositionsZkeep_inference_input_mutationsZ
enable_logr+   r  )pointless_viewr   r  )rN   rY  rL  )r   _guardstracingr&   r,   rI   r  r  r[   Zfx_passes.joint_graphr  rU  r"  opsatenviewdefaultr   r^  r{  rT  r  rO  r   ZCodeGenZ_codegenr  r  )rB   r2   r  r  r  Zmatcher_passrN   r6   r  r9   r    sF    

    
r  zList[torch.fx.node.Argument]r  c                 C  s"   t  }tj| j| jf|j |S r5   )rW   r   r   r   r2   r3   r   )rn   r2   r6   r6   r9   r  ?  s    r  c                   s   t t| j}t  tt }d }|r| } fddt|D }|rZ||d  | q  | |r||j	|k	r||| |}|
t||d q |st t| jkstd S )Nc                   s   g | ]}| kr|qS r6   r6   r   readyr6   r9   rt   X  s      z+stable_topological_sort.<locals>.<listcomp>rZ  r6   )rW   rk   rQ   r^   r   r6  r  r   r  ry   ra   r   rI   )r[   pendingZwaitingcursorr   Zwaiting_forr6   r  r9   stable_topological_sortE  s    

r  zCallable[[], Any])rB   r4   c                   s*   t dt  dd fdd}|S )z0Wrapper around lazy init functions in fx_passes/Nr   rF   c                    sZ   t d  } tjd 0 t  t    }W 5 Q R X W 5 Q R X W 5 Q R X | t d< |S )Nr  )r   copyr   r  r  r   r)   )Zcounters_refrc  r   r6   r9   	lazy_initl  s    $z%init_once_fakemode.<locals>.lazy_init)r   rE  r[  )rB   r  r6   r   r9   init_once_fakemodei  s    r  zCallable[[Match], Any]r   c                   s   ddd fdd}|S )z2Function for extra_check to put pass behind a flagrL   r   ry  c                   s
   t t S r5   )r   r%   rz  r   r6   r9   
flag_check  s    zconfig_flag.<locals>.flag_checkr6   )r   r	  r6   r   r9   config_flag~  s    r
  )input_graphr4   c                 C  s   G dd dt }||  S )Nc                      s$   e Zd Zddd fddZ  ZS )zclone_graph.<locals>.CopyGraphrw   )old_noder4   c                   sD   t  |}t|tjjr@|jj|j | j	j
|jd |j_|S r5   )rV   ra  r   r   r   ZProxyr   r   rb   Z	new_graphZ_graph_namespacerA  r   )r8   r  r  rX   r6   r9   ra    s     z'clone_graph.<locals>.CopyGraph.run_node)r0   r;   r<   ra  r   r6   r6   rX   r9   	CopyGraph  s   r  )r*   Z	transform)r  r  r6   r6   r9   r    s    
r  r  r  )r   
arg_number
kwarg_namer4   c                 C  s$   t | j|kr| j| S | j|S r5   )r   r2   r3   r   )r   r  r  r6   r6   r9   get_arg_value  s    
r  zIterable[torch.fx.Node]rP   )rQ   rB   r4   c                   sB    gt  tjjr0 fdd  D  fdd| D S )Nc                   s   g | ]}t  |qS r6   r   r   r   r6   r9   rt     s     z filter_nodes.<locals>.<listcomp>c                   s   g | ]}|j  kr|qS r6   )r  )rr   r   )r   r6   r9   rt     s     
 )r   r   r   r   ra   r   )rQ   rB   r6   )rB   r   r9   filter_nodes  s    r  ztorch.fx.node.Targetc                 C  s"   | j dkrt| jj| jjS | jS )zFor call_function and call_method, we directly use the target function;
    For call_module, the target is string, and we treat the module class
     as a function.
    r(  )r   r   r[   Zowning_moduler  rY   ro  r6   r6   r9   r     s    
r   )Nr6   )r6   r6   Nr6   )N)r   
__future__r   r   dataclassesr   r  r  r   loggingrm  r  r  r  r   abcr   r   collectionsr   pathlibr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   typing_extensionsr   r   r   Ztorch._guardsZtorch.fxZtorch.utils._pytreer  Z_pytreer   Ztorch._dispatch.pythonr   Ztorch._dynamo.utilsr   Ztorch._prims_commonr   Z"torch.fx.experimental.proxy_tensorr   r   Z%torch.fx.experimental.symbolic_shapesr    Ztorch.fx.immutable_collectionsr!   r"   Z
_functorchr%   r  Z_functorch.aot_autogradr&   r'   Z_functorch.partitionersr(   r  r)   r   r*   r   decompositionr,   Zloweringr-   	getLoggerr0   r}  r  r  ZprimsConstantr   r   r.   r>   r@   rC   r   ZTargetr/   ZFnsTyperD   rG   rL   r   r`   r   r   rS   rM   r   r   r   r   r   Z_SimpleSpecr   r"  r%  r'  r)  r*  r,  r-  r.  r1  r4  r   rF  	dataclassrJ  rW  r^  r|   r{  r  r  r^   r  r=   r  __file__parentr  r  r  r  r  r  r  r  compiler  r  r  r  r  rU  r  r  Zno_gradr   Zenable_gradr  r  r  r  r
  r  r  r  r  r   r6   r6   r6   r9   <module>   s  %L
k*$
Q $/I&6 $" 2I
$9
  !

I    L.$	 
