U
    zh                 
   @   s4  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZ d dlmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlm  mZ d dlmZ d dl m!Z!m"Z" d dl#m$Z$m%Z% d dl&m'Z'm(Z(m)Z)m*Z* d d	l+m,Z, d
dl-m.Z. d
dl/m0Z0 d
dl1m2Z2m3Z3 er8d dl4Z4e.j5Z6e7e8Z9ej:j;Z;ej:j<Z<eG dd dZ=eG dd dZ>eG dd dZ?ej@eAdddZBejCeAdddZDejCeAdddZEej@eFdddZGG dd dZHeH ZIejJeej@ eej@ ejJd d!d"ZKej@eAdd#d$ZLej@eAdd%d&ZMej@eAdd'd(ZNej@eAdd)d*ZOej@eAdd+d,ZPejCeeej@ eej@ f d-d.d/ZQeej@ eRd0d1d2ZSejCeej@ eej@ eFeejCejCf d3d4d5ZTejCeejCejCf d-d6d7ZUeFd8ZVeFeFd9d:d;ZWej@eFdd<d=ZXejJd>d?d@ZYeZddAdB Z[eej@eFf eeej@eFf  dCdDdEZ\ejCejCdFdGdHZ]ejCejCejCeFeejCejCf dIdJdKZ^ejCejCd-dLdMZ_dpejJe>e?dNdOdPZ`dQdR Zae=dSdTdUZbejJd>dVdWZceed eed edeedeeF eeF f dXdYdZZeeed eed edeedeeF eeF f dXd[d\Zfeed eed edeedeeF eeF f dXd]d^Zgeed eed edeedeeF eeF f dXd_d`Zhd dalimjZj dbdc ZkdqejJe>eej@ dddedfZldrejCeejCejCf d-dhdiZmdsejjCeReReAeeeReeR f  eAeeR ddmdndoZndS )t    Ndefaultdict)	dataclassreplace)CallableDictListOptionalSetTupleTYPE_CHECKINGUnion)BackwardState)is_sym_nodepy_sym_types)magic_methodsmethod_to_operator)find_symbol_binding_fx_nodesfree_symbolshint_intis_symbol_binding_fx_node)graph_drawer   )config)get_aot_graph_name)fx_graph_cseget_aten_targetc                   @   s   e Zd ZU dZee ed< ee ed< ee ed< ee ed< ee ed< ejddd	Z	ejdd
dZ
ejdddZejdddZejdddZdS )OpTypesz8Class for keeping track of different operator categoriesfusible_opscompute_intensive_ops
random_opsview_opsrecomputable_opsnodec                 C   s   t || jkS N)r   r   selfr$    r(   O/var/www/html/venv/lib/python3.8/site-packages/torch/_functorch/partitioners.py
is_fusible5   s    zOpTypes.is_fusiblec                 C   s   t || jkS r%   )r   r   r&   r(   r(   r)   is_compute_intensive8   s    zOpTypes.is_compute_intensivec                 C   s   t || jkS r%   )r   r    r&   r(   r(   r)   	is_random;   s    zOpTypes.is_randomc                 C   s   t || jkS r%   )r   r!   r&   r(   r(   r)   is_view>   s    zOpTypes.is_viewc                 C   s   t || jkS r%   )r   r"   r&   r(   r(   r)   is_recomputableA   s    zOpTypes.is_recomputableN)__name__
__module____qualname____doc__r
   r   __annotations__fxNoder*   r+   r,   r-   r.   r(   r(   r(   r)   r   +   s   
r   c                   @   s   e Zd ZU eej ed< eej ed< eej ed< eej ed< eeje	f ed< e
jeej dddZejed	d
dZejed	ddZejed	ddZeje	d	ddZdS )NodeInfoinputs_required_fw_nodesrequired_bw_nodesunclaimed_nodesfw_orderreturnc                    s    t dd  jD  fdddS )Nc                 s   s   | ]
}|V  qd S r%   r(   .0nr(   r(   r)   	<genexpr>R   s     z-NodeInfo.required_fw_nodes.<locals>.<genexpr>c                    s
    j |  S r%   )r;   r@   r'   r(   r)   <lambda>R       z,NodeInfo.required_fw_nodes.<locals>.<lambda>key)sortedr8   rC   r(   rC   r)   required_fw_nodesO   s     
zNodeInfo.required_fw_nodes)r@   r=   c                 C   s
   || j kS r%   )r8   r'   r@   r(   r(   r)   is_required_fwU   s    zNodeInfo.is_required_fwc                 C   s
   || j kS r%   )r9   rJ   r(   r(   r)   is_required_bwX   s    zNodeInfo.is_required_bwc                 C   s
   || j kS r%   )r:   rJ   r(   r(   r)   is_unclaimed[   s    zNodeInfo.is_unclaimedc                 C   s$   || j kstd| d| j| S )NNode z not in fw nodes!)r8   AssertionErrorr;   rJ   r(   r(   r)   get_fw_order^   s    zNodeInfo.get_fw_orderN)r/   r0   r1   r   r4   r5   r3   r
   r   int	functoolscached_propertyrI   boolrK   rL   rM   rP   r(   r(   r(   r)   r6   E   s   
r6   c                   @   s6   e Zd ZU eed< eed< eed< eed< eed< dS )MinCutOptionsban_if_used_far_apartban_if_long_fusible_chainsban_if_materialized_backwardban_if_not_in_allowlistban_if_reductionN)r/   r0   r1   rT   r3   r(   r(   r(   r)   rU   c   s
   
rU   )r$   r=   c                 C   s   | j ddS )N	recomputeF)metagetr#   r(   r(   r)   must_recomputel   s    r^   )fx_gr=   c                 C   s$   d}| j jD ]}t|r dS qdS )NFT)graphnodesr^   )r_   foundr$   r(   r(   r)   has_recomputable_opsp   s
    rc   c                 C   s<   | j jD ].}t|rt|jdrtjj|jjkr dS qdS )NtagsTF)	r`   ra   r^   hasattrtargettorchTagnondeterministic_seededrd   )r_   r$   r(   r(   r)   has_recomputable_rng_opsx   s    
rj   c                 C   s6   t | jd tjtjfrdS t | jd tjs2tdS )Nvalr      )
isinstancer\   rg   SymIntSymBoolSymFloatrO   r#   r(   r(   r)   sym_node_size   s    rq   c                   @   s   e Zd Zdd ZdS )InvalidNodeBasec                 C   s   dS )NzInvalid Noder(   rC   r(   r(   r)   __repr__   s    zInvalidNodeBase.__repr__N)r/   r0   r1   rs   r(   r(   r(   r)   rr      s   rr   )joint_graphr7   outputsr=   c           	         sr  t  }i  |D ] }||j}|j|_| |< q| jD ]}| krHq8q8|jdkr\t |< q8|jdkrtj	|j
|j} fdd|D }t|rt |< q8|| fdd |< q8|jdkr|| fdd |< q8|jd	kr8q8g }|D ]f}t|t jrH| krtd
| dt | tr8td
| d| |  q|| q|| |  |  |S )a  
    Given a graph, extracts out a subgraph that takes the specified nodes as
    inputs and returns the specified outputs.

    This includes specifying non-placeholder nodes as inputs.

    The general strategy is to initialize all inputs with proxies as we
    encounter them, and trace through the graph, only keeping values which take
    in valid proxies. Then, all dead code is eliminated.
    placeholdercall_functionc                    s&   g | ]}t |tjrt  | tqS r(   )rm   r4   r5   rr   )r?   xenvr(   r)   
<listcomp>   s   z6_extract_graph_with_inputs_outputs.<locals>.<listcomp>c                    s    |  S r%   r(   rx   ry   r(   r)   rD      rE   z4_extract_graph_with_inputs_outputs.<locals>.<lambda>get_attrc                    s    |  S r%   r(   r|   ry   r(   r)   rD      rE   outputrN   z couldn't be found in envz was invalid, but is output)r4   Graphrv   namer\   ra   opInvalidNodepytreearg_tree_leavesargskwargsany	node_copyrm   r5   RuntimeErrorrr   rO   appendr~   eliminate_dead_codeZlint)	rt   r7   ru   	new_graphr$   new_nodeZall_argsZoutput_valuesrx   r(   ry   r)   "_extract_graph_with_inputs_outputs   sR    








 

r   c                 C   s,   | j dko*dt| jko*t|  o*t|  S Nrv   tangents)r   strrf   _is_bwd_seed_offset_is_fwd_seed_offsetr#   r(   r(   r)   
_is_primal   s    
r   c                 C   s   | j dkodt| jkS r   r   r   rf   r#   r(   r(   r)   _is_tangent   s    r   c                 C   s&   | j dko$dt| jkp$dt| jkS )Nrv   Zbwd_seedZbwd_base_offsetr   r#   r(   r(   r)   r      s    
r   c                 C   s&   | j dko$dt| jkp$dt| jkS )Nrv   Zfwd_seedZfwd_base_offsetr   r#   r(   r(   r)   r      s    
r   c                 C   s   | j dkot| jdtS )Nrv   rk   )r   rm   r\   r]   r   r#   r(   r(   r)   _is_backward_state   s    r   )joint_moduler=   c                C   s>   t jdd | jjddD  }|d | }||d  }||fS )Nc                 s   s   | ]}|j V  qd S r%   )r   r?   r$   r(   r(   r)   rA      s     z+_extract_fwd_bwd_outputs.<locals>.<genexpr>r~   r   )r   r   r`   
find_nodes)r   num_fwd_outputsru   fwd_outputsbwd_outputsr(   r(   r)   _extract_fwd_bwd_outputs   s    r   )saved_valuesr   c                 C   s&   | D ]}|j |kr| |  q"qd S r%   )r   remove)r   r   Zsaved_valuer(   r(   r)   _remove_by_name   s    

r   )r   r   saved_sym_nodesr   r=   c                C   s  t | |d\}}| jjdd}tt|}tt|}tt|}	tt|}
tt|}t	| j|| | |
 |}|jddD ]@}|j
st||j t||j qt|rt||j |stqt }g }g }|D ]0}t|}|r|| || q|| qt| j}t|||D ]d}d|jkr2qt|jd | }t|dd dD ]"}||krfqT|||  qT||O }q|  |||  t	| j||	 || | }t	| j|| | |
 | |}tj| |}tj| |}||fS )Nr   rv   r   rk   c                 S   s   | j S r%   r   )sr(   r(   r)   rD   <  rE   z*_extract_fwd_bwd_modules.<locals>.<lambda>rF   )r   r`   r   filterr   r   r   r   r   r   usersr   r   rO   setr   addr   r   	itertoolschainr\   r   rH   clearextendr4   Z_lazy_graph_moduleZ_make_graph_module)r   r   r   r   r   r   Zplaceholdersprimal_inputstangent_inputsfwd_seed_offset_inputsZbwd_seed_offset_inputsZbackward_state_inputsZ	bwd_graphr$   Zsaved_symbolsZsaved_sym_nodes_bindingZsaved_sym_nodes_derivedsymbolZsymbol_bindingsZnew_symbolsr   Z	fwd_graphZ
fwd_moduleZ
bwd_moduler(   r(   r)   _extract_fwd_bwd_modules  s     






r   c                   s`  t | rt| ||dS ttt| jj}ttt| jj}|| }t| |d\}}t	| j||}dd |jD  g }	g }
| jjD ]}|j
 krqt|r|
| qd|jkr|jdkr|j}tdd |D st|	| q fdd	|jD }d|jkr td
d |D r |
| q|	| qtt|	 }	tt|
 }
t| |	|
|dS )a  
    Partitions the :attr:`joint_module` in a manner that closely resembles the
    behavior observed in the original ``.forward()`` and ``.backward()`` of the
    callable, i.e., the resulting forward graph contains those operators that
    are executed in the original ``.forward()`` callable passed to
    :func:`aot_function`.

    The default partitioner collects the operators that are between the forward
    inputs and the forward outputs. This helps in finding the tensors which have
    to be stashed for the backward pass. These stashed tensors become the output
    of the generated forward graph. The remaining operators are then placed in
    the backward graph.

    .. warning::
        This API is experimental and likely to change.

    Args:
        joint_module(fx.GraphModule): The joint forward and backward graph. This
            is the result of AOT Autograd tracing.

    Returns:
        Returns the generated forward and backward Fx graph modules.
    r   c                 S   s   h | ]}|j d kr|jqS r~   r   r   r   r(   r(   r)   	<setcomp>  s    
 z$default_partition.<locals>.<setcomp>tensor_metarw   c                 s   s   | ]}|j tjkV  qd S r%   )rf   operatorgetitemr?   userr(   r(   r)   rA     s     z$default_partition.<locals>.<genexpr>c                    s   g | ]}|j  kr|qS r(   r   r>   Zforward_node_namesr(   r)   r{     s    
 z%default_partition.<locals>.<listcomp>c                 s   s   | ]}t |V  qd S r%   r   r>   r(   r(   r)   rA     s    r   r   )rc   #min_cut_rematerialization_partitionlistr   r   r`   ra   r   r   r   r   r   r   r\   r   r   allrO   r   dictfromkeyskeysr   )r   _joint_inputsr   r   r   r7   r   r   forward_only_graphr   r   r$   r   Zbackward_usagesr(   r   r)   default_partitiona  s`       
  


r   g    .A)numelr=   c                 C   s
   | |j  S r%   )itemsize)r   dtyper(   r(   r)   _tensor_nbytes  s    r   c                 C   s   d| j krx| j d }t|tr"dS t|ttfrBtdd |D S t|tjrftt	|
 dd|jS tdt| | jdkrd	S td
d S )Nrk   r   c                 s   s2   | ]*}t |tjrtt| d d|jV  qdS )   fallbackN)rm   rg   Tensorr   r   r   r   r>   r(   r(   r)   rA     s   z_size_of.<locals>.<genexpr>r   r   zUnknown metadata type r}   r   z1We should always have `val` metadata on the nodes)r\   rm   r   r   tuplesumrg   r   r   r   r   r   r   typer   )r$   rk   r(   r(   r)   _size_of  s    



r   )r`   c                 C   s\   ddl m} |t}| jD ]"}|jdkr||jj  d7  < qtt|	 dd dd d S )	Nr   r   rw   r   c                 S   s   | d S Nr   r(   r|   r(   r(   r)   rD     rE   z_count_ops.<locals>.<lambda>TrG   reverse)
collectionsr   rQ   ra   r   rf   r/   printrH   items)r`   r   Zcntr$   r(   r(   r)   
_count_ops  s    

r   c                  C   sl   g } t tjjD ]V}ttjj|}t|tjjs2q| D ]*}t||}tj	j
|jkr:| |  qq:q| S r%   )dirrg   opsatengetattrrm   Z_opsZOpOverloadPacketZ	overloadsrh   Z	pointwiserd   r   )r   	attr_nameZopoverloadpacketoverloadZop_overloadr(   r(   r)   pointwise_ops  s    

r   )	depth_mapr=   c                    s(    fdd| D }t | dd ddS )Nc                    s&   i | ]}t |tjjjr| | qS r(   )rm   rg   r4   r$   r5   )r?   argr   r(   r)   
<dictcomp>  s      zsort_depths.<locals>.<dictcomp>c                 S   s   | d S r   r(   r|   r(   r(   r)   rD     rE   zsort_depths.<locals>.<lambda>Tr   )rH   r   )r   r   Z
arg_depthsr(   r   r)   sort_depths  s    
r   )gmr=   c           
         s   t  i  | jjddD ]}| fdd |< qi t| jjD ]\}}||< qH fdd}ttt	| jj}d}t
j}|D ](}|jD ]}| |k r| }|}qq|dkr| S t| jj| d D ]}|| qtj | }	|	S )a  
    This pass finds the first bwd node in the graph (by looking at users of
    tangents) and then reorders the graph by walking from this node to all the
    way to the end of the graph. At each op in this traveral, we insert this op
    in a new graph and try to bring only the relevant subgraph from the other
    non-bwd edges relevant for this op. This closely mimics the behavior of
    autograd engine.

    Why is this pass required in the first place?

    This is an artifact of how partitioners work today. The starting point of
    partitioner is a joint graph, which is fwd and then bwd graph. In the case
    of checkpointing, we keep portions of fwd graph in their original place in
    the joint graph, while obtaining a bwd graph. As a result, the resulting bwd
    graph has copies of recomputed fwd subgraphs followed by the original bwd
    graph. If we run this naively, this leads to bad memory footprint, because
    the fwd subgraphs are live for way longer duration than necessary. This pass
    reorders the operations such that we prioritize the ops for the original bwd
    graph while only realizing those ops from the fwd graph that are necessary
    at any given point in the graph.
    rv   r   c                    s    |  S r%   r(   r|   ry   r(   r)   rD     rE   z5reordering_to_mimic_autograd_engine.<locals>.<lambda>c                    s   | g}t  }t|dkrH| } | |ks|  kr2q||  || j7 }qt|fddd}|D ]} |  fdd | < q`d S )Nr   c                    s    |  S r%   r(   rB   )orderr(   r)   rD   (  rE   zSreordering_to_mimic_autograd_engine.<locals>.insert_node_in_graph.<locals>.<lambda>rF   c                    s    |  S r%   r(   r|   ry   r(   r)   rD   *  rE   )r   lenpopr   Zall_input_nodesrH   r   )r$   	cur_nodesZinsertable_nodesrz   r   r   r(   r)   insert_node_in_graph  s    
zAreordering_to_mimic_autograd_engine.<locals>.insert_node_in_graphN)r4   r   r`   r   r   	enumeratera   r   r   r   mathinfr   rg   GraphModule)
r   r$   idxr   r   Zfirst_node_in_bwdZminimum_ordertangentr   Znew_gmr(   r   r)   #reordering_to_mimic_autograd_engine  s,    


r   )r   	fw_module	bw_modulenum_sym_nodesr=   c               
   C   s  t  }dd }dd }dd }|| }||}	||}
t }| jjD ]T}t|rFt|jdrFtj	j
|jjkrF||j }|	|j }|
|j }||d||< qFtjjj}tjjj}d }|jjd	d
D ]}d|jkr|} qq|d krtdg }| D ]0\}}|d }|d }|j}||r |jd||jf|j|jd}|jdtj|dfi d}|jdtj|dfi d}|| || || W 5 Q R X |j}||0 dt| }||}||||jd< W 5 Q R X ||: |jd|||jf|j|jd}|| || W 5 Q R X qtt |jjdd
}|jd }t!|| }|d | | ||d   }|j"| |j| |#  |#  ||fS )Nc                 S   sF   i }| j jD ]4}|jdkrt|jdrtjj|jjkr|||j	< q|S )Nrw   rd   )
r`   ra   r   re   rf   rg   rh   ri   rd   r   )ZgmodZrandom_nodesr$   r(   r(   r)   get_rng_ops^  s    
z*functionalize_rng_ops.<locals>.get_rng_opsc                 S   sT   d| j krdS | j d }t|ts(|f}|D ]"}t|tjr,|jjdkr, dS q,dS )zV
        Check the example value of the node outputs to find the device type.
        rk   Ncudacpu)r\   rm   r   rg   r   devicer   )r$   
candidates	candidater(   r(   r)   
get_devicei  s    


z)functionalize_rng_ops.<locals>.get_devicec                 S   s   | dkrt j S t  S )Nr   )rg   r   Zget_rng_state)r   r(   r(   r)   get_sample_rng_state{  s    
z3functionalize_rng_ops.<locals>.get_sample_rng_staterd   )fwdbwdrv   r   r   zaCouldn't find tangent node in graph inputs. This is unexpected, please file a bug if you see thisr   r  rw   )r   r   r   r   Zrng_state_output_rk   r~   )$r   countr   r`   ra   r^   re   rf   rg   rh   ri   rd   r   Z_primsZ	rng_primsZrun_and_save_rng_staterun_with_rng_stater   r   r   Zinserting_beforeZcreate_noder   r   r   r   Zreplace_all_uses_withZ
erase_noder   nextrv   r\   iterr   r~   	recompile) r   r   r   r   uidr   r   r   Zjoint_graph_rng_opsZfw_graph_rng_opsZbw_graph_rng_opsZrecomputable_rng_ops_mapr$   Z	base_nodeZfw_nodeZbw_nodeZrun_and_save_rngr  Zbw_tangent_start_nodeZfw_rng_state_outputsZ	node_pairZfw_graphZfunctional_fw_nodestateZ
rng_outputZbw_graphZ
state_nameZbw_rng_state_nodeZfw_output_nodeZ
fw_outputsZsym_node_start_idxru   r(   r(   r)   functionalize_rng_opsC  s    






	






r	  c                 C   sL   | j jD ]>}t|r|jD ]*}t|r|jd |jd krd|jd< qq| S )a  
    If there are two consecutive checkpointed blocks with no operator in
    between, we would still want to stash the tensor at the boundary of
    checkpointed blocks. The following pass makes the last output node
    non-recomputable to allow for that.
    r[   r   )r`   ra   r^   r   r\   )r   r$   r   r(   r(   r)   cleanup_recompute_tags  s    
r
  )rt   	node_infomin_cut_optionsc           $         s|  d krt  t trLdd | jD }|dd jD  }td| t  fddzdd l}W n, tk
r } ztd|W 5 d }~X Y nX fd	d
fdd}fddt	dfdd}	|
 
t    
fdd}
| jD ]\}|jdkr
q|jkrT|jkr<
j|jd dtjd q
j|jd dtjd t|sht|rp|
| |r||r|
| d|jkrd|jkpd|jkot|jd tj }t|rt	t|}n.|rt|jdtrdntj}n|	|}
j|jd |jd |d |jD ]$}
j|jd |jd tjd q.qttj  t!t!dfdd}j"r>j#D ]}fdd |jD }fd!d |jD }t$|dkr||t%|}t&|jD ]d}|rԈ'||krԈ||r| krqt()d"|'|||'| |
| qԐqj*rNt  }| jD ]}|sfqR'||fg}'|}t$|dkrRt+,|\}}||krq|-| '||d# kr t$|dkr t()d$||'|'| |
| qR|jD ]>}|r||r| krt+.|'||f qqqRz|/
d%d\}}W n@ t0k
r   td& td'1|j2j34
 t5
  Y nX |\}	t  }
fd(d)|D D ]$\}|6	fd*d)|D  qt  }|D ]>\} }!| d d+ |!d d, kst7| d d+ }"|-|" qt8| d-d. t9| jD t:fd/d)|D fd0d1d2}#|# fS )3Nc                 S   s.   h | ]&}|j d krt|jdrt|jjqS )rw   _overloadpacket)r   re   rf   r   r  r   r(   r(   r)   r     s   
 z solve_min_cut.<locals>.<setcomp>c                 S   s   h | ]}t |qS r(   )r   r?   ir(   r(   r)   r     s     z#Ops banned from rematerialization: c                    s&   t |tjkrdS  | o$ |S )NT)r   r   catr*   )ab)op_typesr(   r)   r*     s    z!solve_min_cut.<locals>.is_fusibler   zANeed networkx installed to perform smart recomputation heuristicsc                    sh    | rdS | h}t|dkrd| }|jD ]2}|sL ||sL dS  |r.|| q.qdS )NFr   T)r-   r   r   r   rK   r   )r$   r   curr   )r*   r  r  r(   r)   is_materialized_backwards  s    


z0solve_min_cut.<locals>.is_materialized_backwardsc                    s   | j dkrdS | jtjkrdS | jtjjtjjfkr8dS | j	dd dkrNdS j
rd| s|dS n| sx| r|dS jr | rtd| t| j dS | jdk r| jtjkrdS jrtdd	 | jD }t| }|d
 |k S dS )Nrw   Fr[   r   Tzmaterialized backwards: %s %si  c                 s   s"   | ]}t |tjrt|V  qd S r%   )rm   r4   r5   r   r  r(   r(   r)   rA   L  s     zBsolve_min_cut.<locals>.should_ban_recomputation.<locals>.<genexpr>rl   )r   rf   r   r   r   lift_fresh_copydefaultZ
lift_freshr\   r]   rY   r.   r,   r+   rX   loginfor   r   dist_from_bwr   Zmax_dist_from_bwrZ   r   r   r   )r$   Zinput_tensors_sizeZoutput_size)r  r  r  r(   r)   should_ban_recomputation"  s6    

z/solve_min_cut.<locals>.should_ban_recomputationc                    s*    j dkrdS t fdd jD  S )Nrv   Tc                 3   s   | ]} |V  qd S r%   r(   r   )r*   r$   r(   r)   rA   W  s     z9solve_min_cut.<locals>.is_materialized.<locals>.<genexpr>)r   r   r   r#   )r*   r#   r)   is_materializedS  s    
z&solve_min_cut.<locals>.is_materializedr<   c                    sd   t | }t| jd tr.t| jd tjs.tS t|dtt	| j
dd  } | rX|S |d S d S )Nrk   g?d   r      )r   rm   r\   r   rg   rn   INT_INFrQ   maxminr  )r$   Zmem_sz)r  r(   r)   get_node_weightY  s    z&solve_min_cut.<locals>.get_node_weightc                    sv    | rdS | krdS | jdddkr0dS d| jkrPt| jd tjrPdS  |  jd| jd t	j
d dS )	NFr[   r   rk   source_incapacityT)r-   r\   r]   rm   rg   rp   r   add_edger   r   r   r#   )banned_nodesdont_bannx_graphr  r(   r)   ban_recomputation_if_allowedl  s    

z3solve_min_cut.<locals>.ban_recomputation_if_allowedr~   r$  Zsinkr%  Z_outrk   r           )start_nodes	max_ranger=   c              	      s   g }| D ]}t |||df qt|dkrt |\}}}|sP|S |jD ]<}|rV||krtqVt ||| ||f qVq&|S )z
        Finds the first unfusible node in the chain of nodes starting from
        `start_nodes` and returns its position.
        Tr   )heapqheappushrP   r   heappopr   rK   )r-  r.  Zsorted_nodesr@   _r$   Znode_is_fusibler   )r*   r  r(   r)   find_first_unfusible  s     


z+solve_min_cut.<locals>.find_first_unfusiblec                    s    g | ]}  |r |qS r(   )rK   rP   r   r  r(   r)   r{     s   
z!solve_min_cut.<locals>.<listcomp>c                    s   g | ]}  |r|qS r(   )rK   r   r4  r(   r)   r{     s    
 z1used above/below fusible %s:(%s) -> %s -> %s:(%s)r  ztoo long %s %s %s %sr#  z-Failed to compute min-cut on following graph:
c                 3   s   | ]}| | fV  qd S r%   r(   r>   )r*  r(   r)   rA   .  s     z solve_min_cut.<locals>.<genexpr>c                 3   s   | ]}| kr|fV  qd S r%   r(   )r?   v)non_reachableur(   r)   rA   /  s      c                 S   s   i | ]\}}||qS r(   r(   )r?   r   r$   r(   r(   r)   r   9  s      z!solve_min_cut.<locals>.<dictcomp>c                 3   s   | ]} | V  qd S r%   r(   r   name_to_noder(   r)   rA   ;  s     c                    s    |  S r%   r(   r|   )node_idxr(   r)   rD   ;  rE   zsolve_min_cut.<locals>.<lambda>rF   );r   get_default_op_listAOT_PARTITIONER_DEBUGra   r"   r   networkxImportErrorr   floatZDiGraphr   r9   r7   r'  r   r   r   r   r   rK   r\   rm   rg   r   r   rq   r]   r   r   r   r4   r5   rQ   rV   rI   r   r   r   rP   r  r  rW   r/  r1  r   r0  Zminimum_cut	ExceptionjoinZ	readwriteZedgelistZgenerate_edgelistvisualize_min_cut_graphupdaterO   get_name_to_noder   rH   )$rt   r  r  r)  Zjoint_module_opsZops_ignorednxer  r"  r+  r$   Zis_non_tensor_nodeweightr   r3  Z	used_nodeZordersZfw_usersZfirst_unfusible_usevisitedZ
start_nodeZfusibleZstart_orderr2  r  Z	cut_value	partitionZ	reachableZcutsetZnbrsZ	cut_nodesZnode_inZnode_outZ	node_namer   r(   )r(  r)  r*   r  r  r  r<  r=  r  r7  r*  r  r8  r)   solve_min_cut  s   
1	

$









" 
rM  c                 C   s   dd l }dd l}|j|  }||d }| D ]@}| |  |  d }|	t
| |tdkr6|d q6td |d d S )Nr   r&  r   redz2Visualizing the failed graph to min_cut_failed.svgzmin_cut_failed.svg)r@  pydotZnx_pydotZto_pydotZ	to_stringZgraph_from_dot_dataZ	get_edges
get_sourceZget_destinationZ	set_labelr   rB  Z	set_colorr   Z	write_svg)r*  rH  rO  Z
dot_formatZ	dot_graphedgerJ  r(   r(   r)   rE  @  s    rE  r<   c               K   C   s  t jt jt jt jt jt jt jt jt j	t j
t jt jt jt jt jt jt jt jt jt jt jt jt jt jt jt jt jt jt jt jt jt j t j!t j"t j#t j$t j%t j&t j't j(t j)t j*t j+t j,t j-t j.t j/t j0t j1t j2t j3t j4t j5t j6t j7t j8t j9t j:t j;t j<t j=t j>t j?t j@t jAt jBt jCt jDt jEt jFtGjHt jIt jJt jKt jLgK} t jIt jJt jMg}|t jNt jOt jPtQjRt jSt jTt jUg7 }|}| tQjtQjVt jWt jLt jXtQjYtQj@t jYt jZtQjRt j[t j\t jNt jSt jOt j]t j^t j_t j`t jat jbt jct jdt jet jft jgt jht jTt jit jjt jktQjltQjmg!7 } | t jnt jog7 } | |7 } | tp 7 } | t jqg7 } | dd trD 7 } ts| }t jtt jut jvg}t jwt jxt jyt jzt j{t j|t j}t j~g}|ts|B }tts|ts|ts|ts|ts|S )Nc                 S   s   g | ]}t |qS r(   )r   r?   mr(   r(   r)   r{     s     z'get_default_op_list.<locals>.<listcomp>)r   r   subdivatan2mulr   r!  pow	remainderfmod__and____or____xor__
__lshift__
__rshift__eqnegegtleltabsZbitwise_notceilfloorfracnegZreluroundZsilutruncr  log10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtZrsqrtZ
reciprocalZsigmoidZsoftplus	thresholdZthreshold_backwardclampwhereZlerpZaddcmulZgeluZgelu_backwardr   ZmeanZ_grad_sum_to_sizeZsum_to_sizeZamaxtoZtype_asr   r   ZsqueezeZ	unsqueezeZrsubZ_to_copyaliasviewslicetprimsZbroadcast_in_dimexpandZ
as_stridedZpermuteZconvert_element_typecloneZ	full_likevarZstdselectZ_unsafe_viewZreshapeZbroadcast_tensorsZscalar_tensorZones	new_zerosr  ZarangeZtriuZvar_meanisinfr   fullzerosZargmaxmaximumiotaZ)_low_memory_max_pool2d_offsets_to_indicesindexZgatherr   Z
zeros_liker   r   Znative_dropoutZ	rand_likeZ
randn_likemmZconvolutionZconvolution_backwardZbmmZaddmmZ#_scaled_dot_product_flash_attentionZ'_scaled_dot_product_efficient_attentionZupsample_bilinear2dr   )Zdefault_recomputable_opsZrecomputable_view_opsr!   r"   r    r   r   r(   r(   r)   r>  Q  s(   M	$
r>  c                 C   s   i }| j D ]}|||j< q
|S r%   )ra   r   )r`   r<  r$   r(   r(   r)   rG    s    
rG  )memoryruntimes
max_memoryr=   c           
         s   t }tt|}t| fdddd}d}d}g }g }|D ]B}	| |	  |krx| |	 7 }||	 7 }||	 q@||	 q@|||fS )Nc                    s   |   |   S r%   r(   )r  r  r  r(   r)   rD     rE   z!greedy_knapsack.<locals>.<lambda>Tr   r,  )r   r   rangerH   r   )
r  r  r  r@   r   Ztotal_memoryZtotal_runtimeitems_to_saveitems_to_allow_recomputingr  r(   r  r)   greedy_knapsack  s    r  c                 C   s   dd l }zddlm}m}m} W n tk
r>   tdd Y nX || }||}| }	||||d}
|
g}||	}||	|||ddd}|j	stdg }g }t
|jD ]&\}}|dkr|| q|| q|j ||fS )Nr   )BoundsLinearConstraintmilpzHTo use the ILP for memory budget checkpointing you need to install scipy)AZubr   )cconstraintsintegralityZboundszSomehow scipy solving failed)numpyZscipy.optimizer  r  r  rA  r   arrayZ	ones_likesuccessr   rx   r   Zfun)r  r  r  npr  r  r  Z	np_memoryZnp_runtimesr  Zmemory_constraintr  r  resr  r  r   r  r(   r(   r)   ilp_knapsack  s<    


   r  c                    s  d t j fdd| D t jdd}t j|t jdd}tt|  }t| }t j|d |d ft jdd}td|d D ]}||d  }||d  }	||d d d f ||d d f< |dkr||d d d f |	 ||d d f< qzt 	||d |d f ||d d | f |	 |||d f< qzg }
g }|}t|ddD ]Z}|| | ||d  | kr|

|d  |t||d   8 }n|
|d  q<|
  || |  }||
|fS )	Ni'  c                    s   g | ]}t t|  qS r(   )rQ   rk  rR  Sr(   r)   r{   ;  s     zdp_knapsack.<locals>.<listcomp>r   )r   r   r   r   )rg   ZtensorlongZfloat32rQ   rk  r   r  r  r  r   itemr   )r  r  r  Zquantized_memoryZquantized_max_memoryr@   dpr  Zcurrent_memoryZcurrent_runtimeZsaved_itemsZrecomputable_itemsjZmax_runtimer(   r  r)   dp_knapsack3  sF         &r  c                 C   sT   t j}|dkrt| ||S |dkr.t| ||S |dkrBt| ||S td| d S )NZgreedyZilpr  z,Not aware of memory budget knapsack solver: )r   Zactivation_memory_budget_solverr  r  r  r   )r  r  r  ZSOLVERr(   r(   r)   #_optimize_runtime_with_given_memoryo  s    r  no_dispatchc              
      s   t j}dd }|dkrdS |dkrzddlm} t > t|jjf\ | fdd	}|W  5 Q R  S Q R X nn|d
krddl	m
} t|jjf\ |dd}j  W 5 Q R X | }t|dS td| d S )Nc                    s   t | tjrXt | jd tjrXt| jd j}dd   fdd|D }| jd |S t | tjrt | jd tj	rt
| jd ddS t | tjrt | jd tjrdS t | tjrt | jd tjrd	S | S d S )
Nrk   c                 S   s   t | ddS )Nr   r   )r   )dr(   r(   r)   realize_symbol  s    zAestimate_runtime.<locals>.materialize_arg.<locals>.realize_symbolc                    s   g | ]} |qS r(   r(   )r?   r   r  r(   r)   r{     s     z=estimate_runtime.<locals>.materialize_arg.<locals>.<listcomp>r   r   g      ?T)rm   r4   r5   r\   rg   r   r   shaper  rn   r   rp   ro   )rx   r  r(   r  r)   materialize_arg  s    z)estimate_runtime.<locals>.materialize_argZtestingr   Zprofiler   )do_benchc                      s   j  S r%   )rf   r(   r   r   r$   r(   r)   rD     rE   z"estimate_runtime.<locals>.<lambda>Zflops)FlopCounterModeF)displayz Not aware of runtime estimator: )r   Z*activation_memory_budget_runtime_estimatorZtriton.testingr  r  r   Ztree_mapr   r   Ztorch.utils.flop_counterr  rf   Zget_total_flopsr   r   )r$   ZRUNTIME_MODEr  r  msr  modeZcounted_flopsr(   r  r)   estimate_runtime  s$    
r  )rt   r  r=   c                    s  |dks|dk rt d| ttjtjtjtjtjd}tjrRt	|ddddd}|dkr`j
S t|\}}|dkr||S ttj tddd	j
	|	kr|S 	fd
dttj d	fdd}t	|dddd}t|\}}|||k r|S t	|dd t \}	}
||	|k r:|	S ddlm fddj
D ttj ttj dfdd}||
}t|tddtdkrj
S fddD dd D ddlm
  
fdd}tjrg }tdd d!D ]2}||d \}}||t| ||f qdd lm} d"d |D }d#d |D }|jd$d% |j||d&d' t|D ].\}}|j |d(|| || fd)d*d+d, q|!d- |"d. |#d/ |$d |% }|&  d0t'  d1}|(| t)*d2| ||d3d S )4Nr   r   zJThe valid ranges for memory budget are 0 <= m <= 1. The provided value is )rV   rW   rX   rY   rZ   F)rV   rW   rX   rY   )r   r=   c                 S   s   t dd | D d S )Nc                 S   s   g | ]}t |qS r(   r   r  r(   r(   r)   r{     s     zNchoose_saved_values_set.<locals>.estimate_activations_size.<locals>.<listcomp>    eA)r   )r   r(   r(   r)   estimate_activations_size  s    z:choose_saved_values_set.<locals>.estimate_activations_sizec                    s   | d    S )Nr  r(   )sz)max_act_sizemin_act_sizer(   r)   get_normalized_size  s    z4choose_saved_values_set.<locals>.get_normalized_sizeZactivationsc                    s    |    S r%   r(   r  )r  r  r  r(   r)   get_mem_ratio  s    
z.choose_saved_values_set.<locals>.get_mem_ratio)rV   rW   rX   )rY   get_node_storagec                    s   h | ]} |qS r(   r(   r   r  r(   r)   r     s     z*choose_saved_values_set.<locals>.<setcomp>)r(  r=   c                    s    fdd| D S )Nc                    s*   g | ]"}|j td k r |kr|qS )r  )r  rQ   r  r  input_storagesr(   r)   r{     s   zRchoose_saved_values_set.<locals>.get_recomputable_banned_nodes.<locals>.<listcomp>r(   )r(  r  r(   r)   get_recomputable_banned_nodes  s    z>choose_saved_values_set.<locals>.get_recomputable_banned_nodesTr   c                    s   g | ]} t |qS r(   r  r  )r  r(   r)   r{     s    z+choose_saved_values_set.<locals>.<listcomp>c                 S   s   g | ]}t |qS r(   )r  r   r(   r(   r)   r{     s    r  c              	      sp     t t| d\}}}W 5 Q R X t }|D ]}||  q4|sVtt |\}}||fS )Nr   )r  r   r   r   issubsetrO   rM  )memory_budgetexpected_runtimeZsaved_node_idxsZrecomputable_node_idxsr)  r   r   r2  )aggressive_optionsall_recomputable_banned_nodesrt   memories_banned_nodesr  r  runtimes_banned_nodesr(   r)   get_saved_values_knapsack  s,      
z:choose_saved_values_set.<locals>.get_saved_values_knapsackr  r  c                 S   s   g | ]}|d  qS )r  r(   r?   r  r(   r(   r)   r{   B  s     c                 S   s   g | ]}|d  qS )r   r(   r  r(   r(   r)   r{   C  s     )
      )Zfigsizeo)markerz.2fzoffset points)r   r  center)Z
textcoordsZxytextZhazMemory Budgetz Runtime of Recomputed Componentsz:Pareto Frontier of Memory Budget vs. Recomputation RuntimeZmemory_budget_pareto_z.pngz%Generated Pareto frontier curve at %sr  )+r   rU   r   Zban_recompute_used_far_apartZ!ban_recompute_long_fusible_chainsZ#ban_recompute_materialized_backwardZban_recompute_not_in_allowlistZban_recompute_reductionsZaggressive_recomputationr   r7   rM  r   r4   r5   rB  torch._inductor.fx_utilsr  rH   r   r   torch.utils._mode_utilsr  Zvisualize_memory_budget_paretor  r   r   Zmatplotlib.pyplotZpyplotZfigureZplotr   ZannotateZxlabelZylabeltitlegridZgcfshowr   Zsavefigr  warning)rt   r  r  r  Zruntime_optimized_saved_valuesr2  r  Zmore_aggressive_optionsZmore_aggressive_saved_valuesZ%aggressive_recomputation_saved_valuesr(  r  Zrecomputable_banned_nodesr  optionsZsweep_memory_budgetr   r  ZpltZx_valuesZy_valuesr  txtZfigZfig_namer(   )r  r  r  r  r  r  rt   r  r  r  r  r  r  r)   choose_saved_values_set  s    
    "  







r  inductorc                   s  | j   |   | j }tjr,t|}|| _ | j }t| }t| }|rNt| } fdd}	|	| }
t	|
j
dkr~t| |dS t| j jD ]V}|jdkrtd|_q|
|sd|_qtd|_|jD ]}t|j|jd |_qqtj}|jD ](}t|jdd	tr|jd } qqt||
|d
}ttt|}ttdd |}t| ||d\}}|r||r|t| ||t	|\}}t|}t rddl!m"   fdd|D }t#dt$dd |D d  t%dd |D }dd |j jD }dd |j jD }||@ }t&t}|j jD ]8}|j'|krt(|j)dr|t*|j)j+  d7  < qt#dt	| dt	| dt	|  t#dt%|, dd dd ||fS )ax  
    Partitions the joint graph such that the backward recomputes the forward.
    Recomputing helps in trading off memory bandwidth with computation.

    To create the fwd and bwd graph, we copy the joint graph, manually set the
    outputs to just original forward or backward outputs. And then we run the
    resulting graphs through dead code elimination.

    .. warning::
        This API is experimental and likely to change.

    Args:
        joint_module(fx.GraphModule): The joint forward and backward graph. This
            is the result of AOT Autograd tracing.
        _joint_inputs: The inputs to the joint graph. This is unused.
        compiler: This option determines the default set of recomputable ops.
            Currently, there are two options: ``nvfuser`` and ``inductor``.
        recomputable_ops: This is an optional set of recomputable ops. If this
            is not None, then this set of ops will be used instead of the
            default set of ops.
        num_fwd_outputs: The number of outputs from the forward graph.

    Returns:
        Returns the generated forward and backward Fx graph modules.
    c                    s"  t | j t | jjD ]@}|jdkr:d|jkr:| |kr|jD ]}| qHqtt	t
| jj}tt	t| jj}|| }t| d\}}dd |D  t| j||} fdd|jD fdd| jjD }	d	}
i }| jjD ]}|kr|
||< |
d
7 }
qt||	|S )Nrv   r   r   c                 s   s$   | ]}|d k	r|j dkr|V  qd S )Nr~   r   )r?   r  r(   r(   r)   rA     s     
 zNmin_cut_rematerialization_partition.<locals>.classify_nodes.<locals>.<genexpr>c                    s    h | ]}|j d kr |j qS r   r   r   r;  r(   r)   r     s   
zNmin_cut_rematerialization_partition.<locals>.classify_nodes.<locals>.<setcomp>c                    s    h | ]}|kr| kr|qS r(   r(   r   )r9   rI   r(   r)   r     s    r   r   )rG  r`   r   ra   r   rf   r   r   r   r   r   r   r   rF  r   r6   )r   r$   r   r   r   r7   r   r   r   r:   Zfw_cntr;   r   )r<  r9   rI   r)   classify_nodes  sX    


 

  

    z;min_cut_rematerialization_partition.<locals>.classify_nodesr   r   r~   r  r   r  Nr  c                 S   s
   t |  S r%   r   rB   r(   r(   r)   rD     rE   z5min_cut_rematerialization_partition.<locals>.<lambda>r   r  c                    s   h | ]} |qS r(   r(   r   r  r(   r)   r     s     z6min_cut_rematerialization_partition.<locals>.<setcomp>z Theoretical Activations Stored: c                 s   s   | ]}t |V  qd S r%   r  r  r(   r(   r)   rA     s     z6min_cut_rematerialization_partition.<locals>.<genexpr>c                 S   s   g | ]}t |t|fqS r(   )r   r   r  r(   r(   r)   r{     s     z7min_cut_rematerialization_partition.<locals>.<listcomp>c                 S   s   h | ]}|j d kr|jqS rw   r   r   r(   r(   r)   r     s    
 c                 S   s   h | ]}|j d kr|jqS r  r   r   r(   r(   r)   r     s    
 r  z# remat/fw/bw: /zCount of Ops Rematerialized: c                 S   s   | d S r   r(   r|   r(   r(   r)   rD     rE   Tr   )-r`   r   r  r   Zcser   rc   rj   r
  r   r9   r   reversedra   r   rQ   r  rK   r   r!  Zactivation_memory_budgetrm   r\   r]   rB  r  r   r   r   r   r	  r   r?  r  r  r   r   rH   r   r   re   rf   r   r  r   )r   r   compilerr   r_   Z	cse_graphrt   Zgraph_has_recomputable_opsZgraph_has_recomputable_rng_opsr  r  r$   r   r  r   r   r   r   ZstoragesZsorted_sizesZfw_module_nodesZbw_module_nodesZremat_nodescountsr(   )r  r   r)   r   f  s    !
,  





  
    r   fx_graphTF)tracedfnamefigname
clear_metaprogparse_stack_tracedot_graph_shaper=   c                 C   s   |r0t | j}t| |} | jjD ]
}i |_q$tj	|\}	}
|
sNdt
j }
td|	 |
  tj| |||d}| }t|d|
d }|	 |
 }|d kr|| n|||d d S )N.zWriting FX graph to file: )r  r  Zwrite_)r  )copydeepcopyr`   r4   r   ra   r\   ospathsplitextr   Ztorch_compile_graph_formatr   r   ZFxGraphDrawerZget_main_dot_graphr   lstrip)r  r  r  r  r  r  r  r   r$   baseextgrx   Zwrite_methodr(   r(   r)   
draw_graph  s*    	

r  )N)r   )r  )r  TNFN)or  rR   r/  r   loggingr   r   r  r   r   dataclassesr   r   typingr   r   r   r	   r
   r   r   r   rg   Ztorch._inductor.inductor_primsZtorch.fxr4   Ztorch.utils._pytreeutilsZ_pytreer   Z%torch.fx.experimental._backward_stater   Z"torch.fx.experimental.proxy_tensorr   r   Ztorch.fx.experimental.sym_noder   r   Z%torch.fx.experimental.symbolic_shapesr   r   r   r   Ztorch.fx.passesr    r   Z_aot_autograd.logging_utilsr   Zcompile_utilsr   r   ZsympyZdebug_partitionerr?  	getLoggerr/   r  r   r   r  r   r6   rU   r5   rT   r^   r   rc   rj   rQ   rq   rr   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   	lru_cacher   r   r   r	  r
  rM  rE  r>  rG  rB  r  r  r  r  r  r  r  r  r   r  r(   r(   r(   r)   <module>   s  (
  @	aV

(K    P      %  =.   ;  0     