U
    yhU                  	   @   s   d dl Z d dlmZmZmZmZmZmZ d dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ ddgZe
eZed	d
G dd dZed	d
deejjeegef eeeef  ee ee dddZdS )    N)AnyCallableDictListOptionalSet)OrderedDict)compatibility)GraphModule)Node	Partitionsplit_moduleT)Zis_backward_compatiblec                   @   s(   e Zd ZedddZedddZdS )r   namec                 C   sN   || _ d| | _g | _i | _i | _i | _i | _tjj	
 | _	i | _i | _d S )NZsubmod_)r   submod_name
node_namesinputsoutputsdependencies
dependentstorchfxgraphGraphenvironmenttargets)selfr    r   N/var/www/html/venv/lib/python3.8/site-packages/torch/fx/passes/split_module.py__init__   s    zPartition.__init__)returnc                 C   s4   d| j  d| j d| j d| j d| j d| j S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r   r   r   r   __repr__   s    2zPartition.__repr__N)__name__
__module____qualname__strr   r!   r   r   r   r   r      s   F)mroot_msplit_callbackqualname_mapkeep_original_orderkeep_original_node_namec           5   
      s	  t ttt f tttjjjf dfdd}ddl}i 
i i t tt  d
fdd
fd	d
}tj	j
tj	jtjjg}	t }
t }i }d}t }jjD ]jdkrjdkrjd }dk	rt|tjrt|jj|jr|jj< q| jdkrj|	krjtjjkrntjdks@ttjd tsVt}th|
|< njtj	j
krtdd jD st|  th|< d|< nZjtj	jkrtjdkst|jd    |!jd  |jd < |dk	r4|
|   |D ]}||   q8qtdd |" D sttddd |# D }dd |
# D }
t$%t&j'rt$(d| t$(d|
 t|pt|
}d}jjD ]j)< jdkrq܈jdkr(tjj*jd fdd q|rX}||ksTtd| d| |}j|	krtjj*jfd d tjj*j+fd!d qt,
- }g }
# D ] \}	t	j.s|/| qg }|r2|0 }|/| 
| j1D ],}
| j.0| 
| j.s |/| q qt|t
krLt2d"||
fD ]}|# D ]\}t|dkszt
t|d  j3< |dd D ]T}
t| 		jj4jjt5d#d jD i j6d$}j7 |_|	j3< qq`qT|D ]P}
| 		j8D ]:}	jj9|| j6d%}| j7 |_|	j3| < qqjjD ]Ht:d&rX
j; 		j3tjj*jfd'd} tjj*j+fd(d}!jd)krj}"n|j<d*}#}$|#D ].}%t:|$|%st=d+j d,t>|$|%}$qd-?|#}"|$	j@|"< |dk	r>	jA d*|" }&j||&< t| t5sNtt|!tBs^trjj)nd}'	jj4j|"| |!j6|'d.}j7 |_|	j3< qX|fD ]}tC|D ]| }t|dkst|dd D ]\}
t| 	| }(|(dk	s
td/	jj4|(j|(j	j3 fi |(j6d$}|(j7 |_qqqi i  tjjD i })|sjjD ]| |)\ })qlnjjD ]j)< q|s|n|}*t }+d0d1 jjD |*D ]}
| 	t5	fd2d	jED },t|,}-|-dkr	jF|,d  n|-dkr,	jF|, |rfd3d1	j8D }.D ].|+kr^qL| |)\ }/|+  qL|.D ].|+krq| |)\ })|+  qtjj	j@	j|)	jA< G	jAt5 fd4d	j8D }0t	jE}1|1dk	r4tjjHI|0}2tJ	jED ]\}3}4|2|3 j |4< 	qn|1dkr|0 tKtL	jE< qʈjjD ]6jdk	r\Ftjj*jd  fd5d 	q\tjj|)S )6a  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.
        keep_original_order: Optional[bool]: keep the original order of the GraphModule
            or use the Topological order of the new constructed GraphModule


    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx.symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self):
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    )nodebase_mod_envbase_mod_attrsc                    s  | j dkrt| jdkr"| jd ntjj}rb|tjjkr>dn|f} jd| j|| jd|| j< n j	| j
| j|d|| j< | j || j _nv| j dkr | j
|| j< | j || j _}| j
dD ]*}t||std| j
 d	t||}q||| j
< ||fS )
Nplaceholderr   r   )args	type_expr)r1   default_valueget_attr.zNode target  not found!)oplenr0   inspect	Signatureemptycreate_noder   typer/   targetmetacopyr3   splithasattrAttributeErrorgetattr)r,   r-   r.   r2   r0   Zattr_valatom)base_mod_graphr+   r&   r   r   construct_graph   s,    
  

z%split_module.<locals>.construct_graphr   N)def_nodeuse_nodec           	         s   ddl m} t| dd }t|dd }||kr|d k	r^ | }|j| j |d k	r^|j| |d k	r̈ | }|j| j | j	d }d k	rt
||tdD ]}|j| j q|d k	r|j| d S )Nr   )free_symbols_fx_partitionexample_value)key)Z%torch.fx.experimental.symbolic_shapesrI   rC   r   
setdefaultr   r   r   r>   getsortedr%   r   )	rG   rH   rI   ZdefinedusedZdef_partitionZuse_partitionZdef_vals)
partitionssymbol_to_noder   r   record_cross_partition_use   s"    z0split_module.<locals>.record_cross_partition_usec                    sF   t | } |}|d kr.t|  |< }|j| j || _d S N)r%   rN   r   r   appendr   rJ   )r,   partition_name	partition)rR   r(   r   r   "instantiate_node_partition_mapping   s    
z8split_module.<locals>.instantiate_node_partition_mapping)r/   r3   outputr/   rK   Zcall_function   c                 s   s   | ]}t |t V  qd S rU   )
isinstancer   .0argr   r   r   	<genexpr>  s     zsplit_module.<locals>.<genexpr>c                 s   s   | ]}|d k	V  qd S rU   r   )r^   vr   r   r   r`     s     zautocast must exitc                 S   s   i | ]\}}|t |qS r   rO   r^   kra   r   r   r   
<dictcomp>  s      z split_module.<locals>.<dictcomp>c                 S   s   i | ]\}}|t |qS r   rb   rc   r   r   r   re     s      zautocast_regions: %szgrad_regions: %s)r/   r3   rZ   c                    s
    | d S rU   r   n)rT   r   r   <lambda>,      zsplit_module.<locals>.<lambda>zRautocast or set_grad_enabled require monotonically increasing partitions:highest: z, this node's: c                    s
   |  S rU   r   rG   r,   rT   r   r   ri   ;  rj   c                    s
   |  S rU   r   rk   rl   r   r   ri   >  rj   z cycle exists between partitions!c                 s   s   | ]
}|V  qd S rU   r   r]   r   r   r   r`   ^  s     )r6   r=   r0   kwargsr1   )r1   rJ   c                    s    |  S rU   r   rg   r   r   r   ri   w  rj   c                    s    |  S rU   r   rg   rn   r   r   ri   y  rj   )call_moduler3   r4   zOperator target r5   _)r6   r=   r0   rm   r1   r   zMissing exit nodec                 S   s   g | ]}|j d kr|qS )r/   )r6   )r^   r,   r   r   r   
<listcomp>  s     
 z split_module.<locals>.<listcomp>c                 3   s   | ]}j  |  V  qd S rU   rn   r^   r   )
orig_nodesrX   r   r   r`     s    c                    s   g | ]}|kr | qS r   r   )r^   rL   )orig_mod_envoriginal_orderr   r   rq     s     c                 3   s   | ]} | V  qd S rU   r   rr   r-   r   r   r`     s     c                    s
    | j  S rU   r   rg   rv   r   r   ri     rj   )Mr   r   r%   r   r   Zgraph_moduler
   sympyr   ampZ_enter_autocastZ_exit_autocastZ_CZ_set_grad_enabledr   setr   Znodesr6   r>   rN   r\   ZSymIntr,   exprSymbolr=   r7   r0   AssertionErrorboolalladdremovevaluesitems_LOGGERisEnabledForloggingDEBUGdebugr   Zmap_argrm   listkeysr   rV   popr   RuntimeErrorr   r;   tupler<   r?   r   r/   rA   rJ   r@   rB   rC   joinr   r   dictreversedr   r   rZ   ro   proxyZProxy	enumeratenextiter)5r&   r'   r(   r)   r*   r+   rF   rw   rY   ZGLOBAL_STATE_NODESZgrad_regionsZautocast_regionsZautocast_exitsZactive_gradZactive_autocastsvalaZassert_monotonically_increasingZhighest_partitionpidZoriginal_partition_orderZroot_partitionsrW   Zsorted_partitionsZroot_partitionZ	dependentZregions_mappingZregionsrnew_nodeZinpr/   Zgathered_argsZgathered_kwargsr=   Ztarget_atomsZtarget_attrrD   qualnamer   Z	exit_noder.   Zconstruct_order_partitionsZalready_constructed_attr_nodesZoutput_valsZnum_output_valsZorig_mod_attr_nodesZbased_mod_attrsZ
output_valZnum_outputsZoutput_val_proxyiZoutput_namer   )r-   rE   r   r+   r&   r,   rt   rs   ru   rX   rR   rT   r(   rS   r   r   *   s   i
 
	




 
 
  


 






  	



  
   



)NFF)r8   typingr   r   r   r   r   r   collectionsr   r   r   Ztorch.fx._compatibilityr	   Ztorch.fx.graph_moduler
   Ztorch.fx.noder   __all__	getLoggerr"   r   r   nnModuleintr%   r}   r   r   r   r   r   <module>   s.    
   