U
    zht                     @   sx  d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZm	Z	m
Z
 d dlZddlmZmZ ddlmZmZmZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZ dd
lm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 dd Z5dd Z6ee7ej8j9dddZ:dd Z;G dd de.Z<G dd de4Z=G dd de=Z>dS )    N)contextmanagernullcontext)AnyDictList   )trace_rules	variables)unimplementedUnspecializeRestartAnalysisUnsupported)GuardBuilderinstall_guard)GenerationTracker)
AttrSourceFSDPNNModuleSourceGetItemSourceNNModuleSourceNotNNModuleSource)get_custom_getattrget_fake_valueis_lazy_moduleis_namedtupleis_safe_constantistensoristypennmodule_has_hooksobject_has_getattributeproxy_args_kwargsset_example_value   )MutableLocaltypestrVariableTracker)invoke_and_store_as_constant)SliceVariable)UserDefinedObjectVariablec                    s`   t |dr\ fdd t||\}} fdd|D } fdd| D }|||| dS )	a~  
    Fairly coupled helper used by NNModuleVariable and UnspecializedNNModuleVariable.

    Used to cause lazy module to be initialized (and delete its init hook) before tracing. Especially
    useful now that 'allowed' modules graph-break on hooks, calling this first ensures there is no hook
    by the time we trace __call__ and thus no graph-break for lazy allowed modules.
    Z_initialize_hookc                    s   t | r"t|  fdd| D  S t| trB fdd|  D S t| tttfrlt|  fdd| D S t| tj	j
rt| jS | S d S )Nc                 3   s   | ]} |V  qd S N .0elemconvert_to_faker(   S/var/www/html/venv/lib/python3.8/site-packages/torch/_dynamo/variables/nn_module.py	<genexpr>6   s     zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>c                    s   i | ]\}}| |qS r(   r(   r*   kvr,   r(   r.   
<dictcomp>8   s      zCinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<dictcomp>c                 3   s   | ]} |V  qd S r'   r(   r)   r,   r(   r.   r/   :   s     )r   type
isinstancedictitemslisttuplesettorchfxZProxyr   node)xr-   txr(   r.   r-   4   s    
z/initialize_lazy_module.<locals>.convert_to_fakec                    s   g | ]} |qS r(   r(   )r*   argr,   r(   r.   
<listcomp>A   s     z*initialize_lazy_module.<locals>.<listcomp>c                    s   i | ]\}}| |qS r(   r(   r0   r,   r(   r.   r3   B   s      z*initialize_lazy_module.<locals>.<dictcomp>N)hasattrr   r7   Z_infer_parameters)r@   modargskwargs
proxy_argsproxy_kwargsZ	fake_argsZfake_kwargsr(   r?   r.   initialize_lazy_module*   s    
rI   c                 C   sD   t | tr@t | jtr@t | jjtr@| jjdkr@t| jj| jS | S )N_modules)r5   r   baser   r   memberindexsourcer(   r(   r.   "cleanup_source_for_nn_module_stackF   s    

rP   )
module_keyrD   c                 c   s:   t |}| }z||jf|j| < d V  W 5 |j| = X d S r'   )rP   nameZnn_module_stack	__class__)rQ   rO   r@   rD   Zsource_for_nn_module_stackZfully_qualified_namer(   r(   r.   record_nn_module_stackb   s    
rT   c                 C   sx   | rtd|j krZt|j d rZ|j d }t| d}t|tjkrHt|d}t|tj	 nt| t
jtjdd d S )Nforward__func__)attr)__dict__callabler   r4   types
MethodTyper   
make_guardr   ZCLOSURE_MATCH	functoolspartialZNOT_PRESENT_IN_GENERIC_DICT)rO   rD   ZfwdZforward_sourcer(   r(   r.   &guard_to_detect_forward_monkeypatchingm   s    


 r_   c                       s   e Zd ZdddhejZeeejj	d fddZ
dd Zd	d
 Zdd ZeddddZdd Zdd Zdd Zdd Zdd ZddddddZd#dddd fd!d"Z  ZS )$NNModuleVariablemodule_typerQ   module)ra   rQ   rb   c                    s.   t  jf | || _|| _|| _| js*td S r'   )super__init__ra   rQ   rb   rO   AssertionError)selfra   rQ   rb   rF   rS   r(   r.   rd      s
    zNNModuleVariable.__init__c                 C   s   | j S r'   )ra   )rf   r(   r(   r.   python_type   s    zNNModuleVariable.python_typec                 O   s   d S r'   r(   )rf   r@   rO   submodZ	key_extraoptionsr(   r(   r.   _wrap_submodule   s    z NNModuleVariable._wrap_submodulec                 C   s   |j | j}t|tjjrng }| D ]@\}}tj	
|}|j j|| j|tt| j|d || q(|S t|tjjtjjtjjfstt|| jstg }t|D ]0\}}||j j|| j|tt| j|d q|S NrN   )outputget_submodulerQ   r5   r;   nn
ModuleDictr7   r	   ConstantVariablecreateregister_attr_or_moduler   r   rO   append
ModuleListParameterList
Sequentialre   r"   	enumerate)rf   r@   rK   resultrR   ri   Zname_varidxr(   r(   r.   unpack_var_sequence   s>     
z$NNModuleVariable.unpack_var_sequencer#   )rR   returnc                 C   s@   |j | j}t||}ttt| j|t	j
 tj|S r'   )rm   rn   rQ   rC   r   r   r   rO   r\   r   ZHASATTRr	   rq   rr   )rf   r@   rR   rD   ry   r(   r(   r.   call_hasattr   s    
zNNModuleVariable.call_hasattrc                 C   s   |j | j}t|ddS )NZtrainingF)rm   rn   rQ   getattrrf   r@   rD   r(   r(   r.   is_training   s    zNNModuleVariable.is_trainingc                 C   s:   |j | j}t| |jjdkr2tt| t	dS )zIRestart analysis treating this module as an UnspecializedNNModuleVariablerd   N)
rm   rn   rQ   r   tagf_codeco_nameZmark_class_dynamicr4   r   r   r(   r(   r.   convert_to_unspecialized   s
    
z)NNModuleVariable.convert_to_unspecializedc                 C   sd   |j | j}t|rtd |j j| |rP|j jj| |dd}t|t	j
 S t|d}||kS )Nz-NNModuleVariable with custom __getattribute__T)Z
deleted_okrX   )rm   rn   rQ   r   r
   side_effectsZhas_pending_mutation_of_attrZ	load_attrr5   r	   ZDeletedVariableobject__getattribute__)rf   r@   keyrK   Zmutated_attr	base_dictr(   r(   r.   has_key_in_generic_dict   s    z(NNModuleVariable.has_key_in_generic_dictc                 C   s\   t |rtd t|}|dkr$dS t|tjs8td tj|| f||tj	
|gi S )zDCheck for a __getattr__ and handle it specially if it is implementedz6torch.nn.Module with a custom __getattribute__ definedNz6torch.nn.Module with a non-function custom __getattr__)r   r
   r   r5   rZ   FunctionTyper	   UserMethodVariablecall_functionrq   rr   )rf   rK   r@   rR   rj   Z
getattr_fnr(   r(   r.   _custom_getattr_fallback   s      z)NNModuleVariable._custom_getattr_fallbackc              	   C   s  ddl m} | jr t| j|}nd }|j| j}t|d}d}t	 }t
|jD ]}	||	j  qT| jsxtd |dkrtj| ||dS ||kr|| }
nd|kr||d kr||kr|d | }
nd|kr||d kr|d | }
n|d	|kr||d	 kr|d	 | }
nVzt
||}
d
}W n@ tk
rh   | j|||d|id}|d k	rb| Y S  Y nX |dkrt| j| |dkr|stj|j|dS |r||t||
S t|
tr| jrtt| jd|}t|d}tj|
j|d|| gi S t|
tr,tj|
j t!t"||dS t|
t#rLtj|
$||dS t|
t%j&rjtj|
| |dS t'|
s~t(|
r||t||
S td| dt)| dt)|
  tj| ||dS )Nr    VariableBuilderrX   TzGETATTR with no sourcerN   rJ   _parameters_buffersFrO   )rK   r@   rR   rj   rU   rS   fgetzclass property z -  )*builderr   rO   r   rm   rn   rQ   r   r   r:   inspectgetmrorS   updaterX   keysr
   r	   ZGetAttrVariablegetattr_staticAttributeErrorr   r_   ZUserDefinedClassVariabler   r   propertyUserFunctionVariabler   r   classmethodr   rV   r&   r4   staticmethod__get__rZ   r   r   r   r"   )rf   r@   rR   r   rO   rK   r   Zobject_memberZall_class_attribute_namesr>   Zsubobjry   r(   r(   r.   var_getattr   s    

   


   zNNModuleVariable.var_getattrList[VariableTracker]Dict[str, VariableTracker]rE   rF   r|   c                 C   s&  |j | j}t| j| j|| t|}t|tjj	r|j
jtjj	jkrt|r^| | |rjtd|rrt|\}|j D ]>\}}||j j|| j|tt| j|d|gi  | }q|W  5 Q R  S |r|jd k	r|j| _t|||| |j  rr|jdrr|jdkrrt|dddr6| | ddlm}	 |	||j jd	| jft|| d
W  5 Q R  S | jstdt|tj j!r|j}
t| jd}n|j"}
t| jd}t#|
t$j%r|
j&}
t|d}| g| }nt#|
t$j'st|(t)j*|
|d||W  5 Q R  S W 5 Q R X d S )Nz3Expected lazy sequential isn't a valid combination?rN   )z	torch.nn.z	torch.ao.ztorch.nn.utils.parametrizeT)Zcheck_forward_hooksZcheck_backward_hooksr    wrap_fx_proxyZcall_moduler@   proxyzsMust provide a valid source in order to inline, since inlined function may have default args which must be guarded.rU   
_call_implrV   )+rm   rn   rQ   rT   rO   r   r5   r;   ro   rw   rS   rU   r   r   re   rJ   r7   r   rs   r   r   popcls_to_becomera   rI   Zis_root_tracer
__module__
startswithr   r   create_proxyr   r<   ZGraphModuler   r   rZ   r[   rV   r   inline_user_function_returnr	   r   )rf   r@   rE   rF   rD   Zis_lazyrA   Z
child_nameri   r   fn	fn_sourcer(   r(   r.   r   R  s    





  
	

zNNModuleVariable.call_functionFc              
      sv  ddl m mm 
jj
fdd}	dkrP
S 	dkrt
j
j	 |	W  5 Q R  S Q R X 	dkrt
tjjr dS 	d	kr.d  sttd
 std
 jd j }t|t r|S |jj}jj|tt
j	dS |rbt	}	jj d	 d	t|		S 	fdd	fdd}

fdd} 
fdd}dd 	dkr"jjt
j	d   srtg }! D ]\	}|"|	| q|t# dS 	dkrjjt
j	d   g }j$f |
ddD ]\	}|"|	| q\|t# dS 	dkrjjt
j	d    g }j%f |
ddd!D ]\	}|"|	| q|t# dS 	d"krRjjt
j	d   g }j&f |
d#dd!D ]\	}|"|	| q(|t# dS 	d$krjjt
j	d   srt|! S 	d%krjjt
j	d   |& S 	d&krjjt
j	d   |j$f |
dS 	d'kr2jjt
j	d    |j%f |
dS 	d(kr~sHrLtg }' D ]	|" 	 qX|t# dS 	d)krsrt| S 	d*krsrtg } D ]\	}|"|	| qʈ|t# dS 	d+krs
rt t(S 	d,krjtt)j*j+t)j*j,frjrjd
  rj d
 - j.kS 	d-krst(dkstt)j*j+j/t)j*j0j/t)j*j,j/t)j*j1j/t)j*j2j/f}t3j/|kr\td
 t4jstt5d
 d
 - tt6t7fs
tt	j8}	t|	t9j:s(ttt
j		d.};t4j<|	|d
gt= S 
j	shttd
 t>r2g }g }t=t?t(d
 -  }t@d
 -  D ]D\}}|| tt
j	}|"jj||d |"| qt)j*j2| }jj|
 d/tt
j	d
 - d}|S dd0lAmB} td
 |r`d
 Cjnd
 -  }jj|
jtt
j	dS 	d1kstt)j*jDjEjFr	d2kstt)j*jDjEjGr	d3krt	j8}	tt
j		d.};t4j<|	|d
g S 	jjHkr`tIjjH	 r`tJd4d5 tKLM D r`|	S tN O	S d S )6Nr    )rq   ListIteratorVariableTupleVariablec                    s^   j djt i }t|j t \}}ddlm} |j jd| |f||ddS )NZget_attrr    r   call_method)rE   rF   r   )	rm   r   rQ   r9   r   r=   r   r   r   )rR   Z	mod_proxyrG   rH   r   )rE   rF   rb   rf   r@   r(   r.   generic_call_method_helper  s$    z@NNModuleVariable.call_method.<locals>.generic_call_method_helperr   Z_wrapped_call_implrU   _check_input_dimTZ_get_item_by_idxr   rN   __resultc                      s0   t dd t  D s,td  d S )Nc                 s   s   | ]}|  V  qd S r'   )is_python_constantr*   r>   r(   r(   r.   r/     s    zUNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const.<locals>.<genexpr>znon-const NNModule method )all	itertoolschainvaluesr
   r(   )rE   rF   rR   r(   r.   assert_all_args_kwargs_const  s    zBNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_constc                     sZ     t }t|jdd D dd  D      j  fdd| D S )Nc                 S   s   g | ]}|  qS r(   as_python_constantr   r(   r(   r.   rB     s     zDNNModuleVariable.call_method.<locals>.get_kwargs.<locals>.<listcomp>c                 S   s   i | ]\}}||  qS r(   r   r0   r(   r(   r.   r3     s      zDNNModuleVariable.call_method.<locals>.get_kwargs.<locals>.<dictcomp>c                    s   i | ]}| | qS r(   r(   )r*   r1   Z
bound_argsr(   r.   r3     s      )r~   r   	signaturebindr7   apply_defaults	arguments)namesr   )rE   r   rF   rb   rR   r   r.   
get_kwargs  s    

z0NNModuleVariable.call_method.<locals>.get_kwargsc                    sF   g }| D ].\}}| jj||tj|d q |t dS )NrN   Zmutable_local)rt   rm   rs   r   rO   r!   )r7   ry   rR   ri   )r   
gen_sourcer   rf   r@   r(   r.   wrap_values  s    z1NNModuleVariable.call_method.<locals>.wrap_valuesc              
      s.     | jj|| tj| dgS rl   )rr   rm   rs   r   rO   )rR   obj)rq   r   r   r   rf   r@   r(   r.   named_embed$  s    z1NNModuleVariable.call_method.<locals>.named_embedc                 S   s@   | d}|d dkr| S t|dkr<|d}t| |} q| S )N.r    )splitlenr   r   )rO   rR   Z
name_splitr>   r(   r(   r.   r   1  s    

z0NNModuleVariable.call_method.<locals>.gen_sourcenamed_childrenrJ   r   named_parametersr   prefixrecursenamed_buffersr   Zremove_duplicatenamed_modulesmemochildrenmodules
parametersbuffersr   r   r7   __len____contains____getitem__rV   z.__getitem__(slice))SymNodeVariableZ_get_abs_string_indexZ_conv_forwardZ_output_paddingc                 s   s   | ]}t |tjV  qd S r'   )r5   r	   TensorVariabler   r(   r(   r.   r/     s   z/NNModuleVariable.call_method.<locals>.<genexpr>)Pr   rq   r   r   rQ   rm   rn   r   rT   rO   r   Zis_torch_inline_allowedr   getfilerS   r   rr   r   re   r5   r7   valueUnspecializedNNModuleVariablers   r   r   r~   __name__r$   Zguard_on_key_orderaddr   rR   r   rt   r!   r   r   r   r   r   r;   ro   rp   ZParameterDictr   rJ   r   ru   rv   rw   r4   r	   r"   strintrV   rZ   r   r   r   r8   r%   rangerx   Ztensorr   Zevaluate_exprr   convZ_ConvNdZ_ConvTransposeNdrX   rY   r   r   r   r   rc   r   )rf   r@   rR   rE   rF   Zconstantr   Zmod_varri   r   r   r   r   ry   parambufferZbuiltin_supportedsrcZsubmodsr   rz   Z
new_moduleZnew_module_variabler   r   rg   )rq   r   r   rE   r   r   r   rF   rb   rR   rf   r@   r.   r     s   


	















"
zNNModuleVariable.call_method)F)r   r   __qualname__r#   _nonvar_fieldsr4   r   r;   ro   Modulerd   rh   rk   r{   r}   r   r   r   r   r   r   r   __classcell__r(   r(   rg   r.   r`      s:     	 

[o r`   c                       st   e Zd ZddhejZ fddZeeddd Z	 fdd	Z
d
dddddZd
ddd fddZ  ZS )r   
value_typeis_state_mutatedc                    sj   t |tjjjkrtdd|krJt|d dd }t ||krJt ||d< t jf d|i| d| _	d S )NzScriptModules aren't supported in UnspecializedNNModuleVariable becuase their .forward function isn't a static member of their typer   r   r   F)
r4   r;   Zjit_scriptZRecursiveScriptModuler   r~   rc   rd   r   )rf   r   rF   Zlazy_value_to_becomerg   r(   r.   rd     s    z&UnspecializedNNModuleVariable.__init__Nc                      s(   t jjjh  fddt jjj D S )Nc                    s(   h | ] }t |d r| krt|jqS )__code__)rC   idr   r   	supportedr(   r.   	<setcomp>  s   
 zFUnspecializedNNModuleVariable._nn_module_method_ids.<locals>.<setcomp>)r;   ro   r   __setattr__rX   r   r(   r(   r   r.   _nn_module_method_ids  s    
z3UnspecializedNNModuleVariable._nn_module_method_idsc              
      s   ddl m  ztjd}W n( tk
rF } z
t|W 5 d }~X Y nX |tjj	j
tjjj
tjjj
fkrjsrt fddtjD S t S )Nr    r   __iter__c                    s(   g | ] \}} t j|d |qS )rN   )r   rO   )r*   rz   itemr   rf   r@   r(   r.   rB   '  s   zEUnspecializedNNModuleVariable.unpack_var_sequence.<locals>.<listcomp>)r   r   r   r   r   r   NotImplementedErrorr;   ro   ru   r   rv   rw   rO   re   rx   r   rc   r{   )rf   r@   r   erg   r   r.   r{     s    
z1UnspecializedNNModuleVariable.unpack_var_sequencer   r   r#   r   c           	   
   C   s   | j }t|r.|jd k	r |j| _t|||| d}t| j|}| jrXtt| jd|}nd }t| j| | jrt	t
t|| j||nt }|0 tj||d|| gt| |W  5 Q R  S Q R X d S )Nr   rS   rN   )r   r   r   r   rI   r~   rO   r   r_   rT   r   r   r   r	   r   r   r8   )	rf   r@   rE   rF   rD   rR   r   rO   ctxr(   r(   r.   r   .  s*    
  z+UnspecializedNNModuleVariable.call_functionc                    sV  ddl m} |dkr`t| j|}| jr:tt| jd|}nd }tj||d|| gt	| |S |t| j
di krDztt| j
|}W n tk
r   d }Y nX |tjjjkr,|r|st|jj| rtd t| jtj g }	| j
 D ]$\}}
|	||t| j||
 qtj|	t dS t |t!rjttt| jd|d	}|"tj|j#|d||S t$|d
rt%|j&| ' krtd|  |dkrDd| j
j(krD|d ) }|d }
t |
tj*r|
+ tjj,ks|| j
j(d krd| _-nF|| j
j(d krd| _-n,t |
tj.tj/fs>|| j
j(d krDd| _-t0 1||||S )Nr    r   r   rS   rN   rX   z)Module.parameters() with pending mutationr   rV   r   z&UnspecializedNNModuleVariable missing r   r   r   Tr   rJ   )2r   r   r~   r   rO   r   r	   r   r   r8   r   r   r   r4   r   r;   ro   r   r   re   rm   r   Zhas_pending_mutationr
   r   r\   r   ZNN_MODULE_PARAM_NAMESr   rt   r   r!   r5   r   r   rV   rC   r   r   r   rX   r   r   rh   	Parameterr   r`   r   rc   r   )rf   r@   rR   rE   rF   r   r   rO   methodr7   r   	attr_namerg   r(   r.   r   J  s      
  

z)UnspecializedNNModuleVariable.call_method)r   r   r   r&   r   rd   r   r]   	lru_cacher   r{   r   r   r   r(   r(   rg   r.   r     s$   	  r   c                       sB   e Zd ZdZ fddZedd Zeedd fdd	Z	  Z
S )
FSDPManagedNNModuleVariablea  
    Tracing behavior: trace into submodules and treat them as Unspecialized, do not
    register parameters to the top-level, treat them as function inputs.

    Guards behavior: if 'skip_fsdp_guards', many guards that would be installed
    by a vanilla UnspecializedNNModuleVariable are simply dropped, on the basis
    that a user wrapping their model in FSDP(model) is already opting into a
    requirement to not modify internal model state, which would already break FSDP without
    compilation.
    c                    s<   | dd }|d k	stdt jf d|i| || _d S )NrO   zMFSDPManagedNNModule depends on having an accurate source to control guarding.r   )getre   rc   rd   rO   )rf   r   rF   rO   rg   r(   r.   rd     s    z$FSDPManagedNNModuleVariable.__init__c                 C   s2   t | ttfs*tjjjr t| S t| S n| S d S r'   )r5   r   r   r;   Z_dynamoconfigZskip_fsdp_guardsrN   r(   r(   r.   _wrap_source  s
    

z(FSDPManagedNNModuleVariable._wrap_sourceN)rR   r   r|   c                    s    |dkrt |}t ||S )NrO   )r  r
  rc   r   )rf   rR   r   rg   r(   r.   r     s    
z'FSDPManagedNNModuleVariable.__setattr__)r   r   r   __doc__rd   r   r
  r   r   r   r   r(   r(   rg   r.   r    s
   	

r  )?r]   r   r   rZ   
contextlibr   r   typingr   r   r   Ztorch.nnr;   r   r   r	   excr
   r   r   Zguardsr   r   Zmutation_guardr   rO   r   r   r   r   r   utilsr   r   r   r   r   r   r   r   r   r   r   rK   r!   r"   r#   Z	functionsr$   listsr%   Zuser_definedr&   rI   rP   r   ro   r   rT   r_   r`   r   r  r(   r(   r(   r.   <module>   s:   4
!    c @