U
    yh                     @   s,  d dl Z d dlZd dlmZmZ d dlmZmZmZm	Z	m
Z
mZmZ d dlZd dlm  mZ d dlmZ d dlmZ d dlmZmZmZmZ ejedZdd	 Z G d
d dej!Z"G dd deZ#e j$dd Z%e# fe#dddZ&G dd deZ'G dd de'Z(G dd de'Z)G dd de'Z*dS )    N)ABCabstractmethod)AnyCallableContextManagerDictOptionalTupleUnion)$_functionalization_reapply_views_tls)_get_dispatch_mode_pre_dispatch)_detect_infra_mode_disable_infra_modereturn_and_correct_aliasingTorchDispatchModenot_implementedc                     s    fdd}|S )Nc                    s   | j || S N)toselfargskwargsextra_kwargs U/var/www/html/venv/lib/python3.8/site-packages/torch/_subclasses/functional_tensor.py_$   s    z&_conversion_method_template.<locals>._r   )r   r   r   r   r   _conversion_method_template#   s    r   c                       s  e Zd ZU dZejed< ejjj	Z
ejjejjjZejjjjejjjjejjjjejjjjejjjjejjjjejjjjejjjjejjjjejjjjejjjjejjjjejjjjejj j!jgZ"ejjj#jejjj$jejjj%jejjj&jejjj'jejjj(jgZ)dd Z*d!ddZ+d	d
 Z,e-dd Z.dd Z/ddddZ0ddddZ1ddddZ2ddddZ3e4dddZ5 fddZ6d"ddZ7e8ej9dZ:e8e!dd Z;e8ej<dZ<e8ej=dZ>e8ej?dZ@e8ejAdZBe8ejCdZCe8ejDdZEe8ejFdZGe8ejHdZI  ZJS )#FunctionalTensoraF  
    Functional tensors represent tensors that will remove mutations
    from a program. If you perform a mutable operation on a functional tensor,
    it will re-dispatch to the functional variant of that operation.

    Historically, functionalization is implemented in C++ in the dispatcher.
    This class is a lightweight python shim around the C++ functionalization logic.

    FunctionalTensor is required to be used with a corresponding
    FunctionalTensormode active, because it relies
    on using the mode for dispatch (which can properly handle factory functions).
    elemc                 C   sn   t |sttjt j|@ }t j| |j	|
 | d |j|j|jd|jddd|}t j| ||_|S )NFsizes)torch_is_functional_tensorAssertionErrorr   _extra_dispatch_keys_CZ_dispatch_keysTensorZ_make_wrapper_subclassshapestridestorage_offsetdtypeZlayoutdeviceZrequires_gradZ_set_throw_on_mutable_data_ptrr   )clsr   Zextra_dispatch_keysoutr   r   r   __new__f   s,    zFunctionalTensor.__new__r   Nc                 C   s   dd |D }|r"t d| tS |d kr.i }|tjkrt|dksHt|tjj	j
jtjj	jjfkrt|dkr~t|d tst||d j|d S t|dkrt|d tst||d jS tdd S )Nc                 S   s$   g | ]}|t jt jjtfkr|qS r   )r!   r&   _subclasses
FakeTensorr   .0tr   r   r   
<listcomp>   s   z7FunctionalTensor.__torch_dispatch__.<locals>.<listcomp>.FunctionalTensor unrecognized subclass(es): %sr         zqAttempting to use FunctionalTensor on its own. Instead, please use it with a corresponding FunctionalTensorMode())not_implemented_logdebugNotImplementedr   metadata_fnslenr#   r!   opsatenis_strides_like_formatdefaultis_contiguousmemory_format
isinstancer   RuntimeError)r   functypesr   r   unrecognized_typesr   r   r   __torch_dispatch__   s0     


z#FunctionalTensor.__torch_dispatch__c                 C   s   dt | j dS )NzFunctionalTensor())reprr   r   r   r   r   __repr__   s    zFunctionalTensor.__repr__c              	   C   sf   t | rtt | }tt jjj}|d k	s2t|& t | | t	|}t || W 5 Q R X |S r   )
r!   r"   r#   Z_to_functional_tensorr   r%   _TorchDispatchModeKey
FUNCTIONALZ_mirror_autograd_meta_tor   )xZx_functionalZfunctional_moder-   r   r   r   to_functional   s    
zFunctionalTensor.to_functionalc                 C   s   t |  t | jS r   )r!   _sync_from_functional_tensorr   rK   r   r   r   from_functional   s    
z FunctionalTensor.from_functionalreturnc                 C   s   t | j| d S r   )r!   _functionalize_replacer   )r   outputr   r   r   replace_   s    zFunctionalTensor.replace_c                 C   s   t | j d S r   )r!   _functionalize_commit_updater   rK   r   r   r   commit_update   s    zFunctionalTensor.commit_updatec                 C   s   t | j d S r   )r!   _functionalize_syncr   rK   r   r   r   sync   s    zFunctionalTensor.syncc                 C   s   t | j d S r   )r!   1_functionalize_mark_mutation_hidden_from_autogradr   rK   r   r   r   "mark_mutation_hidden_from_autograd   s    z3FunctionalTensor.mark_mutation_hidden_from_autogradc                 C   sJ   | j  dkr| j  S | j  dkr6dd | j D S dd | j D S d S )Nr   r7   c                 S   s   g | ]}|  qS r   )itemr2   r   r   r   r   r4      s     z+FunctionalTensor.tolist.<locals>.<listcomp>c                 S   s   g | ]}|  qS r   )tolistr`   r   r   r   r4      s     )r   dimr_   rK   r   r   r   ra      s
    
zFunctionalTensor.tolistc                    sJ   t tjjjjr<tdd |D dkr<t j||ddiS t j||S )Nc                 S   s   g | ]}t |tr|qS r   )rC   bool)r2   argr   r   r   r4      s     
 z'FunctionalTensor.to.<locals>.<listcomp>r7   copyT)	r   r!   r%   rM   rN   exportr<   superr   r   	__class__r   r   r      s    zFunctionalTensor.toc                 O   sD   |pt j }t|dkr,| j|f||S | jf d|i|S d S )Nr   r+   )r!   cudaZcurrent_devicer<   r   )r   r+   r   r   r   r   r   rj      s    zFunctionalTensor.cuda)r*   cpu)r+   )r   N)N)K__name__
__module____qualname____doc__r!   r&   __annotations__r%   rM   rN   	_mode_keyZ,_additional_keys_to_prop_for_wrapper_tensorsaddDispatchKeyZ
ZeroTensorr$   r=   r>   rA   r@   rB   r?   Zis_non_overlapping_and_densesizeZsym_sizer(   Z
sym_strider)   Zsym_storage_offsetZnumelZ	sym_numelrb   primr+   r;   dropoutZ
batch_normZnative_batch_normZ_batch_norm_impl_indexZcudnn_batch_normZmiopen_batch_normmaybe_aliasing_or_mutating_opsr.   rH   rL   staticmethodrP   rS   rX   rZ   r\   r^   r   ra   r   rj   r   Zint8charrk   Zbfloat16Zuint8byteZfloat64doubleZfloat32floatrc   Zfloat16ZhalfZint32intZint64long__classcell__r   r   rh   r   r   *   sf   


	



















	/
+

r   c                       s<   e Zd ZdddZ fddZ fddZdd
dZ  ZS )FunctionalTensorModeFc                 C   sH   || _ d| _g | _tjjj| _|| _|r2tjj	j
nd | _i | _|| _d S )NF)rf   is_on_stackenter_stackr!   r%   rM   rN   rq   pre_dispatchrs   PreDispatch_dispatch_key_tokens_allow_token_discovery)r   r   rf   r   r   r   r   __init__  s    	zFunctionalTensorMode.__init__c                    s@    fdd}| d kr, j d t  S  j d  S d S )Nc                      s0    j tjjjkrttjjjS tjtjjjS r   )	r   r!   r%   rs   r   r   rM   rN   Z_get_dispatch_moder   rK   r   r   _get_prev_mode'  s    z6FunctionalTensorMode.__enter__.<locals>._get_prev_modeTF)r   appendrg   	__enter__)r   r   rh   rK   r   r   &  s    	

zFunctionalTensorMode.__enter__c                    s"   | j  }|rt ||| d S r   )r   poprg   __exit__)r   abcr   rh   r   r   r   7  s    
zFunctionalTensorMode.__exit__r   Nc              
      s  |d kri }dd |D }|r.t d| tS  fdd}|tjkr||rtj| r * |j	||}|tk	r|W  5 Q R  S W 5 Q R X dd }dd	 }	d
d }
ddl
m}m} ||rtj| tjjjs||||S ddlm}m} ||||r4tj| tjjjr t| j j|||S tt|
||f\}}tjtjjj}tjtjjj}|s~|r~ttj tjtjjjB }tj tjjjtj }tj|| zt!d}|tjkr|||}ttj"|	|}nX|j#tjjjf||} j$rB|tj%j&j'j(tj%j&j)j(fkrBt*| ttj"|	|}W 5 t   t!| X W 5 Q R X tjtjjj}tjtjjj}|s|rtt+dd t,|D r|tj%j&j-j(kr|S t.||||S )Nc                 S   s,   g | ]$}t |tjjs|tjtfkr|qS r   )
issubclassr!   r/   r0   r&   r   r1   r   r   r   r4   @  s   z;FunctionalTensorMode.__torch_dispatch__.<locals>.<listcomp>r5   c                    s    j r| tjjjjkrdS  jd k	r|| tjkr2dS t	dd | j
jD }|dkpV| j
j}|sx| jdkrxtd|  d |S dS )	NFTc                 S   s   g | ]}|j d k	r|qS r   )
alias_info)r2   ir   r   r   r4   e  s     
 zSFunctionalTensorMode.__torch_dispatch__.<locals>._can_decompose.<locals>.<listcomp>r   )r>   ru   zAt pre-dispatch tracing, we will assume that any custom op that is marked with CompositeImplicitAutograd and functional are safe to not decompose. We found z to be one such op.)rf   r!   r=   r>   rv   r@   r   r   rw   r<   Z_schema	argumentsZ
is_mutable	namespacewarningswarn)rE   r   Zshould_decomposerK   r   r   _can_decomposeL  s     



z?FunctionalTensorMode.__torch_dispatch__.<locals>._can_decomposec                 S   s   t | std S r   )r!   r"   r#   rO   r   r   r   assert_is_functional  s    zEFunctionalTensorMode.__torch_dispatch__.<locals>.assert_is_functionalc                 S   s0   t | trtt | tjr,t| r,t| S | S r   )rC   r   r#   r!   r&   r"   r   r   r   r   wrap  s    z5FunctionalTensorMode.__torch_dispatch__.<locals>.wrapc                 S   s   | j S r   )r   r   r   r   r   unwrap  s    z7FunctionalTensorMode.__torch_dispatch__.<locals>.unwrapr   )can_auto_functionalizedo_auto_functionalize)handle_effectshas_effectsTc                 s   s   | ]}t |tV  qd S r   )rC   r   )r2   rO   r   r   r   	<genexpr>  s   z:FunctionalTensorMode.__torch_dispatch__.<locals>.<genexpr>)/r8   r9   r:   r   r;   r!   r%   Z_dispatch_has_kernelnameZ	decomposeZ*torch._higher_order_ops.auto_functionalizer   r   Z%_dispatch_has_kernel_for_dispatch_keyrs   FunctionalizeZtorch._higher_order_ops.effectsr   r   r#   r   r   pytreetree_map_onlyZ&_dispatch_tls_is_dispatch_key_includedZ&_dispatch_tls_is_dispatch_key_excludedZ_dispatch_tls_local_include_setDispatchKeySetZ_dispatch_tls_local_exclude_setremover$   Z_ForceDispatchKeyGuardZ_disable_functionalizationZ#_functionalize_enable_reapply_viewsr&   Z_op_dkrf   r=   r>   rv   r@   Z_to_copyZ_freeze_functional_tensoranyZtree_leavesZ
lift_freshr   )r   rE   rF   r   r   rG   r   rr   r   r   r   r   r   r   Zargs_unwrappedZkwargs_unwrappedZis_includedZis_excludedZinclude_to_setZexclude_to_setZold_apply_viewsZouts_unwrappedZouts_wrappedr   rK   r   rH   <  s     (	  
      
	

  	


  
z'FunctionalTensorMode.__torch_dispatch__)FFF)r   N)rl   rm   rn   r   r   r   rH   r   r   r   rh   r   r     s   
r   c                   C   s   t tjjjS r   )r   r!   r%   rM   rN   r   r   r   r   disable_functional_mode  s    r   )modec                    s&   dd dd   fdd}|S )Nc                 S   s   t | tjrt| S | S r   )rC   r!   r&   r   rP   r3   r   r   r   to_fun  s    
z&dispatch_functionalize.<locals>.to_func                 S   s>   t | ts(t | tjr$t| r$t| S t|  t| jS r   )	rC   r   r!   r&   r"   r#   rQ   rR   r   r   r   r   r   from_fun  s    

z(dispatch_functionalize.<locals>.from_func                     s   t jt jt jjj}|f V tt j| }tt j|}||}tt	 |}|W  5 Q R  W  5 Q R  S Q R X W 5 Q R X d S r   )
r!   r%   _ExcludeDispatchKeyGuardr   rs   r   r   r   r&   r   )r   r   Zdisable_aboveZ	func_argsZfunc_kwargsZfunc_outputsoutputsr   rE   r   r   r   r   inner   s    
z%dispatch_functionalize.<locals>.innerr   )rE   r   r   r   r   r   dispatch_functionalize  s    	r   c                   @   s   e Zd Zeee ee dddZeeej	eej	df f eej	eej	df f dddZ
eeeddd	Zeed
ddZedd
ddZedd
ddZedd
ddZedd
ddZdS )BaseFunctionalizeAPIr   rU   c                 C   s   d S r   r   r   r   r   r   r   wrap_tensors0  s    z!BaseFunctionalizeAPI.wrap_tensors.c                 C   s   d S r   r   r   r   r   r   unwrap_tensors4  s    z#BaseFunctionalizeAPI.unwrap_tensorsinner_frU   c                 C   s   d S r   r   r   r   r   r   r   functionalize:  s    z"BaseFunctionalizeAPI.functionalizerT   c                 C   s   d S r   r   rK   r   r   r   redispatch_to_next>  s    z'BaseFunctionalizeAPI.redispatch_to_nextNc                 C   s   d S r   r   r   Zinput_tensorZoutput_tensorr   r   r   replaceB  s    zBaseFunctionalizeAPI.replacec                 C   s   d S r   r   r   Ztensorr   r   r   rZ   F  s    z"BaseFunctionalizeAPI.commit_updatec                 C   s   d S r   r   r   r   r   r   r\   J  s    zBaseFunctionalizeAPI.syncc                 C   s   d S r   r   r   r   r   r   r^   N  s    z7BaseFunctionalizeAPI.mark_mutation_hidden_from_autograd)rl   rm   rn   r   r	   r   r   r
   r!   r&   r   r   r   r   r   r   rZ   r\   r^   r   r   r   r   r   /  s$   r   c                       s   e Zd Zdee edd fddZee ee dddZ	e
ejeejd	f f e
ejeejd	f f dd
dZeedddZedddZddddZddddZddddZddddZ  ZS )PythonFunctionalizeAPINF)r   r   rU   c                    s$   t    |r|nt | _|| _d S r   )rg   r   r   r   r   )r   r   r   rh   r   r   r   T  s    
zPythonFunctionalizeAPI.__init__r   c              
   C   s4   | j $ tjjtjtj|W  5 Q R  S Q R X d S r   )r   r!   utils_pytreer   r&   r   rP   r   r   r   r   r   [  s      z#PythonFunctionalizeAPI.wrap_tensors.c                 C   s   t jjttj|S r   )r!   r   r   r   r   rS   r   r   r   r   r   a  s
      z%PythonFunctionalizeAPI.unwrap_tensorsr   c                 C   s   t || jS r   )r   r   r   r   r   r   r   h  s    z$PythonFunctionalizeAPI.functionalizerT   c                 C   s   t  S r   )
contextlibnullcontextrK   r   r   r   r   k  s    z)PythonFunctionalizeAPI.redispatch_to_nextc                 C   s*   t |tstt |trt|| d S r   )rC   r   r#   rX   r   r   r   r   r   s  s    zPythonFunctionalizeAPI.replacec                 C   s   t |tst|  d S r   )rC   r   r#   rZ   r   r   r   r   rZ   x  s    z$PythonFunctionalizeAPI.commit_updatec                 C   s   t |tst|  d S r   )rC   r   r#   r\   r   r   r   r   r\   |  s    zPythonFunctionalizeAPI.syncc                 C   s   t |tst|  d S r   )rC   r   r#   r^   r   r   r   r   r^     s    z9PythonFunctionalizeAPI.mark_mutation_hidden_from_autograd)NF)rl   rm   rn   r   r   rc   r   r	   r   r   r
   r!   r&   r   r   r   r   r   r   rZ   r\   r^   r   r   r   rh   r   r   S  s"       r   c                   @   s   e Zd Zee ee dddZeejeejdf f eejeejdf f dddZ	e
e
ddd	Zed
ddZdd
ddZdd
ddZdd
ddZdd
ddZdS )CppFunctionalizeAPIr   c                 C   s   ddl m} ||ddS Nr   )_wrap_all_tensors_to_functional)level)!torch._functorch.eager_transformsr   r   r   r   r   r   r   r     s    z CppFunctionalizeAPI.wrap_tensors.c                 C   s   ddl m} ||t dS Nr   )#_unwrap_all_tensors_from_functional)Zreapply_views)r   r   _reapply_viewsr   r   r   r   r   r   r     s    z"CppFunctionalizeAPI.unwrap_tensorsr   c                 C   s   t j|S r   )r!   rE   r   r   r   r   r   r     s    z!CppFunctionalizeAPI.functionalizerT   c                 C   s   t jt jt jjjS r   )r!   r%   r   r   rs   r   rK   r   r   r   r     s    z&CppFunctionalizeAPI.redispatch_to_nextNc                 C   s   t || d S r   r!   rV   r   r   r   r   r     s    zCppFunctionalizeAPI.replacec                 C   s   t | d S r   r!   rY   r   r   r   r   rZ     s    z!CppFunctionalizeAPI.commit_updatec                 C   s   t | d S r   r!   r[   r   r   r   r   r\     s    zCppFunctionalizeAPI.syncc                 C   s   t | d S r   r!   r]   r   r   r   r   r^     s    z6CppFunctionalizeAPI.mark_mutation_hidden_from_autograd)rl   rm   rn   r	   r   r   r
   r!   r&   r   r   r   r   r   r   rZ   r\   r^   r   r   r   r   r     s   	r   c                   @   s   e Zd Zdd Zee ee dddZeej	eej	df f eej	eej	df f dddZ
eed	d
dZedddZddddZddddZddddZddddZdS )FunctorchFunctionalizeAPIc                 C   s
   || _ d S r   )interpreter)r   r   r   r   r   r     s    z"FunctorchFunctionalizeAPI.__init__r   c                 C   s   ddl m} ||| j dS r   )r   r   r   r   r   r   r   r   r     s    z&FunctorchFunctionalizeAPI.wrap_tensors.c                 C   s   ddl m} ||| j dS r   )r   r   r   functionalize_add_back_viewsr   r   r   r   r     s
     z(FunctorchFunctionalizeAPI.unwrap_tensorsr   c                 C   s   t jj|| j rdnddS )NZmutations_and_viewsZ	mutations)r   )r!   rE   r   r   r   r   r   r   r   r     s    z'FunctorchFunctionalizeAPI.functionalizerT   c                 C   s
   | j  S r   )r   lowerrK   r   r   r   r     s    z,FunctorchFunctionalizeAPI.redispatch_to_nextNc                 C   s   t || d S r   r   r   r   r   r   r     s    z!FunctorchFunctionalizeAPI.replacec                 C   s   t | d S r   r   r   r   r   r   rZ     s    z'FunctorchFunctionalizeAPI.commit_updatec                 C   s   t | d S r   r   r   r   r   r   r\     s    zFunctorchFunctionalizeAPI.syncc                 C   s   t | d S r   r   r   r   r   r   r^     s    z<FunctorchFunctionalizeAPI.mark_mutation_hidden_from_autograd)rl   rm   rn   r   r	   r   r   r
   r!   r&   r   r   r   r   r   r   rZ   r\   r^   r   r   r   r   r     s   r   )+r   r   abcr   r   typingr   r   r   r   r   r	   r
   r!   Ztorch.utils._pytreer   r   r   Ztorch._Cr   r   Z
torch._opsr   Ztorch.utils._python_dispatchr   r   r   r   Z_loggingZgetArtifactLoggerrl   r8   r   r&   r   r   contextmanagerr   r   r   r   r   r   r   r   r   r   <module>   s*   $ d w
$2$