U
    MhTQ                 y   @   s@$  U d dl Z d dlZd dlmZ d dlmZ d dlmZmZ d dl	m
Z
mZ d dl	Z	d dlZd dlm  mZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z' d dl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6 d dl7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z= d dl>m?Z? d dl@mAZAmBZBmCZCmDZDmEZE d dlFZFe jjGe jHjjIjGe jHjjJjGe jHjjKjGe jHjjKjGgZLeAe? eMd< e jjNe jjOe jjPe jHjjKjQe jHjjKjQhZReDeC eMd< eSe
dd eLD  ZTeAeC eMd< dd eTD ZTi ZUeEeCeVf eMd< eLD ]PZWeWjXD ]BZYeZeWeYZ[eWj\]dd]ddZ^e[eUkr6e^ deY eUe[< q6q,eddZ_G dd deZGdd  Z`G d!d" d"ZaG d#d$ d$ZbG d%d& d&eZcG d'd( d(ZdG d)d* d*Zed+d, Zfd-d. Zgd/d0 Zhd1d2 Zid3d4 Zjd5d6 Zkd7d8 Zld9d: Zmd;d< Znd=d> Zod?d@ ZpdAdB ZqdCdD ZrdEdF ZsdGdH ZtdIdJ ZudKdL ZvdMdN ZwdOdP ZxdQdR ZydSdT ZzdUdV Z{dWdX Z|dYdZ Z}d[d\ Z~d]d^ Zd_d` Zdadb Zdcdd Zdedf Zdgdh Zdidj Zdkdl Zdmdn Zdodp Zdqdr Zdsdt Zdudv Zdwdx Zdydz Zd{d| Zd}d~ Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZddĄ ZddƄ ZddȄ Zddʄ Zdd̄ Zdd΄ ZddЄ Zdd҄ ZddԄ Zddք Zdd؄ Zddڄ ZdOdd݄Zdd߄ Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Ze'ejddeoHe; d de'ejddeohe; d de'ejddeoe; d de'ejddeoe;e jfd de'e%dgddd de'e%dgd	d
d dfZѐdd ZҐdd ZӐdd ZԐdd ZՐdd Z֐dd Zeee jjeve'e$fdeee jje:ewe'e$e'ejddeFڐddfdeee jje:exe'eܐddde'e$fdeee jjeydeee jje:ezdeee jje:e{e'eܐddde'e$fdeee jjesdeee jjete'ejddeFڐdd de'e9de jgdfdeee jjeue:e'eܐddde'e$fd eee jjd!e|e'e$e'ejϐd"d#eFڐdde'ejϐd"d$eFڐddfd%eee jjd!e}e'e$e'ejϐd"d#eFڐdde'ejϐd"d$eFڐddfd%eee jjd!e~e'e$e'ejϐd"d#eFڐdde'ejϐd"d$eFڐddfd%eee jjee'ejdd&d'e jgd(fdeee jjeed)dېd*e:d!e'e d+d,dde'e!dde jgde'e9ddd'e jgd(e'e9ddd'e jgd(fe'e"e jd-iddfd.eee jjeed/dېd*e:d!e'e d+d,dde'e!dde jgde'ejddd e jgd(e'ejddd'e jgd(e'e9ddd'e jgd(e'e9ddd'e jgd(fe'e"e jd-iddfd.eee jjeed0dېd*e:d!e'e d1d,dde'e!dde jgde'e$e'ejddfe'e"e jd-iddfd.eee jjeed)dېd!d2e:d!ee je'e d+d,dde'e!dde jgde'ejdd3e jfd d4e'e9ddd'e jgd(e'e9ddd'e jgd(fe'e"e jd-idde'e"e jd5iddfd6eee jjeed/dېd!d2e:d!ee je'e d+d,dde'e!dde jgde'ejdde je je jfde'ejddd e je jgd(e'ejddd'e jgd(e'ejdd3e jfd d4e'e9ddd'e jgd(e'e9ddd'e jgd(fe'e"e jd-idde'e"e jd5iddfd6eee jjeed0dېd!d2ee je:d!e'e d1d,dde'e!dde jgde'e$e'ejdde'ejddd e je jge;d7e'ejdd3e jfd d4fe'e"e jd-idde'e"e jd-idd3e'e"e jd5iddfd8eee jjee'eܐdddfdeee jjee'ejdd&d'e jgd(fdeee jjee:e'e$e'eܐdddfd eee jjee:e'e$e'eܐdddfd eee jjee'eܐdddfdeee jjee'eܐddde'e9dde jgdfdeee jjeed)d!d*e:d!e'e d+d,dde'e!dde jgde'e#e'e9ddd'e jgd(e'e9ddd'e jgd(fe'e"e jd-iddfd.eee jjeed/d!d*e:d!e'e d+d,dde'e!dde jgde'e#e'ejddd e jgd(e'ejddd'e jgd(e'e9ddd'e jgd(e'e9ddd'e jgd(fe'e"e jd-iddfd.eee jjeed0d!d*e:d!e'e d1d,dde'e!dde jgde'e#e'e$e'ejddfe'e"e jd-iddfd.eee jjeed)d!d!d2e:d!e'e d+d,dde'e!dde jgde'e#e'e9ddd'e jgd(e'e9ddd'e jgd(fe'e"e jd-iddfd.eee jjeed/d!d!d2e:d!e'e d+d,dde'e!dde jgde'e#e'ejddd e jgd(e'ejddd'e jgd(e'e9ddd'e jgd(e'e9ddd'e jgd(fe'e"e jd-iddfd.eee jjeed0d!d!d2e:d!e'e d1d,dde'e!dde jgde'e#e'e$e'ejddfe'e"e jd-iddfd.eee jj efe'eܐdddfdeee jjege'ee jed-d-d9e jed-d-d9idd:d;dge'eܐddde'ejdd:d'e jgd(fd<eee jjee'eܐddde'eܐdddfdeee jjee'eܐddde'eܐddde'ejddeFڐdd'dfdeee jjee'eܐddde'eܐddde'eܐddde'e9fdeee jjedeee jjQedeee jjee:e'e$fd eee jjehe'eܐddde'eܐddd3e'eܐddde'eܐdddfdeee jjele'eܐddde'e9dde jgde'ejdd:d'e jgd(fdeee jj	ee'eܐdddfdeee jj
ee'eܐddde'e9de'eܐdddfdeee jjee'eܐddde'e9de'eܐdddfdeee jjee'eܐddde'ejdd:d'e jgd(fdeee jjee'eܐdddfdeee jjeie'eܐddde'ejdd:d'e jgd(fdeee jjeje'eܐdddfdeee jjeke'eܐdddfdeee jjee'eܐdddfdeee jjee'eܐddde'ejdd:d'e jgd(fdeee jjee'eܐddde'e9de jgdfdeee jjee'eܐddde'e9de jgdfdeee jjeed!dېd=e'ejdde'ee jed>d?d9idd:e jgd;d4e'ejdd3e jgd d4fd@eee jjee'eܐddde'e9de'eܐddde'eܐddde'eܐdddfdeee jjee'ejdd:d'e jgd(fdeee jjedeee jjeed!d!dAe'eܐddd3e'ee jed-d-d9iddd;de'eܐdddd de'eܐdddd'de'eܐddde;d dfdBeee jjee'e$fdeee jjee'ejddeFڐdd'dfdېdCeee jjedeee jjeed)dDd!e'eܐdddfdEeee jjeed/dDd!e'eܐdddfdEeee jjeed0dDd!e'e$e'eܐdddfdEeee jj ee'e$fdeee jj!ee'eܐdddfdeee jj"edeee jj#d!ee'eܐdddd;dge'eܐddde'ejddFfdGeee jj$d!ee'ee jed-d-d9iddd;e<dHe'eܐdddd;dge'eܐdddfdGeee jj%ee'eܐdddd;dge'eܐdddfd<eee jj&ee'eܐdddd;dge'eܐdddfd<eee jj'd!ee'eܐdddfd%eee jj(ee'ee jed-d-d9iddd'dge'eܐdddfd<eee jj)ee'ejddeFڐdd'dfdeee jj*edeee jj+ee'e$fdeee jj,ee'e$fdeee jj-eed!dIeҐdJeee jj.eeҐdJeee jj/eeӐdJeee jj0ee'ejddeFڐdd'dfdeee jj1ee'ejdd:d'e jgd(fdeee jj2edeee jj3edeee jj4ee'eܐddde'ejdd:d'e jgd(fdeee jj5ee'eܐddde'ejdd:d'e jgd(fdeee jj6ee'eܐdddfdeee jj7ee'e$fdeee jj8ee'e$fdeee jj9edeee jj:ee'ejddeFڐdd'dfdeee jj;ee'ejddeFڐdd'dfdeee jj<ee'e$fdeee jj=ee'e$fdeee jj>d!eed!dIeeѐdKeee jj?d!eedېdIeeѐdKeee jj@d!eee'e$feѐdLeee jjAedeee jjBee:e'eܐdddd de'eܐdddd'dfd eee jjCee:e'eܐdddd de'eܐdddd'dfd eee jjDeÐdeee jjEee:e'eܐdddd de'eܐdddd'dfd eee jjFee:e'eܐdddd de'eܐdddd'dfd eee jjGee'e$fdeee jjHeƐdeee jjIee'ejddd'dfdeee jjJee'ejddd'dfdeee jjKeeՐdJeee jjLee֐dJeee jjMeee'ejddfdMeee jjNeɐdeee jjOee'ejddd'dfdeee jjPee'ejddd'dfdgqZQeAee eMdN< dS (P      N)deepcopy)Enum)wrapspartial)chainproduct)pack_padded_sequence)make_tensor)
TEST_CUDNN)floating_typesfloating_and_complex_types_andget_all_fp_dtypes)
_TestParametrizer_update_param_kwargstoleranceOverridetolskipCUDAIfCudnnVersionLessThanskipCUDAIfRocmprecisionOverrideskipMetaskipMPSskipCUDAVersionIn)DecorateInfo)cosineembeddingloss_referencecross_entropy_loss_referencectcloss_referencehingeembeddingloss_referencehuberloss_referencekldivloss_referencemarginrankingloss_referencemultimarginloss_referencemultilabelmarginloss_referencenllloss_referencenlllossNd_referencesmoothl1loss_referencesoftmarginloss_referenceget_reduction)freeze_rng_state	skipIfMpsGRADCHECK_NONDET_TOLTEST_WITH_ROCM
IS_WINDOWSskipIfTorchDynamo)
ModuleType)ListTupleTypeSetDictMODULE_NAMESPACESMODULES_TO_SKIPc                    s    g | ]  fd d j D qS )c                    s   g | ]}t  |qS  )getattr).0module_name	namespacer5   X/var/www/html/venv/lib/python3.8/site-packages/torch/testing/_internal/common_modules.py
<listcomp>5   s     z<listcomp>.<listcomp>)__all__)r7   r5   r9   r;   r<   4   s   r<   MODULE_CLASSESc                 C   s   g | ]}|t kr|qS r5   )r4   )r7   clsr5   r5   r;   r<   7   s      MODULE_CLASS_NAMESztorch. z.modules.TrainEvalMode)
train_only	eval_onlytrain_and_evalc                   @   s2   e Zd ZdZdejdfddZdd Zdd	 ZdS )
moduleszQ PROTOTYPE: Decorator for specifying a list of modules over which to run a test. NTc                 C   s0   t || _|d k	rt|nd | _|| _|| _d S N)listmodule_info_listsetallowed_dtypestrain_eval_modeskip_if_dynamo)selfZmodule_info_iterablerL   rM   rN   r5   r5   r;   __init__M   s    
zmodules.__init__c                 C   s^   g }| j tjks| j tjkr&|d | j tjks>| j tjkrH|d |jsZ|d d }|S )NTF   )rM   rC   rD   rF   appendrE   train_and_eval_differ)rO   module_infotraining_flagsr5   r5   r;   _get_training_flagsT   s    



zmodules._get_training_flagsc                 #   sL  |d krt d| jD ].}t||j}| jd k	rB|| j}| |}t||D ]\}}|j	}	t
|dkr|	d|r|dnd 7 }	d|i}
t|
d| t|
d| z\t  fd	d
}| jrtjjjjstd|}t|j|j j|j|}||	|
|fV  W qV tk
rB } z td|	 d|j d |W 5 d }~X Y qVX qVqd S )NzThe @modules decorator is only intended to be used in a device-specific context; use it with instantiate_device_type_tests() instead of instantiate_parametrized_tests()rQ   _Z
train_modeZ	eval_moderT   dtypetrainingc                     s
    | |S rH   r5   argskwargstestr5   r;   test_wrapper~   s    z/modules._parametrize_test.<locals>.test_wrapperz/Policy: we don't run ModuleInfo tests w/ DynamozFailed to instantiate z for module !)RuntimeErrorrJ   rK   supported_dtypesdevice_typerL   intersectionrV   r   formatted_namelenr   r   rN   torchZtesting	_internalZcommon_utilsZTEST_WITH_TORCHINDUCTORr,   r   get_decorators__name__	Exceptionprintname)rO   r^   Zgeneric_clsZ
device_clsrT   dtypesrU   rY   rX   	test_nameparam_kwargsr_   Zdecorator_fnexr5   r]   r;   _parametrize_testd   s8    


  zmodules._parametrize_test)	rj   
__module____qualname____doc__rC   rF   rP   rV   rr   r5   r5   r5   r;   rG   J   s    
rG   c                 C   s   | t krt |  S | jS d S rH   )r@   rj   )
module_clsr5   r5   r;   get_module_common_name   s    rw   c                   @   s    e Zd ZdZddgZdd ZdS )FunctionInputz: Contains args and kwargs to pass as input to a function. r[   r\   c                 O   s   || _ || _d S rH   rZ   )rO   r[   r\   r5   r5   r;   rP      s    zFunctionInput.__init__Nrj   rs   rt   ru   	__slots__rP   r5   r5   r5   r;   rx      s   rx   c                   @   s&   e Zd ZdZddddgZd
dd	ZdS )ModuleInputzA Contains args / kwargs for module instantiation + forward pass. constructor_inputforward_inputdescreference_fnNrA   c                    s>   || _ || _|| _ | _ d k	r:t  fdd}|| _d S )Nc                    s,   t |t | }} | t|  f||S rH   )r   rI   
parameters)mr[   r\   r   r5   r;   copy_reference_fn   s    z/ModuleInput.__init__.<locals>.copy_reference_fn)r|   r}   r~   r   r   )rO   r|   r}   r~   r   r   r5   r   r;   rP      s    zModuleInput.__init__)NrA   Nry   r5   r5   r5   r;   r{      s   r{   c                   @   s   e Zd ZdZdZdZdS )ModuleErrorEnumz7 Enumerates when error is raised when testing modules. r   rQ   N)rj   rs   rt   ru   CONSTRUCTION_ERRORFORWARD_ERRORr5   r5   r5   r;   r      s   r   c                   @   s.   e Zd ZdZddddgZejedddZd	S )
ErrorModuleInputzw
    A ModuleInput that will cause the operation to throw an error plus information
    about the resulting error.
    module_error_inputerror_on
error_typeerror_regex)r   r   c                C   s   || _ || _|| _|| _d S rH   )r   r   r   r   )rO   r   r   r   r   r5   r5   r;   rP      s    zErrorModuleInput.__init__N)	rj   rs   rt   ru   rz   r   r   ra   rP   r5   r5   r5   r;   r      s
   r   c                
   @   s`   e Zd ZdZdde ejejfdddddd	dd	Zd
d Z	dd Z
edd Zedd ZdS )
ModuleInfoz+ Module information to be used in testing. r5   NT        F)	skips
decoratorsrn   dtypesIfMPSsupports_gradgradgradcheck_nondet_tolmodule_memformat_affects_outrS   module_error_inputs_funcc       
         C   sh   || _ || _|r|ng |r|ng | _|| _|| _|| _|| _|	| _|
| _|| _	t
|tjjjj| _d S rH   )rv   module_inputs_funcr   rn   r   r   r   r   rS   r   
issubclassrg   nnrG   lazyZLazyModuleMixinZis_lazy)rO   rv   r   r   r   rn   r   r   r   r   rS   r   r5   r5   r;   rP      s    zModuleInfo.__init__c                 C   sH   g }| j D ]8}t|tr8||||||rB||j  q
|| q
|S rH   )r   
isinstancer   Z	is_activeextendrR   )rO   Z
test_classro   devicerX   rp   result	decoratorr5   r5   r;   ri      s    

zModuleInfo.get_decoratorsc                 C   s   |dkr| j S | jS d S )Nmps)r   rn   )rO   rc   r5   r5   r;   rb      s    zModuleInfo.supported_dtypesc                 C   s
   t | jS rH   )rw   rv   rO   r5   r5   r;   rm      s    zModuleInfo.namec                 C   s   | j ddS )NrB   rW   )rm   replacer   r5   r5   r;   re     s    zModuleInfo.formatted_name)rj   rs   rt   ru   r   rg   float16float32rP   ri   rb   propertyrm   re   r5   r5   r5   r;   r      s"   


r   c                 K   s|   t t|||d}ttddt|dddd dttddd	d
t|dddd dttddt|dddd dg}|S )Nr   rX   requires_grad
         r   )inputc                 S   s,   t ||d  |d dddd S )Nr   rQ   r   r   )rg   mmtviewexpand)r   pr   r5   r5   r;   <lambda>      z/module_inputs_torch_nn_Linear.<locals>.<lambda>r|   r}   r   Fbiasno_biasc                 S   s   t ||d  S )Nr   )rg   r   r   r   r   ir5   r5   r;   r     r   r|   r}   r~   r         no_batch_dimc                 S   s*   t |dd|d  d|d  S )NrQ   r   r   )rg   r   r   r   r   r5   r5   r;   r     r   r   r	   r{   rx   rT   r   rX   r   rY   r\   
make_inputmodule_inputsr5   r5   r;   module_inputs_torch_nn_Linear  s"    



r   c              	      s   t t|||d}ddd ttdddt|d|d	 d
ttdddddt|d|d	d fdddttdddt|d|dd fdddg}|S )Nr   Tc                 S   sj   t d||d |}|rf|jd dkr:|d|d  }n,||d dd|jd |d jd  }|S )Nzbn,anm,bm->bar   rQ   r   )rg   Zeinsumshaper   r   )r   r   x1x2r   r   r5   r5   r;   bilinear_reference_fn  s    ,z>module_inputs_torch_nn_Bilinear.<locals>.bilinear_reference_fn   r   r   )r   r   )r   r   r   Fr   r   c                    s    | |||ddS )NFr   r5   r   r   r   r   r   r5   r;   r   .  r   z1module_inputs_torch_nn_Bilinear.<locals>.<lambda>r   r   c                    s    | || dd| ddS )NrQ   r   )r   r   r   r5   r;   r   2  r   )Tr   r   r5   r   r;   module_inputs_torch_nn_Bilinear  s$    



r   c              	   K   s  t t|||d}di fdddifdddifddd	ifd
d
difg}g }|D ]\}	}
|
fdd}|d }|d
dr~|dn
|d }|ttf |
t|||	|d |d }|d
dr|dn
|d }|ttf |
t||d|	 |d qJ|S )Nr   rA   reduction_sum	reductionsumZreduction_batchmeanZ	batchmeanreduction_nonenoneZ
log_targetTc                 S   s   t ||f|S rH   )r   r   r   r   r   constructor_kwargsr5   r5   r;   r   E  s    z6module_inputs_torch_nn_KLDivLoss.<locals>.reference_fnr   r   Fr   r5   scalar_)r   r	   loggetrR   r{   rx   )rT   r   rX   r   rY   r\   r   casesr   r~   r   r   r   targetZscalar_inputZscalar_targetr5   r5   r;    module_inputs_torch_nn_KLDivLoss8  s:    



 
 
r   c                 K   s  |||fdd}t t||dd}di fdddifd	dd
ifdddifdd|d ifd|d ddfd|d ddfg}g }	|D ]H\}
}|fdd}|	ttf |t|dtjd|d 	d
  |
|d |fdd}|	ttf |t|dtjddd|d 	d
  d|
 |d |	ttf |t|dtjddddd|d 	d
  d |
 |d |	ttf |t|d!tjdd|d 	d
  d"|
 |d q|	S )#Nc                 S   s   t | ||ddjdd|S )NFr   rQ   dim)r	   log_softmaxrequires_grad_)r   r   rX   r   r5   r5   r;   r   ^  s    z2module_inputs_torch_nn_NLLLoss.<locals>.make_inputFr   rA   r   r   r   r   r   ignore_indexr   weightsweightr   Zweights_ignore_index)r   r   Zweights_ignore_index_negr   c                 S   s   t ||f|S rH   )r"   r   r5   r5   r;   r   t  s    z4module_inputs_torch_nn_NLLLoss.<locals>.reference_fn)   r   r   r   r   c                 S   s   t ||f|S rH   )r#   r   r5   r5   r;   nd_reference_fn  s    z7module_inputs_torch_nn_NLLLoss.<locals>.nd_reference_fn)r   r   r   r   r   Znd_)r   r   r   r   r   r   higher_dim_)r   r   r   3d_)r   r	   absrR   r{   rx   rg   emptyuniform_mulfloorlong)rT   r   rX   r   rY   r\   r   make_weightr   r   r~   r   r   r   r5   r5   r;   module_inputs_torch_nn_NLLLoss]  sl    




"	
&	
 	r   c              
   K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifg}g }	|D ]<\}
}|	ttf |t|d|d|d |
td qP|	S )Nr   FrA   r   r   r   reduction_meanmeanr   r   r   rQ   r   )r   r	   rR   r{   rx   r   no_batch_dim_reference_fn)rT   r   rX   r   rY   r\   r   make_targetr   r   r~   r   r5   r5   r;   &module_inputs_torch_nn_GaussianNLLLoss  s(    




	r   c              
      s   t t|||d}t t||dd}di fdddifdddifd	dd
ifdddifdddifddddfg}ddd g }	|D ]p\}
}|f fdd	}|dd}|r|dn|d d}|	ttf |t||d 	 |
|d qz|	S )Nr   FrA   r   r   r   r   r   r   r   fullTZno_log_input	log_inputZfull_no_log_input)r   r   :0yE>c                 S   s   |r|   ||  }n| || |   }|rj||| | ddtj |    |dkd7 }|dkrv|S |dkr| |   S | S d S )N      ?       @rQ   r   r   r   )expr   r   mathpiZmasked_fillr   numel)r   r   r   r   r   epsr   r5   r5   r;   poissonnllloss_reference_fn  s    8zJmodule_inputs_torch_nn_PoissonNLLLoss.<locals>.poissonnllloss_reference_fnc                    s    ||f|S rH   r5   r   r   r5   r;   r     s    z;module_inputs_torch_nn_PoissonNLLLoss.<locals>.reference_fnr   r   r   r   MbP?r   )TFr   r   )
r   r	   r   r   addrR   r{   rx   Zfloor_Zabs_)rT   r   rX   r   rY   r\   r   r   r   r   r~   r   r   r   r   r5   r   r;   %module_inputs_torch_nn_PoissonNLLLoss  s4    







r   c              
   K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifg}ddd}	g }
|D ]r\}}|
ttf |t|d|d|t |	f|d |
ttf |t|d|d| dt |	f|d qZ|
S )Nr   FrA   r   r   r   r   r   r   r   c                 S   sN   |dkr||  dS |dkr8||  d |  S ||  d S d S )Nr   r   r   )powr   r   )r   r   r   r   r   r5   r5   r;   mse_loss_reference_fn  s
    z=module_inputs_torch_nn_MSELoss.<locals>.mse_loss_reference_fnr   r   r5   Z_scalar)r   r   r	   rR   r{   rx   )rT   r   rX   r   rY   r\   r   r   r   r  r   r~   r   r5   r5   r;   module_inputs_torch_nn_MSELoss  s:    







r  c              	      s   fdd}|ddrdnd |dd}|d	d
}|dk	rt |tsHt D ].\}}||krP|dk	rP|| }	||	|< qP fdd|D }
t  | |
 }W 5 Q R X |rt| }|dkr|dS |S )a  Reference function for modules supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.

    Currently it only supports modules which return a single Tensor as output.
    You can bind the following kwargs.
    Kwargs:
        batch_first[bool] : If True, all the Tensors in `args` while be unsqueezed at dim `0` .
                        and output will be squeezed at dim `0` else dim `1` for both.
        kwargs_to_batchify[dict] : Dictionary specifying the name of the argument and dimension to unsqueeze.
                               Useful if there are few arguments whose batch dimension are different
                               from the ones selected by `batch_first`.
        is_criterion[bool] : Specify if the module is a criterion and handle the reduction for output accordingly.
    c                    s"     | |}|  kr |  |S rH   )r   pop)keydefaultv)r\   r5   r;   get_and_pop"  s    
z.no_batch_dim_reference_fn.<locals>.get_and_popbatch_firstTr   rQ   kwargs_to_batchifyNis_criterionFc                    s   g | ]}|  qS r5   	unsqueezer7   r   	batch_dimr5   r;   r<   3  s     z-no_batch_dim_reference_fn.<locals>.<listcomp>r   )r   dictAssertionErroritemsr  r'   squeezer&   )r   r   r[   r\   r	  r  r  kr  Zbdimsingle_batch_input_argsoutputr   r5   )r  r\   r;   r     s$    


r   c              
      s   | ddrdnd d|kr&|d d|krL|d dk	rL|d d|d<  fdd|D }t 4 | ||}|d  |d dfW  5 Q R  S Q R X dS )	a  Reference function for MultiheadAttention supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    r
  Tr   rQ   key_padding_maskNc                    s   g | ]}|  qS r5   r  r  r  r5   r;   r<   K  s     z.no_batch_dim_reference_mha.<locals>.<listcomp>)r   r  r  r'   r  )r   r   r[   r\   r  r  r5   r  r;   no_batch_dim_reference_mha>  s    

r  c           	   
   O   s   t |dkr|\}d}nt |dkr6|\}}|d}|d rBdnd}|d ||}||f}t 4 | ||}|d ||d dfW  5 Q R  S Q R X dS )a  Reference function for RNN and GRU supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    rQ   Nr   r
  r   rf   r  r  r'   r  	r   r   r[   r\   inphr  r  r  r5   r5   r;   no_batch_dim_reference_rnn_gruQ  s    



r  c           	   
   O   s   t |dkr|\}d}n0t |dkrH|\}}|d d|d df}|d rTdnd}|d ||}||f}t J | ||}|d ||d d d|d d dffW  5 Q R  S Q R X dS )a  Reference function for LSTM supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    rQ   Nr   r   r
  r  r  r5   r5   r;   no_batch_dim_reference_lstmi  s    


r   c           	   
   O   sn   |\}\}}| d| d| dff}t 4 | ||}|d d|d dfW  5 Q R  S Q R X dS )zReference function for LSTMCell supporting no batch dimensions.

    The module is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    r   rQ   N)r  r'   r  )	r   r   r[   r\   r  r  cr  r  r5   r5   r;   no_batch_dim_reference_lstmcell  s
    
r"  c                    s    fdddD S )Nc              	      s>   g | ]6}t t|d t d dttddd| dqS ))r   r   r   Tr  no_batch_dim_r|   r}   r   r~   )r{   rx   r   r   )r7   r   r   r5   r;   r<     s   
z8generate_regression_criterion_inputs.<locals>.<listcomp>)r   r   r   r5   r'  r5   r'  r;   $generate_regression_criterion_inputs  s    
r(  c                 K   s|   t t|||d}ttddt|ddtdttdt|ddttd	d	t|dd
dttdddt|dddgS )Nr   r   )kernel_size)r      r   r   )r   r   r*  r|   r}   r   strider|   r}   r~   rQ   
stride_padr   r	   r{   rx   r   rT   r   rX   r   rY   r\   r   r5   r5   r;    module_inputs_torch_nn_AvgPool1d  s$    





r2  c                 K   s   t t|||d}ttdt|ddtdttdt|ddttddt|ddd	ttddd
t|ddd	ttdddt|ddd	ttddddt|ddd	ttddd
ddt|ddd	gS )Nr   r   r   r   r*  r*  r   r   r   r   r*  r*  r+  r-  r.  rQ   rQ   r/  rQ   Zdivisor_overridedivisordivisor_stridedivisor_stride_padr0  r1  r5   r5   r;    module_inputs_torch_nn_AvgPool2d  s<    







r;  c                 K   s  t t|||d}ttdt|ddtdttdt|ddttddt|d	d
dttdddt|d	ddttdddt|d	ddttdddt|dddttdddt|dddttdddt|dddttdddt|dddttddddt|d	ddttdddddt|d	ddttdddddt|d	ddttdddddt|dddttdddddt|dd dttdddddt|dd!dgS )"Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r+  r   r   r   r   r   r   r-  r.  rQ   rQ   rQ   r/  r   )rQ   r   rQ   Zstride_pad_gpu_fixedkw_output)r   r   r   rQ   )rQ   rQ   r   )r   r   r   r   r   Zstride_pad_gpu_general_outputr   r   Zstride1_pad0_gpu_inputZstride_pad_gpu_input_nooverlapr7  r8  r9  r:  Z%divisor_stride_pad_gpu_fixedkw_outputZ%divisor_stride_pad_gpu_general_outputZdivisor_stride1_pad0_gpu_inputZ&divisor_stride_pad_gpu_input_nooverlapr0  r1  r5   r5   r;    module_inputs_torch_nn_AvgPool3d  s|    















rA  c                 K   s^   t t|||d}ttdt|dddttdt|dtddttd	t|dd
dgS )Nr   r   rQ   r   r   singler.  r   r   r   r&  rQ   Z
one_outputr0  r1  r5   r5   r;   (module_inputs_torch_nn_AdaptiveAvgPool1d  s    


rE  c              	   K   s   t t|||d}ttdt|dddttdt|dtddttd	t|dd
dttdt|dddttdt|dddgS )Nr   r   rQ   r   r   r*  rC  r.  r   r   r*  r   r&  rQ   Zsingle_1x1outputr   r   tupler   N
tuple_noner0  r1  r5   r5   r;   (module_inputs_torch_nn_AdaptiveAvgPool2d
  s.    




rL  c              	   K   s   t t|||d}ttdt|dddttdt|dtddttd	t|d
ddttdt|d
ddttdt|dddgS )Nr   r   )r   r   r   r      rC  r.  )r   r   r   rM  r   r&  r   r   r   )r   r   r   r   rM  rI  )Nr   r   rK  )r   r   r   )rQ   rQ   r   r   r*  Zlast_dimr0  r1  r5   r5   r;   (module_inputs_torch_nn_AdaptiveAvgPool3d  s.    




rO  c                 K   sF   t t|||d}ttdt|dddttdt|dtddgS )	Nr   r   rB  rC  r.  rD  r   r&  r0  r1  r5   r5   r;   (module_inputs_torch_nn_AdaptiveMaxPool1d5  s    

rP  c                 K   sv   t t|||d}ttdt|dddttdt|dtddttd	t|dd
dttdt|dddgS )Nr   r   rF  rC  r.  rG  r   r&  rH  rI  rJ  rK  r0  r1  r5   r5   r;   (module_inputs_torch_nn_AdaptiveMaxPool2dB  s&    



rQ  c              
   K   s   t t|||d}ttdt|dddttdt|dtddttd	t|dd
dttdt|dddttdt|dddttd	t|dddgS )Nr   r   )r   r   r   r*  rM  rC  r.  )r   r   r*  rM  r   r&  rN  rI  )r   Nr   rK  )r   r      	   r   Zsingle_nonatomic)r   r   r*  r   r   Ztuple_nonatomicr0  r1  r5   r5   r;   (module_inputs_torch_nn_AdaptiveMaxPool3dU  s6    





rT  c                 K   s   t t|||d}ttdt|dddttdt|dddttdd	d t|dd
dttdd	ddt|dddttdd	dddt|dddttdd	ddt|dddttdd	ddt|dddgS )Nr   r   r   affiner.  r   )r   r   r   3d_inputr   Zaffine_simple_average333333?F
not_affineTnot_tracking_statsZ3d_input_not_affine)r   r   rS  
zero_batchr   r1  r5   r5   r;   "module_inputs_torch_nn_BatchNorm1dn  s<    






r[  c                 K   s   t t|||d}ttdt|ddttddd t|dddttdddt|dd	dttdddd
t|dddttddddd
t|dddttdddd
t|dddgS )Nr   r   r5  r+  r   Z2d_simple_averager.  g?momentumFrX  TrY  r   rW  )r   r   r   r   rZ  r   r1  r5   r5   r;   "module_inputs_torch_nn_BatchNorm2d  s2    





r]  c                 K   s   t t|||d}ttdt|ddttddd t|dddttdddt|dd	dttdddd
t|dddttddddd
t|dddttdddd
t|dddgS )Nr   r   r>  r+  r   Z3d_simple_averager.  ffffff?r\  FrX  TrY  r   rW  )r   r   r   r   r   rZ  r   r1  r5   r5   r;   "module_inputs_torch_nn_BatchNorm3d  s2    





r_  c           	         s   |d }| dd| dd}tt|||d|r:i gn
i ddig}d\  ftd	d
 t|D  d  fddtddg|D S )NNr   F
transposedr   paddingZsamerN  c                 s   s   | ]}|d  V  qdS rJ  r5   r7   r   r5   r5   r;   	<genexpr>  s     z0module_inputs_torch_nn_ConvNd.<locals>.<genexpr>r,  c                    s^   g | ]V\}}t r tf|nt f|t|r:n|rHd nd|rRdntdqS )rA   r   Nr   )r{   rx   r   )r7   Z
with_batchZconv_kwargsZC_inZC_outinput_batch_shapeinput_no_batch_shaper)  r   r   r5   r;   r<     s   


z1module_inputs_torch_nn_ConvNd.<locals>.<listcomp>T)r   r   r	   rI  range	itertoolsr   )	rT   r   rX   r   rY   r\   r`  ra  Zconv_kwargs_listr5   re  r;   module_inputs_torch_nn_ConvNd  s    
rj  c              
   K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifdddifg}g }	|D ]H\}
}|fdd}|	ttf |t|d|d|d |
|d qZ|	S )Nr   FrA   r   r   r   r   r   r   r   marginr^  c                 S   s   t |||f|S rH   )r   r   r   i1i2r   r   r5   r5   r;   r     s    z@module_inputs_torch_nn_CosineEmbeddingLoss.<locals>.reference_fnr   r   )r   r   r   r	   rR   r{   rx   signrT   r   rX   r   rY   r\   r   r   r   r   r~   r   r   r5   r5   r;   *module_inputs_torch_nn_CosineEmbeddingLoss  s*    





rs  c                 K   s~   t t|||d}ttddt|ddd dttddt|dd	d
tt t|ddtdttddt|ddd
gS )Nr   r   alphar   r   r   c                 S   s   t |dk|d| d  S )Nr   r   rQ   rg   wherer   r   r5   r5   r;   r     r   z,module_inputs_torch_nn_ELU.<locals>.<lambda>r   r5   scalarr.  r   r   r   r   r   r   r   4d_inputr0  r1  r5   r5   r;   module_inputs_torch_nn_ELU  s&    






r}  c                 K   sn   t t|||d}ttddt|ddd dttddt|dd	d d
dttddt|ddtdgS )Nr   r   rt  rv  c                 S   s"   t |dk|dd|  d  S Nr   r   r   rQ   rw  r   r5   r5   r;   r      r   z-module_inputs_torch_nn_CELU.<locals>.<lambda>r   r5   c                 S   s"   t |dk|dd|  d  S r~  rw  r   r5   r5   r;   r     r   ry  r&  rz  r   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_CELU  s     





r  c                 K   sX   t t|||d}tt t|ddttdt|dddtt t|dd	td
gS )Nr   )r   r*  r+  rQ   r   r*  rM  r   r.  r#  r   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_GLU  s    


r  c                 K   sf   t t|||d}ttdt|ddd ddttdt|dd	d d
tt t|ddtdgS )Nr   r   r5   c                 W   s    |d dt |td   S Nr         ?r   rg   erfr   sqrtr   r   xrW   r5   r5   r;   r      r   z-module_inputs_torch_nn_GELU.<locals>.<lambda>ry  r&  rv  c                 W   s    |d dt |td   S r  r  r  r5   r5   r;   r   $  r   r   rz  r   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_GELU  s     


r  c                 K   sn   t t|||d}tt t|dddtt t|dtddtt t|dd	dtt t|d
ddgS Nr   r5   ry  r.  r   r   r&  r   Zchannels_last_mem_format)r   r   r   r   r   Zchannels_last_3d_mem_formatr0  r1  r5   r5   r;   module_inputs_torch_nn_ReLU+  s&    



r  c                 K   sn   t t|||d}tt t|dddtt t|dtddtt t|dd	dtt t|d
ddgS r  r0  r1  r5   r5   r;   module_inputs_torch_nn_ReLU6>  s&    



r  c              	   K   s   t t|||d}tt t|ddtt t|dtddttdt|ddd	ttd
t|ddd	ttdt|ddd	gS )Nr   rv  r+  r   r   r&  r   Zwith_negvalr.  r   r   Zwith_zero_negvalr5   Zwith_negval_scalarr0  r1  r5   r5   r;    module_inputs_torch_nn_LeakyReLUQ  s,    




r  c                 K   s   t t|||d}tt t|dddtt t|dtddtt t|dd	d
 ddttdt|ddd
 ddtt t|ddd
 ddttdt|ddd
 ddtt t|ddd
 ddttdt|ddd
 ddgS )Nr   r5   ry  r.  r   r   r&  r   r   r   c                 S   s(   t j|ddt j|dd|d d   S Nr   min)maxrg   clampr   r5   r5   r;   r   s  r   z.module_inputs_torch_nn_PReLU.<locals>.<lambda>1dr   c                 S   s(   t j|ddt j|dd|d d   S r  r  r   r5   r5   r;   r   w  r   Z1d_multiparamr   c                 S   s(   t j|ddt j|dd|d d   S r  r  r   r5   r5   r;   r   {  r   Z2dc                 S   s(   t j|ddt j|dd|d d   S r  r  r   r5   r5   r;   r     r   Z2d_multiparam)r   r   r   r   r*  c                 S   s(   t j|ddt j|dd|d d   S r  r  r   r5   r5   r;   r     r   Z3dc                 S   s(   t j|ddt j|dd|d d   S r  r  r   r5   r5   r;   r     r   Z3d_multiparamr0  r1  r5   r5   r;   module_inputs_torch_nn_PReLUf  sR    







r  c                 K   sV   t t|||d}tt t|ddtt t|dtddtt t|ddd	gS )
Nr   rv  r+  r   r   r&  r5   ry  r.  r0  r1  r5   r5   r;   module_inputs_torch_nn_SELU  s    


r  c                 K   sb   t t|||d}tt t|ddd ddtt t|dtddtt t|d	d
d dgS )Nr   r5   c                 W   s   |t | S rH   rg   sigmoidr  r5   r5   r;   r     r   z-module_inputs_torch_nn_SiLU.<locals>.<lambda>ry  r&  r   r   r  c                 W   s   |t | S rH   r  r  r5   r5   r;   r     r   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_SiLU  s     


r  c                 K   sh   t t|||d}ttdt|ddd dttdt|dd	d d
dttdt|dtddgS )Nr   rQ   r      c                 S   s&   t |t |ddddS NrQ   Tr   r  )rg   r   divr   r   r   r5   r5   r;   r     r   z0module_inputs_torch_nn_Softmax.<locals>.<lambda>r   r   r5   c                 S   s   t |t |ddS )Nr   Trg   r   r  r   r   r5   r5   r;   r     r   ry  r&  r   r   r   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_Softmax  s     


r  c                 K   sF   t t|||d}tt t|ddd dtt t|dtddgS )	Nr   rQ   r   r   r  c                 S   s   t |t |ddS NrQ   Fr  r   r5   r5   r;   r     r   z2module_inputs_torch_nn_Softmax2d.<locals>.<lambda>r   rN  r   r&  r0  r1  r5   r5   r;    module_inputs_torch_nn_Softmax2d  s    

r  c              	   K   s   t t|||d}ttdt|ddd dttdt|ddd d	d
ttdt|ddd dd
ttdt|dtdd
gS )Nr   rQ   r  c                 S   s*   t |t |dddd S r  )rg   r   div_r   r   log_r   r5   r5   r;   r     r   z3module_inputs_torch_nn_LogSoftmax.<locals>.<lambda>r   r  c                 S   s"   t |t |dd S r  rg   r   r  r   r  r   r5   r5   r;   r     r   Z
multiparamr&  r   r5   c                 S   s"   t |t |dd S )Nr   Fr  r   r5   r5   r;   r     r   Zmultiparam_scalarr   r  r   r0  r1  r5   r5   r;   !module_inputs_torch_nn_LogSoftmax  s*    



r  c              	   K   st   t t|||d}ttdt|ddttdt|dddttdt|d	d
dttdt|dtddgS )Nr   rQ   r  r+  )r   r   r   r   Zmultidimr.  r   r5   ry  r   )r   r   r   r   r&  r0  r1  r5   r5   r;   module_inputs_torch_nn_Softmin  s$    



r  c              
   K   s   t t|||d}tt t|ddd dttdt|ddd dd	ttdd
t|ddd dd	ttdd
t|ddd dd	tt t|dtdd	gS )Nr   r  c                 S   s   t dt | S NrQ   rg   r   r   r   r5   r5   r;   r     r   z1module_inputs_torch_nn_Softplus.<locals>.<lambda>r   r   c                 S   s   dt dt d|   S )Nr   rQ   r   r  r   r5   r5   r;   r     r   betar&  c              	   S   sH   |d dk || |d dk |d d tdtd|    S Nr   r  r  r   rQ   Ztype_asrg   r   r   r   r5   r5   r;   r     s   0Zbeta_thresholdr5   c              	   S   sH   |d dk || |d dk |d d tdtd|    S r  r  r   r5   r5   r;   r     s   0Zbeta_threshold_scalarr   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_Softplus  s4    






r  c              	   K   sp   t t|||d}tt t|ddttdt|dddttdt|dddtt t|d	td
dgS )Nr   rv  r+  rQ   lambdar.  r5   Zlambda_scalarr   r   r&  r0  r1  r5   r5   r;   !module_inputs_torch_nn_Softshrink  s$    



r  c                 K   sb   t t|||d}tt t|ddd dtt t|ddd dd	tt t|d
tdd	gS )Nr   rv  c                 S   s   | dt| S r  r  rg   r   r   r5   r5   r;   r   %  r   z1module_inputs_torch_nn_Softsign.<locals>.<lambda>r   r5   c                 S   s   | dt| S r  r  r   r5   r5   r;   r   (  r   ry  r&  r   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_Softsign  s     


r  c                 K   sV   t t|||d}tt t|ddtt t|dddtt t|dtdd	gS 
Nr   r   r+  r5   ry  r.  r   r   r&  r0  r1  r5   r5   r;   module_inputs_torch_nn_Tanh0  s    


r  c                 K   sV   t t|||d}tt t|ddtt t|dddtt t|dtdd	gS r  r0  r1  r5   r5   r;   !module_inputs_torch_nn_Tanhshrink@  s    


r  c              	   K   s~   t t|||d}ttddt|dddttddt|dddttddt|d	d
dttddt|dtddgS )Nr   r   r  r   Zthreshold_valuer.  g      $@Zlarge_valuer5   Zthreshold_value_scalarr   r   r&  r0  r1  r5   r5   r;    module_inputs_torch_nn_ThresholdO  s&    







r  c                 K   sb   t t|||d}tt t|ddd dtt t|ddd dd	tt t|d
tdd	gS )Nr   r  c                 S   s   |t t| S rH   rg   tanhFZsoftplusr   r5   r5   r;   r   h  r   z-module_inputs_torch_nn_Mish.<locals>.<lambda>r   r5   c                 S   s   |t t| S rH   r  r   r5   r5   r;   r   k  r   ry  r&  r   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_Mishb  s     


r  c                 K   s^   t t|||d}tt t|d|ddd dtt t|d|ddd dd	gt| S )
Nr   r  c                 S   s$   d|   tdd t||D  S )Nr  c                 s   s"   | ]\}}||    V  qd S rH   )r   r   )r7   abr5   r5   r;   rd  z  s   zBmodule_inputs_torch_nn_L1Loss.<locals>.<lambda>.<locals>.<genexpr>)r   r   zipr   r   r   r   r5   r5   r;   r   z  s   z/module_inputs_torch_nn_L1Loss.<locals>.<lambda>r   r5   c                 S   s   d|   ||    S )Nr  )r   r   r   r  r5   r5   r;   r   ~  r   ry  r&  )r   r	   r{   rx   r(  r1  r5   r5   r;   module_inputs_torch_nn_L1Losss  s    	r  c              	   K   s   t t|||d}di fdddifdddifddd	ifg}g }|D ]n\}	}
|
fd
d}|ttf |
t|d|d|	|d |ttf |
t|d|dd|	 |d q@|S )Nr   rA   r   r   r   r   r   r   r   c                 S   s   t ||f|S rH   )r$   r   r5   r5   r;   r     s    z9module_inputs_torch_nn_SmoothL1Loss.<locals>.reference_fnr   r   r   r5   r   r  rT   r   rX   r   rY   r\   r   r   r   r~   r   r   r5   r5   r;   #module_inputs_torch_nn_SmoothL1Loss  s8    




r  c                 K   s  t t|||d}t t||dd}t t||dd}di fdddifdddifd	dd
ifdd|difg}	ddd}
g }|	D ]L\}}|ttf |t|dddd|dd||t |
f|d qx|d}|tt|dt|dddd|dd|dt |
|dd |S )Nr   FrA   r   r   r   r   r   r   r   r   r   r   c                 S   sb   ||   d| d|      }|d k	r2|| }|dkr>|S |dkrV| |  S | S d S )NrQ   r   r   )r   r   r   r   r   r   r   r   r   r   r5   r5   r;   bce_loss_reference_fn  s    "z=module_inputs_torch_nn_BCELoss.<locals>.bce_loss_reference_fnro  {Gz?Gz?lowhighr   r   r5   )r   scalar_weight)r   Nr   r	   rR   r{   rx   gtto)rT   r   rX   r   rY   r\   r   r   r   r   r  r   r~   r   r  r5   r5   r;   module_inputs_torch_nn_BCELoss  s@    







r  c                 K   s   t t|||d}t t||dd}t t||dd}di fdddifdddifd	dd
ifdd|difdd|difg}	ddd}
g }|	D ]L\}}|ttf |t|dddd|dd||t |
f|d q|S )Nr   FrA   r   r   r   r   r   r   r   r   r   r  Zscalar_weightsr5   c                 S   s   | j dd}d| |||  | |   }|d k	rR|| }|dkr^|S |dkrv| |  S | S d S )Nr   r  rQ   r   r   )r  mul_Zadd_Zexp_r  r   r   )r   r   r   r   r   r   Zmax_valr   r5   r5   r;   bce_withlogitsloss_reference_fn  s    4zQmodule_inputs_torch_nn_BCEWithLogitsLoss.<locals>.bce_withlogitsloss_reference_fnro  r  r  r  r   r   )r   Nr  )rT   r   rX   r   rY   r\   r   r   r   r   r  r   r~   r   r5   r5   r;   (module_inputs_torch_nn_BCEWithLogitsLoss  s.    


	


r  c                 K   s(  t t|||d}t t|tjdd}t t||dd}dddg}	di fdd|d	ifd
d
difdddifddddfg}
g }t|	|
D ]\}\}}||fdd}|ttf d|i|t|d|ddddd| d| |d |ttf d|i|t|d|ddddd| d| |d |ttf d|i|t|d|ddddd | d| |d |ttf d|i|t|d!|d"dddd#| d| |d |d
d d kr|ttf d|i|t|d$|d$j	dd%d&| d| |d |ttf d|i|t|d'|d'j	dd%d(| d| |d |ttf d|i|t|d)|d)j	dd%d*| d| |d |ttf d|i|t|d!|d!j	dd%d+| d| |d |ttf d|i|t|d	|d,dddd-| d| t t
d.d/d q|S )0Nr   Fr   r   r   rA   r   r   rz  r   rQ   label_smoothingg333333?Zignore_index_label_smoothing)r   r  c                 S   s   t ||fd|i|S )Nr   )r   )r   r   r   r   r   r   r5   r5   r;   r     s    z=module_inputs_torch_nn_CrossEntropyLoss.<locals>.reference_fnr   )r   r   r   r   )r   r   r   r   r   r  Z4d_rW   r   )r   r   r   )r   r   r   r   r   r   Z2d_)r   r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   r   r   Z4d_prob_target_)r   r   r   Z3d_prob_target_)r   r   Z2d_prob_target_Zhigher_dim_prob_target_r5   r%  Tr$  )r   r	   rg   r   r   rR   r{   rx   r   Zsoftmaxr   )rT   r   rX   r   rY   r\   r   r   r   Z
reductionsr   r   r   r~   r   r   r5   r5   r;   'module_inputs_torch_nn_CrossEntropyLoss  s    



r  c                 K   s  t t|||d}t t|dd}di fdddifddd	ifd
ddifdddifg}tjtjg}	g }
t|	|D ]t\}\}}|fdd}|dd}|dkrdnd}|dkrdnd}|
tt	f |t	|d
d|d|||ddd| d|d |
tt	f |t	|d
d|d|||dtjd|dtjd|d| d|d |
tt	f |t	|d
d|d|||ddd| d|d |
tt	f |t	|d
d|d|||dtjd|dtjd|d| d|d qj|
S ) Nr   F)r   r   rA   r   r   r   r   r   r   r   blank   c                 S   s   t ||||f|S rH   )r   )r   r   r   r   iltlr   r5   r5   r;   r   g  s    z4module_inputs_torch_nn_CTCLoss.<locals>.reference_fnr   rQ   r   )2   r   r   r   )r      )rX   r  r  )r  r  r  )r     r  Z_lengths_intlistsr   r   Z_lengths_tensors)K   Z_1d_target_lengths_intlistsZ_1d_target_lengths_tensors)r   r	   rg   intr   r   r   rR   r{   rx   r   tensor)rT   r   rX   r   rY   r\   r   r   r   Ztarget_dtypesr   Ztarget_dtyper~   r   r   r  r  r  r5   r5   r;   module_inputs_torch_nn_CTCLossX  s    



 	
 	r  c                 K   s6  t t|||d}ttdddt|dddttdddt|d	d
dttdddt|dddttddddt|dddttddddt|dddttdddt|dddttdddt|dddttddddt|dddttddddt|dddttddddt|dddg
S ) Nr   r   r*  r   )r   r*  r   Z	1d_affiner.  rR  )r   rR  Z1d_affine_GNrQ   )   r*  Z1d_affine_large_batchr   Fr   r   r   Z1d_no_affine_INr   r   Z1d_no_affine_LN)r   r*  r   r   Z	2d_affine)r   r*     r  Z2d_affine_large_feature3   gh㈵>)r   r  r  r  Z2d_no_affine_large_feature)r   r   r   r   Z2d_no_affine_INZ2d_no_affine_LNr   r1  r5   r5   r;    module_inputs_torch_nn_GroupNorm  sh    














r  c                 K   sZ   t t|||d}ttdt|ddttdt|dddtt t|dtd	d
gS )Nr   r   )r   r   r   r   r+  r5   ry  r.  r   r   r&  r0  r1  r5   r5   r;   !module_inputs_torch_nn_Hardshrink  s"    


r  c                 K   sB   t t|||d}tt t|dtddtt t|dddgS )Nr   r   r   r&  r{  r|  r.  r0  r1  r5   r5   r;    module_inputs_torch_nn_Hardswish  s    

r  c                 K   sb   t t|||d}tt t|ddd dtt t|ddd dd	tt t|d
tdd	gS )Nr   rv  c                 S   s   | ddS Nr   rQ   r  r   r5   r5   r;   r     r   z1module_inputs_torch_nn_Hardtanh.<locals>.<lambda>r   r5   c                 S   s   | ddS r  r  r   r5   r5   r;   r     r   ry  r&  r   r   r0  r1  r5   r5   r;   module_inputs_torch_nn_Hardtanh  s&    


r  c              
   K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifdddifg}g }	|D ]\}
}|fdd}|	ttf |t|d|dd|dd|
|d |	ttf |t|d|dd|ddd|
 |d qZ|	S )Nr   FrA   r   r   r   r   r   r   r   rk  r   c                 S   s   t ||f|S rH   )r   r   r5   r5   r;   r     s    z?module_inputs_torch_nn_HingeEmbeddingLoss.<locals>.reference_fnr  r   r   rQ   r   r5   r   )	r   r	   rR   r{   rx   r  r  r  Zsub_rr  r5   r5   r;   )module_inputs_torch_nn_HingeEmbeddingLoss  s<    





r  c              	   K   s   t t|||d}di fdddifdddifddd	ifg}g }|D ]>\}	}
|
fd
d}|ttf |
t|d|d|	|d q@|S )Nr   rA   r   r   r   r   r   r   r   c                 S   s   t ||f|S rH   )r   r   r5   r5   r;   r   5  s    z6module_inputs_torch_nn_HuberLoss.<locals>.reference_fnr  r   r  r  r5   r5   r;    module_inputs_torch_nn_HuberLoss)  s&    



r  c              
   K   s  t t|||d}|dd}|d }d\}	}
}}}dddd	}|| }d
| }t|r^t|
|n
t|	|
|t||dt|rt|
|||nt|	|
|||t||ddt|rt|
|n
t|	|
|t||tddt|rt|
|||nt|	|
|||t||tddgS )Nr   r   Fr`  )r   r   rW  FT)r   r   r4  r=  rQ   r   r   r#  r+  Ztracking_statsr.  Ztracking_stats_no_batch_dimr&  r   )r   r	   r   r{   rx   r   )rT   r   rX   r   rY   r\   r   r   r`  Znum_featuresr   r\  rU  Ztrack_running_statsZinput_no_batch_shape_dictrg  rf  r5   r5   r;   %module_inputs_torch_nn_InstanceNormNdC  s>    



r  c                 K   s   t t|||d}ttdgdt|dddttdgdt|dddttdgdd	t|dd
dttdddgdt|dddttdddgdd	t|dddttdgdt|dddttdddgddd	dt|dddgS )Nr   r   r   r  1d_elementwise_affiner.     r   r   !1d_elementwise_affine_large_batchF1d_no_elementwise_affiner   r   r   r   r   3d_elementwise_affine3d_no_elementwise_affiner   r   1d_empty_elementwise_affineT)Zelementwise_affiner   Z3d_elementwise_affine_no_biasr   r1  r5   r5   r;    module_inputs_torch_nn_LayerNormj  sJ    









r  c                 K   s   t t|||d}dd }ttdgdt|dd|dttdgdt|d	d
|dttdgddt|dd|dttdddgdt|dd|dttdddgddt|dd|dttdgdt|dd|dgS )Nr   c                    s   | j }|d krt|jj }|j | j}| j} fddtt|D }|t	|
dj|dd| j   }|d k	r|||9 }|S )Nc                    s   g | ]} | d  qS )rQ   r5   rc  ndimr5   r;   r<     s     zQmodule_inputs_torch_nn_RMSNorm.<locals>.rms_norm_reference_fn.<locals>.<listcomp>r   T)r   Zkeepdim)r   rg   ZfinforX   r  normalized_shaper   rh  rf   Zrsqrtr  r   )r   r   r   r   r  r   Zdimsr   r5   r  r;   rms_norm_reference_fn  s    $z=module_inputs_torch_nn_RMSNorm.<locals>.rms_norm_reference_fnr   r   r  r  r   r  r  Fr  r   r  r  r  r  r  r   )rT   r   rX   r   rY   r\   r   r  r5   r5   r;   module_inputs_torch_nn_RMSNorm  sN    








r   c                 K   sb   t t|||d}ttdt|dddttdt|dddttd	d
ddt|dddgS )Nr   r   )rQ   r   rM  r  r.  r   )rQ   r   rM  rM  Z2d_uneven_padrQ   r  r   r   )rQ   r   rM  rM  rM  Z3d_custom_paramsr   r1  r5   r5   r;   (module_inputs_torch_nn_LocalResponseNorm  s"    


r  c                 K   sf   t t|||d}ttddt|dddttdddt|ddttdddt|d	td
dgS )Nr         ?r   )rQ   r   rM  normr.  r   r+  )r   rM  r   r&  r0  r1  r5   r5   r;   module_inputs_torch_nn_LPPool1d  s"    




r  c                 K   sf   t t|||d}ttdddt|ddttdddt|dtddttddt|dd	d
gS )Nr   r   rQ   r   rM  rM  r+  r   rM  rM  r   r&  r  r  r.  r0  r1  r5   r5   r;   module_inputs_torch_nn_LPPool2d  s"    




r  c                 K   sf   t t|||d}ttdddt|ddttdddt|dtddttddt|dd	d
gS )Nr   r   )rQ   r   rM  rM  rM  r+  )r   rM  rM  rM  r   r&  r  r  r.  r0  r1  r5   r5   r;   module_inputs_torch_nn_LPPool3d  s"    




r  c                 K   sb   t t|||d}ttdt|dddttddt|dddttdddt|dd	dgS )
Nr   r   )r   r   r   rV  r.  r-  Treturn_indicesr
  r   r1  r5   r5   r;    module_inputs_torch_nn_MaxPool1d  s"    



r  c              	   K   sl   t t|||d}ttdddt|dddttdddt|dd	dttdddd
dt|dddgS )Nr   r   r   r3  r6  r  rV  r.  r  r|  Tr	  r
  r   r1  r5   r5   r;    module_inputs_torch_nn_MaxPool2d  s"    




r  c              
   K   s   t t|||d}ttdt|ddttddt|dddttdddt|dd	dttdddd
dt|dddgS )Nr   r<  r?  r+  r   r-  r.  r@  Zstride_paddingTr	  r
  r   r1  r5   r5   r;    module_inputs_torch_nn_MaxPool3d0  s*    




r  c              
      s   t t ||d} fdd}ttdd| dt|ddd	ttd
d| dt|ddd	ttdd| ddt|ddd	ttdd| dt|dtddttd
d| dt|dtddgS )Nr   c                      s   t jdt j d S )N)rQ   r   r   rX   r   rg   r   doubler   r5   r   r5   r;   make_random_samplesI  s    zGmodule_inputs_torch_nn_FractionalMaxPool2d.<locals>.make_random_samplesr   r   output_ratio_random_samples)rQ   r   r   rM  ratior.  r  )r   r   Zoutput_sizer  )rQ   r   rM  r*  sizeTr  r  r
  ratio_return_indices)r   r   rM  ratio_no_batch_dimr&  )r   rM  r*  size_no_batch_dimr0  rT   r   rX   r   rY   r\   r   r  r5   r   r;   *module_inputs_torch_nn_FractionalMaxPool2dF  sF    

   


r  c                    s   t t ||d} fdd}ttdd| dt|ddd	ttd
d| dt|ddd	ttdd| dt|ddd	ttdd| ddt|ddd	ttdd| dt|dtddttd
d| dt|dtddgS )Nr   c                      s   t jdt j d S )N)r   r   r   r  r  r5   r   r5   r;   r  k  s    zGmodule_inputs_torch_nn_FractionalMaxPool3d.<locals>.make_random_samplesr   r   r  )r   r   r   r   r   r  r.  r<  )r   r   r   r  )r   r   rM  rM  rM  r  )r   r   r   )r   r   r   )r   r      rM  r   ZasymsizeTr  r  )r   r   r   r   r  r&  )r   rM  rM  rM  r  r0  r  r5   r   r;   *module_inputs_torch_nn_FractionalMaxPool3dh  sP    


   


r   c                 K   sn   t t|||d}tt t|dddtt t|dtddtt t|dd	dtt t|d
ddgS r  r0  r1  r5   r5   r;   module_inputs_torch_nn_Sigmoid  s.    



r!  c                 K   sb   t t|||d}tt t|ddd ddtt t|ddd d	tt t|d
tddgS )Nr   r5   c                 S   s   |   S rH   r  r   r   r5   r5   r;   r     r   z3module_inputs_torch_nn_LogSigmoid.<locals>.<lambda>ry  r&  r  c                 S   s   |   S rH   r"  r   r5   r5   r;   r     r   r   r   r   r0  r1  r5   r5   r;   !module_inputs_torch_nn_LogSigmoid  s&    


r#  c              
   K   s   t t|||d}t t|tjdd}di fdddifdddifd	dd
ifdddifg}g }	|D ]H\}
}|fdd}|	ttf |t|d|d|d |
|d q\|	S )Nr   FrA   r   r   r   r   r   r   r   rk  r   c                 S   s   t |||f|S rH   )r   rl  r5   r5   r;   r     s    z>module_inputs_torch_nn_MarginRankingLoss.<locals>.reference_fn)r  r   )r   r	   rg   r   rR   r{   rx   rq  rr  r5   r5   r;   (module_inputs_torch_nn_MarginRankingLoss  s*    





r$  c                 K   s   t t|||d}t t|tjdd}di fdddifdddifd	dd
ifg}g }	|D ]z\}
}|fdd}|	ttf |t|d|ddddd|
 |d |	ttf |t|d|dddd|
|d qR|	S )Nr   FrA   r   r   r   r   r   r   r   c                 S   s   t ||f|S rH   )r!   r   r5   r5   r;   r     s    zAmodule_inputs_torch_nn_MultiLabelMarginLoss.<locals>.reference_fnr  r   r   r  Z1d_r   r  r   r	   rg   r   rR   r{   rx   rr  r5   r5   r;   +module_inputs_torch_nn_MultiLabelMarginLoss  s:    




r&  c                 K   s   t t|||d}t t|tjdd}t t||dd}di fdddifdddifd	dd
ifdddifdddifdd|difg}	g }
|	D ]D\}}|fdd}|
ttf |t|d|dddd||d q|
S )Nr   FrA   r   r   r   r   r   r   r   r   r   rk  r   r   r   r   c                 S   s   t ||f|S rH   )r    r   r5   r5   r;   r   	  s    z<module_inputs_torch_nn_MultiMarginLoss.<locals>.reference_fnr  r   r   r  r   r%  )rT   r   rX   r   rY   r\   r   r   r   r   r   r~   r   r   r5   r5   r;   &module_inputs_torch_nn_MultiMarginLoss	  s0    






r'  c                 K   s   t t|||d}t t|tjdd}t t||dd}di fdddifdddifd	dd
ifdd|difg}	ddd}
g }|	D ]@\}}|ttf |t|d|dddd|t |
f|d qz|S )Nr   FrA   r   r   r   r   r   r   r   r   r   c                 S   s|   ||    d| |      }|d k	r6||9 }| | d |d }|dkr`|S |dkrp| S | S d S )NrQ   r   r   r   )r  r   r   r   r  r   r  r5   r5   r;   &multilabelsoftmargin_loss_reference_fn-	  s    &z_module_inputs_torch_nn_MultiLabelSoftMarginLoss.<locals>.multilabelsoftmargin_loss_reference_fnr  r   r   r  r   )r   Nr%  )rT   r   rX   r   rY   r\   r   r   r   r   r(  r   r~   r   r5   r5   r;   /module_inputs_torch_nn_MultiLabelSoftMarginLoss 	  s,    





r)  c              	   K   s   t t|||d}t t||dd}di fdddifdddifd	dd
ifg}g }	|D ]B\}
}|fdd}|	ttf |t|d|d |
|d qP|	S )Nr   FrA   r   r   r   r   r   r   r   c                 S   s   t ||f|S rH   )r%   r   r5   r5   r;   r   T	  s    z;module_inputs_torch_nn_SoftMarginLoss.<locals>.reference_fn)r   r   r   rp  rr  r5   r5   r;   %module_inputs_torch_nn_SoftMarginLossG	  s(    




r*  c                 K   s   g }t d ||||D ]|}|jj|jj }}	||	d< ||	d< tjj||	}
d}|j}d|jkrt|jd |jd< |jd= |t	t
|
|||jd q|S )Nr   rX   r   src_maskmaskr.  ).module_inputs_torch_nn_TransformerEncoderLayerr|   r[   r\   rg   r   TransformerEncoderLayerr}   rR   r{   rx   r~   )rT   r   rX   r   rY   r\   samplesZlayer_module_inputZl_argsZl_kwargsZencoder_layer
num_layersr}   r5   r5   r;   )module_inputs_torch_nn_TransformerEncoderb	  s2        

r1  c                 K   s  t t|||d}ttddddt|dddttddd	dtjt|dd
dttddd	dddt|dddg}d tjdddg|tjdf}d tjdddg|tjd	df}	t
|	|dddD ]X\}
}}}}|ttddd	d|||dt|d|
|dt t|ddidd| d qdd }|rt
ddD ]P\}}|ttddd	dd||dt|d|rp|nd d| d| d q:|S )Nr   r   r   r  r   r  relu_activationr.  r   gelu_activationFr   r   Tr   rX   r  TFd_modelnheaddim_feedforwarddropoutr
  
norm_firstr   rH  )r+  src_key_padding_maskr<  r   r
  r  no_batch_dim_batch_first_r&  c              	   _   s@   | j s
t| d t  | ||}W 5 Q R X | d |S )NFT)rY   r  trainrg   Zno_grad)moduler   r[   r\   r  r5   r5   r;   fast_path_reference_fn	  s    



zNmodule_inputs_torch_nn_TransformerEncoderLayer.<locals>.fast_path_reference_fn)r:  r
  r;  r   Z	fastpath_Z_norm_first_)r   r	   r{   rx   r  gelurg   r  boolr   ri  r   rR   r   )rT   r   rX   r   rY   r\   r   r/  key_padding_masks
attn_masksr+  r<  r;  r
  r   rA  r5   r5   r;   r-  {	  s    "           r-  c                 K   s  t t|||d}ttddddt|d|dddttddd	dtjt|d|dd
dttddd	dddt|d|dddg}d tjdddg|tjdf}d tjdddg|tjd	df}	t
|	|dddD ]\}
}}}}|
}|}|ttddd	d|||dt|d|d|
|||dt t|ddddd| d |d|d }}|sx|dd|dd }}|d k	r|	ddfd \}}|ttddd	d|||dt|||
|||dd| d| d| d q|S )Nr   r   r   r  r   r  r2  r.  r   r3  Fr   r   Tr4  r  r5  r6  rH  )tgt_maskmemory_masktgt_key_padding_maskmemory_key_padding_maskr   )rH  rI  r=  r>  r&  rQ   r   Znorm_first_Z_batch_first_Z_bias_)r   r	   r{   rx   r  rB  rg   r  rC  r   ri  r   rR   r   	transpose)rT   r   rX   r   rY   r\   r   r/  rD  rE  rF  rH  r;  r   r
  rG  rI  srctgtr5   r5   r;   .module_inputs_torch_nn_TransformerDecoderLayer	  s       "      
      rM  c                 K   sx  t t|||d}g }d tjdddg|tjdf}d tjdddg|tjddf}	t|	|dddD ]\}
}}}}|
fd \}}|fd \}}|t	t
ddd	d
d
d|||d	t
|d|d||||dt t|ddddd| d |d|d }}|s|dd
}|dd
}|d k	r:|ddfd \}}|t	t
ddd	d
d
d|||d	t
||||||dd qd|S )Nr   FTr4  r  r5  r   r   r   rQ   r   )	r7  r8  r9  Znum_encoder_layersZnum_decoder_layersr:  r
  r;  r   rH  )rF  r+  rH  r<  r   )rH  r<  r=  r>  r&  r  r   r+  )r   r	   rg   r  rC  r   ri  r   rR   r{   rx   r   rJ  )rT   r   rX   r   rY   r\   r   r/  rD  rE  r,  r  r;  r   r
  r+  rF  r<  rH  rK  rL  r5   r5   r;   "module_inputs_torch_nn_Transformer
  sx    "        
        
rN  c                 K   sf   t tj|tjdd}ttdddt|ddddttdddt|dd	dd
d	ddgS )NFr   r   r   )Znum_embeddingsZembedding_dimr   r+  rQ   i   rM  Zdiscontiguousr.  )r   rg   r   r   r{   rx   Zrandom_r   )rT   r   rX   r   rY   r\   Z
make_emptyr5   r5   r;    module_inputs_torch_nn_Embedding6
  s    

rO  c                 K   s  t t|||d}g }d}d tjdddg|tjdf}	d tjdddg|tjddf}
t||||	|
}|D ]\}}}}}|t	t
ddd|||dt
|d	|d	|d	||d
td |t	t
ddd|||dt
|d	|d	|d	||d
t tddd ql|S )Nr   r5  FTr4  )r   r   r   r   )Z	embed_dimZ	num_headsr
  r   add_bias_kvadd_zero_attnr  )r  	attn_maskr   r
  )r   r	   rg   r  rC  r   ri  r   rR   r{   rx   r  )rT   r   rX   r   rY   r\   r   r/  Z	bool_valsrD  rE  Zproductsr   rP  rQ  r  rR  r5   r5   r;   )module_inputs_torch_nn_MultiheadAttentionE
  sH    "   	   

rT  c           	   	   K   s   t t|||d}ttddt|d|dtdttddddt|d|dtdg}|dd}|r|ttdddd	d
t|d|dtd |S )Nr   r   r   r   Tr   is_rnnFrelu)r   nonlinearity)r   r	   r{   rx   r   r   rR   )	rT   r   rX   r   rY   r\   r   r/  rU  r5   r5   r;   #module_inputs_torch_nn_RNN_GRU_Celld
  s,    rX  c                 K   sl   t t|||d}ttddt|d|d|dftdttddddt|d|d|dftdf}|S )Nr   r   r   r   Tr   )r   r	   r{   rx   r"  rT   r   rX   r   rY   r\   r   r/  r5   r5   r;   module_inputs_torch_nn_LSTMCell
  s    rZ  c                 C   s*   | j }| d t| |}|j| |S )NF)r   r   r   data)r  Zbatch_sizesZrequired_gradseqr5   r5   r;   make_packed_sequence
  s
    

r]  Fc                 K   s  t t|||d}|d }d}	d}
d}d}g }|r@t|	|
||}nt|
||}|D ]<}|rh|\}}}}n
|\}}}ddd|||d}ddd|||d}|r||d< ||d< |ttf |t|d	t t|d
d |ttf |t|d	||rdnddft t|d
d |rP|ttf |tt|dt	ddgt t|d
d |ttf |tt|dt	dddddgt t|d
d qP|S )Nr   rU  )rV  r  FTr   )
input_sizehidden_sizer0  r
  r   bidirectionalr   rW  r   r   rS  r   r   )r   r   r   r   )r   r   r   )
r   r	   r   rR   r{   rx   r  r]  rg   r  )rT   r   rX   r   rY   Zwith_packed_sequencer\   r   rU  rW  r   r
  ra  r/  prod_genr[   nlr  b_fbidir	cons_argscons_args_hiddenr5   r5   r;   module_inputs_torch_nn_RNN_GRU
  sr    

    



 
ri  c              
   K   s  t t|||d}d}d}d}	d}
g }t|||	|
}|D ]}|\}}}}d}d|d||||d}d|d||||d}|ttf |t|dt t|dd	 |d
kr|n|}||rdnd|f||rdnd|ff}|ttf |t|d|t t|dd	 q6|S )Nr   r^  )r   r   r   r   )r_  r`  r0  	proj_sizer
  r   ra  r3  rS  r   r   r   rb  )r   r	   r   rR   r{   rx   r   )rT   r   rX   r   rY   r\   r   r   r
  ra  Z
proj_sizesr/  rc  r[   r  re  rf  rj  r`  rg  rh  Zh_outhxr5   r5   r;   module_inputs_torch_nn_LSTM
  sJ        

(
	rl  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS )Nr   rQ   r  r   rQ   r   r  r+  r0  r1  r5   r5   r;   &module_inputs_torch_nn_ReflectionPad1d  s    

rn  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS Nr   rQ   rN  r   rQ   r   r   r   r   r   r   r*  r+  r0  r1  r5   r5   r;   &module_inputs_torch_nn_ReflectionPad2d  s    

rr  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS )Nr   rQ   r   r   rQ   r   rQ   r   rQ   r   )r   r   r   r   r   r+  r0  r1  r5   r5   r;   &module_inputs_torch_nn_ReflectionPad3d#  s    

rt  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS Nr   rQ   rH  r   rm  rN  r+  r0  r1  r5   r5   r;   'module_inputs_torch_nn_ReplicationPad1d2  s    

rv  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS ro  r0  r1  r5   r5   r;   'module_inputs_torch_nn_ReplicationPad2dA  s    

rw  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS )Nr   rQ   rq  r   rQ   r   r   r   r   r*  )r   r   r   r*  rM  r+  r0  r1  r5   r5   r;   'module_inputs_torch_nn_ReplicationPad3dP  s    

ry  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS ru  r0  r1  r5   r5   r;    module_inputs_torch_nn_ZeroPad1d_  s    

rz  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS )Nr   rQ   r  r   rp  r+  r0  r1  r5   r5   r;    module_inputs_torch_nn_ZeroPad2dn  s    

r{  c                 K   sB   t t|||d}ttdt|dtdttdt|ddgS )Nr   rQ   rq  r   rx  )rQ   r   r   r   r   r+  r0  r1  r5   r5   r;    module_inputs_torch_nn_ZeroPad3d}  s    

r|  c                 K   sF   t t|||d}ttddt|dtdttddt|dd	gS )
Nr   rQ   r   rH  r   rm  r   rN  r+  r0  r1  r5   r5   r;   $module_inputs_torch_nn_ConstantPad1d  s    

r}  c                 K   sF   t t|||d}ttddt|dtdttddt|ddgS )	Nr   rQ   r   rN  r   rp  r   r+  r0  r1  r5   r5   r;   $module_inputs_torch_nn_ConstantPad2d  s    

r~  c                 K   sF   t t|||d}ttddt|dtdttddt|dd	gS )
Nr   rQ   r   rq  r   rx  rM  )rQ   r   rQ   r   rQ   r+  r0  r1  r5   r5   r;   $module_inputs_torch_nn_ConstantPad3d  s    

r  c              	      s   t t|||d}dd  ttdt|dtdttdt|d fd	d
dttdt|d fdd
dttdt|d fdd
dgS )Nr   c              	   S   sJ   t j| dddd|d  df | | ddddd|d f gddS )z input:
                [[[0., 1., 2.],
                  [3., 4., 5.]]]
                pad: (1, 2)
                output:
                    [[[2., 0., 1., 2., 0., 1.],
                      [5., 3., 4., 5., 3., 4.]]]
            Nr   rQ   r   r   rg   catr  padr5   r5   r;   padding1d_circular_ref  s    	zDmodule_inputs_torch_nn_CircularPad1d.<locals>.padding1d_circular_refrQ   rH  r   rm  r  c                    s    || j S rH   rb  r   r  r5   r;   r     r   z6module_inputs_torch_nn_CircularPad1d.<locals>.<lambda>)r   rQ   c                    s    || j S rH   r  r   r  r5   r;   r     r   r  c                    s    || j S rH   r  r   r  r5   r;   r     r   r0  r1  r5   r  r;   $module_inputs_torch_nn_CircularPad1d  s.    






r  c              	      s   t t|||d}dd  ttdt|dtdttdt|d fd	d
dttdt|d fdd
dttdt|d fdd
dgS )Nr   c              
   S   s   t j| dddd|d  df | | ddddd|d f gdd} t j| dddddd|d  df | | ddddddd|d f gddS )aS  input:
                [[[[0., 1., 2],
                   [3., 4., 5.]]]]
                pad: (1, 2, 2, 1)
        output:
            [[[[2., 0., 1., 2., 0., 1.],
               [5., 3., 4., 5., 3., 4.],
               [2., 0., 1., 2., 0., 1.],
               [5., 3., 4., 5., 3., 4.],
               [2., 0., 1., 2., 0., 1.]]]]
        Nr   r   r   r   rQ   r  r  r5   r5   r;   padding2d_circular_ref  s    JzDmodule_inputs_torch_nn_CircularPad2d.<locals>.padding2d_circular_refrQ   rN  r   )rQ   r   r   rQ   )rQ   rQ   r   r   c                    s    || j S rH   r  r   r  r5   r;   r     r   z6module_inputs_torch_nn_CircularPad2d.<locals>.<lambda>)r   r   r   r   c                    s    || j S rH   r  r   r  r5   r;   r     r   )r   r   r   rQ   )rQ   rQ   r   r   c                    s    || j S rH   r  r   r  r5   r;   r     r   r0  r1  r5   r  r;   $module_inputs_torch_nn_CircularPad2d  s.    






r  c              	      s   t t|||d}dd  ttdt|dtdttdt|d fd	d
dttdt|d fdd
dttdt|d fdd
dgS )Nr   c                 S   s  t j| dddd|d  df | | ddddd|d f gdd} t j| dddddd|d  df | | ddddddd|d f gdd} t j| dddddddd|d  df | | ddddddddd|d f gddS )	aN  input:
                [[[[[ 0.,  1.,  2.],
                    [ 3.,  4.,  5.]],
                   [[ 6.,  7.,  8.],
                    [ 9., 10., 11.]]]]]
            pad: (1, 2, 2, 1, 1, 2)
            output: [[[[[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]],

                       [[ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.]],

                       [[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]],

                       [[ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.]],

                       [[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]]]]]
        Nr   r   r   r   r   r   rQ   r  r  r5   r5   r;   padding3d_circular_ref  s    %JVzDmodule_inputs_torch_nn_CircularPad3d.<locals>.padding3d_circular_refrQ   rq  r   rs  )rQ   rQ   r   r   r   c                    s    || j S rH   r  r   r  r5   r;   r   =  r   z6module_inputs_torch_nn_CircularPad3d.<locals>.<lambda>)r   r   r   rQ   rQ   r   c                    s    || j S rH   r  r   r  r5   r;   r   B  r   )r   r   r   rQ   r   r   c                    s    || j S rH   r  r   r  r5   r;   r   G  r   r0  r1  r5   r  r;   $module_inputs_torch_nn_CircularPad3d  s.    *






r  Z
TestModuleZ	test_gradcuda)	active_ifrc   Ztest_gradgradZtest_non_contiguous_tensors)r  rn   rc   )   rM  ZTestExpandedWeightModuleZtest_module)rc   Z
TestDecompZtest_rnn_decomp_modulec                 K   s@  t t|||d}tttddt|dd|dddtjtddtttddt|dd|dd	dtjtd
dtttddt|dd|dddtjtddtttddt|dd|dddddtjtddtttdddt|dd|dd	dtjtd
dtttdddt|dd|dd	dtjtd
dg}|S )Nr   r   r  r   r  r+  5input has inconsistent input_size: got 11 expected 10r   r   r      9hidden0 has inconsistent hidden_size: got 21, expected 20r   5Input batch size 3 doesn't match hidden0 batch size 5rQ   z.Expected hidden to be 1D or 2D, got 4D insteadrV  r  	r   r	   r   r{   rx   r   r   ra   
ValueErrorrY  r5   r5   r;   )module_error_inputs_torch_nn_RNN_GRU_Cellp  sr    				
	
8r  c                 K   s  t t|||d}tttddt|dd|dd|ddfdtjtddtttddt|dd|dd	|dd	fdtjtd
dtttddt|dd|dd|ddfdtjtddtttddt|dd|dddd|ddddfdtjtddg}|S )Nr   r   r  r   r  r+  r  r  r  r  r   r  rQ   z/Expected hx\[0\] to be 1D or 2D, got 4D insteadr  rY  r5   r5   r;   %module_error_inputs_torch_nn_LSTMCell  sN    			&&r  c                 K   sD   t ttddddtjtddt ttddddtjtddg}|S )Nr   r   rQ   )r|   z%hidden_size must be greater than zeror  z$num_layers must be greater than zero)r   r{   rx   r   r   r  )rT   r   rX   r   rY   r\   r/  r5   r5   r;   $module_error_inputs_torch_nn_RNN_GRU  s    r  c                 K   sR   t t|||d}|dd}tt|r.tddntdt|ddtjtdd	gS )
Nr   is_constantFrQ   r   r   r+  z(expected 2D or 3D input \(got 4D input\)r  	r   r	   r   r   r{   rx   r   r   r  rT   r   rX   r   rY   r\   r   r  r5   r5   r;   "module_error_inputs_torch_nn_Pad1d  s    
r  c                 K   sR   t t|||d}|dd}tt|r.tddntdt|ddtjtdd	gS )
Nr   r  FrQ   r   r  r+  z(expected 3D or 4D input \(got 2D input\)r  r  r  r5   r5   r;   "module_error_inputs_torch_nn_Pad2d  s    
r  c                 K   sR   t t|||d}|dd}tt|r.tddntdt|ddtjtdd	gS )
Nr   r  FrQ   r   r  r+  z(expected 4D or 5D input \(got 2D input\)r  r  r  r5   r5   r;   "module_error_inputs_torch_nn_Pad3d  s    
r  )r   r   Ztest_memory_formatrY   )r  )r   r   r   zSkipped!)r   )r   r   )rn   )r   r   r   TZTestEagerFusionModuleInfoZ,test_aot_autograd_symbolic_module_exhaustiveZ#test_aot_autograd_module_exhaustive)rS   r   r   Ztest_check_inplacer   )rc   rn   rQ   )r`  r   i  )versiong-C6?)r   r   r   r   r   r   r   iE  )r`  r   ra  Ztest_cpu_gpu_parity)rn   rc   g{Gzt?)r   r   r   rn   r   r   )rc   rn   r  )r   rn   r   r   r   r   )ZatolZrtolZtest_forwardcpu)r   r   r   )include_halfinclude_bfloat16gQ?r   )r   rn   r   )r  r  )r   rn   r   )r   r   r   )r`  )r   rS   r   Ztest_factory_kwargs)rS   r   r   r   )rc   r  )rU  )r   r   )rS   r   r   r   )rS   r   r   r   r   )r   r   r   	module_db)F(R  rg   Zunittestcopyr   enumr   	functoolsr   r   ri  r   r   r   Ztorch.nn.functionalr   Z
functionalr  Ztorch.nn.utils.rnnr   Ztorch.testingr	   Z#torch.testing._internal.common_cudar
   Z$torch.testing._internal.common_dtyper   r   r   Z*torch.testing._internal.common_device_typer   r   r   r   r   r   r   r   r   r   Z2torch.testing._internal.common_methods_invocationsr   Z!torch.testing._internal.common_nnr   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   Z$torch.testing._internal.common_utilsr'   r(   r)   r*   r+   r,   typesr-   typingr.   r/   r0   r1   r2   operatorrG   ZaoZqatZquantizableZ	quantizedr3   __annotations__Module	ContainerZ	NLLLoss2dZ	MaxPool2dr4   rI   r>   r@   strr:   r=   r8   r6   rv   rj   r   Znamespace_namerC   rw   rx   r{   r   r   r   r   r   r   r   r   r   r  r   r  r  r   r"  r(  r2  r;  rA  rE  rL  rO  rP  rQ  rT  r[  r]  r_  rj  rs  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r   r!  r#  r$  r&  r'  r)  r*  r1  r-  rM  rN  rO  rT  rX  rZ  r]  ri  rl  rn  rr  rt  rv  rw  ry  rz  r{  r|  r}  r~  r  r  r  r  ZexpectedFailurefloatZ#rnn_gru_lstm_module_info_decoratorsr  r  r  r  r  r  ZAdaptiveAvgPool1dZAdaptiveAvgPool2d
itemgetterZAdaptiveAvgPool3dskipZAdaptiveMaxPool1dZAdaptiveMaxPool2dZAdaptiveMaxPool3dZ	AvgPool1dZ	AvgPool2dr   Z	AvgPool3dZBatchNorm1dZBatchNorm2dZBatchNorm3dZCELUZConv1dr   ZConv2dZfloat64ZConv3dZConvTranspose1dZchalfZConvTranspose2dZ	complex32Z	complex64Z
complex128ZConvTranspose3dZCosineEmbeddingLossZELUZFractionalMaxPool2dZFractionalMaxPool3dZL1LossZSmoothL1LossZ
LazyConv1dZ
LazyConv2dZ
LazyConv3dZLazyConvTranspose1dZLazyConvTranspose2dZLazyConvTranspose3dZLinearZBilinearZLPPool1dZLPPool2dZLPPool3dZ	MaxPool1dZ	MaxPool3dZ	KLDivLossZMSELossZMarginRankingLossZMultiLabelMarginLossZMultiMarginLossZSoftMarginLossZMultiLabelSoftMarginLossZNLLLossZGaussianNLLLossZPoissonNLLLossZHingeEmbeddingLossZ	HuberLossZBCELossZBCEWithLogitsLossZCrossEntropyLossZCTCLossZGELUZGLUZ	GroupNormZ
HardshrinkZ	HardswishZHardtanhZInstanceNorm1dZInstanceNorm2dZInstanceNorm3dZLocalResponseNormZ	LayerNormZRMSNormZTransformerEncoderr.  ZTransformerDecoderLayerZTransformerZMultiheadAttentionZ	EmbeddingZReLUZ	LeakyReLUZReLU6ZPReLUZRNNCellZGRUCellZLSTMCellZSigmoidZ
LogSigmoidZSiLUZSoftmaxZ	Softmax2dZ
LogSoftmaxZSoftminZSoftplusZ
SoftshrinkZSoftsignZTanhZ
TanhshrinkZ	ThresholdZMishZRNNZGRUZLSTMZReflectionPad1dZReflectionPad2dZReflectionPad3dZReplicationPad1dZReplicationPad2dZReplicationPad3dZSELUZ	ZeroPad1dZ	ZeroPad2dZ	ZeroPad3dZCircularPad1dZCircularPad2dZCircularPad3dZConstantPad1dZConstantPad2dZConstantPad3dr  r5   r5   r5   r;   <module>   sH	   0@ 






	




E	8%C0'.
4%#0*Y@/#'"2"&#'JC.>*%)I  
   
   
     
  
  #<+

	 
 
 
 
 
 
 
 
   
 
  
 
 
  
 
 "
   
 
   
 
 
 
  
 
 

    	 








 





 	

 	



 	


  

 

   







	




  




 	





























	
	

	
	









        