U
    MhU                    @   s.	  U d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZ d dl	m
Z
mZ d dlmZmZmZmZmZmZmZmZmZmZmZ d dlZd dlm  mZ d dlm  mZ d dlm  mZ d dl m  m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 d d	l2m3Z3m4Z4m5Z5m6Z6 d d
l7m8Z8m9Z9m:Z:m;Z; d dl<m=Z=m>Z>m?Z?m@Z@ d dlAmBZB d dlCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[ d dl\m]Z]m^Z^m_Z_m`Z` d dlambZb d dlcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZk d dllmmZmmnZn edZoG dd denZpedddgZqi areeseqf etd< i aueesees f etd< devees eeeI  eseweqdddZxdevesees dd d!Zyejzdd"esees d#d$d%Z{dddd&eseseeeI  eweev eqd'd(d)Z|eeP dd*d+d,Z}esesd-d.d/Z~eeE eeI d0d1d2ZePeEeeI d3d4d5ZePeeE ees d3d6d7ZeeE ees d0d8d9Zee,e.f e,esd:d;d<Zee,e.f ePeEesd=d>d?Zee,e.f ePeeE esd@dAdBZee,e.f ePeeE esd@dCdDZe
dEdFG dGdH dHZe
dEdFG dIdJ dJZe
dEdFG dKdL dLZe
dEdFG dMdN dNZe
dEdFG dOdP dPZe5ePesdQdRdSZeQees dTdUdVZePebewdWdXdYZe
dEdFG dZd[ d[Zevesd\d]d^Zesevd-d_d`ZeYesdadbdcZeeZ ees dddedfZePeeeesesf  eesesf f dQdgdhZe+ewees ees eesesf evdidjdkZeDewees ees eesesf evdldmdnZe5ePevdQdodpZePewdQdqdrZe6ePeeIeEf esdsdtduZeees  ees ebdvdwdxZeeP eeePeRf  dydzd{ZeeP eeePeQf  dyd|d}Ze#jd~eeePeQf  eeIeEf eeeQePf eEgees f eesees f dddZeesees f ees dddZe#jd~eeePeQf  eeIeEf eeeQePf eEgees f ees dddZeePeQf eEesdddZegeeePeQf  eIeEebewewewewees d
ddZeeePeQf  eIeEebewewees dddZeeP ebeees esf dddZeeP eeePeQf  eeQ eeE ebeeIeEf egegeeI eeI ewddddZeeP eeePeQf  eeE ebeeIeEf egegegeeI eeI ewddddZeeP ees eeePeQf  eeQ eeE ebeeIeEf egegegegeeI eeI ewewddddZeeP eeePeQf  eeQ eeR ebeeE eeIeEf egegegegegeeI eeI ewewewewewddddZegeeP ddddZejdddZddddZedk	r*e  dS )    N)defaultdict
namedtupleOrderedDict)	dataclassfield)AnyCallableDictListLiteralOptionalSequenceSetTupleTypeVarUnion)inductor_fallback_ops)cpp)	translate)BindingCppSignatureCppSignatureGroupDispatcherSignature
NamedCTypeNativeSignatureSpecialArgName)method_with_native_functionnative_function_managerwith_native_function with_native_function_and_indices)gen_aoti_c_shim*gen_static_dispatch_backend_call_signatureget_fallback_op_nameget_header_for_aoti) gen_functionalization_definition"gen_functionalization_registration.gen_functionalization_view_inverse_declarationGenCompositeViewCopyKernelgen_all_vmap_plumbing)ArgumentBackendIndexBackendMetadataBaseOperatorNameDEFAULT_KERNEL_NAMESPACEDispatchKeyFRAGMENT_NAMESPACESFunctionSchemais_cuda_dispatch_keyis_generic_dispatch_keyis_ufunc_dispatch_keyLocationNativeFunctionNativeFunctionsGroupNativeFunctionsViewGroupOperatorNameOptionalType
SchemaKindSelfArgumentSTRUCTURED_DISPATCH_KEYSTensorOptionsArgumentsTypeVariantViewSchemaKind)add_generated_native_functionsgen_composite_functional_kernelgen_composite_out_kernelpre_group_native_functions)SelectiveBuilder)assert_never	concatMapcontextFileManagermake_file_managermapMaybeNamespaceHelperTarget)
YamlDumper
YamlLoaderTc                       s   e Zd Zd fdd	Z  ZS )
LineLoaderFc                    s$   t  j||d}|jjd |d< |S )N)deep   __line__)superconstruct_mappingZ
start_markline)selfnoderS   mapping	__class__ >/var/www/html/venv/lib/python3.8/site-packages/torchgen/gen.pyrW      s    zLineLoader.construct_mapping)F)__name__
__module____qualname__rW   __classcell__r^   r^   r\   r_   rR      s   rR   
ParsedYamlnative_functionsbackend_indices_GLOBAL_PARSE_NATIVE_YAML_CACHE_GLOBAL_PARSE_TAGS_YAML_CACHE<stdin>F)es
valid_tagsignore_keyspathskip_native_fns_genreturnc              
      s"  t | tstg }tt}| D ]}t |ts:td| t |dtsRt|t||d |d  d k	std| t fdd0 t	
|||\}}	|| t||	 W 5 Q R X qt| tdd }
|st|| | D ]"\}}t|dd	t||d
|
|< qt||
S )Nzexpected to be dict: rU   funczmissed 'func' in c                      s   d d  S Nzin z:
  r^   r^   funcslocr^   r_   <lambda>       z*parse_native_yaml_struct.<locals>.<lambda>c                   S   s   t tjdddi dS )NTFdispatch_keyZuse_out_as_primaryZexternaldevice_guardindex)r+   r/   Z	Undefinedr^   r^   r^   r_   ru      s   TFrw   )
isinstancelistAssertionErrorr   dictgetintr5   rI   r6   Z	from_yamlappendr+   Z
grow_indexerror_check_native_functionsrB   itemsr2   rd   )rj   rk   rl   rm   rn   rsbserp   mindiceskvr^   rr   r_   parse_native_yaml_struct   s8    


r   )rj   rm   ro   c              
      s   t | tstt }| D ]}t |dts4t|t||d  |dt fdd: | }|	d}|	dd}|dkst|
| W 5 Q R X q|S )NrU   tagc                      s   d  d S rq   r^   r^   rt   tagsr^   r_   ru      rv   z(parse_tags_yaml_struct.<locals>.<lambda>desc )r{   r|   r}   setr   r   r5   rI   copypopadd)rj   rm   r   r   Ze_inamer   r^   r   r_   parse_tags_yaml_struct   s    

r   )maxsize)rm   ro   c              	   C   sB   | t kr:t| $}tj|td}t|| dt | < W 5 Q R X t |  S )NLoader)rm   )rh   openyamlloadrR   r   )rm   frj   r^   r^   r_   parse_tags_yaml   s
    
r   )rn   loaded_yaml)rm   tags_yaml_pathrl   rn   r   ro   c             	   C   s^   | t krVt|}|d kr<t| }tj|td}W 5 Q R X n|}t|||| |dt | < t |  S )Nr   )rm   rn   )rg   r   r   r   r   rR   r   )rm   r   rl   rn   r   rk   r   rj   r^   r^   r_   parse_native_yaml   s    	

r   )rs   ro   c                 C   sF  i }t t}| D ]$}|||jj< ||jjj | q| D ]}|jd k	r||j}|d k	st|jj d|j d|j d|jst|jj d|j d|j dd|j	kr:t
|jjdkr:t
|jjdkr:t
|jjjdkr:|jjj}|jst|jj d	t|jd
|j}t|| dks:t|jj d| dq:d S )Nz0 is marked as a structured_delegate pointing to z, but z is missing.zY is not marked as structured. Consider adding 'structured=True' to the delegated operatorZinplace_viewZresize_Z
resize_as_set_z is marked with tag: inplace_view, but it doesn't follow the naming convention for inplace ops - the codegen expects the base name to have a trailing underscore. Fr   zw is marked with tag: inplace_view. The codegen expects there to be a corresponding out-of-place view op with the name 'z/' and matching schema, but it didn't find one. )r   r|   rp   r   r   structured_delegater   r}   
structuredr   strinplacer-   baseZdunder_methodlen)rs   Zfunc_mapZbase_func_mapr   Zdelegate_func	base_nameZout_of_place_base_namer^   r^   r_   r   	  sD    




  r   )sro   c                 C   sl   |  dd} |  dd} |  dd} |  dd} |  d	d
} |  dd} |  dd} |  dd} d|  dS )z1Convert a python string into a c++ string literal\z\\"z\"z\az\bz\f
z\nz\v	z\t)replacer   r^   r^   r_   
cpp_string4  s    r   )backendsro   c                 C   s6   t | dkrg S dd | D tjtjtjtjg S d S )Nr   c                 S   s   g | ]
}|j qS r^   )rx   ).0backendr^   r^   r_   
<listcomp>R  s     z(static_dispatch_keys.<locals>.<listcomp>)r   r/   CompositeImplicitAutograd%CompositeImplicitAutogradNestedTensorCompositeExplicitAutograd&CompositeExplicitAutogradNonFunctionalr   r^   r^   r_   static_dispatch_keysN  s    r   )r   backend_indexro   c                 C   sN   | j d k	s|| r|jS | jr&tjS | jr2tjS | jr>tj	S | j
rJtjS d S N)r   
has_kernelrx   &has_composite_explicit_autograd_kernelr/   r   5has_composite_explicit_autograd_non_functional_kernelr   &has_composite_implicit_autograd_kernelr   4has_composite_implicit_autograd_nested_tensor_kernelr   )r   r   r^   r^   r_   get_static_dispatch_backendZ  s    r   c                 C   sZ   |d ks| j rd S g }|D ]4}t| |}|d k	r|d| j d|  d qd|S )N#include <ATen/ops/__dispatch.h>r   )manual_kernel_registrationr   r   	root_namelowerjoin)r   r   outputrz   rx   r^   r^   r_   static_dispatch_ops_headern  s    
r   c                 C   s   dd t | D S )Nc                 S   s   g | ]}d | dqS )#include <ATen/Functions.h>r^   )r   rx   r^   r^   r_   r     s   z1static_dispatch_extra_headers.<locals>.<listcomp>)r   r   r^   r^   r_   static_dispatch_extra_headers~  s    r   )sigcpp_sigro   c                 C   sr   t t t t ddd}t|  }t| }|D ]}|jjtjkr4||} qTq4t||}d	dd |D S )N)input_bindingsro   c                 S   sT   g }| D ]F}|j dkrDtttj|jj|j |j|jd}|	| q|	| q|S )NZmemory_format)nctyper   defaultargument)
r   r   r   r    possibly_redundant_memory_formatr   typer   r   r   )r   Zoutput_bindingsZbindingZspl_mem_format_bindingr^   r^   r_   add_spl_memory_format_binding  s    
	z5translate_args.<locals>.add_spl_memory_format_binding, c                 s   s   | ]}|j V  qd S r   exprr   ar^   r^   r_   	<genexpr>  s     z!translate_args.<locals>.<genexpr>)
r
   r   r|   	argumentsr   r   r   r   r   r   )r   r   r   Zsrc_bindingsZgoal_bindingsargexprsr^   r^   r_   translate_args  s    
r   )r   r   r   ro   c           	   	   C   sj   t | |}| }t| |}||}|r6|jr6|jnt}|dd}d| d|j  d| d| d	S )N::nativer   return ::();)	r!   r   r   
get_kernelcpp_namespacer.   r   rx   r   )	r   r   r   r   r   r   backend_metadataZ	kernel_nsnsr^   r^   r_   %generate_static_dispatch_backend_call  s    


r   )r   r   rf   ro   c              	   C   s(  t j|ddd}| jr(|j r(|j}n|j}|d k	s:t| }t	| |}t
dd}|jrd| dtj  d| d| d	S |jrd| dtj  d| d| d	S |jrd| dtj  d| d| d	S |jrd| dtj  d| d| d	S d	| d
ddd |D  dS d S )NFmethodZfallback_bindingr   r   r   r   r   r   z5TORCH_CHECK(false, "Static dispatch does not support z forr   c                 S   s   g | ]}t |jqS r^   )r   rx   )r   rz   r^   r^   r_   r     s     z:generate_static_dispatch_fallback_call.<locals>.<listcomp>z ");)r   from_native_functionsymintrp   
has_symintZsymint_signature	signaturer}   r   r   r.   r   r   r/   r   r   r   r   r   r   r   r   r   )r   r   rf   Zcpp_sigsr   r   r   r   r^   r^   r_   &generate_static_dispatch_fallback_call  s.      
$$$$r   c                    sH  t |dks jrdS  fdd|D }t |dkrDt|  |d S t |dkr\t|  |S dd |  D }d|} jjj}g }g }|dk	r|d	 |dkr|d
| d |dd| d |d g }	|D ]2}
|	d|
j	 d |	dt|  |
 d qt|  |}d}d|| d||	 d| dS )a  
    For a given `NativeFunction`, find out the corresponding backend and dispatch to it. If more than one
    backends exsit, fallback to static dispatch by determining dispatch key from inputs.
    Arguments:
        sig: A CppSignature or DispatcherSignature for this native function we want to use.
        f: NativeFunction to generate static dispatch.
        backend_indices: All available backends.
    Return:
        C++ code to call backend-specific functions, e.g., "return at::cpu::add(self, other, scale);"
    r   r   c                    s.   g | ]&}|  s& jd k	r|jtkr|qS r   )r   r   rx   r=   )r   br   r^   r_   r     s
   


z#static_dispatch.<locals>.<listcomp>rT   c                 S   s6   g | ].}t |jts,t |jtr|jj r|jqS r^   )r{   r   r<   r*   r   is_tensor_liker   r   r^   r^   r_   r     s
   r   Nz>DispatchKeySet(c10::computeDispatchKey(dtype, layout, device))z$c10::detail::multi_dispatch_key_set()zDispatchKeySet _dk_set = z | ;z=DispatchKey _dk = c10::highestPriorityBackendTypeId(_dk_set);zcase DispatchKey:::r   z
		
    z
    switch (_dk) {
        z
        default:
            z
    }
    )
r   r   r   r   r   r   rp   tensor_optionsr   rx   )r   r   rf   keysnative_tensor_argstensor_argsZtensor_optsZstmtsZsubexprsZdispatch_coderz   fallbackZ	connectorr^   r   r_   static_dispatch  sN    
	


r  T)frozenc                   @   sH   e Zd ZU eed< eedZee	e
f ed< eeee	 dddZdS )RegisterSchemaselector)default_factory
known_tagsr   ro   c                 C   s   | j |sd S dddd t|jD  d }|dkrPdtt|j dS d	}|| jkrt	| j}|| j|< d
| d| d}| dtt|j d| j|  dS )N{r   c                 s   s   | ]}d | V  qdS )z	at::Tag::Nr^   )r   r   r^   r^   r_   r   /  s     z*RegisterSchema.__call__.<locals>.<genexpr>}z{}zm.def(z, {});
r   z const std::vector<at::Tag> tags_z = ;
z, tags_z);
)
r  is_native_function_selectedr   sortedr   r   r   rp   r	  r   )rY   r   r   Z
maybe_tagsidxr^   r^   r_   __call__+  s    "


zRegisterSchema.__call__N)r`   ra   rb   rF   __annotations__r   r~   r	  r	   r   r   r   r6   r   r  r^   r^   r^   r_   r  &  s   
r  c                   @   sB   e Zd ZU eejejf ed< ee	 ed< e
eedddZdS )ComputeOperatorstargetstatic_dispatch_backend_indicesr
  c                 C   s  t |j}|jj }| jtjkrd| d|  d|jjj d|jjj	 dt
t|j d|jddd	 d
|jddd	 dS | jtjkrd| d|jjj d| d|jjj	 d| dt
t|j d|j d| d| d| d| d| d}dD ]}|r(ddgdd | D  }d}nddd | D }d}|}| d | }	d!| d"| d#| d$}
|st| jd%krt||| jd&}
|d'|j d(|j|	|d	 d)|
 d*7 }q|S t| j d S )+Nz
struct TORCH_API z {
  using schema = z;
  using ptr_schema = schema*;
  // See Note [static constexpr char* members for windows NVCC]
  STATIC_CONSTEXPR_STR_INL_EXCEPT_WIN_CUDA(name, "aten::z>")
  STATIC_CONSTEXPR_STR_INL_EXCEPT_WIN_CUDA(overload_name, "z:")
  STATIC_CONSTEXPR_STR_INL_EXCEPT_WIN_CUDA(schema_str, z)
  static callF)r   is_redispatching_fnz;
  static Z
redispatchTz;
};z+
STATIC_CONST_STR_OUT_OF_LINE_FOR_WIN_CUDA(z, name, "aten::z-")
STATIC_CONST_STR_OUT_OF_LINE_FOR_WIN_CUDA(z, overload_name, "z, schema_str, z)

// aten::z.
static C10_NOINLINE c10::TypedOperatorHandle<z::schema> create_zQ_typed_handle() {
  return c10::Dispatcher::singleton()
      .findSchemaOrThrow(z::name, z::overload_name)
      .typed<z::schema>();
}
)FTr   dispatchKeySetc                 S   s   g | ]
}|j qS r^   r   r   r^   r^   r_   r   }  s     z-ComputeOperators.__call__.<locals>.<listcomp>c                 S   s   g | ]
}|j qS r^   r  r   r^   r^   r_   r     s     r   z
    static auto op = create_z_typed_handle();
    return op.r   r   r   rf   

// aten::r    {
    z
}
)r   from_schemarp   r   unambiguous_namer  rN   DECLARATIONr   overload_namer   r   defn
DEFINITIONr   r   r   r  r  rG   )rY   r   r   r   Zdefnsr  Zdispatcher_exprs_strZmethod_baseZdispatcher_callmethod_nameZfn_bodyr^   r^   r_   r  D  s    	  zComputeOperators.__call__N)r`   ra   rb   r   rN   r  r"  r  r
   r+   r   r6   r   r  r^   r^   r^   r_   r  ?  s   
r  c                   @   s$   e Zd Zeeee dddZdS )ComputeFunctionr
  c           
      C   s   t j|d|jd}|j }d}| D ]}t|j}t|	 |	 }d
dd |D }|jrjd}	nd}	tj|jkr|d	|j d
|  d|jj  d| d	7 }|r(|d|	 d|jdd d|jj  d| d	7 }q(|S )NFr   r   r   c                 S   s   g | ]
}|j qS r^   r   r   r   r^   r^   r_   r     s     z,ComputeFunction.__call__.<locals>.<listcomp>zc10::SymIntZint64_tr  
inline  {
    return at::_ops::::call(z);
}zX
namespace symint {
  template <typename T, typename = std::enable_if_t<std::is_same<T, z>::value>>
  T)Zsuppress_symint_suffixz	);
  }
}
)r   r   manual_cpp_bindingrp   r   
signaturesr   r  r   r   r   r   r@   functionvariantsdeclr   r  )
rY   r   	sig_groupr   resultr   
target_sigr   	exprs_strZ	intlike_tr^   r^   r_   r    sF      




zComputeFunction.__call__Nr`   ra   rb   r   r6   r   r   r  r^   r^   r^   r_   r$    s   r$  c                   @   sF   e Zd ZU eejejf ed< ee	 ed< e
eee dddZdS )ComputeTensorMethodr  r  r
  c                 C   s  t j|jkrd S |j rt|jjjd k	s0ttj	|d|j
d}| jtjkrvd}| D ]}||  d7 }qZ|S | jtjk	rt| j d}| D ]n}t|j}t| | dd}ddd |D }|d	|j d
|jdd d|jj  d| d	7 }q|S )NTr   r   z const;
)r   r   c                 S   s   g | ]
}|j qS r^   r   r%  r^   r^   r_   r     s     z0ComputeTensorMethod.__call__.<locals>.<listcomp>r  r&  zTensor::)prefixz const {
    return at::_ops::r(  );
}
)r@   r   r,  rp   	is_out_fnr}   r   Zself_argr   r   r)  r  rN   r  r*  r-  r"  rG   r   r  r   r   r!  r   r  rY   r   r.  r/  r   r0  r   r1  r^   r^   r_   r    s>      


zComputeTensorMethod.__call__N)r`   ra   rb   r   rN   r  r"  r  r
   r+   r   r6   r   r   r  r^   r^   r^   r_   r3    s   
r3  c                   @   s$   e Zd Zeeee dddZdS )ComputeRedispatchFunctionr
  c                 C   s   t j|d|jd}d}| D ]p}t|j}t| | }d	dgdd |D  }|d|j d	|j
d
d d|jj  d| d	7 }q|S )NFr   r   r   r  c                 S   s   g | ]
}|j qS r^   r   r   r^   r^   r_   r     s     z6ComputeRedispatchFunction.__call__.<locals>.<listcomp>r  r&  T)r  r'  z::redispatch(r5  )r   r   r)  r*  r   r  rp   r   r   r   r-  r   r  r7  r^   r^   r_   r     s(      

z"ComputeRedispatchFunction.__call__Nr2  r^   r^   r^   r_   r8    s   r8  r
  c                 C   s   d| j jj d| j jj dS )Nz{"aten::z", "z"},)rp   r   r   r   r^   r^   r_   compute_aten_op  s    r9  gro   c                 C   s  | j s
d S t| j~ t| }t | }ddd |D }| jj}|d krTd}d}| j rf| jjnd }|rV|j	
 |jf}dd |D }dd |D }	dd	d |	D }
d
|
 d}dd |D }ddd |D }g }t|D ]\}}d|	d | dg |	|d d   }d| d}|| jdd}| d|j d| d}d|j d}d|	|  d| d}g }|| d t|D ]F\}}||kr|d|j d n|d|j d|j d  q|d! d"|}|d#| d$| d%| d& qd"|}ddgt|	 }d'| d(}d)}d*| d+| d#| d,}nd-}d-}d.| d/| d0| d1| d1| d2| d3W  5 Q R  S Q R X d S )4Nr   c                 s   s   | ]}|  V  qd S r   )r-  r   r^   r^   r_   r   )  s     z4compute_meta_function_declaration.<locals>.<genexpr>zat::impl::MetaBasevoidc                 S   s   g | ]}|D ]}|qqS r^   r^   )r   Zreplace_listelemr^   r^   r_   r   5  s      z5compute_meta_function_declaration.<locals>.<listcomp>c                 S   s   g | ]}|j  qS r^   )r   upperr   r=  r^   r^   r_   r   8  s    c                 s   s   | ]}d | dV  qdS )zbool z = falseNr^   )r   paramr^   r^   r_   r   ;  s    z
template <>c                 S   s   g | ]}t j||jd qS ))binds)r   argument_typer   r?  r^   r^   r_   r   A  s   r  c                 s   s&   | ]}|j d d d|j V  qdS )TZ	strip_ref N)cpp_typer   r?  r^   r^   r_   r   F  s   truerT   zprecompute_out<TrD  z set_r   z value)r   z already set"zstatic_assert(z == false, r   z ret;zret.z	 = value;z	 = this->r   zreturn ret;r   z
                    z {
                        z
                        z'
                    }
                z'using meta_return_ty = precompute_out <z>;Zmeta_return_tyz
                zG
                struct TORCH_API precompute_out {
                    z;
            };r   zstruct TORCH_API structured_z
 : public r  r   z meta(z);
};
)r   r   outmetar   Zmeta_argumentsr   Zstructured_inheritsprecomputedr   valuesr   	enumeraterF  r   r   )r;  r   argsargs_strZparent_classZmeta_returnrJ  Zprecomputed_valuesZprecomputed_elementsZprecomputed_template_parametersZprecomputed_template_params_strZprecompute_template_declZ#precomputed_elements_with_cpp_typesZprecomputed_elements_declZsetter_methodsir=  Zreturn_ty_templatesZ	return_tyZelem_cpp_tyr   Z
assert_msgZassert_stmtZconstruction_stmtsjZconstruction_blockZsetter_methods_declZmeta_return_template_paramsZmeta_return_typedefZprecomputed_declr^   r^   r_   !compute_meta_function_declaration#  s    









rQ  )r   r  ro   c                 C   sB   t | jjj}|ds"|dr&dS | jjjd kr8dS || S )NZ_likeZnew_F)r   rp   r   endswith
startswithr   r   r  )r   r  r   r^   r^   r_   needs_backend_select  s    rT  c                   @   sB   e Zd ZU eejejf ed< eed< e	e
ee dddZdS )ComputeBackendSelectr  r  r
  c                 C   s4  t || jsd S t|j}t|jdd}dd | D }t|j}|}|	 }d}| j
tjkr|r|jj sztddd |D }	d	| d
|	 d}
n|jj rtd| d}
d|j d|| d|
 d|jj  dddd |D  dS | j
tjkr&d|jj d| dS t| j
 d S )NTr   c                 S   s(   g | ] }t |jtr|jj r|qS r^   )r{   r   r*   r   r   r   r^   r^   r_   r     s    z1ComputeBackendSelect.__call__.<locals>.<listcomp>z.c10::computeDispatchKey(dtype, layout, device)r   c                 s   s   | ]}|j V  qd S r   r  r   r^   r^   r_   r     s     z0ComputeBackendSelect.__call__.<locals>.<genexpr>z-DispatchKeySet _dk_set = c10::DispatchKeySet(z() | c10::detail::multi_dispatch_key_set(z);
DispatchKeySet _dk_mask = c10::DispatchKeySet(DispatchKeySet::FULL_AFTER, DispatchKey::BackendSelect);
DispatchKeySet _dk = c10::impl::computeDispatchKeySet(_dk_set, _dk_mask);z)DispatchKeySet _dk = c10::DispatchKeySet(r   z	// aten::z
C10_ALWAYS_INLINE
z {
  z
  return at::_ops::z::redispatch(
      _dk, c                 s   s   | ]}|j V  qd S r   r   r   r^   r^   r_   r     s     r5  zm.impl("aten::z", TORCH_FN(z));)rT  r  nativer   rp   r   r   r   r  r   r  rN   r"  Zhas_tensor_argr}   r   r!  r  REGISTRATIONrG   )rY   r   r   Z
native_sigr  Zdispatcher_sigr   Zdispatcher_exprsrx   r  Z
compute_dkr^   r^   r_   r    sJ    


	zComputeBackendSelect.__call__N)r`   ra   rb   r   rN   r"  rX  r  rF   r   r6   r   r   r  r^   r^   r^   r_   rU    s   
rU  )dataro   c                 C   s:   dd t _tttddd}t t| tj| dt ddS )	Nc                 S   s   dS )NTr^   )rY   rY  r^   r^   r_   ru     rv   zformat_yaml.<locals>.<lambda>)dumperrY  ro   c                 S   s   |  | S r   )Zrepresent_dictr   )rZ  rY  r^   r^   r_   dict_representer  s    z%format_yaml.<locals>.dict_representerFg    eA)Zdefault_flow_styleZDumperwidth)rO   Zignore_aliasesr   Zadd_representerr   r   dump)rY  r[  r^   r^   r_   format_yaml  s    
r^  c                 C   sh   | dkrdS | dkrdS z
t | W S  tk
rb   zt| W  Y S  tk
r\   |  Y  Y S X Y nX d S )NrG  TfalseF)r   
ValueErrorfloatr   r^   r^   r_   pythonify_default  s    
rb  )tro   c                 C   s:   t | trt| jS t| dkr$dS tj| dddd S )NTensorz
at::TensorF__placeholder__)mutablerB  r   )r{   r:   dynamic_typer=  r   r   Zargumenttype_typerF  )rc  r^   r^   r_   rg    s    

   rg  )r,  ro   c                 C   s2   dg}t j| kr|d t j| kr.|d |S )Nr?   rd  	namespace)r@   r   r   r+  )r,  	method_ofr^   r^   r_   compute_method_of_yaml(  s    



rj  c                 C   s   i }t | }g }tt| jj|D ]f\}\}}t|j|t j|dd	 d}|j
r|j
|d< | j r|j
|| jjj| j
< || q$||fS )NFrV  )rg  r   r   
field_name)r   Zreturn_namesrL  ziprp   returnsrg  r   return_typerF  r   r6  r   rH  r   )r   name_to_field_namenamesrm  rO  rr   retr^   r^   r_   compute_returns_yaml3  s    *


rs  )cpp_aschema_orderkwarg_only_setout_arg_setro  ro   c                C   sp   t | jtr:d dd| j| jdd}| jd k	r6| j|d< |S t | jtrLtn t | jtrlt	| j||||dS d S )Nzat::TensorOptionsFT)
annotationrg  is_nullabler   r   
kwarg_onlyr   ru  rv  rw  ro  )
r{   r   r>   r   r   r   r<   r}   r*   compute_argument_yaml)rt  ru  rv  rw  ro  r   r^   r^   r_   compute_cpp_argument_yamlv  s*    

r}  )r   ru  rv  rw  ro  ro   c             	   C   s   | j rt| j nd t| j| j | jtj| ddd d}| j	d k	rdt
tj| j	| jdd|d< | j|krvd|d< | j|krd|d	< d|d
< | j|kr|| j |d< | j }|d k	r|jd k	rt|jdkr|j|d< |S )Nre  F)rB  r   )rx  rg  ry  r   r   rV  r   Trz  r   allocaterk  boolsize)rx  r   rg  r   ry  r   r   rC  rF  r   rb  Zdefault_exprZis_list_liker  r=  )r   ru  rv  rw  ro  r   lr^   r^   r_   r|    s*    	




 
r|  c                    s  t | \}dd | jjjD  dd | jjjD tj| ddd}|j } fdd|D }t| j	 } fdd|D }d	d |D }t
j| jjdd
 }| dd| d}	tdd |D otj| jk}
tdt
| jfdt| jjjfdt| jjjfd| jfd| jd k	r,| jndfdd| j fd|fd|	fd|fdt| jfdd| jd krpdn| jfd|fd| jjjjfd |
fd!| jfd"| jfd#d$d%| jfgS )&Nc                 S   s   h | ]
}|j qS r^   r  r   r^   r^   r_   	<setcomp>  s     z+compute_declaration_yaml.<locals>.<setcomp>c                 S   s   h | ]
}|j qS r^   r  r   r^   r^   r_   r    s     Fr   c              	      s   g | ]}t |d  dqS )Fr{  )r}  )r   rt  rv  ro  rw  r^   r_   r     s   z,compute_declaration_yaml.<locals>.<listcomp>c              	      s   g | ]}t |d  dqS )Tr{  )r|  r   r  r^   r_   r     s   c              
   S   s0   g | ](}t j|d t d d d dD ]
}|jq qS )F)r   Zcpp_no_default_argsZfaithfulr   Zhas_tensor_options)r   r   r   r   )r   r   rq  r^   r^   r_   r     s    rV  z (r   r   c                 s   s   | ]}t |jtV  qd S r   )r{   r   r>   r   r^   r^   r_   r     s     z+compute_declaration_yaml.<locals>.<genexpr>r   Zoperator_namer   r   category_overrider   Zschema_stringaten::r   schema_order_cpp_signatureschema_order_argumentsri  )moderW  python_modulerm  r   is_factory_methodZabstractry   )Zwith_gilF)
deprecatedFZhas_math_kernel)rs  rp   r   Zflat_kwarg_onlyrH  r   r   r   r|   r  r   returns_typerm  rF  r   anyr@   r   r,  r   r   r   r   r   r  rj  r  r   Zis_abstractry   r   )r   rm  r.  Zcpp_argsr   Zschema_order_jit_argumentsr  Zcpp_schema_order_typesZcpp_returnsr  r  r^   r  r_   compute_declaration_yaml  sb      

r  c                 C   s0   | j s| jd k	o.| j tjkp.| j tjkS r   )r   r   rp   kindr;   
functionalr   r   r^   r^   r_   "has_autogenerated_composite_kernel  s    r  )r   rf   ro   c              	      s   t  j}t  jj }t  j}ddd |D }d j t fdd|	 D t
jhko fdd|	 D t
jt
jhkt jpt d}| d	| d
| dt| dS )Nr   c                 s   s   | ]}|   V  qd S r   )Z
no_defaultZdecl_registration_declarationsr   r^   r^   r_   r   #  s     z4compute_registration_declarations.<locals>.<genexpr>r  c                    s   h | ]\}}|  r|qS r^   r   r   r   r   r   r^   r_   r  (  s     
 z4compute_registration_declarations.<locals>.<setcomp>c                    s   h | ]\}}|  r|qS r^   r  r  r   r^   r_   r  *  s     
 )schemadispatchr   rE  r   z); // r   )
dispatcherr   rp   r  rm  Z"cpp_type_registration_declarationsr   r   r   r   r/   r   r   Zhas_composite_kernelr  jsondumps)r   rf   r   r  rM  rN  Zcomment_datar^   r   r_   !compute_registration_declarations  s(    
	r  )"provided_op_registration_allowlistop_selection_yaml_pathro   c                 C   sd   | d k	r|d k	rt dd }| d k	r,t| }|d k	rDt|dd}n|d k	rXt|}nt }|S )NzhBoth provided_op_registration_allowlist and op_selection_yaml_path can NOT be provided at the same time.TF)r}   r   rF   Z&from_legacy_op_registration_allow_listZfrom_yaml_pathget_nop_selector)r  r  Zop_registration_allowlistr  r^   r^   r_   get_custom_build_selector=  s&    	r  )re   ro   c                 C   s   t tttf tf ttttf  ddd}tt}| D ]x}|j	
 }|j}|tjkr~|j	 }||| kspt||| |< q8||| kst| d||   ||| |< q8tt|| S )Ndro   c                 S   s\   g }t j| krJ| t j}| t jd }| tjd }|t|||d || 	  |S )N)view	view_copyview_inplace)
rA   Zaliasingr   Zaliasing_inplacer;   r  r   r8   extendrK  )r  rs   r  r  r  r^   r^   r_   maybe_create_view_group_  s    
	zEget_grouped_by_view_native_functions.<locals>.maybe_create_view_groupz already in )r	   r   rA   r;   r6   r
   r8   r   r~   rp   Zview_signatureZview_schema_kindZnon_aliasingr  r}   r   r|   rH   rK  )re   r  Zgrouped_by_viewsr   r  Z	view_kindr  r^   r^   r_   $get_grouped_by_view_native_functions\  s$    



r  c                 C   sB   t ttf ttttf  ddd}t| }tt|t|	 S )Nr  c                 S   sB   t | }|d kr8tdd |  D r,tt|  S |gS d S )Nc                 s   s   | ]}d |j kV  qdS )	generatedN)r   r   r   r^   r^   r_   r     s     zJget_grouped_native_functions.<locals>.flatten_pre_group.<locals>.<genexpr>)r7   	from_dictr  rK  r}   r|   )r  rq  r^   r^   r_   flatten_pre_group  s
    
z7get_grouped_native_functions.<locals>.flatten_pre_group)
r	   r;   r6   r   r   r7   rE   r|   rH   rK  )re   r  Zpre_grouped_native_functionsr^   r^   r_   get_grouped_native_functions  s    
r  )native_function_decl_gen)grouped_native_functionsrf   r  ro   c                 C   s   t t}| D ]}t }t }| D ]j\}}||}	|	rV|	j}
|| ||
 nt}
t|dkszt	d| d| ||
 
||| q$q|S )NrT   z6Codegen only supports one namespace per operator, got z from )r   r|   r   r   r   r   r   r.   r   r}   r  )r  rf   r  ns_grouped_kernelsr   Znative_function_namespacesdispatch_keysrx   backend_idxr   rh  r^   r^   r_   get_ns_grouped_kernels  s&    


r  )r  ro   c              
   C   sh   g }d}|   D ]R\}}t|ddd}tt|}|d|j d|| d|j d	| q|S )Nr   r      namespace_strentity_nameZ	max_level	
        )
r   rM   r|   r   fromkeysr  prologuer   epiloguesplit)r  declarationsnewlinerh  kernels	ns_helperordered_kernelsr^   r^   r_   8get_native_function_declarations_from_ns_grouped_kernels  s*    	r  c                 C   s   t | ||d}t|dS )a  
    Generate kernel declarations, in `NativeFunction(s).h`.
    :param grouped_native_functions: a sequence of `NativeFunction` or `NativeFunctionGroup`.
    :param backend_indices: kernel collections grouped by dispatch key.
    :param native_function_decl_gen: callable to generate kernel declaration for each `NativeFunction`.
    :return: a list of string, from the string with all declarations, grouped by namespaces, split by newline.
    r  rf   r  )r  )r  r  )r  rf   r  r  r^   r^   r_    get_native_function_declarations  s    r  )r   r  ro   c                 C   s^   | | }|rPd|jksPtdt| tr.| jjn| jjj d|j d|j d|rZ|jS t	S )Nr   zThe kernel for function z with dispatch key z has a namespace z% and it's not ending with '::native'.)
r   r   r}   r{   r6   rp   r   r  rx   r.   )r   r  r   r^   r^   r_   get_kernel_namespace  s    
2r  )
fmr  rx   r  r  rocmr   skip_dispatcher_op_registrationgen_dispatch_helpersro   c        	            s  g }	t tt t t t}
d}tjtj|||d d}tjtj|||d d}tjtj|||d d}|D ]}t	|d
dd ||   || t|tr|jn|jj}||
 krt t|
< |
 | || qrD ]t dkrqtdd|
 D ]B}|
 | s>q(d| d	 d
||
 |  d7 q(|	| d f	dd| q|	S )Nr   r  r   class_method_namer  r   r  r   r   r   )r  z
TORCH_LIBRARY_IMPL(r   z, m) {
    
};zRegisterDispatchDefinitions.inic                	      s<   j jrtng   r&dnd  dS )Nr   )Zns_prologueZns_epiloguedispatch_helpersZdispatch_anonymous_definitionsZ"static_init_dispatch_registrationsZdeferred_dispatch_registrationsdispatch_namespaceZdispatch_namespaced_definitions)r  r  destZgen_registration_helpersr   r^   	Zanonymous_definitionsr  rx   r  Zkernel_namespaceZns_definitionsr  Zregistration_bodyr  r^   r_   ru   S  s    z1get_native_function_definitions.<locals>.<lambda>)r   r|   r~   r  RegisterDispatchKeyrN   ZNAMESPACED_DEFINITIONZANONYMOUS_DEFINITIONrX  r  r   r  r{   r6   rh  r  r   rM   r   Zsubstitute_with_templater  )r  r  rx   r  r  r  r   r  r  definitionsZregistrationsr  Zns_genZanonymous_genZreg_genr   rh  r^   r  r_   get_native_function_definitions  s    			 
r  )r  rx   r  r  r  r   ro   c              
   C   s   g }t t}d}tj|tj||d d|d}	| D ].}
t|
|dd| }|| 	|	|
 q.|
 D ]`\}}t|dkr|qft|ddd	}tt|}|	d|j d|| d|j d
| qf|S )Nr   F)r  r  r  r   r  rW  r   r      r  r  )r   r|   r  r  rN   NAMESPACED_DECLARATIONr  r   r   r  r   r   rM   r   r  r  r   r  r  )r  rx   r  r  r  r   r  r  r  rp   r   rh  r  r  r  r^   r^   r_   get_namespaced_declarationk  sR    		   	r  )re   schema_selectorro   c              
   C   s   t t}| D ]}||j | qd}g }d }| D ]b\}}ttt||}	|dkr^|	}q6|}d}
|tkrrdnd}|d| d| d|
|	 d	7 }q6||fS )
Nr   Zatenr   ZTORCH_LIBRARY_FRAGMENTZTORCH_LIBRARYr   r   z	, m) {
  r  )	r   r|   rh  r   r   rL   r  r0   r   )re   r  Zns_native_functionsZnative_functionschema_registrationsaten_schema_registrationscustom_namespacerh  rs   Zschema_registrations_bodytabZtorch_library_macror^   r^   r_   (get_native_function_schema_registrations  s4    r  )re   r  structured_native_functionsstatic_dispatch_idxr  rf   cpu_fmcuda_fmfunctions_keysr  r  ro   c              
      s  | dfdd dd D fddD | d
fdd | d	
fd
d | d
fdd t d| dfdd |	D ]ntr|n|}|krd d| ddfdd | dd 	fdd ~qd S )NzNativeMetaFunctions.hc                      s   g t tt dS )N)ZNativeMetaFunctions_includesZ NativeMetaFunctions_declarationsr|   rL   rQ  r^   )r  r^   r_   ru     s
    z(gen_aggregated_headers.<locals>.<lambda>c                 S   s   g | ]}t j|jkr|qS r^   r@   r   r,  r   fnr^   r^   r_   r     s     z*gen_aggregated_headers.<locals>.<listcomp>c                    s   g | ]}| kr|qS r^   r^   r  )method_native_functionsr^   r_   r     s     MethodOperators.hc                      s   g t tttjd dS )Nr  ZMethodOperators_includesZMethodOperators_declarationsr|   rL   r  rN   r  r^   )r  r  r^   r_   ru     s    zOperators.hc                      s    dgt tttjd dS )Nz!#include <ATen/MethodOperators.h>r  )ZOperators_includesZOperators_declarationsr  r^   )non_method_native_functionsr  r^   r_   ru     s    Functions.hc                      s   t dgttt  dS )Nz#include <ATen/Operators.h>)r   ZFunctions_includesZFunctions_declarations)r   r|   rL   r$  r^   re   r  r^   r_   ru      s    )r  rf   zNativeFunctions.hc                      s   dg dS )Nz%#include <ATen/NativeMetaFunctions.h>)ZNativeFunctions_includesZNativeFunctions_declarationsr^   r^   )r  r^   r_   ru     s    r   Functions_inl.h>DispatchKeyFunctions.hc                      s   t  dS N)rx   Zinline_headersr   r^   rx   inl_headersr^   r_   ru   !  s    Functions_inl.hDispatchKeyFunctions_inl.hc                
      s$   g   t  dddS )NT)r  rx   r  r  r  r   )!DispatchKeyFunctions_inl_includesr   dispatch_namespaced_declarations)r   r  r^   )rf   rx   r  r  r  r^   r_   ru   )  s    )writer  r2   write_with_template)re   r  r  r  r  rf   r  r  r  r  r  r  r^   )rf   r  rx   r  r  r  re   r  r  r  r  r  r_   gen_aggregated_headers  sZ    
	


r  )re   r  r  r  rf   r  r  ops_fmr  r  r  ro   c                    s`  t t| D ]}|j | qt t}|D ]}|j	|	 | q. D ]\	|	 dd
fdd |	 dd	
fdd |	g }dd	 |D td
kr|	 ddfdd t||t	j
d|	 dd	fdd qPdD ](\ |  d fdd q|	D ]|krTq@ g  D ]~\	|	g }ttt	j| tj||
dd dd|td
krqh	 |	 d ddfdd qhtr|n|}d d| ddfdd | d d!fd"d ~q@|d#fd$d d S )%Nz_ops.hz
Operator.hc                      s   dt tttjd iS )Nr  r  r  r^   )	functionsr  r^   r_   ru   Y  s     z*gen_per_operator_headers.<locals>.<lambda>.hz
Function.hc                      s2   t tfdd d dt tt  dS )Nc                    s   t |  dS )Nr   )r   r  r  r^   r_   ru   l  s    z<gen_per_operator_headers.<locals>.<lambda>.<locals>.<lambda>r   _ops.h>)Zstatic_dispatch_ops_headersZoperator_includesZfunction_definitions)r|   rL   r$  r^   )r  r   r  r^   r_   ru   i  s    

c                 S   s    g | ]}t |tr|jr|qS r^   )r{   r7   r   r  r^   r^   r_   r   }  s   
 z,gen_per_operator_headers.<locals>.<listcomp>r   z_meta.hzNativeMetaFunction.hc                      s   dt tt iS )NZmeta_function_declarationsr  r^   )structured_functionsr^   r_   ru     s      r  z	_native.hzNativeFunction.hc                      s   rd dng  dS )Nr   z_meta.h>)Zextra_includesZnative_function_declarationsr^   r^   )r  is_structuredr   r^   r_   ru     s    ))Z	Functionsr   )Z	OperatorsZ_ops)ZNativeMetaFunctions_meta)ZNativeFunctionsZ_nativec                      s.     dfddt  D   dg iS )NZ	_includesc                    s   g | ]}d |   dqS )r   .h>r^   r   r   )suffixr^   r_   r     s   >gen_per_operator_headers.<locals>.<lambda>.<locals>.<listcomp>Z_declarations)r  r   r^   )categoryfunctions_by_root_namer  r^   r_   ru     s     

 TFr  r   z_dispatch.hzDispatchKeyFunction.hc                      s
    dS )N)r  r  r^   r^   )r  r  r^   r_   ru     s    r   r  r  r  c                      s   t  dS r  r  r^   r  r^   r_   ru     s    r  r  c                      s   fddt  D g dS )Nc                    s   g | ]}d | d  dqS )r   r   r   r^   r  )r  r^   r_   r     s   r  )r  r  r  )r  r^   )dispatch_namesr  r^   r_   ru     s    
r  c                      s   t dd   D g dS )Nc                 s   s0   | ](\}}t d d |D rd| dV  qdS )c                 s   s   | ]}t j|jkV  qd S r   r  r  r^   r^   r_   r     s     zGgen_per_operator_headers.<locals>.<lambda>.<locals>.<genexpr>.<genexpr>r   r  N)r  )r   r   r  r^   r^   r_   r     s   z=gen_per_operator_headers.<locals>.<lambda>.<locals>.<genexpr>r  )r  r   r^   )r  r^   r_   ru     s
    )r   r|   r   r   r   r  r   r   r  r  #compute_native_function_declarationr  r   rH   r  rN   r  r2   )re   r  r  r  rf   r  r  r  r  r  r  r  Zgrouped_functions_by_root_namegroupZgrouped_functionsr  r^   )r  r  rx   r	  r  r  r  r  r  r   r  r   r  r_   gen_per_operator_headers:  s    


	
	
r  )re   rk   r  r  r  r  rf   core_fmr  r  r  r  r  r  per_operator_headersro   c                    s   |r$t || ||	|
|||d nt||| ||	|||d |dfdd |dfdd |d fd	d |d
fdd tttf dfdd}|d| tttf dfdd}|d| d S )N)re   r  r  r  rf   r  r  r  r  r  r  )re   r  r  r  r  rf   r  r  r  r  r  zTensorBody.hc                      s2   t tttjd t tttjd dS )N)r  r  )Ztensor_method_declarationsZtensor_method_definitions)r|   rL   r3  rN   r  r"  r^   r  r^   r_   ru   -  s&    	zgen_headers.<locals>.<lambda>zRedispatchFunctions.hc                      s   dt tt  iS )NZfunction_redispatch_definitions)r|   rL   r8  r^   re   r^   r_   ru   E  s
     
zRegistrationDeclarations.hc                      s   d fddD iS )NZregistration_declarationsc                    s   g | ]}t | qS r^   )r  r  r  r^   r_   r   O  s   z1gen_headers.<locals>.<lambda>.<locals>.<listcomp>r^   r^   )rf   re   r^   r_   ru   N  s
     
zVmapGeneratedPlumbing.hc                      s   t  S r   r(   r^   r  r^   r_   ru   W  rv   ro   c                     s   t  } t  } D ]D}|t|jjj ||jjjj | dd |j D  q|ddddddd	d
dddh8 }ddd t	|D ddd t	| D dS )Nc                 s   s   | ]}|j V  qd S r   r  )r   r   r^   r^   r_   r   c  s     zAgen_headers.<locals>.gen_aten_interned_strings.<locals>.<genexpr>andZand_eqZbitandZbitorZcomplnotZnot_eqorZor_eqxorZxor_eqz \
c                 S   s   g | ]}d | dqS )z_(aten, r   r^   r  r^   r^   r_   r   w  s     zBgen_headers.<locals>.gen_aten_interned_strings.<locals>.<listcomp>c                 S   s   g | ]}d | dqS )z_(attr, r   r^   r  r^   r^   r_   r   z  s     )Zaten_symbolsZattr_symbols)
r   r   r   rp   r   r   updater  r   r  )attrsrp  rp   r  r^   r_   gen_aten_interned_stringsZ  s4    z.gen_headers.<locals>.gen_aten_interned_stringszaten_interned_strings.hc                      s   dd t iS )NZenum_of_valid_tagsz,
)r   r  r^   )rk   r^   r_   gen_tags_enum  s    z"gen_headers.<locals>.gen_tags_enumz
enum_tag.h)r  r  r  r	   r   )re   rk   r  r  r  r  rf   r  r  r  r  r  r  r  r  r  r  r^   )rf   re   r  rk   r_   gen_headers  s^    
	
 
$r  )re   r  r  view_groupsr  r  rf   aoti_fmr  r  
cpu_vec_fmr  r  r  r  force_schema_registrationr  r  update_aoti_c_shimro   c           '         sD  drd|D ]ft r"|n|	}rJtt d
fddntt d
fdd tt}D ]*}t|tr|jn|jj}|| 	| qvt
 tjk}t|d|d	|d	 d
dfdd D ]ډjjr tsq jjjjtjkr||	ksBt|d d
dfdd |
d d
dfdd nRtjkrdrd|d ddfdd ntd dq t D ]2}| D ]"}|jd k	r||j<  qqqtjtjfkrvt }D ] }t|}|tkr6|||< q6tdd t| D 	d
  d}t	dd d!|r| |fd"d npz>t!t"j#$|j%| }|& }|kstd#W 5 Q R X W n0 t'k
r    t(t"j#$|j%| d$ Y nX td	fd%d&t rHnd | d
  d
	fd'd ~qt)ttt f dfd(d)}|	 d*| } |rt*+ } t,| d+\ |	 d, fd-d t-tt.t/f td.d/d0}!|	j0d1|!fd2dd3t1id4d5d6d3hd7 |	 d8t | d9t | d:fd;d t-tt.t/f t)ttt f d<fd=d>}"tt }#d?d@ t2dAd D }$dBd@ t2dCd D }%D ],}&|&jj|$kr|&jj|%kr|#	|& q|	j0dD|#|!|"dEdFdGdHdIdJhdK |	 dLfdMd |	 dNfdOd d S )PNz#include <c10/cuda/CUDAGuard.h>
#include <ATen/cuda/ATenCUDAGeneral.h>
#include <ATen/cuda/CUDADevice.h>
#include <ATen/cuda/CUDAContext.h>z#include <ATen/hip/impl/HIPGuardImplMasqueradingAsCUDA.h>
#include <ATen/hip/ATenHIPGeneral.h>
#include <ATen/hip/HIPDevice.h>
#include <ATen/hip/HIPContext.h>r  c                     s   g } D ]}d}  |r d}nDt|trJt fdd| D rJd}n|jrdtjtjfkrdd}|sjq| 	d|j
 d tjkr| 	d|j
 d kr| 	d|j
 d d	 qtt| S )
NFTc                 3   s   | ]}  |V  qd S r   r  r  r  r^   r_   r     s    z=gen_source_files.<locals>.operator_headers.<locals>.<genexpr>r   
_native.h>r  r   r   )r   r{   r7   r  r  r   r/   Metar   r   r   r  r   )headersr;  Zis_registered)r   rx   r  r  r  r^   r_   operator_headers  s6    
z*gen_source_files.<locals>.operator_headersc                     s8   dg}  t jkr| d  kr4| d d | S )Nz!#include <ATen/NativeFunctions.h>z#include <ATen/Functions.h>r   r   )r/   r   r   )r!  )rx   r  r^   r_   r"    s    

T)	r  r  rx   r  r  r  r   r  r  ZRegisterz.cppzRegisterDispatchKey.cppc                      s*   t rnddt  ddS )Nr   )extra_cuda_headersZexternal_backend_headersZdispatch_headersops_headersr  dispatch_definitions)r2   r  Zgen_registration_headersr^   )r   r%  rx   r#  r"  r  r  r^   r_   ru     s      z"gen_source_files.<locals>.<lambda>Z	UfuncCPU_zUfuncCPU.cppc                      s"   t t  tdS )N)meta_declarationnative_declarationnative_definitions)rQ  r  r
  Zcompute_ufunc_cpur^   )rf   rx   r;  r^   r_   ru   	  s     ZUfuncCPUKernel_zUfuncCPUKernel.cppc                      s   t  dS )N)r   r(  )r  Zcompute_ufunc_cpu_kernelr^   )r;  r   r^   r_   ru   !	  s    z%#include <ATen/native/cuda/Loops.cuh>z$#include <ATen/native/hip/Loops.cuh>Z
UfuncCUDA_z.cuzUfuncCUDA.cuc                      s&   t t  tdS )N)r   cuda_headersr&  r'  r(  )rQ  r  r
  Zcompute_ufunc_cudar^   )rf   r)  rx   r;  r   r^   r_   ru   -	  s     zunrecognized z
 for ufuncc                 s   s   | ]\}}|V  qd S r   r^   )r   r   valuer^   r^   r_   r   G	  s    z#gen_source_files.<locals>.<genexpr>Zc_shim_r  r   headerZincludesc                      s    S r   r^   r^   )
new_headerr^   r_   ru   X	  rv   a  

WARNING: The generated AOTInductor C shim header files have unexpectedly changed. This
indicates an AOTInductor fallback operator ABI backward compatibility breakage!!!
Only in a limited number of situations, this is allowed:

1. You added a fallback op to the inductor_fallback_ops list in torchgen/aoti/fallback_ops.py.
If that's the case, run `python torchgen/gen.py --update-aoti-c-shim` to update the existing
C shim header files.

2. You added a new default argument to an existing fallback op. This is clearly a BC breaking
change in the AOTInductor land. In this case, you need to keep a manual copy of that existing
fallback op in a file, e.g. torch/csrc/inductor/aoti_torch/c/shim.h, bump up the version
number of that fallback op in the newly generated C shim files, and update the cpp wrapper
codegen to generate the correct cpp call for this op. Contact AOTInductor team for assistance.

                        z
 not foundc                     s@   g } D ]$}t | }|d k	r| | qdtt| S )Nr   )r#   r   r   r  r   )r!  rp   r,  )rf   rx   fallback_native_functionsstructured_func_group_dictr^   r_   headers_for_aotiy	  s       z*gen_source_files.<locals>.headers_for_aotic                      s   t  d d  dS )NFr   r+  )r    r^   )rf   rx   extra_headersr.  r0  r/  r^   r_   ru   	  s   c                     sL   fdd D } dd | D t tttj| t tttj| dS )Nc                    s   g | ]}t | r|qS r^   )rT  r  r  r^   r_   r   	  s    
 z@gen_source_files.<locals>.gen_backend_select.<locals>.<listcomp>c                 S   s   g | ]}d |j  dqS )r   r  r   r  r^   r^   r_   r   	  s    )r$  Z!backend_select_method_definitionsZ%backend_select_function_registrations)r|   rL   rU  rN   r"  rX  )Zrelevant_fns)re   r  r^   r_   gen_backend_select	  s&    

 
 z,gen_source_files.<locals>.gen_backend_selectzRegisterBackendSelect.cpp)re   r  zRegisterSchema.cppc                      s   rg n rg ndS )N)r  r  r^   r^   )r  r  r  r^   r_   ru   	  s    )r  ro   c                 S   s   | j S r   r3  r  r^   r^   r_   key_func	  s    z"gen_source_files.<locals>.key_funczOperators.cppc                    s&   d| j  dgttj d| gdS )Nr   r  r  )r"  r  )r   r  rN   r"  r  r  r^   r_   ru   	  s    r      r"  r  )key_fnenv_callableZbase_env
num_shardssharded_keyszFunctions.cppzTensorMethods.cppzATenOpList.cppc                      s   dt tt iS )NZaten_ops)r|   rL   r9  r^   r  r^   r_   ru   	  s     r:  c                    sB   t tttf tt ddd}|| t| t|  tj	 dS )Nr:  c                 S   s&  t | tr\d| jj dd| jj dg}| jd k	rX|d| jj dd| jj dg7 }|S t | trd| jj dd| jj dd| jj dd| jj dg}| jd k	r|d| jj dd| jj dg7 }| j	d k	r|d| j	j dd| j	j dg7 }|S d| j dd| j dgS d S )Nr   r  r  )
r{   r8   r  r   r  r7   r  rH  r   rf  )r;  r!  r^   r^   r_   gen_op_headers	  s<    


zPgen_source_files.<locals>.functionalization_env_callable.<locals>.gen_op_headers)r$  func_definitionsfunc_registrations)
r   r6   r7   r8   r
   r   r$   r%   r/   r   )r;  r;  )rf   r  r^   r_   functionalization_env_callable	  s    (z8gen_source_files.<locals>.functionalization_env_callablec                 S   s   i | ]}|j j|qS r^   rp   r   r  r^   r^   r_   
<dictcomp>-
  s    z$gen_source_files.<locals>.<dictcomp>c                 S   s   t |  S r   r|   r  r;  r^   r^   r_   ru   /
  rv   c                 S   s   i | ]}|j j|qS r^   r?  r  r^   r^   r_   r@  1
  s     c                 S   s   t |  S r   rA  rB  r^   r^   r_   ru   2
  rv   zRegisterFunctionalization.cppr  r$  r<  r=  Zfunc_add_back_views_definitionsZ!func_add_back_views_registrations)r7  r8  r9  r:  zFunctionalInverses.hc                      s   dt t fddiS )NZview_inverse_declarationsc                    s
   t  | S r   )r&   rB  r2  r^   r_   ru   L
  s    z4gen_source_files.<locals>.<lambda>.<locals>.<lambda>)r|   rL   r^   )r  r  r^   r_   ru   I
  s     
zCompositeViewCopyKernels.cppc                      sN   dd D dd D  t tt tj t ttt ttdS )Nc                 S   s:   g | ]2}d  dd |jdkr$|jgn
|j|jgD qS )r   c                 s   s$   | ]}d |j  d|j  dV  qdS )r   z_ops.h>
#include <ATen/ops/r  Nr3  r  r^   r^   r_   r   i
  s   @gen_source_files.<locals>.<lambda>.<locals>.<listcomp>.<genexpr>N)r   r  r  r   r;  r^   r^   r_   r   h
  s   
z6gen_source_files.<locals>.<lambda>.<locals>.<listcomp>c                 S   s,   g | ]$}d  dd |j|j|jfD qS )r   c                 s   s.   | ]&}|d k	rd|j krd|j dV  qd S )Nr  r   r  )r   r   r  r^   r^   r_   r   v
  s    
rC  )r   r   rf  r  rD  r^   r^   r_   r   u
  s   
)r$  Z#CompositeViewCopyKernel_DefinitionsZ(GeneratedCompositeFunctional_DefinitionsZ!GeneratedCompositeOut_Definitions)r|   rL   r'   r/   r   rC   rD   r^   )rf   r  r  r^   r_   ru   g
  s<    
)3r2   r
   r   r   r|   r{   r6   rh  r  r   r   r/   r   r  r  rH  Zufunc_inner_loopr4   rp   r   CPUr}   CUDAr~   r  r   r"   r   tupler  r   r    r  r   osrm   r   install_dirreadFileNotFoundErrorprintr	   rF   r  r  r   r7   r8   Zwrite_shardedr   rH   )'re   r  r  r  r  r  rf   r  r  r  r  r  r  r  r  r  r  r  r  r  Zns_grouped_native_functionsZgrouped_native_functionrh  r  Z
func_grouprp   Z	fallbacksZop_nameZheader_file_nameZold_fileZ
old_headerr4  r  r5  r>  Z
all_groupsZstructured_mapZview_mapr   r^   )r  r   rf   r)  r%  rx   r  r#  r1  r.  r  r;  r  r0  r   re   r-  r"  r  r  r  r  r  r  r/  r  r  r_   gen_source_files  s|   
 *








  

 
9
rM  )r  re   ro   c                    s   |  d fdd d S )NzDeclarations.yamlc                      s   t dd  D S )Nc                 S   s   g | ]}t |qS r^   )r  r  r^   r^   r_   r   
  s     z;gen_declarations_yaml.<locals>.<lambda>.<locals>.<listcomp>)r^  r^   r  r^   r_   ru   
  rv   z'gen_declarations_yaml.<locals>.<lambda>)r  )r  re   r^   r  r_   gen_declarations_yaml
  s    
rN  r  c                   C   s   t tj S )z
    If you're depending on torchgen out-of-tree, you can use the root to figure
    out the path to native_functions.yaml
    )pathlibPath__file__parentresolver^   r^   r^   r_   get_torchgen_root
  s    rT  c            !         s:  t jdd} | jddddd | jdd	d
d | jdddd | jdddd | jdddddd | jddddd | jdddd | jdddd | jdd d!d"d# | jd$d%d&d | jd'd(d!d)d# | jd*d+d!d,d# | jd-d.dd/d | jd0d1dd2d | jd3td!d4d5d6gd4d5d6gd7d8 | jd9dd:d |  tjj}tj	
jd;}tj	
jd<}d=d>lm} t }js|tj tj|kr||tj= t|||}t| }|j|j } t|}	d?d@ |	D }
t|}dAd@ |D }j dB}t|jdCdCdD j dE}t|jdCdCdD j }t|jdCdCdD t|dF}tdG}tdG}tdG}t|dF}t|dF}tj tj!tj"tj#tj$tj%tj&h}jr|tj j'rfdHd@|D }g }j(r, fdId@j(D }j(D ]$}t)|}||kr|| qd5j*krpt+||	|
||| |||||||j,j-j.j/j0dJ d4j*krt1|||	|
|| ||||||j,j.dK d6j*krt2||dL j3r6tj34 }|j5}|j6}|dMf|dNf|dOf|dPf|dQffD ]0\}}|| }|j7||  } |8|t|  qd S )RNzGenerate ATen source files)descriptionz-sz--source-pathz!path to source directory for ATenzaten/src/ATen)helpr   z-oz--output-dependenciesz:output a list of dependencies into the given file and exit)rV  z	--dry-run
store_truez5run without writing any files (still updates outputs))actionrV  z--per-operator-headersz2generate separate headers per operator in ATen/opsz-dz--install-dirz--install_dirzoutput directoryzbuild/aten/src/ATenz--aoti-install-dirz--aoti_install_dirz%output directory for AOTInductor shimz(torch/csrc/inductor/aoti_torch/generatedz--rocmz=reinterpret CUDA as ROCm/HIP and adjust filepaths accordinglyz--mpsz'Generate MPS registration code when setz--op-registration-whitelistz--op_registration_whitelist*zfilter op registrations by the whitelist (if set); each item is `namespace`::`operator name` without overload name; e.g.: aten::empty aten::conv2d ...)nargsrV  z--op-selection-yaml-pathz--op_selection_yaml_pathaD  Provide a path to the operator selection (for custom build) YAML that contains the information about the set of selected operators and their categories (training, ...). Each operator is either a full operator name with overload or just a bare operator name. The operator names also contain the namespace prefix (e.g. aten::)z--backend-whitelistz--backend_whitelistzRfilter dispatch backend by the whitelist (if set), e.g.: CPU CUDA QuantizedCPU ...z--static-dispatch-backendz--static_dispatch_backendz?generate static dispatch code for the specific backend (if set)z!--skip-dispatcher-op-registrationz!--skip_dispatcher_op_registrationz0Avoid registering operators into the dispatcher.z--force-schema-registrationz--force_schema_registrationz}force it to generate schema-only registrations for all ops, includingthose that are not listed on --op-registration-whitelistz
--generater!  sourcesZdeclarations_yamlzGenerate only a subset of files)r   rZ  choicesr   rV  z--update-aoti-c-shimzUpdate AOTInductor C shim after adding an entry to inductor_fallback_ops in torchgen/aoti/fallback_ops.py. WARNING: Do not use this unless you are sure what you are doing!!!znative/native_functions.yamlznative/tags.yamlr   )r  c                 S   s   g | ]}t |tr|qS r^   )r{   r7   rD  r^   r^   r_   r   0  s    
 zmain.<locals>.<listcomp>c                 S   s   g | ]}t |tr|qS r^   )r{   r8   rD  r^   r^   r_   r   6  s   
z/coreT)parentsexist_okz/ops)optionsrI  r_  c                    s&   g | ]}t |st| jkr|qS r^   )r3   r   backend_whitelist)r   r   r`  r^   r_   r   c  s    c                    s   g | ]} t | qS r^   )r/   parse)r   keyr  r^   r_   r   k  s   )re   r  r  r  r  r  rf   r  r  r  r  r  r  r  r  r  r  r  r  )re   rk   r  r  r  r  rf   r  r  r  r  r  r  r  r  )re   r  r   Zcpu_vec_Zcore_Zcuda_Zops_)9argparseArgumentParseradd_argumentr   
parse_argsr  Zop_registration_whitelistr  rH  rm   r   source_pathtorchgen.modelr  r   Zmpsr   r/   ZMPSrz   r   rh   re   rf   r  r  rI  rO  rP  mkdiraoti_install_dirrK   rE  rF  r   r   r   r   r   ra  Zstatic_dispatch_backendrb  generaterM  r  r  r  r  r  r  rN  Zoutput_dependenciesrS  r   stemrR  Zwrite_outputs)!parserr  Znative_yaml_pathr   r  rl   Zparsed_yamlrk   re   r  r  Z!native_functions_with_view_groupsr  Zcore_install_dirZops_install_dirrk  r  r  r  r  r  r  r  r  rc  Zdp_keyZdepfile_pathZdepfile_nameZdepfile_stemr  r4  varnamerm   r^   )rf   r_  r_   main
  s   	


	




rp  __main__)Nri   F)ri   )N)rd  	functoolsr  rH  rO  collectionsr   r   r   dataclassesr   r   typingr   r   r	   r
   r   r   r   r   r   r   r   r   Ztorchgen.api.dispatcherapir  Ztorchgen.api.metarI  Ztorchgen.api.nativerW  Ztorchgen.api.structuredr   Ztorchgen.destr  Ztorchgen.aoti.fallback_opsr   Ztorchgen.apir   Ztorchgen.api.translater   Ztorchgen.api.typesr   r   r   r   r   r   r   Ztorchgen.contextr   r   r   r   Ztorchgen.gen_aoti_c_shimr    r!   r"   r#   Z#torchgen.gen_functionalization_typer$   r%   r&   r'   Ztorchgen.gen_vmap_plumbingr)   ri  r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   Z#torchgen.native_function_generationrB   rC   rD   rE   Z!torchgen.selective_build.selectorrF   Ztorchgen.utilsrG   rH   rI   rJ   rK   rL   rM   rN   Ztorchgen.yaml_utilsrO   rP   rQ   rR   rd   rg   r   r  rh   objectr  r   r   	lru_cacher   r   r   r   r   r   r   r   r   r   r   r  r  r  r$  r3  r8  r9  rQ  rT  rU  r^  rb  rg  rj  rs  r}  r|  r  r  r  r  r  r  r
  r  r  r  r  r  r  r  r  r  r  rM  rN  rP  rT  rp  r`   r^   r^   r^   r_   <module>   s    4$	h(
!	   
0
 
 +  
$


L`/-|DD
!
$Y 
#
 0
"

 i4&
{
 E
 
     	  
