U
    zh5S                    @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZmZmZmZmZmZmZmZ d dlZd dlZd dlZd dlZd dlm  mZ d dlm Z  d dlm!Z!m"Z"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d d	l*m+Z+m,Z,m-Z- d d
l.m/Z/ d dl0m1Z1 ddl2m3Z3mZ4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z= ddl>m>Z> ddl?m@Z@ ddlAmBZB ddlCmDZDmEZEmFZFmGZGmHZH ddlImJZJmKZK ddlLmMZM ddlNmOZOmPZP ddlQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^ ddlm_Z_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZn ddlompZp ddlqmrZrmsZsmtZtmuZumvZv ddlwmxZx ddlymzZz ddl{m|Z| ddl}m~Z~mZmZmZ ddlmZ erd d lmZ eeZejed!Zejed"Zejed#Zejed$Zed%d&G d'd( d(ZG d)d* d*Zedd+d, ZeG d-d. d.Zd/d0 ZG d1d2 d2ejjZG d3d4 d4Zeeef ZG d5d6 d6Zd7Zd8d9 Ze ZG d:d; d;e jZdS )<    N)	dataclass)	AnyCallableDictListOptionalSetTupleTYPE_CHECKINGUnion)fx)GlobalContextCheckpointStateSourceTracingContext)signpost_event)_make_graph_module)BackwardState)free_symbolsis_symbolicShapeEnv)insert_deferred_runtime_asserts)is_traceable_wrapper_subclass   )configlogging	variables)
CompiledFn
CompilerFn)create_call_functioncreate_instructionInstruction	unique_id)code_context)	PyCodegen)enter_new_scope)BackendCompilerFailed!exceptions_allowed_to_be_fallback	SkipFrameunimplementedunimplemented_with_warning)GuardBuilderinstall_guard)is_dynamic_nn_module)AttributeMutationExistingSideEffects)
AttrSourceBackwardStateSourceConstantSourceGetItemSourceGlobalStateSourceis_constant_sourceis_from_local_sourceLocalSourceParamBufferSourceShapeEnvSourceSyntheticLocalSourceTensorPropertyTensorPropertySource)checkpoint_paramsCleanupHookclone_inputscount_callscountersdynamo_timedget_instruction_source_311get_locals_to_stealget_static_address_typegraph_break_reasonsincrement_op_countlazy_format_graph_code
LazyStringnn_module_proxysameset_example_value)VariableTracker)BackwardStateGraphArgGraphArgTrackedFakeVariableBuilderwrap_fx_proxy)BaseListVariable)NullVariable)NNModuleVariable)NumpyNdarrayVariableSymNodeVariableTensorVariableUnspecializedPythonVariable)TensorWithTFOverrideVariable)InstructionTranslatorBasegraphZ
graph_codeZgraph_sizesZ
trace_callT)frozenc                   @   s   e Zd ZU eed< eed< dS )VariableTrackerCacheKeyZvt_idsourceN)__name__
__module____qualname__int__annotations__r    rd   rd   L/var/www/html/venv/lib/python3.8/site-packages/torch/_dynamo/output_graph.pyr]   u   s   
r]   c                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )VariableTrackerCachec                 C   s
   i | _ d S N)cacheselfrd   rd   re   __init__   s    zVariableTrackerCache.__init__c                 C   s&   t t||}|| jkrd S | j| S rg   r]   idrh   )rj   valuer^   keyrd   rd   re   lookup   s    
zVariableTrackerCache.lookupc                 C   s   t t||}|| j|< d S rg   rl   )rj   rn   r^   vtro   rd   rd   re   add   s    zVariableTrackerCache.addc                 C   s   t  }|j| j |S rg   )rf   rh   update)rj   Z	new_cacherd   rd   re   clone   s    zVariableTrackerCache.clonec                 C   s   | j   d S rg   )rh   clearri   rd   rd   re   ru      s    zVariableTrackerCache.clearN)r_   r`   ra   rk   rp   rr   rt   ru   rd   rd   rd   re   rf   ~   s
   rf   c                   C   s
   t tS rg   )torchdynamo_loggingZget_step_loggerlogrd   rd   rd   re   _step_logger   s    rx   c                   @   s<   e Zd ZU dZeed< eej ed< dZ	e
ed< dd ZdS )	GraphCompileReasonzOStores why a given output graph was compiled; i.e. what caused the graph break.reasonZ
user_stackTgraph_breakc                 C   s   | j rt|  d S rg   )r{   rE   appendri   rd   rd   re   __post_init__   s    z GraphCompileReason.__post_init__N)r_   r`   ra   __doc__strrc   r   	tracebackFrameSummaryr{   boolr}   rd   rd   rd   re   ry      s
   
ry   c                    s    fdd}|S )Nc                      s   dd  D S )Nc                 S   s   g | ]\}}}|||qS rd   rd   ).0fnargskwargsrd   rd   re   
<listcomp>   s     zE_get_gen_rand_values_fn.<locals>._gen_rand_values.<locals>.<listcomp>rd   rd   random_callsrd   re   _gen_rand_values   s    z1_get_gen_rand_values_fn.<locals>._gen_rand_valuesrd   )r   r   rd   r   re   _get_gen_rand_values_fn   s    r   c                       s:   e Zd ZdZeeejjf d fddZ	dd Z
  ZS )FakeRootModulez'Trick the constructor of fx.GraphModule)
nn_modulesc                    s,   t    | D ]\}}t| || qd S rg   )superrk   itemssetattr)rj   r   kv	__class__rd   re   rk      s    
zFakeRootModule.__init__c                 C   s   dS )NzFakeRootModule(...)rd   ri   rd   rd   re   __repr__   s    zFakeRootModule.__repr__)r_   r`   ra   r~   r   r   torchnnModulerk   r   __classcell__rd   rd   r   re   r      s   r   c                   @   s4   e Zd ZedddZejjeej	 dddZ
dS )WrapperBackendbackendc                 C   s
   || _ d S rg   r   )rj   r   rd   rd   re   rk      s    zWrapperBackend.__init__)gmexample_inputsc                 C   s   t || _|| _t| j}| ||| _| jd ksB| j| jjkrJ| jjS tj	sV| jS zvzP| jjt
| }| jt
| }t||r| jW W BS td|  | jjW W &S  tk
r   td  Y nX W 5 |   X d S )Nzincorrect results of backend zerror in verify_correctness)r<   Zrestorer   copydeepcopyr   	candidateforwardr   verify_correctnessr>   rJ   RuntimeError	Exceptionrw   	exception)rj   r   r   Zcopy_gmZcorrectresultrd   rd   re   __call__   s&    


zWrapperBackend.__call__N)r_   r`   ra   r   rk   r   r   GraphModuler   Tensorr   rd   rd   rd   re   r      s   r   c                       s6  e Zd ZdZeeef ee e	e
e
d fddZdd Zdedd	d
Zdd Zdd Zdd Zeg ef dddZdd Zedd Zedd Zdd Zedd Zejdd Zedd  Zed!d" Zd#d$ Zd%d& Zd'd( Ze j!d)d* Z"ed+d, Z#ed-d. Z$ed/d0 Z%ee&j'j(d1d2d3Z)eeeef d1d4d5Z*dd7d8Z+d9d: Z,d;d< Z-ed=d> Z.e/d?d@dAZ0dBdC Z1dDdE Z2dFdG Z3ddIdJZ4dKdL Z5e6dMdN Z7e8e&j9j:e&j;ef dOdPdQZ<dRdS Z=dee> dUdVdWZ?dXdY Z@dZd[ ZAd\d] ZBed^d_d`ZCe j!dadb ZDe&j'jEF dcdd ZGeeHeIjJ d1dedfZKeeHe/ d1dgdhZLeMdidjeIjNeOdkdldmZPeHe&j; d1dndoZQd6d1dpdqZReHeS d6drdsdtZTd6d1dudvZUed1dwdxZVed1dydzZWd6d1d{d|ZXe	d6d}d~dZYeeIjNgd6f d6dddZZe&jIjJdddZ[  Z\S )OutputGrapha  
    Wrapper class to hold outputs of InstructionTranslator.  Mainly the
    generated fx.Graph.

    OutputGraph is 1:1 with a frame being processed. Each frame is associated
    with some root InstructionTranslator. When user code calls a function,
    we construct a InliningInstructionTranslator that continues to write into
    the root InstructionTranslator's OutputGraph.
    )code_optionscompiler_fnexportlocal_scopeglobal_scopec
              	      s  t    t| |dg| _i | _|| _|| _|| _i | _g | _	t
t| _t | _|	j|	j|	jd| _g | _t | _t| jtjtjtjtj| jd}
dd lm  m} |jdd$ tjj |
| jrdnd| jd}W 5 Q R X t!|| _"| #  t$%t&| _'t( | _)t* | _+t, | _-t./ | _0t(|| _1g | _2d| _3g | _4|| _5|| _6|| _7|| _8i | _9g | _:g | _;d| _<i | _=tj>? | _@d| _Ati | _Bti | _C| D  i | _Eg | _Fd | _Gg | _Hi | _Id | _Jd | _K| L | _Mt | _Nd S )	N)export_root)co_nameco_filenameco_firstlineno)tracked_fakesZallow_scalar_outputsZallow_dynamic_output_shape_ops+prefer_deferred_runtime_asserts_over_guards(_allow_complex_guards_as_runtime_asserts	co_fieldsr   FZ(fake_tensor_allow_unsafe_data_ptr_accessT)	shape_envZallow_non_fake_inputsr   )Or   rk   SubgraphTracertracersZinput_source_to_varr   export_constraintsframe_stateZinput_source_to_sizes_stridescleanup_hooksnext_compile_id_counter
compile_idsetinstalled_globalsr   r   r   r   r   bound_symbolsr   r   Zcapture_scalar_outputsZ capture_dynamic_output_shape_opsr   r   torch._functorch.config
_functorchpatchr   _subclassesFakeTensorModer   tracing_contextinit_ambient_guardscollectionsdefaultdictlistZtracked_fakes_id_to_sourcedictparam_name_to_sourcer.   side_effectsrf   variable_tracker_cache	itertoolscountunique_var_idr   output_instructions	timestampregister_finalizer_fnsr   r   r   root_txsource_to_user_stacks_current_txcleanupsshould_exitZunspec_variable_map_CZ_is_torch_function_enabledtorch_function_enabled!has_user_defined_allowed_in_graphnon_compliant_opscompliant_custom_opssave_global_state dynamo_flat_name_to_original_fqnr   random_values_varpregraph_bytecodebackward_statebackward_state_proxybackward_state_var!install_builtins_dict_in_fglobalsZ%name_of_builtins_dict_key_in_fglobalsZguard_on_key_order)rj   r   r   r   r   r   r   r   r   f_coder   _config	fake_moder   rd   re   rk      s    


	



zOutputGraph.__init__c                 C   s&   | j d }t|ts|j}| d|S )N__builtins__Z__builtins_dict__)r   
isinstancer   __dict__install_global)rj   
f_builtinsrd   rd   re   r     s    

z-OutputGraph.install_builtins_dict_in_fglobalshook)r   c                 C   s6   | t | j }|| jks t|| j|< ||  fS rg   )lenr   AssertionErrorget_backward_state_proxy)rj   r   prefixnamerd   rd   re   add_backward_state_hook  s    
z#OutputGraph.add_backward_state_hookc                 C   s^   | j d krX| jrtd | jjdtt d| _ t | j jj	d< t
| j jt  |  | _| j S )Nz&backward_state does not support exportZdynamo_backward_state)r^   grapharg)r   r   r(   root_tracercreate_graph_inputr   r0   rM   nodemetarK   new_varr   ri   rd   rd   re   r     s    
  
z$OutputGraph.get_backward_state_proxyc                 C   s   | j t tj | j t tj | j t tj | j t tj	 | j t tj
 tjj }|d k	r| j t tj d S rg   )guardsrr   r8   
make_guardr*   Z	SHAPE_ENVr3   ZDETERMINISTIC_ALGORITHMSZ	GRAD_MODEZDEFAULT_DEVICEZTORCH_FUNCTION_STATEr   r   r   Zpeek_interpreter_stackZFUNCTORCH_STACK_MATCH)rj   cird   rd   re   r     s    zOutputGraph.init_ambient_guardsc                 C   s   || }|   }t| j}||j|j |ttj	j
| |t|d || | j|  t|}t| j||}t jj| |S )z]
        call fn(*args) before the graph runs and turn the result into a fake input.
        T)r   r#   r   Zload_import_fromr`   r_   Zforeachmapr   ZConstantVariablecreatecall_functionr   storer   extendget_instructionsr9   rP   r   getguards_contextdynamo_guardsZremove_guards_with_source)rj   r   r   example_valuevarnamecgr^   r   rd   rd   re   synthetic_graph_input  s"    

z!OutputGraph.synthetic_graph_input)r   c                 C   s   | j | d S rg   )r   r|   )rj   r   rd   rd   re   add_cleanup_hook  s    zOutputGraph.add_cleanup_hookc                 C   s$   t | jD ]
}|  q
| j  d S rg   )reversedr   ru   )rj   r   rd   rd   re   call_cleanup_hooks  s    zOutputGraph.call_cleanup_hooksc                 C   s
   | j d S Nr   r   ri   rd   rd   re   r     s    zOutputGraph.root_tracerc                 C   s
   | j d S Nr  ri   rd   rd   re   current_tracer  s    zOutputGraph.current_tracerc                 C   s   t | jdkS )Nr   )r   r   ri   rd   rd   re   is_root_tracer  s    zOutputGraph.is_root_tracerc                 C   s   | j jS rg   r  r[   ri   rd   rd   re   r[     s    zOutputGraph.graphc                 C   s   || j _d S rg   r  )rj   rn   rd   rd   re   r[     s    c                 C   s   | j jS rg   )r  input_name_to_proxyri   rd   rd   re   r    s    zOutputGraph.input_name_to_proxyc                 C   s   | j jS rg   )r  real_value_cacheri   rd   rd   re   r    s    zOutputGraph.real_value_cachec                 O   s   | j j||S rg   )r  create_proxyrj   r   r   rd   rd   re   r    s    zOutputGraph.create_proxyc                 O   s   | j j||S rg   )r  create_noder  rd   rd   re   r    s    zOutputGraph.create_nodec                 O   s   | j j||S rg   )r  remove_noder  rd   rd   re   r     s    zOutputGraph.remove_nodec                 c   sp   t  }zJ|r|j| jkst|  |r,|nt| | j|d}| j	| |V  W 5 |d d d  | j  X d S )N)parentsource_target)
r$   __exit__r   popr!  r  r   	__enter__r   r|   )rj   r"  Zprior_tracerZnew_scope_ctxtracerrd   rd   re   	subtracer"  s"      
zOutputGraph.subtracerc                 C   s   | S rg   rd   ri   rd   rd   re   output7  s    zOutputGraph.outputc                 C   s   | j jS rg   )r   r   ri   rd   rd   re   r   ;  s    zOutputGraph.fake_modec                 C   s
   | j jjS rg   )r   r   r   ri   rd   rd   re   r   ?  s    zOutputGraph.shape_env)returnc                 C   s
   | j jjS rg   )r   r  r  ri   rd   rd   re   r  C  s    zOutputGraph.guardsc                 C   s
   | j jjS rg   )r   Zmodule_contextr   ri   rd   rd   re   r   G  s    zOutputGraph.nn_modulesNc                 C   s   |dk	r|n| j jj}| j| jf|d< tjt f|d< t	tj
dtdf|d< t	tj
dtdf|d< t	tjdtdf|d< t	tjdtdf|d	< tjt f|d
< dS )zc
        Saves to out if it is provided. Else saves to the tracing context's global_state.
        Nr   grad_enabledcudaZautocast_enabledcpuZautocast_cpu_enabledZautocast_gpu_dtypeZautocast_cpu_dtypeZautocast_cache_enabled)r   global_contextglobal_stateset_torch_function_stater   r   Zset_grad_enabledis_grad_enabled	functoolspartialZset_autocast_enabledZis_autocast_enabledZset_autocast_dtypeZget_autocast_dtypeZset_autocast_cache_enabledZis_autocast_cache_enabled)rj   outr.  rd   rd   re   r   K  s*    zOutputGraph.save_global_statec                 C   s   | j | d S rg   )r   r|   )rj   txrd   rd   re   push_txt  s    zOutputGraph.push_txc                 C   s
   | j  S rg   )r   r$  ri   rd   rd   re   pop_txw  s    zOutputGraph.pop_txc                 C   s   | j s| jS | j d S r  )r   r   ri   rd   rd   re   
current_txz  s    zOutputGraph.current_txargc                    sD   j r
d S |jd k	stfdd  fdd|j|j d S )Nc                    s   t | rt| jjtjsd S | jj}| jkr2d S  j| t	d| |
   jjt|tjd|d}t|j|  t|| dd dd|jjd< d S )Nzbind_symint %s %sT)beforer^   F)Zpass_arg_as_tensorfake_tensorZ	is_tensorr   )r   r   r   exprsympySymbolr   rr   rw   debugr   r   r   r   r   SymIntrK   rN   r   )sprops0proxyri   rd   re   bind_symint  s*    
z4OutputGraph.add_symbol_bindings.<locals>.bind_symintc                    sB  t |  D ]\}} |t|tj| q| jtjkrxt |  D ]\}} |t|tj	| qB | 
 t|tj n| jtjkr|  | |  | n`| jtjtjhkr҈|  | |  | n0| jtjtjhkr|  | |  | t| r>|  \}}|D ] }t| |}|t|| qd S rg   )	enumeratesizer;   r:   ZSIZEZlayoutr   ZstridedstrideZSTRIDEstorage_offsetZSTORAGE_OFFSETZ
sparse_cooZ_indicesZ_valuesZ
sparse_csrZ
sparse_bsrZcrow_indicesZcol_indicesZ
sparse_cscZ
sparse_bscZccol_indicesZrow_indicesr   Z__tensor_flatten__getattrr/   )tsrcirA  attrsctxattrZinner_t)rE  handle_tensorrd   re   rQ    s.    


z6OutputGraph.add_symbol_bindings.<locals>.handle_tensor)r   r;  r   r^   rj   r9  rd   )rE  rQ  rj   re   add_symbol_bindings~  s    zOutputGraph.add_symbol_bindingsc                 C   s
   t | jS rg   )r?   r[   ri   rd   rd   re   r?     s    zOutputGraph.count_callsc                 C   s   t t| jjdkS r  )r   r   r[   nodesri   rd   rd   re   is_empty_graph  s    zOutputGraph.is_empty_graphc                 C   s@   |st | j}|dD ]"}t|tr0|| }qt||}q|S N.)r   r   splitr   r   rJ  )rj   keysobjr   rd   rd   re   get_submodule  s    

zOutputGraph.get_submoduletmpc                 C   sH   t | jd }| dt| j }||kr| jd  |f7  < |S qd S )Nco_varnames_)r   r   r   r   )rj   r   existingvarrd   rd   re   r     s
    zOutputGraph.new_varc                 C   s&   || j d kr"| j d  |f7  < dS )z/Ensure self.code_options.co_names contains nameco_namesN)r   )rj   r   rd   rd   re   update_co_names  s    zOutputGraph.update_co_namesc                  G   sV   d tt| }tdd|}tdd|}tdd|}|rJ|d  sRd| }|S )	Nr^  z^[GL]\['?(.*?)'?\]$z\1z	\[(\d+)\]z_\g<1>z[^a-zA-Z0-9]r   sub)joinr  r   rerc  isalpha)namesr   rd   rd   re   module_key_name  s    zOutputGraph.module_key_nametargetc                    s  t jjrtjfS tdks0td ttrFttt	j
r|j sfjfdd}ntt	jjrtt	jjstrttj fdd}nfdd}n2tt	jt	jfrfdd}nfdd}j D ] \}}|kr||  S qtj|   }t D ]} jkrڈj < tt	jjr· fdd	}	td
r D ]\}
}|	|
 qtdrΈ D ]\}
}|	|
 q|   S | d|  qDtdd S )Nr^   c                    s   j d k	stj | < jjjkr4jjj S tdkrRttj	 nt
sjttj tjd| t i fdi }jjj|}|S )NZguardedget_attrr  )r   r   r   r(  r   rD   r+   r  r*   ZID_MATCHr4   ZTENSOR_MATCHrQ   r  tupleZtrack_object_existing)
module_keyrq   )optionsrj   r^   rj  r&  rd   re   	wrap_name
  s$    
	z6OutputGraph.register_attr_or_module.<locals>.wrap_namec                    s   t t| f S rg   )rT   typerm  rn  rj  rd   re   ro  *  s    c                    s   t jf S rg   )r   UnspecializedNNModuleVariablerq  rr  rd   re   ro  2  s    c                    s&   t jd| t i fdi S )Nrk  Zsym_num)rV   r  r  rl  rq  )rn  rj   rj  rd   re   ro  =  s    c                    s*    j |   j| < t t| dS )N)Zsource_name)r(  rb  r   rP   r1   rq  )rj   rj  rd   re   ro  H  s
    
c                    sR   j d k	stt| }  d|  }|j |< ttrN| jt| < d S rV  )	r   r   r7   r   r6   r   r   rh  r   )	leaf_nameZ
new_sourcenew_name)r   rj   r^   rd   re   register_leaf_name\  s    


z?OutputGraph.register_attr_or_module.<locals>.register_leaf_name_parameters_buffersr^  Zunreachable)r,   r   r   r   rs  r   r   r   r7   r   r   r  r  r   r   r   r+   r  r*   Z	NN_MODULEr@  ZSymFloatr   r   r   rh  r   r   hasattrZnamed_parametersZnamed_buffers)rj   rj  rg  rn  ro  r   r   baserM  rv  rt  r^  rd   )r   rn  rj   r^   rj  r&  re   register_attr_or_module  sN    


z#OutputGraph.register_attr_or_modulec                 C   s  | j d}t|}|sg S g }i }|j|j | jj }|r|	 }t
|trpt
|jtsdt||j7 }q>t
|ttfr>t
|jtr>t
|jjtr>|jjj|ksq>|jjj}||krg ||< || | q>i }	| jD ]}
t
|
jtrt
|
jtr|
jj|ksq|
jj}|| jd ks$t|| D ]l}|jj}||	kr| | d}||	|< |td|dtd|dtdtd|dg t|	| |_q,q|S )	Nrj   r]  Z_ref	LOAD_FASTargval
LOAD_CONSTBINARY_SUBSCR
STORE_FAST)r   r
  rC   stacksymbolic_localsvaluesr   Zstore_attr_mutationsrY  r$  r   rR   r   r   r   rL   r-   r^   r2   rz  r6   
local_namer|   	graphargs_exampler   r   indexr   r  r   )rj   r4  Zmaybe_gmZstolen_list_namesZalias_instsZneeds_aliasqueuexZstolen_namevisitedr9  Z	list_nameZlist_idxZ
alias_namerd   rd   re   handle_aliases_for_stolen_listst  sr    













z+OutputGraph.handle_aliases_for_stolen_listsF)rz   c                    s  |dk	st ddlm} |_|_d_td| tdd j	D sRt
d g  tjd	krĈjD ]\}|jd
kr td
|jd qf|jdkr tdtjd d qf t| qfjr؈jrt d j    fdd}tj	D ]}| q    tj}|D ]}	|	  q<dd j ! D }
t"|
}g }i }|rt ||d < j#! D ]~\}}t$|j%t&r|j%j'|krqtjdkrt()t*|rqnt()t*|rt ||krt ||< || | q|+ D ],}|||  ||gt||   qtj,dkr|  g }-d_.|t/j,}0d|}t1|}||2|d |t3dd ||4j5j. 6| |rtdd |D rtdd |D rtt7|t|krj89 rtj:dksj;s|  6<tt||tdt|dg  6fddt|D  n-d}t1||}=|| t1||d d |j>! D d!}=|| d}g }t?j@dkst|jAdkrP|<|B | t|jAdkrB||4| d}n|td" |  6||C   6fd#dt|D  |r6t1D|g dS )$zw
        Generate a subgraph to continue execution on user code.
        Automatically restore live variables.
        Nr   disableTzCOMPILING GRAPH due to %sc                 s   s   | ]}|  V  qd S rg   )Zcan_restore)r   blockrd   rd   re   	<genexpr>  s     z/OutputGraph.compile_subgraph.<locals>.<genexpr>z&compile_subgraph with block_depth != 0      Z	MAKE_CELLr}  ZCOPY_FREE_VARSco_freevarsr8  z)export does not support pregraph_bytecodec                      s         d S rg   )add_output_instructionsru   rd   )prefix_instsrj   rd   re   append_prefix_insts  s    
z9OutputGraph.compile_subgraph.<locals>.append_prefix_instsc                 S   s   i | ]\}}|t |qS rd   )rI   )r   r   modrd   rd   re   
<dictcomp>  s     z0OutputGraph.compile_subgraph.<locals>.<dictcomp>r  )r     r   Zrandom_valuesZ__gen_rand_valuesFc                 s   s8   | ]0}t |tttf o.t |to,| tk V  qd S rg   )r   rX   rU   rY   rV   Zpython_typefloat)r   r   rd   rd   re   r  5  s   
c                 s   s   | ]}t |tV  qd S rg   )r   rW   r   r  rd   rd   re   r  A  s     UNPACK_SEQUENCEc                    s   g | ]}t  |qS rd   r#   create_storer   r`  r4  rd   re   r   O  s     z0OutputGraph.compile_subgraph.<locals>.<listcomp>Z	graph_outc                 S   s   i | ]\}}|d kr|dqS )r   Nrd   )r   valr   rd   rd   re   r  [  s       )ZtempvarsPOP_TOPc                    s   g | ]}t  |qS rd   r  r  r  rd   re   r   p  s     )Er   
decoratorsr  partial_convertcompile_subgraph_reasonr   rw   r?  allZblock_stackr(   sysversion_infor  opnamer|   r   r~  r   r   r   r   r   r  r  r  exitcleanup_graphZprune_dead_localsr   r  Zrealizer   r   r   r  r   r^   r6   r  rp  __instancecheck__rS   rY  r   r   r   r   r   r#   Zload_function_namer   r  r(  r  r   r   Zis_emptydebug_localsr   compile_and_call_fx_graphcodegen_suffixZusesr?   r[   Zgraph_outputsZgraph_output_varsr	  Zcreate_delete)rj   r4  r  rz   r  instr  r  stack_valuesrn   Znn_modules_proxiesrootZrestore_varsZval_to_namesr   r   Zrandom_calls_instructionsZrand_fnZrand_fn_namecodegenZgraph_output_varZpass1Zpass2Zstored_graph_output_varr(  rd   )r  rj   r4  re   compile_subgraph  s   



 







 zOutputGraph.compile_subgraphc           	      C   s   | j rH| jrt| j  D ],\}}|| ||| j || q| j	| | j
| |jD ]F\}}|| |D ]}|| qz|tt|d |tdg qf|j||j d | j| d S )NTr  )Zvalue_from_source)r   r   r   r   Zappend_outputZcreate_loadr   Z
store_attrr   Zcodegen_hooksZcodegen_save_tempvarsr  Zextend_outputr   r   r   Zrestore_stackZcodegen_update_mutated)	rj   r4  r  r  r   r  Z	debug_varr   r9  rd   rd   re   r  x  s     

zOutputGraph.codegen_suffixc                 C   s   | j s
tt| jj}|D ]}|jdd qt }t	||dd D ]\}}|j
tjjkrHt|j| fkrH|jsH|jd }|j
tjjkrHt|j| fkrH|jsH|jd }| j| | j| qHdS )z
        Remove "creation_timestamp" from node meta

        Remove this pattern from the graph:
            torch._C._set_grad_enabled(False)
            torch._C._set_grad_enabled(True)
        creation_timestampNr   r   )r   r   r   r[   rT  r   r$  r   r0  ziprj  r   Z_set_grad_enabledrl  r   Z_erased
erase_node)rj   rT  r   r*  Znode1Znode2rd   rd   re   r    s,    


zOutputGraph.cleanup_graphc                 C   sN   i }| j jD ]<}|jdd }t|tjjr| }dd |D ||j	< q|S )Nr  c                 S   s"   g | ]}t |tr|nt|qS rd   )r   rb   repr)r   rA  rd   rd   re   r     s     z:OutputGraph.get_graph_sizes_structured.<locals>.<listcomp>)
r[   rT  r   r
  r   r   r   
FakeTensorrG  r   )rj   retr   r  rG  rd   rd   re   get_graph_sizes_structured  s    z&OutputGraph.get_graph_sizes_structured)r   c           	      C   s   d}|d| d7 }| j jD ]}|jdd }t|tjjr| }||j	 dt
| d7 }g }d}|D ]>}t|tr|| qjt|tjrd}||jj qj qqj|r||j	 d	t
| d7 }q|S )
NzTRACED GRAPH TENSOR SIZES
z===== z =====
r  z: 
FTz (concrete): )r[   rT  r   r
  r   r   r   r  rG  r   rl  rb   r|   r@  r   hint)	rj   r   Zgraph_sizes_strr   r  rG  Zconcrete_sizeZ
has_symintszrd   rd   re   get_graph_sizes  s*    
zOutputGraph.get_graph_sizesc              
   c   sN   | j j }i }| j|d z| j j| dV  W 5 | j jt| X dS )zj
        Momentarily restores the global state to what it was prior to tracing the current output
        )r3  N)r   r-  Zcopy_graphstater   Zrestore_graphstater   )rj   Zprior_global_stateZcurrent_global_staterd   rd   re   restore_global_state  s    
z OutputGraph.restore_global_statec              	      sF  ddl m} jsttd}t|ts,tt|ts:tddj	
tdd |D fi  tjstt|jj|   tj}td d  |7  < j  t|j jD ]}|  qj _j  jd	< td
t | ddd t!j"j#dfdd fddd $  j%j&}j's|ddl(m)  m}	 |	j*dd t!j+j,|jd}
W 5 Q R X |
j%_&-  . }W 5 Q R X ddl/m0} t||stt1|dd|r|j2dkrt||r|n|j3}|4| t||s|j5}||}td d  d7  < 6|| t7|}|8| |9 S )zr
        Generate code from self.graph and return the Instruction()s to
        call that generated code.
        r   r  Z__compiled_fnr(  c                 s   s   | ]}|  V  qd S rg   )Zas_proxyr  rd   rd   re   r    s     z8OutputGraph.compile_and_call_fx_graph.<locals>.<genexpr>statsZcalls_capturedr   %sT)include_strideinclude_deviceZdynamo_output_graphc                      s   d   iS )Nsizes)r  rd   ri   rd   re   <lambda>      z7OutputGraph.compile_and_call_fx_graph.<locals>.<lambda>c                      s    j ddddS )NFT)Zprint_outputr  r  )Zprint_readablerd   )r   rd   re   r    s     )Z
payload_fnr   NFr   )r   )_LazyGraphModule__self__Z_lazy_forwardZunique_graphs):r  r  r   r   r!   r   r   r   r  r  Z
create_argrl  r   Zdo_not_emit_runtime_assertsr   r   r   r[   r   remove_unused_graphargsr?   r@   r  ru   r   r   r  r   r   r   graph_code_logr?  rG   r   _loggingZtrace_structuredr  r   r   r   r   r   r   r   r   r  call_user_compilertorch.fx._lazy_graph_moduler  rJ  r_   r  Zforce_recompiler   install_global_unsafer#   Zmake_call_generated_coder	  )rj   r4  rvr  r  r   Zncallsregister_finalizerZold_fake_moder   Zbackend_fake_modecompiled_fnr  Zlazy_gmr  rd   )r   rj   re   r    s    









z%OutputGraph.compile_and_call_fx_graphc                 C   s   | j jddS )Nplaceholder)op)r[   Z
find_nodesri   rd   rd   re   placeholdersI  s    zOutputGraph.placeholdersc                 C   s   dd | j D S )Nc                 S   s   g | ]}|j d  qS )r   r   )r   r   rd   rd   re   r   O  s     z)OutputGraph.graphargs.<locals>.<listcomp>)r  ri   rd   rd   re   r  M  s    zOutputGraph.graphargsZbackend_compile)Z
phase_name)r   r)  c              
   C   s  | j d k	std}g }|jjD ]*}|jdkr4|d7 }|jdkr|| qt| |D ]}|jd }|j|_	qV| j
|_| j|_zvt| j dr| j jnd}t tjd|  | j }tjrt|}|||  }	t tjd	|  t|	std
W n tk
rX }
 zD| jr&t| j |
|
jd d| j  d}t|
| jj | W 5 d }
~
X Y n` t!k
r~ }
 z|
W 5 d }
~
X Y n: t"k
r }
 zt| j |
|
jd W 5 d }
~
X Y nX t#dd| j$|t%|jjt%|d |	S )Nr   r  call_methodcall_moduler   r  r   r_    zcalling compiler function zdone compiler function z#compiler_fn did not return callablez9Backend compiler failed with a fake tensor exception at 
zAdding a graph break.ZdynamoOutputGraph.call_user_compiler)Zop_countZ
node_countZinput_count)&r   r   r[   rT  r  r|   rF   r   r^   Z_dynamo_sourcer   Z_param_name_to_sourcer   Z_source_to_user_stacksry  r_   rx   r   INFOr   r   r   r   callabler&   r   r%   with_traceback__traceback__r   Zformat_frame_summaryr)   r   r'   r   r   r   r   )rj   r   Ztotr  r   plr9  r   r   r  emsgrd   rd   re   r  Q  sl    





 
r  c                 C   s    g }| j D ]}||j q
|S rg   )r  r|   example)rj   r   r9  rd   rd   re   r     s    
zOutputGraph.example_inputsc                    sP  j s
ttjjddd}tjjddd  fdd}d	d
 }ttjjD ]t}t	t|j
dkrR|jdks|jdkr|jtjks|jdkr|jtjkr||jd s||s||rR| qRdd }fdd}t ttjtjf dddg }jD ]}||d k	}|r2|j
s|| n|j
sVt|jd tsV|| n|jd }	t|	trpqt|jd jtjr|jd j}
|jd j}t|
  }|! D ]2}t"|j#|}t$%tjtjffdd| qq|	j&d k	r|	j&n|	j}| q|D ]6}||}|d k	r|kr>|| n
'| qd S )N)b_nodec                 S   sf   | dkrdS t | tjsdS | jd}|d kr4dS |dkr@dS t |tjrb|j  }d k	rb|S dS )NTFr  )	r   r   Noder   r
  r   ZSymBoolr   Zmaybe_as_bool)r  brrd   rd   re   is_static_true  s    
z;OutputGraph.remove_unused_graphargs.<locals>.is_static_true)ac                 S   sB   ddl m} t| tttfr dS t| tjr>t| j	d|S dS )Nr   SymTypesTr  F)
torch.fx.experimental.sym_noder  r   rb   r  r   r   r  r   r
  )r  r  rd   rd   re   is_symnode_arg  s    z;OutputGraph.remove_unused_graphargs.<locals>.is_symnode_argc                    sp   ddl m} | jdkrdS t| jd|s0dS t fdd| jD sLdS t fdd| j	 D sldS d	S )
Nr   r  r  Fr  c                 3   s   | ]} |V  qd S rg   rd   r   r  r  rd   re   r    s     zWOutputGraph.remove_unused_graphargs.<locals>.is_symnode_compute_node.<locals>.<genexpr>c                 3   s   | ]} |V  qd S rg   rd   r  r  rd   re   r    s     T)
r  r  r  r   r   r
  r  r   r   r  )r   r  r  rd   re   is_symnode_compute_node  s    
zDOutputGraph.remove_unused_graphargs.<locals>.is_symnode_compute_nodec              	   S   s   | j dkr2t| jd jdtjr2| jdkr2dS | j dkr| jtjj	j
tjj	j
jtjj	j
jtjj	jtjj	jjtjj	jjtjj	jtjj	jjfkrdS dS )Nr  r   r  )rG  rH  rI  itemTr  F)r  r   r   r   r
  r   r   rj  ZopsatenZsym_sizedefaultrb   Z
sym_strideZsym_storage_offsetr   rd   rd   re   is_accessor_node  s&    





z=OutputGraph.remove_unused_graphargs.<locals>.is_accessor_noder   rk  r  c                 S   s8   | j d }|j}t|tjr4t|jjtjr4|jjS d S Nr   )	r   r  r   r   r@  r   r<  r=  r>  )r   r9  r  rd   rd   re   placeholder_binds_symbol  s    
 zEOutputGraph.remove_unused_graphargs.<locals>.placeholder_binds_symbolc                    s<   t d| jd j  | jd=  |   j| d  d S )NzREMOVE UNUSED GRAPHARG %sr   )rw   r?  r   r^   r   r   r  r$  r  ri   rd   re   remove_unused  s    
z:OutputGraph.remove_unused_graphargs.<locals>.remove_unused)fakec                 S   s   | t |O } d S rg   )r   )used_symbolsr  rd   rd   re   update_used_symbols  s    z@OutputGraph.remove_unused_graphargs.<locals>.update_used_symbolsr   c                    s
    | S rg   rd   )rK  )r  r  rd   re   r  +  r  z5OutputGraph.remove_unused_graphargs.<locals>.<lambda>)(r   r   r   r   ZArgumentr  r   r[   rT  r   usersr  rj  operatorgetitemr   _checkr   r   r   r   r@  r   r  r|   r   r   rM   r  ZScriptObjectZexample_strong_refr   Z__obj_flatten__rY  rJ  Zwrapped_objpytreeZtree_map_onlyr;  remove)rj   r  r  r  r   r  r  Zrecheck_placeholdersZbinds_symbolr9  Zreal_script_objZfake_script_objZ	flat_dictrP  Zfake_attr_valr  symbolrd   )r  rj   r  r  re   r    s~    



		
 





z#OutputGraph.remove_unused_graphargs)r   r)  c                 C   s   | j | d| _dS )zt
        We call this on the creation of a new compiled subgraph that is inserted
        before user code.
        TN)r   r  r   )rj   r   rd   rd   re   r  >  s    z#OutputGraph.add_output_instructionsc                 C   s6   || j kst| j | | jt| j|| dS )a`  
        WARNING: prefer the safer `install_global_by_id/install_global`.
        torch.compile instances should be independent of each other;
        one footgun is to have one instance depend on the existence of
        a global installed by another instance. This can happen if we mangle
        a global the same way across both instances.
        N)r   r   rr   r   r|   r=   r  r   )rj   r   rn   rd   rd   re   r  F  s    z!OutputGraph.install_global_unsafec                 C   s8   | dt | d| j }|| jkr(|S | || |S )z
        Installs a global if it hasn't been installed already.
        This is determined by (prefix, id(value)) pair.

        Returns the name of the newly installed global.
        r^  Z_c)rm   r   r   r  rj   r   rn   r   rd   rd   re   install_global_by_idR  s
    	
z OutputGraph.install_global_by_idc                 C   s   t |}| || |S )z~
        Installs a global, generating a unique name for it.

        Returns the name of the newly installed global.
        )r!   r  r  rd   rd   re   r   a  s    zOutputGraph.install_globalc                 C   s   d | _ | j  d | _| jjD ]}d|jkr|jd= q| j  | j  | j	  | j
  | j  | j  | j  d S r  )r   r   ru   r   r[   rT  r   r  r  r   r   r   r   r   rj   r   rd   rd   re   cleanupl  s    








zOutputGraph.cleanup)enabledr)  c                 C   s
   || _ d S rg   )r   )rj   r  rd   rd   re   r/  ~  s    z$OutputGraph.set_torch_function_state)r  r)  c                 C   s   | j | d S rg   )r   r|   )rj   r  rd   rd   re   add_graph_finalizer  s    zOutputGraph.add_graph_finalizerr  c                 C   s0   |j dkr|jd jS |j dks$t| j|j S )z#Extract the non-fake example tensorr  r   rk  )r  r   r  r   r   rj  r  rd   rd   re   example_value_from_input_node  s    
z)OutputGraph.example_value_from_input_node)r   )N)r\  )FN)]r_   r`   ra   r~   r   r   r   r   r   r   Scoperk   r   rL   r   r   r   r  r   r  r  propertyr   r  r  r[   setterr  r  r  r  r   
contextlibcontextmanagerr'  r(  r   r   r   Z_guardsZ	GuardsSetr  r   r   r5  r6  r7  rN   rS  r?   rU  r[  r   rb  staticmethodrh  r   r   r   r   r{  r  ry   r  r  r  r  r  r  r   Zclear_framer  r   r   r  r  r  rA   r   r   r  r   r  r    r  r  r  r   r  r/  r	  r
  r   rd   rd   r   re   r      s   
 .





	




)
@

	
 	J    <	


i@ 'r   a  With the current config, we will graph break (and fall back to eager-mode PyTorch) on all ops that have do not have the 'pt2_compliant_tag'. Please see the following doc for how to mark this op as PT2 compliant https://pytorch.org/tutorials/advanced/custom_ops_landing_page.htmlc              
      sp  |dkrd S  fdd} fdd}t |tjjrbtjj|jkrL|| d S ||d| d d S t |tjjrlt|	 }t
|dkrt||d	 }tjj|jkr|| d S ||d
| d d S tjj j||fd\}}ztjj|jf||}	W n0 tk
r. }
 ztt|
 W 5 d }
~
X Y nX t||	}tjj|jkrT|| n||d| d|	 d d S )Nr  c                    s   | j dkrd S  j|  d S )N>   primsprimr  )	namespacer   rr   ri  output_graphrd   re   encountered_compliant_op  s    
z8check_pt2_compliant_op.<locals>.encountered_compliant_opc                    s&    j |  tjr"t|d t  d S )N )r   rr   r   Zonly_allow_pt2_compliant_opsr(   err_epilogue)rj  r  r  rd   re   encountered_non_compliant_op  s    z<check_pt2_compliant_op.<locals>.encountered_non_compliant_opz%Encountered the torch.ops.OpOverload z that is not PT2 compliant.r   r   z:Encountered the non-overloaded torch.ops.OpOverloadPacket z that is not PT2 compliant. Fz+Encountered the torch.ops.OpOverloadPacket z! which resolves to the overload (z) that is not PT2 compliant.)r   r   Z_opsZ
OpOverloadTagZpt2_compliant_tagtagsZOpOverloadPacketrl  	overloadsr   rJ  Z_dynamoutilsZget_fake_values_from_nodesr7  r   Z_jit_resolve_packetZ_qualified_op_namer   r(   r   )r  kindrj  r   r   r  r  r  r  overloadr  rd   r  re   check_pt2_compliant_op  s^    

  

r   c                       s`   e Zd ZdZd fdd	Zd fdd	Zd fdd		Zd
d ZdddZdd Z	dd Z
  ZS )r   a  
    Holds an FX graph that is being traced. OutputGraph owns a SubgraphTracer
    and the separation of responsibilities is that SubgraphTracer is
    responsible for building the graph while OutputGraph is responsible for
    compiling and executing the graph.
    NFc                    s   t    t|| _tj | _|r2|d ks2t	|| _
i | _i | _|| _i | _d | _d | _d | _d | _d | _| jd krg | _n| jj| j||fg | _d S rg   )r   rk   weakrefrD  r  r   r   ZGraphr[   r   r   r  r  r!  lifted_freevars	prev_inst	_cur_code_orig_gm_meta_orig_gm_lineno_map_orig_gm_firstlinenosource_fn_stackZ_target_to_str)rj   r  r!  r   r"  r   rd   re   rk     s(    

zSubgraphTracer.__init__c              	      s  | j d k	rNt||f\}}	g }
|D ]}| |}|
| q$t|
|	\}}t |||||||| jj	}t
jdkr|dkr|j  | jk	r jd k	r jjd k	r|j|j jjd fdd}tdt|  | _d}|j| jk	r`t|jdd	d
  }t|tjjrNd}dd |jjD | _|j | _!|j"j#j$| _%nd | _d | _!d | _%|j&}|r||' j(j)d< |dkr| j*j(j+|fg j(j)d< nJ|dkr| j d k	rt,d | j*j(j+j(j)d | d fg j(j)d< | jr| j!r| j%r|jj-}d }|d k	r0| j!|| j% d }|d k	r| j| }tj.j/D ] }||krL|| j(j)|< qLd|kr|d j(j)d< |s8dj(j)kr|j&}|r|' j(j)d< dj(j)kr8|dkr| j*j(j+|fg j(j)d< nJ|dkr8| j d k	rt,d | j*j(j+j(j)d | d fg j(j)d< dj(j)krg }|rn||0  t1|dd }qJ|2  t3j45|6 }d7|j(_8S )Nr  r  )linenoc                     s(   t   } djj d d|  S )NzTRACE FX call z from r  )rB   rstripr   r   )lineZcur_instheaderr  Ztx_coderd   re   get_trace_call_log_str`  s    z;SubgraphTracer.create_proxy.<locals>.get_trace_call_log_strr  FZorig_graphmodulec                   S   s   d S rg   rd   rd   rd   rd   re   r  k  r  z-SubgraphTracer.create_proxy.<locals>.<lambda>Tc                 S   s   g | ]
}|j qS rd   r  )r   ndrd   rd   re   r   o  s    z/SubgraphTracer.create_proxy.<locals>.<listcomp>nn_module_stack>   r  r  r(  r  z0Invoking an nn.Module inside HigherOrderOperatorr   stack_tracer!  r  )9r!  r  Ztree_flatten#maybe_lift_tracked_freevar_to_inputr|   Ztree_unflattenr   r  r  r7  r  r  Zcurrent_instructionr#  Z	positionsr)  r   Zget_line_of_code_headertrace_call_logr?  rH   r$  r"   Zget_contextr
  r   r   r   r   r[   rT  r%  Z_lineno_mapr&  r   __code__r   r'  r0  r   r   r   r(  r   r(   starts_linerD  Z_COPY_META_FIELDSframe_summaryrJ  reverser   StackSummary	from_listformatrd  r1  )rj   r  rj  r   r   r   	type_exprZproxy_factory_fn	flat_argsZ	tree_specZnew_flat_argsr9  Zmaybe_new_argr4  r.  Zis_retracingZorig_graphmodule_mayber0  r)  Znode_idxr   fieldZframe_summariesZmsgsr   r,  re   r    s    +

      
 


	
 






zSubgraphTracer.create_proxyc           
         s~   t | j|||| | jd k	rVtj||}|D ](}t|tjjs@q,|j	| j	ks,t
dq,t ||||||}	| jj|	jd< |	S )Nz2create_node using arg not from this SubgraphTracerr  )r   r  r!  r  Zarg_tree_leavesr   r   r   r  r[   r   r   r  r   r   )
rj   r  rj  r   r   r   r;  r<  r9  r   r   rd   re   r    s    

zSubgraphTracer.create_nodec                 C   sz   t |jdkrZg }|j D ]&}|j| jkr|tt|jj q|D ]}|j| qH| j| | j	
|jd  d S r  )r   r  rY  r[   r  r  r   rT  r  r  r$  r   )rj   r   Zuser_graph_nodesuserZother_graph_noderd   rd   re   r     s    zSubgraphTracer.remove_nodec              
   C   sX  t d||d k	r| nd |d kr8| jd k	s8td| jrdt|ddsd| jj	|g 
t  || jkrt D ]$}| d| }|| jkrv|} qqv| jrtt| j}| j| j}|r| j|}	q| j|}	n| jd }	|	b | jd|di |d	}
| jr4|r4| j \}}|
| j|< || j|< n
|
| j|< |
W  5 Q R  S Q R X d S )
Nzcreate_graph_input %s %sz(none)zByou are required to provide a source for inputs on the root tracerF)Zallow_cell_or_freevarr^  r  rd   )r;  )rw   r?  r   r!  r   r   r5   r  r   
setdefaultr|   r   extract_stackr  r   r   r   r  r   r[   Zinserting_beforeZinserting_afterr  popitem)rj   r   r;  r:  r^   rM  Zcandidate_nameZ	prev_namer   rO  rD  r   r   rd   rd   re   r     sH    



z!SubgraphTracer.create_graph_inputc                 C   sx   | j d k	std|| jkr&| j| S | |jj}t|j|jjd  || j|< | j d k	rt|j| j krt| j 	| |S )NzIlift_tracked_freevar_to_input should not be called on root SubgraphTracerr  )
r!  r   r"  r   r   r   rK   r   r&  lift_tracked_freevar_to_input)rj   rD  Z	new_proxyrd   rd   re   rB  %  s    


z,SubgraphTracer.lift_tracked_freevar_to_inputc                 C   s*   t |tjjs|S |j| kr |S | |S )z
        If arg is a free variable, then lift it to be an input.
        Returns the new lifted arg (if arg was a freevar), else the
        original arg.
        )r   r   r   ZProxyr&  rB  rR  rd   rd   re   r2  7  s
    
z2SubgraphTracer.maybe_lift_tracked_freevar_to_input)NFN)NNN)NNNN)NFN)r_   r`   ra   r~   rk   r  r  r   r   rB  r2  r   rd   rd   r   re   r     s&        :    3       
5r   )r   r  r   r1  r   r   r  re  r  r   r!  dataclassesr   typingr   r   r   r   r   r   r	   r
   r   r=  Ztorch._guardsr   Ztorch._loggingZtorch.nnZtorch.utils._pytreer  Z_pytreer  r   r   r   r   Ztorch._utils_internalr   r  r   Z%torch.fx.experimental._backward_stater   Z%torch.fx.experimental.symbolic_shapesr   r   r   Ztorch.fx.passes.runtime_assertr   Ztorch.utils._python_dispatchr   r  r   rv   r   Zbackends.registryr   r   Zbytecode_transformationr   r   r    r!   r"   r  r#   Zcurrent_scope_idr$   excr%   r&   r'   r(   r)   r  r*   r+   Zmutation_guardr,   r   r-   r.   r^   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   Zvariables.baserL   Zvariables.builderrM   rN   rO   rP   rQ   Zvariables.listsrR   Zvariables.miscrS   Zvariables.nn_modulerT   Zvariables.tensorrU   rV   rW   rX   Zvariables.torch_functionrY   Ztorch._dynamo.symbolic_convertrZ   	getLoggerr_   rw   r  ZgetArtifactLoggerZgraph_tabular_logr  Zgraph_sizes_logr3  r]   rf   	lru_cacherx   ry   r   r   r   r   r   r   objectr  r   r  r   r   r   ZTracerr   rd   rd   rd   re   <module>   s   ,<H

#           7	@