U
    zhT                     @   sR  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZmZmZ d dlmZ d dlZd dlmZmZmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	l m!Z!m"Z" d d
l#m$Z$ d dl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1 e2e3Z4ee Z5e 6dddgZ7ddddgZ8e9de:dddZ;d7ee+ dddZ<ee+ ej=ddd Z>d8e5ee?e?f ee? e@d!d"d#ZAee?e?f d$d%d&ZBejje5dd'd(d)ZCejDd*d+ ZEG d,d- d-ZFG d.d/ d/ZGejHG d0d1 d1ZIeJ ZKd2d3 ZLe?d4d5d6ZMdS )9    N)AnyDictListOptional)patch)
draw_graphget_aot_graph_nameget_graph_being_compiled)fx)save_graph_reprowrap_compiler_debug)get_debug_dir)GraphModule)_extract_tensor_metadataTensorMetadata)legalize_graph)tree_map   )configir)BaseSchedulerNodeFusedSchedulerNodeNopKernelSchedulerNode
OutputNodeSchedulerNode)VBufMetanameZn_origindotz-Gnslimit=2z-Gnslimit1=2z-Gmaxiter=5000)returnc                   C   s8   zt jddgt jd W dS  t jk
r2   Y dS X d S )Nwhichr   )stderrTF)
subprocesscheck_outputPIPESubprocessError r&   r&   G/var/www/html/venv/lib/python3.8/site-packages/torch/_inductor/debug.pyhas_dot.   s
    r(   Fnodesc           	   	   C   s   t  std dS |dkr"t }t| }|jD ]~}d|jkr@q0|jd j}t|t	rxt|d t
rp|d f}n|d }d}t|tjr|jj}t||ddddd}||jd< q0|rt| ti |}t| |j  t||dtjjd dS )z$
    Draw a graph in fname.svg.
    z*draw_buffers() requires `graphviz` packageNfusion_metar   Ztensor_metaF)
clear_metadot_graph_shape)r(   logwarningr	   create_fx_from_snodesr*   metagroup
isinstancetupleintr   ZComputedBufferdatadtyper   printr   r   graphZlintr   r   tracer-   )	r*   Zprint_graphfnamer9   noder2   r7   metadatagmr&   r&   r'   draw_buffers7   s<    





   r?   )snodesr   c                    s.  dd }t ddddg}i }tj }d}g }d}| D ]>}| rPd}	|	}nZ| rbd	}	|	}nHt|trvd
}	|	}n4t|t	rd}	|j
}nt|trd}	|j
}ntdtjj| d}
|	 d|
 }||}i }t|drd| i}|j|d|d} fdd  |r|| | }||_||||	|jd< t|trd|jD ]}||| < qP|||< |dkr8|}q8| D ]}| }|jj}|| }g }|D ]T}|j|kr||j }n,|| ||j}|||j< W 5 Q R X || qt||_q~|t |dkr |d nt| |S )B
    Creates a FX Graph from a list of SchedulerNode objects.
    c                 S   s   dd }| |_ |S )Nc                  W   s   dS )Nr   r&   )argsr&   r&   r'   func1g   s    z;create_fx_from_snodes.<locals>.get_fake_func.<locals>.func1)__name__)r   rC   r&   r&   r'   get_fake_funcf   s    z,create_fx_from_snodes.<locals>.get_fake_func
FusionMetar2   snodetypeNZexterntemplateZnopZcomputeZfusedzUnknown node typeZoriginal_atenz: 
get_devicedevicer&   rB   kwargsc                    s6   t | tr"t fdd| jD S tdd | jD S )Nc                 3   s   | ]} |V  qd S Nr&   ).0x	in_outputr&   r'   	<genexpr>   s     z;create_fx_from_snodes.<locals>.in_output.<locals>.<genexpr>c                 s   s   | ]}t |jtV  qd S rN   )r3   r<   r   )rO   userr&   r&   r'   rS      s     )r3   r   anyr@   Zusers)rG   rQ   r&   r'   rR      s    
z(create_fx_from_snodes.<locals>.in_outputr+   r   r   )!collections
namedtupletorchr
   GraphZ	is_externZis_templater3   r   r   r2   r   RuntimeErrorZ	_inductorutilsZget_fused_kernel_name	get_nodeshasattrrJ   Zcall_functionappendget_namer   r1   r@   Zread_writesZreadsZinserting_beforeplaceholderr4   rB   outputlen)r@   rE   rF   Zbuf_to_fx_noder9   Z
first_nodeoutputsr2   rG   Z	node_typeZ
fused_name	func_nameZ	node_funcrM   Zfx_noder   rP   depsnew_argsdepZdep_noder&   rQ   r'   r0   a   sz    




 



$r0   )r*   node_name_to_buf_nameparent_buf_name	n_originsc           
      C   s   | d krd S | D ]}|  }| }|d k	rTt|dkrTt|||d krJ|n| qnt|dkrl|d |kspt|j}|d ks|jd krq|jD ]&}|j}	|	|kr|d kr|n|||	< qqd S )Nr   r   )r_   r\   rb   $update_orig_fx_node_name_to_buf_nameAssertionErrorr<   Zoriginsr   )
r*   rh   ri   rj   r<   buf_nameZchildren_nodesZir_nodeorigin	node_namer&   r&   r'   rk      s*    
rk   )rh   c                 C   sl   i }|   D ]*\}}||kr(|h||< q|| | qi }|   D ]"\}}t|| }t||||< qD|S rN   )itemsaddrb   r   )rh   Zbuf_name_to_n_nodero   rm   node_name_to_buf_metaZn_noder&   r&   r'   get_node_name_to_buf_meta   s    rs   )r>   r@   r   c                 C   sP   i }t || |dkrdS t|}| jjD ] }|j|kr*||j|jd< q*dS )rA   NZbuf_meta)rk   rs   r9   r*   r   getr1   )r>   r@   rh   rr   r<   r&   r&   r'   annotate_orig_fx_with_snodes   s    

ru   c               	   c   s   t jdddk} dd l}t|jjj}t	
 }| sPz
d V  W 5 |  X d S |tdd t jt d}t j|dd tt j|d	t  d
}|tj |td || z
d V  W 5 || |  X d S )NZTORCH_COMPILE_DEBUG01r   z*functorch.compile.config.debug_partitionerTtorchinductor)exist_okZaot_z
_debug.log3[%(filename)s:%(lineno)d %(levelname)s] %(message)s)osenvironrt   Ztorch._functorch.aot_autogradlogging	getLoggerZ
_functorchZaot_autogradrD   
contextlib	ExitStackcloseenter_contextr   pathjoinr   makedirsFileHandlerr   setLevelDEBUGsetFormatter	Formatter
addHandlerremoveHandler)Zcompile_debugrX   r.   stackr   fhr&   r&   r'   enable_aot_logging   s6    




r   c                   @   s   e Zd Ze Zedd Zeee	e dddZ
dd Zedd	d
Zd"eedddZejd#eedddZedddZdd Zdd ZeedddZdd Zdd Zdd  Zd!S )$DebugContextc                    s"   t   fdd}t|ddS )Nc               
      s(   t    | |W  5 Q R  S Q R X d S rN   )r   rL   fnr&   r'   inner-  s    z DebugContext.wrap.<locals>.innerinductor)Zcompiler_name)	functoolswrapsr   )r   r   r&   r   r'   wrap+  s    zDebugContext.wrap)folder_namer   c                 C   sV   t jjpt }tjD ]<}tj|d|  d| }tj	|st
| |  S qd S )Nrx   .)r   r:   	debug_dirr   r   _counterr{   r   r   existsr   )r   r   ndirnamer&   r&   r'   create_debug_dir4  s    


zDebugContext.create_debug_dirc                 C   s   d | _ d | _t | _d S rN   )_prof_pathr   r   _stack)selfr&   r&   r'   __init__B  s    zDebugContext.__init__)new_pathc              	   C   s   | j s
d S |dst|ddlm} zB|| d* tj|rPt	| t
| j | W 5 Q R X W n$ tk
r   td| j | Y nX d S )Nz.debugr   )FileLockz.lockz(Failed to copy debug files from %s to %s)r   endswithrl   filelockr   r{   r   r   shutilrmtreecopytreeOSErrorr.   r/   )r   r   r   r&   r&   r'   copyG  s    
  zDebugContext.copyw)filename
write_modec                 O   s(   | j s
tttj| j ||f||S rN   r   rl   openr{   r   r   )r   r   r   rB   rM   r&   r&   r'   fopenW  s    
zDebugContext.fopenc              	   o   s>   | j s
tttj| j ||f||}|V  W 5 Q R X d S rN   r   )r   r   r   rB   rM   fr&   r&   r'   fopen_context[  s    
 zDebugContext.fopen_context)suffixc                 C   s   | j s
ttj| j |S rN   )r   rl   r{   r   r   )r   r   r&   r&   r'   r   a  s    
zDebugContext.filenamec              	   C   s   t jjd k	r|dd l}| jsttj| jtj	| j d}|
|d }|j| jtj	| jd W 5 Q R X t j| d S )Nr   z.tar.gzzw:gz)arcname)r   r:   
upload_tartarfiler   rl   r{   r   r   basenamer   rq   )r   r   Ztar_filetarr&   r&   r'   r   e  s    
 $zDebugContext.upload_tarc                    s   t jr<td  j} tj  fdd}| j|| | j	t
|  t jjsZd S | t | _t jjr~| dtj t jjr| dtj d S )Nztorch._dynamoc                    s     |  d S rN   )r   )levelr.   r&   r'   reset_log_levelw  s    z/DebugContext.__enter__.<locals>.reset_log_levelz	debug.logzinfo.log)r   debugr}   r~   r   r   r   r   callbackr   r   Zset_debug_handlerr:   enabledr   r   r   Z	debug_log_setup_log_captureZinfo_logINFO)r   Z
prev_levelr   r&   r   r'   	__enter__q  s    
zDebugContext.__enter__)r   r   c                 C   sp   t d}| j| |}t |}|| |t d |	| |t
|j| | j|j| d S )Nztorch._inductorrz   )r}   r~   r   r   r   StreamHandlerr   r   r   r   minr   r   r   )r   r   r   r.   fdchr&   r&   r'   r     s    



zDebugContext._setup_log_capturec                 C   sF   | j r| j   |   | jr8|   tdt | j | j	  d S )Nz%s debug trace: %s)
r   disable_save_profile_datar   r   r.   r/   r	   r   r   )r   exc_typeexc_valexc_tbr&   r&   r'   __exit__  s    
zDebugContext.__exit__c              	   C   sv   | j s
t| j | d | dF}tj| j |d}|  |d |	d |d |	d W 5 Q R X d S )Nzcompile.profzcompile.stats)streamZcumtimed   Ztottime)
r   rl   Z
dump_statsr   r   pstatsZStatsZ
strip_dirsZ
sort_statsZprint_stats)r   r   statsr&   r&   r'   r     s    



zDebugContext._save_profile_datac                 C   sZ   t jjrJtt j|rJztt| |W S  tk
rF   tjddd Y qVX ndd }|S d S )Nz Ignoring exception in debug codeT)exc_infoc                  _   s   d S rN   r&   rL   r&   r&   r'   ignored  s    z)DebugContext.__getattr__.<locals>.ignored)r   r:   r   getattrDebugFormatter	Exceptionr.   r/   )r   r   r   r&   r&   r'   __getattr__  s    zDebugContext.__getattr__N)r   )r   )rD   
__module____qualname__	itertoolscountr   staticmethodr   strr   r   r   r   r   r   contextmanagerr   r   r   r   r5   r   r   r   r   r&   r&   r&   r'   r   (  s"   

r   c                   @   s   e Zd Zdd Zejjeej dddZ	ejjeej dddZ
edd	d
ZedddZeedddZedddZejjedddZdd Zeeej edef eedddZdS )r   c                 C   s"   |j | _ |j| _|j| _|| _d S rN   )r   r   r   handler)r   r   r&   r&   r'   r     s    zDebugFormatter.__init__)r>   inputsc              	   C   sP   |  d}t|||d W 5 Q R X |  d}||jdd W 5 Q R X d S )Nzfx_graph_runnable.pyr   zfx_graph_readable.pyFZprint_output)r   r   writeprint_readabler   r>   r   r   r&   r&   r'   fx_graph  s    zDebugFormatter.fx_graphc              	   C   s,   |  d}||jdd W 5 Q R X d S )Nzfx_graph_transformed.pyFr   )r   r   r   r   r&   r&   r'   fx_graph_transformed  s    z#DebugFormatter.fx_graph_transformedr)   c                 C   s   |  d| d S )Nzir_pre_fusion.txt	_write_irr   r*   r&   r&   r'   ir_pre_fusion  s    zDebugFormatter.ir_pre_fusionc                 C   s   |  d| d S )Nzir_post_fusion.txtr   r   r&   r&   r'   ir_post_fusion  s    zDebugFormatter.ir_post_fusion)r   r*   c              	   C   sJ   |  |6}td|j |D ]}||  |d qW 5 Q R X d S )NzWriting debug ir to  %sz


)r   r.   infor   r   Z	debug_str)r   r   r*   r   r<   r&   r&   r'   r     s
    zDebugFormatter._write_irc                 C   s   t || dd d S )Nzgraph_diagram.svg)r;   )r?   r   r   r&   r&   r'   graph_diagram  s    zDebugFormatter.graph_diagram)r>   r*   c                 C   s,   t || t|| ddtdtjjd d S )Nzorig_fx_graph_diagram.svgFT)r;   r,   progZparse_stack_tracer-   )ru   r   r   GRAPHVIZ_COMMAND_SCALABLEr   r:   r-   )r   r>   r*   r&   r&   r'   draw_orig_fx_graph  s    
z!DebugFormatter.draw_orig_fx_graphc                 C   s   t || d d S )Nzoutput_code.py)r   r   r   )r   r   r&   r&   r'   output_code  s    zDebugFormatter.output_codeZChoiceCaller)r   input_nodestimingselapseprecompile_elapsec              	      s   dd l }ddlm  tjd fdd|tj tj fdd|D ||d	}| jd
dddL}|	 D ]<\}	}
t
|	 }|| |
|d< ||| |d qpW 5 Q R X d S )Nr   r   )FixedLayout)r<   c                    sz  t | dr| j}nd}|t| jd}z|  }t| rd}zt|j}W nB tk
r   zt	j
jj|jdd}W n tk
r   Y nX Y nX  |j|jtt	j
j|jtt	j
j|j|d}t||d< nt|  |d< W n$ tk
r
 } zW 5 d }~X Y nX zt|  |d< W n$ tk
rD } zW 5 d }~X Y nX zt|  |d	< W n$ tk
r~ } zW 5 d }~X Y nX ztt	j
j|  |d
< W n$ tk
r } zW 5 d }~X Y nX ztt	j
j|  |d< W n$ tk
r } zW 5 d }~X Y nX ztt	j
j|  |d< W n$ tk
rJ } zW 5 d }~X Y nX t | drvt| jtjrv| j|d< |S )Nr    )r   rH   r   )fallback)r7   sizestrideoffsetlayoutr7   rK   r   r   Znumelr6   )r]   r   rH   rD   Z
get_layoutr3   r5   r   r   r   r9   ZsizevarsZ	size_hintrK   r7   listZ
size_hintsr   r   r   Z	get_dtyperJ   Z
get_strideget_sizeZ	get_numelr6   r   IRNode)r<   ro   Z	node_infor   r   Zstatic_layouter   build_node_infor&   r'   r    sp    

 
z>DebugFormatter.log_autotuning_results.<locals>.build_node_infoc                    s   g | ]} |qS r&   r&   )rO   r<   )r  r&   r'   
<listcomp>5  s     z9DebugFormatter.log_autotuning_results.<locals>.<listcomp>)Zop_nameZcuda_device_nameZcuda_device_countr   Zautotuning_timeZprecompile_timezautotuning_result_json_list.txtatzutf-8)encodingZbenchmark_result
)jsonr   r   r   rX   cudaZget_device_nameZdevice_countr   rp   dict	info_dictupdatedumpr   )r   r   r   r   r   r   r  Zgeneral_propertiesr   Zcallertimer  r&   r  r'   log_autotuning_results  s,    =  
z%DebugFormatter.log_autotuning_resultsN)rD   r   r   r   rX   r
   r   r   Tensorr   r   SchedulerNodeListr   r   r   r   r   r   r   r   r   r   floatr  r&   r&   r&   r'   r     s"    
r   c                   @   s    e Zd ZU eed< ejed< dS )TensorMetadataHoldertensor_metadatarK   N)rD   r   r   r   __annotations__rX   rK   r&   r&   r&   r'   r  D  s   
r  c            
   	   O   s   d}t j|st | dd }t|| |f\}}d}| d| dtt d}t|d}t	||f| W 5 Q R X t
tjrd	| d
|d}	t|	 dS )z
    This function is used to save arguments for a compile_fx_inner function call
    to the file system.  Later on one can replay the compile_fx_inner call
    with the saved arguments using load_args_and_run_compile_fx_inner.
    z/tmp/inductor_saved_argsc                 S   s$   t | tjrtt| | jS | S dS )z
        Pickle FakeTensor will result in error:
        AttributeError: Can't pickle local object 'WeakValueDictionary.__init__.<locals>.remove'

        Convert all Tensor to metadata. This may also makes pickle faster.
        N)r3   rX   r  r  r   rK   rP   r&   r&   r'   handle_tensorX  s    z5save_args_for_compile_fx_inner.<locals>.handle_tensorcompile_fx_inner/_z.pklwbz3
Arguments for a compile_fx_inner call is saved to z. To replay the call,
run the following:

from torch._inductor.debug import load_args_and_run_compile_fx_inner
load_args_and_run_compile_fx_inner(z
)
        N)r{   r   r   mkdirr   nextsave_args_cntr   pickler  r.   isEnabledForr}   r   r8   )
rB   rM   folderr  Zargs_to_saveZkwargs_to_savefn_namer   r   messager&   r&   r'   save_args_for_compile_fx_innerM  s     
r$  )r   c                 C   s   ddl m} t| d}t|\}}W 5 Q R X dd }tjjdd}|N t	dd	6 t
|||f\}}|||W  5 Q R  W  5 Q R  S Q R X W 5 Q R X d S )
Nr   )r  rbc                 S   s4   t | tr,tjj| jj| jj| jj	| j
S | S d S rN   )r3   r  rX   Z_dynamoZtestingZrand_stridedr  shaper   r7   rK   r  r&   r&   r'   r    s    
z9load_args_and_run_compile_fx_inner.<locals>.handle_tensorT)Zallow_non_fake_inputsZ	save_argsF)Ztorch._inductor.compile_fxr  r   r  loadrX   Z_subclassesZFakeTensorModer   r   r   )r   r  r   rB   rM   r  Z	fake_moder&   r&   r'   "load_args_and_run_compile_fx_inner{  s    r(  )FN)Nr   )NrV   r   dataclassesr   r   r}   r{   os.pathr  r   r   r"   typingr   r   r   r   Zunittest.mockr   rX   Zfunctorch.compiler   r   r	   r
   Ztorch._dynamo.repro.after_aotr   r   Ztorch._dynamo.utilsr   Ztorch.fx.graph_moduler   Ztorch.fx.passes.shape_propr   r   Ztorch.fx.passes.tools_commonr   Ztorch.utils._pytreer   r   r   r   Z	schedulerr   r   r   r   r   Zvirtualizedr   r~   rD   r.   r  rW   r   r   	lru_cacheboolr(   r?   rY   r0   r   r5   rk   rs   ru   r   r   r   r   	dataclassr  r   r  r$  r(  r&   r&   r&   r'   <module>   sr   
*_  
# 
(  .