U
    Mh9                     @   s(  d dl Z d dlZd dlmZmZmZmZmZmZ d dl	Z	d dl
Z
d dlZd dlmZ d dlZd dlmZmZ d dlZd dlZeeZdd Zdd Ze Zd	d
 Zeeeedee fZ!dZ"dd Z#G dd deZ$dddddZ%dd Z&dd Z'dd Z(dd Z)dZ*dZ+dd Z,dd  Z-d!d" Z.dS )#    N)AnyDictList
NamedTupleOptionalTuple)NamedTemporaryFile)_frames_fmt_block_extrac                    sD   d  fdd}t |  fddtj fdd}|S )NTc                      s   d d S )NF r   )enabledr   I/var/www/html/venv/lib/python3.8/site-packages/torch/utils/viz/_cycles.pydisable   s    z observe_garbage.<locals>.disablec                    sV   sd S | dkrt t j n4| dkrRt dg fdd}t| d S )NstartstopFc                     s   d sdd< nt  d zjd dkr6t  tj tj  td tj	 }t  tj	 }||krt
d||  W 5 d X d k	r| |S d S )Nr   TFZ
generation   z.CUDA Memory changed during GC, %d bytes freed.)sys
setprofilegcZcollectgarbageclear	set_debugtorchcudaZmemory_allocatedloggerwarning)argskwargsbeforeafter)r   infoobserver
orig_traceself_returnr   r   
do_collect#   s$    






z8observe_garbage.<locals>.gc_callback.<locals>.do_collect)r   r   ZDEBUG_SAVEALLr   
getprofiler   )Zphaser    r$   )r   r!   )r    r"   r#   r   gc_callback   s    z$observe_garbage.<locals>.gc_callbackc                      s   t j  d S N)r   	callbacksremover   )r&   r   r   r)   E   s    zobserve_garbage.<locals>.remove)atexitregisterr   r(   append)r!   r   r)   r   )r   r&   r!   r   observe_garbage   s    
)r-   c                  C   s   ddd} t |  jd S )Nc                    s    fddS )Nc                      s    S r'   r   r   xr   r   <lambda>Z       z+_get_cell_type.<locals>.f.<locals>.<lambda>r   r.   r   r.   r   fY   s    z_get_cell_type.<locals>.fr   )N)type__closure__)r2   r   r   r   _get_cell_typeX   s    
r5   c                    s  i fddfdd  fdd} fdd}fd	d
}fdd}fdd} fdd}fdd} fdd} fdd}	t |t|t|t|t|tj|tj|t|tj	|t
j|tj|	i}
tjD ]}||
kr|
|   qڈ dd ttr d S )a%  
    Return known information about references held by the given object.

    Returns a mapping from referents to lists of descriptions.  Note that there
    may be more than one edge leading to any particular referent; hence the
    need for a list.  Descriptions are currently strings.

    c                    s     t|g |  d S r'   )
setdefaultidr,   )nameobj)
referencesr   r   add_referencej   s    z+annotated_references.<locals>.add_referencec                     s(   | D ]}t |r |t| qd S r'   )hasattrgetattr)attrsattrr;   r9   r   r   	add_attrsm   s    
z'annotated_references.<locals>.add_attrsc                      s&   z d W n t k
r    Y nX d S )Ncell_contents)
ValueErrorr   rA   r   r   add_cell_referencesr   s    z1annotated_references.<locals>.add_cell_referencesc                
      s    ddddddddd		 d S )
N__defaults__r4   __globals____code____name__
__module__Z__doc____qualname____annotations____kwdefaults__r   r   rD   r   r   add_function_references|   s    z5annotated_references.<locals>.add_function_referencesc                     s(   t D ]\} } d|  d| qd S )N[])	enumerate)positionitemr@   r   r   add_sequence_references   s    z5annotated_references.<locals>.add_sequence_referencesc                     s6     D ](\} } d|   dt|  d| qd S )NkeyrN   rO   )itemsrepr)rT   valuer@   r   r   add_dict_references   s    
z1annotated_references.<locals>.add_dict_referencesc                     s   D ]}  d|  qd S )Nelementr   )eltr@   r   r   add_set_references   s    z0annotated_references.<locals>.add_set_referencesc                      s    ddd d S )N__self____func__Zim_classr   r   rD   r   r   add_bound_method_references   s    z9annotated_references.<locals>.add_bound_method_referencesc                     s:   t tjkr6t} t| dkr6| d } d| d S )N   r   __callback__)r3   weakrefrefr   get_referentslen)Z	referentstargetr@   r   r   add_weakref_references   s
    
z4annotated_references.<locals>.add_weakref_referencesc                     sL   j }  dddddd t| tkrHj  D ]\}}d| | q.d S )Nf_backf_code
f_builtins	f_globalsf_tracef_localszlocal )rl   r3   dictrU   )rl   r8   local)rA   r;   r9   r   r   add_frame_references   s
    z2annotated_references.<locals>.add_frame_referencesc                      s    ddd d S )N__objclass__rI   __doc__r   r   rD   r   r    add_getset_descriptor_references   s    z>annotated_references.<locals>.add_getset_descriptor_references__dict__	__class____mro__)tuplelistrm   set	frozensettypesFunctionType	FrameTypeCellType
MethodTypera   rb   GetSetDescriptorTyper3   ru   
isinstance)r9   rE   rM   rS   rX   r[   r^   rf   ro   rr   Ztype_based_referencestype_r   )rA   r;   r9   r:   r   annotated_references_   sT    	


           
r       c                 C   s  dd }t | trt| S t| jdkr4d| j S t | tjrpz| jj}W n t	k
rd   d}Y nX d| S t | t
rd||  dS t | trd	||  d
S t | trdt|  dS t | tjrd| j S t | t rd| j S t | tjr"|  }|dkrdS dt|dS nht | tjrn| jj}t|tkr\d|td  d  }d| d| j S dt| j dt| j S dS )zx
    Return a string to be used for Graphviz nodes.

    The string should be short but as informative as possible.
    c                 S   sD   d dd ttd| D }t| dkr@| dt| d  }|S )N,c                 s   s.   | ]&\}}t |trt|nt|jV  qd S r'   )r   
BASE_TYPESrV   r3   rI   ).0ir/   r   r   r   	<genexpr>   s     z=object_annotation.<locals>.format_sequence.<locals>.<genexpr>   z, ...)joinziprangerd   )r9   bodyr   r   r   format_sequence   s    z*object_annotation.<locals>.format_sequencefunctionz	function
z<anonymous>zinstancemethod
rN   rO   ()zdict[zmodule
ztype
Nzweakref (dead referent)zweakref to id 0xr/   z...   zframe
:zobject
.)r   r   rV   r3   rI   rz   r~   r]   __qualname__AttributeErrorrw   rv   rm   rd   
ModuleTypera   rb   r7   r|   rh   co_filenameFRAME_FILENAME_LIMITf_linenorJ   )r9   r   	func_nameZreferentfilenamer   r   r   object_annotation   s@    






r   c                   @   s>   e Zd ZU eed< ee ed< eed< eeee	f  ed< dS )Nodelabelcontextroot
referrentsN)
rI   rJ   r   strrK   r   boolr   r   intr   r   r   r   r     s   
r   r   filterc                   s   d krt   d krt fdd| D }dd | D }dd t| D }| D ]}|t| }|| }t|}	t|D ]`}
t|
}||d }|d krq||| }|	|dg}|| | |D ]}|j	||f qq|qRdd t|D }t
 }|r0| }||krq|| || }|| qi g }t|D ]*\}}||kr@t|< || q@|D ]$}fdd|j	D |j	d d < qp|S )	Nc                    s&   g | ]}t t| ||g qS r   )r   r   r   r9   r   r   r   
<listcomp>  s     z create_graph.<locals>.<listcomp>c                 S   s   g | ]}g qS r   r   r   r   r   r   r     s     c                 S   s   i | ]\}}t ||qS r   )r7   )r   r   r9   r   r   r   
<dictcomp>  s      z create_graph.<locals>.<dictcomp>?c                 S   s   g | ]\}}|j r|qS r   )r   )r   r   nr   r   r   r      s      c                    s$   g | ]\}}| kr| | fqS r   r   )r   r   idx)id_to_filtered_idr   r   r   0  s   )cuda_allocation_contextis_cuda_tensorrP   r7   r   r   rc   getr,   r   rx   popaddextendrd   )objectsr   r   nodesZnode_referrersZ
id_to_noder9   Zfidxr2   r:   Z	referrentZridZtidxtlabelsr   Z	to_searchZto_keepr   Z	referrersfilteredr   r   r   )r   r   r   r   create_graph  sR    



r   c                 C   s
   t | S r'   )jsondumps)r   r   r   r   escape5  s    r   c                 C   s"   t | tjo | jo t | tjj S r'   )r   r   ZTensorZis_cudaZ_subclassesZ
FakeTensor)r9   r   r   r   r   9  s    r   c                     sp   t jj } i  | d D ]F}|d }|d D ]0}|d dkrPt|\}}| |< ||d 7 }q,q fdd}|S )	NsegmentsaddressblocksstateZactive_allocatedsizec                    s<   t | r8|   } |}|d k	r8dt|ddS d S )N
T)Zfull_filename)r   Zuntyped_storageZdata_ptrr   r   r	   )r9   addrframesZaddr_to_framer   r   object_contextG  s    
z/cuda_allocation_context.<locals>.object_context)r   r   memoryZ	_snapshotr
   )Zsnapshotsegr   Zblkr   Z	real_sizer   r   r   r   r   <  s    r   c              
   C   s   dddg}t | D ]4\}}|| dt|j d|jr:dnd d qt | D ]8\}}|jD ](\}}|| d	| d
t| d q^qP|d d|S )Nzdigraph GraphName {znode [shape=rect];zrankdir=LR;z [label=z, color=redblackz];z -> z
 [label = rO   z}
r   )rP   r,   r   r   r   r   r   )r   linesr   r   r2   r   jr   r   r   to_dotP  s    
.$
r   az  
<!DOCTYPE html>
<html>
<head>
  <style>
    body {
      margin: 0;
      padding: 0;
      overflow: hidden;
    }

    #container {
      display: flex;
      flex-direction: column;
      height: 100vh;
    }

    #main {
      flex: 2;
      overflow: auto;
    }

    #preContainer {
      flex: 1;
      overflow: auto;
    }

    svg {
        overflow: scroll;
    }

    pre {
      margin: 0;
      padding: 10px;
    }
  </style>
</head>
<body>
  <div id="container">
    <div id="main">
    </div>
    <div id="preContainer">
      <pre id="stacktrace">Mouse over tensor objects to see where they were allocated.</pre>
    </div>
  </div>
<script src='https://cdnjs.cloudflare.com/ajax/libs/viz.js/1.8.0/viz-lite.js'></script>
<script>
let dot = $DOT
let image = Viz(dot, {format: 'svg'});
document.getElementById('main').innerHTML = image
$LISTENERS
</script>
</body>
</html>
z
document.getElementById('node{id}').addEventListener('mouseover', function(event) {{
  document.getElementById("stacktrace").textContent = {stack}
}})
c                 C   sz   g }t | D ]F\}}|jd kr qtjt|d t|j d|j d}|| qt| }t	
dt|
dd|S )Nr_   z:
)r7   stackz$DOTz
$LISTENERSr   )rP   r   _listener_templateformatr   r   r   r,   r   	_templatereplacerV   r   )r   Z	listenersr   r   sdotr   r   r   to_html  s    
(r   c                    s$   t jjjdd  fdd}t|S )Ni )Zmax_entriesc                    s8   | r4t dd | D s$td d S  tt|  d S )Nc                 s   s   | ]}t |V  qd S r'   )r   r   r   r   r   r     s     z:observe_tensor_cycles.<locals>.observer.<locals>.<genexpr>z No CUDA Tensors found in garbage)anyr   r    r   r   )r   callbackr   r   r!     s
    
z'observe_tensor_cycles.<locals>.observer)r   r   r   Z_record_memory_historyr-   )r   r!   r   r   r   observe_tensor_cycles  s    r   c                  C   s   t d dd } t| S )a  
    Install a warning that reports whenever a cycle that is holding CUDA memory is observed.

    The warning produces an .html file that visualizes the cycle,
    and links it to the stack frame that allocted the CUDA tensor.

    Reference cycles are freed by the cycle collector rather than being cleaned up
    when the objects in the cycle first become unreachable. If a cycle points to a tensor,
    the CUDA memory for that tensor will not be freed until garbage collection runs.
    Accumulation of CUDA allocations can lead to out of memory errors (OOMs), as well as
    non-deterministic allocation behavior which is harder to debug.
    z2Watching Python reference cycles for CUDA Tensors.c              	   S   s6   t dddd}||  td|j W 5 Q R X d S )Nwz.htmlF)suffixdeletezDReference cycle includes a CUDA Tensor see visualization of cycle %s)r   writer   r   r8   )htmlr2   r   r   r   write_and_log  s    
z)warn_tensor_cycles.<locals>.write_and_log)r   r    r   )r   r   r   r   warn_tensor_cycles  s    
r   )/r   r   typingr   r   r   r   r   r   rz   ra   r   tempfiler   r   Ztorch.cuda._memory_vizr	   r
   r*   logging	getLoggerrI   r   r-   r5   r}   r   r   floatcomplexr3   r   bytesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   s:    
Im2.7
