U
    yh,                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z
 d dlmZmZmZmZ edZe ZedZee ee eej de_G dd	 d	eZdee ed
ddZdee ed
ddZ ee edddZ!G dd dZ"dee" eedddZ#dS )    N)Lock)default_timer)AnyListOptionalSequenceZstrobelight_function_profilerzB%(name)s, line %(lineno)d, %(asctime)s, %(levelname)s: %(message)sFc                   @   s   e Zd ZdZdS )StrobelightCLIProfilerErrorzC
    Raised when an error happens during strobelight profiling
    N)__name__
__module____qualname____doc__ r   r   Z/var/www/html/venv/lib/python3.8/site-packages/torch/_strobelight/cli_function_profiler.pyr      s   r   )pidreturnc                 C   s    d}| pt  } t || S )zFReturns the link to the process's namespace, example: pid:[4026531836]z/proc/{}/ns/pid)osgetpidreadlinkformat)r   ZPID_NAMESPACE_PATHr   r   r   _pid_namespace_link!   s    r   c                 C   s.   | p
t  } t| }t||dd d S )z"Returns the process's namespace id[   )r   r   r   intfind)r   linkr   r   r   _pid_namespace(   s    r   )commandr   c                 C   s
   d | S )N )join)r   r   r   r   _command_to_string/   s    r    c                   @   s   e Zd ZdZe Zddddddddddd	
eeee	eee
ee	  e
ee	  eed	
d
dZddddZdeddddZddddZddddZeddddZedddZeeeedddZdS ) StrobelightCLIFunctionProfilera  
    Note: this is a Meta only tool.

    StrobelightCLIFunctionProfiler can be used to profile a python function and
    generate a strobelight link with the results. It works on meta servers but
    does not requries an fbcode target.
    When stop_at_error is false(default), error during profiling does not prevent
    the work function from running.

    Check function_profiler_example.py for an example.
    FiX  g    cAzpytorch-strobelight-ondemand<   N   )
stop_at_errormax_profile_duration_secsample_eachrun_user_nametimeout_wait_for_running_sectimeout_wait_for_finished_secrecorded_env_variablessample_tagsstack_max_lenasync_stack_max_lenc       
         C   s:   || _ || _|| _|| _|| _|| _d | _d | _|| _d S N)	r$   r%   r&   r'   r(   r)   current_run_idprofile_resultr+   )selfr$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r   r   r   __init__C   s    z'StrobelightCLIFunctionProfiler.__init__)r   c                 C   s   t  }t|}ddddddddt| j d	t| jd
  d| d| g}| jrn|d |d| j t	
dt| tj|dd}|jd}t	
d| |jdkrtd| td| }rt|d| _d S td| d S )Nstrobeclientrunz
--profilerZpyperfz--eventZcyclesz--asyncz--sample-intervalz--duration-msi  z--pid:z--sample-tags,running command: %sTcapture_outpututf-8output:
{%s}r   z:failed to start strobelight profiling, error in run_async:zINFO Run Id: (-?\d+)r   z9failed to start strobelight profiling, unexpected result )r   r   r   r   r&   r%   r+   appendr   loggerdebugr    
subprocessr4   stderrdecode
returncoder   researchgroupr/   )r1   Z	processId	namespacer   resultoutputmatchr   r   r   
_run_async]   sB    


z)StrobelightCLIFunctionProfiler._run_asyncr   )counterr   c                 C   s   |dkrt dddd| j g}tdt| tj|dd}|jd	}td
| |j	dkrnt d| t
d| }r|d}|dkrd S |dkrtd | |d  d S t d| dt d| dd S )N   z*wait_for_running called more than 20 timesr3   getRunStatus--run-idr7   Tr8   r:   r;   r   zAfailed to start strobelight profiling, error in wait_for_running:zProfile run status: (.*)r   ZRUNNINGZ	PREPARING
   zunexpected z phaseunexpected output
: r   )r   r/   r=   r>   r    r?   r4   r@   rA   rB   rC   rD   rE   timesleep_wait_for_running)r1   rK   r   rG   rH   rI   current_statusr   r   r   rS      s.    


z0StrobelightCLIFunctionProfiler._wait_for_runningc                 C   s   dddt | jg}tdt| tj|dd}|jd}td| |j	d	kr`t
d
| td| }r|d}|drd S t
d| dt
d| dd S )Nr3   ZstopRunrN   r7   Tr8   r:   r;   r   z<failed to stop strobelight profiling, return code is not 0 :INFO ::1:(.*)r   zSuccess!z*failed to stop strobelight profiling, got z resultrP   r   )strr/   r=   r>   r    r?   r4   r@   rA   rB   r   rC   rD   rE   __contains__)r1   r   rG   rH   rI   rT   r   r   r   	_stop_run   s"    



z(StrobelightCLIFunctionProfiler._stop_runc                 C   s   dddt | jg}tdt| tj|dd}|jd}td| |j	d	kr`t
d
| td| }r|d}|drtd |   d S |dst
d| g | _td|D ]$}|  j|d	 7  _t|d	  qd S )Nr3   rM   rN   r7   Tr8   r:   r;   r   z<failed to extract profiling results, return code is not 0 : rU   r   zProfile run status: PROCESSINGrO   z!Profile run finished with SUCCESSz9failed to extract profiling results, unexpected response zF(Total samples(.*)|GraphProfiler(.*)|Icicle view \(python stack\)(.*)))rV   r/   r=   r>   r    r?   r4   r@   rA   rB   r   rC   rD   rE   rW   rQ   rR   _get_resultsr0   findallinfo)r1   r   rG   rH   rI   rT   itemr   r   r   rY      s4    




z+StrobelightCLIFunctionProfiler._get_results)collect_resultsr   c              
   C   sh   z2|    td td |s(W d S |   W n0 tk
rb } ztjddd W 5 d }~X Y nX d S )Nzstrobelight profiling stoppedzcollection stoppedzerror during stop_strobelightTexc_info)rX   r=   r[   r>   rY   	Exceptionwarning)r1   r]   errorr   r   r   _stop_strobelight_no_throw   s    

z9StrobelightCLIFunctionProfiler._stop_strobelight_no_throwc              
   C   s   d}z2|    d}td| j |   td W dS  tk
r| } z(tjddd |rf| jdd W Y dS d }~X Y nX d S )NFTzstrobelight run id is: %szstrobelight profiling runningzerror during start_strobelight:r^   r]   )rJ   r=   r[   r/   rS   r`   ra   rc   )r1   Zstrobelight_startedrb   r   r   r   _start_strobelight   s    
z1StrobelightCLIFunctionProfiler._start_strobelight)work_functionargskwargsr   c              
   O   s"  d | _ d | _tjd }r|sD| jr0tdtd |||S | 	 }|s| jrhtj
  td|||}tj
  |S zPtd t }|||}t }|| }	td|	 | jdd tj
  |W S  tk
r }
 z,tjddd	 | jdd tj
  |
W 5 d }
~
X Y nX d S )
NFzconcurrent runs not supportedz%failed to start strobelight profilingzcollection startedzwork function took %s secondsTrd   zwork function throw exceptionr^   )r/   r0   r!   _lockacquirer$   r   r=   ra   re   releaser>   timerr[   rc   r`   )r1   rf   rg   rh   lockedstartedrG   startend
total_timerb   r   r   r   profile   sB    








z&StrobelightCLIFunctionProfiler.profile)r   )r	   r
   r   r   r   ri   boolr   floatrV   r   r   r2   rJ   rS   rX   rY   rc   re   r   rr   r   r   r   r   r!   3   sB   

)!r!   )profilerrh   r   c                    s&    st f | ttd fdd}|S )N)rf   r   c                    s&   t  tttd fdd}|S )N)rg   rh   r   c                     s    j f| |S r.   )rr   )rg   rh   )ru   rf   r   r   wrapper_function1  s    z@strobelight.<locals>.strobelight_inner.<locals>.wrapper_function)	functoolswrapsr   )rf   rv   ru   )rf   r   strobelight_inner0  s    z&strobelight.<locals>.strobelight_inner)r!   r   )ru   rh   rz   r   ry   r   strobelight*  s    
r{   )N)N)N)$rw   loggingr   rC   r?   rQ   	threadingr   Ztimeitr   rl   typingr   r   r   r   	getLoggerr=   StreamHandlerZconsole_handler	Formatter	formattersetFormatter
addHandlersetLevelINFO	propagater`   r   r   rV   r   r   r    r!   r{   r   r   r   r   <module>   s:   


 y  