U
    T?h+                     @   sH   d dl Z d dlmZ dd Zdd Zdd Zdd	 Zd
d Zdd Z	dS )    Nc                  C   s   dt jkrHtt jd  tt jd } tt jd }tjdd|| d nTdt jkrtt jdd tt jd	d} tt jd
d}tjdd|| d n d S )NZ
LOCAL_RANKZRANKZ
WORLD_SIZEZncclztcp://127.0.0.1:7645)Zinit_method
world_sizerankZOMPI_COMM_WORLD_LOCAL_RANKr   ZOMPI_COMM_WORLD_RANKZOMPI_COMM_WORLD_SIZE   ztcp://127.0.0.1:7647)osenvironintdistZinit_process_groupget)r   r    r
   e/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/transformers/models/llama/dist_settings.py	init_dist   s    

r   c                  C   s4   zddl m}  | j}|W S  tk
r.   Y d S X d S )Nr   )MPI)Zmpi4pyr   Z
COMM_WORLDImportError)r   commr
   r
   r   	_get_comm   s    r   c                  C   s   t  } | d k	r|  S dS Nr   )r   ZGet_rankr   r
   r
   r   get_rank'   s    r   c                  C   s   t  } | d k	r|  S dS )Nr   )r   ZGet_sizer   r
   r
   r   get_size,   s    r   c                  C   s   t  } | d k	r|   d S )N)r   Barrierr   r
   r
   r   barrier1   s    r   c                  G   s   t  dkrt|   d S r   )r   print)argsr
   r
   r   	print_out7   s    
r   )
r   Ztorch.distributeddistributedr   r   r   r   r   r   r   r
   r
   r
   r   <module>   s   
