U
    yh                     @   s   d dl mZmZ d dlZd dlm  mZ dd Zdd Z	dd Z
d	d
 Zdd Zejeeee edddZeej eeee edddZejeee  eee  eee edddZejejdddZdS )    )ListOptionalNc                 C   s   t ||}tjj| ||S N)c10d$_resolve_group_name_by_ranks_and_tagtorchops_c10d_functional	broadcast)inputsrctagranks
group_size
group_name r   `/var/www/html/venv/lib/python3.8/site-packages/torch/distributed/_functional_collectives_impl.py
_broadcast   s    r   c                 C   s   t ||}tjj| ||S r   )r   r   r   r   r	   Z
all_reducer   	reduce_opr   r   r   r   r   r   r   _all_reduce   s    r   c                 C   s   t ||}tjj| ||S r   )r   r   r   r   r	   Zall_reduce_coalescedinputsr   r   r   r   r   r   r   r   _all_reduce_coalesced   s    r   c                 C   s   t ||}tjj| ||S r   )r   r   r   r   r	   Zall_gather_into_tensorr   r   r   r   r   r   r   r   _all_gather_into_tensor(   s    r   c                 C   s   t ||}tjj| ||S r   )r   r   r   r   r	   Z all_gather_into_tensor_coalescedr   r   r   r   !_all_gather_into_tensor_coalesced1   s    r   )r   r   r   r   r   c                 C   s    t ||}tjj| |||S r   )r   r   r   r   r	   Zreduce_scatter_tensorr   r   r   r   _reduce_scatter_tensor:   s    r   )r   r   r   r   r   c                 C   s    t ||}tjj| |||S r   )r   r   r   r   r	   Zreduce_scatter_tensor_coalescedr   r   r   r    _reduce_scatter_tensor_coalescedJ   s    r   )r   output_split_sizesinput_split_sizesr   r   r   c                 C   s`   |d ks|d kr@|d kr |d ks(t d| jd | g| }|}t||}tjj| |||S )Nz^output_split_sizes and input_split_sizes must either be specified together or both set to Noner   )AssertionErrorshaper   r   r   r   r	   Zall_to_all_single)r   r   r    r   r   r   r   r   r   r   _all_to_all_singleZ   s    r#   )tensorreturnc                 C   s   t jj| S r   )r   r   r	   Zwait_tensor)r$   r   r   r   _wait_tensors   s    r&   )typingr   r   r   Z"torch.distributed.distributed_c10ddistributedZdistributed_c10dr   r   r   r   r   r   ZTensorstrintr   r   r#   r&   r   r   r   r   <module>   s6   				


