U
    yh"                     @   s   d dl mZmZ d dlmZmZ d dlZd dlmZ d dl	m
Z
mZ ddlmZ eejddd	Ze ejejd
ddZdS )    )contextmanagernullcontext)AnyTupleN)'_checkpoint_without_reentrant_generator_DEFAULT_DETERMINISM_MODE   )contractmodulec              
   c   s6   t | j}dt | _z
dV  W 5 |t | _X dS )zs
    Disable hooks installed by checkpoint to avoid unintentional recursion
    during backward recomputation.
    FN)
checkpointstateenable_hook)r   Zorig_enable_hook r   e/var/www/html/venv/lib/python3.8/site-packages/torch/distributed/_composable/checkpoint_activation.py_no_hook   s
    
r   )r   returnc                 C   sp   t jd tjttdf dddd}tjttdf ttddd	}d
t| _	| 
| | j|d
d
d | S )a  
    This is a composable activation checkpointing API. Unlike functional
    activation checkpointing APIs, this one does not require changing model
    source code. Unlike ``nn.Module`` wrapper activation checkpointing APIs,
    this one does not modify model structure or fully-qualified names either.
    Under the hood, it registers activation checkpointing logic as pre- and
    post-forward hooks. Hence, this API can be easily applied to any model or
    sub-modules in the model.

    Args:
        module (nn.Module): the target model or sub-module to apply activation
            checkpointing.

    Example::
        >>> # xdoctest: +SKIP
        >>> import torch.nn as nn
        >>>
        >>> class MyModel(nn.Module):
        >>>     def __init__(self):
        >>>         super().__init__()
        >>>         self.l1 = nn.Linear(10, 10)
        >>>         self.l2 = nn.Linear(10, 10)
        >>>
        >>>     def forward(self, x):
        >>>         return self.l2(self.l1(x))
        >>>
        >>> model = MyModel()
        >>> checkpoint(model.l1)  # apply activation checkpointing only to l1
        >>> model(torch.zeros(2, 10)).sum().backward()

    ztorch.distributed.checkpoint.N)r   inputsr   c                    sJ   t  jrF fdd}t d|tdf| t  _tt  j d S )Nc                      s   t  t fS )N)r   r   r   r
   r   r   context_fnsC   s    z9checkpoint.<locals>.forward_pre_hook.<locals>.context_fnsTF)r   r   r   r   r   _ac_generatornext)r   r   r   r   r
   r   forward_pre_hook@   s        z$checkpoint.<locals>.forward_pre_hook)r   r   outputr   c                 S   sN   t | jr>ztt | j W n tk
r4   Y n
X tdd t | _d S )NzWExpected non-reentrant activation checkpoint generator to be exhausted, but it was not!)r   r   r   r   r   StopIterationRuntimeError)r   r   r   r   r   r   forward_hookM   s    z checkpoint.<locals>.forward_hookT)prependZalways_call)torchZ_CZ_log_api_usage_oncennModuler   r   r   r   r   Zregister_forward_pre_hookZregister_forward_hook)r   r   r   r   r   r   r      s    !
r   )
contextlibr   r   typingr   r   r   Ztorch.nnr   Ztorch.utils.checkpointr   r   r	   r   r   r   r   r   r   r   <module>   s   