U
    yh5                     @   s   d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	m
Z
mZ d dlZd dlmZ d dlmZ d dlmZ ddlmZmZmZmZmZ d	gZejhZd
ddgZd	gZG dd	 d	e jZdS )    N)defaultdict)AnyDictOptionalSetTupleListType)nn)parametrize)type_before_parametrizations   )module_contains_paramswap_moduleFakeSparsityget_arg_info_from_tensor_fqnmodule_to_fqnBaseSparsifiermodule
module_fqntensor_namec                       sf  e Zd ZdZd)eeeef  d fddZeeef dddZ	eeeeef f dd	d
dZ
dd Zeeef dddZd*eeef edddZefejee ddddZdd Zdd Zd+eeedf  eeeeedf f  dddZddefejeeeej eej f  eeej d d!d"Zd,edd#d$d%Zejejed&d'd(Z  ZS )-r   a'  Base class for all sparsifiers.

    Abstract methods that need to be implemented:

    - update_mask: Function to compute a new mask for all keys in the
        `groups`.

    Args:
        - model [nn.Module]: model to configure. The model itself is not saved
            but used for the state_dict saving / loading.
        - config [list]: configuration elements should be a dict map that includes
            `tensor_fqn` of tensors to sparsify
        - defaults [dict]: default configurations will be attached to the
            configuration. Only the keys that don't exist in the `config` will
            be updated.

    Example::

        >>> # xdoctest: +SKIP("Can't instantiate abstract class BaseSparsifier with abstract method update_mask")
        >>> config = [{'tensor_fqn': 'layer1.weight', 'tensor_fqn': 'linear2.weight2', 'sparsity_level': 0.5}]
        >>> defaults = {'sparsity_level': 0.7}
        >>> # model.layer1.weight will have `sparsity_level` = 0.7 (getting default)
        >>> sparsifier = BaseSparsifier(config, defaults)
    N)defaultsc                    s.   t    |pi | _tt| _g | _d| _d S )NT)super__init__r   r   dictstategroupsenable_mask_update)selfr   	__class__ ]/var/www/html/venv/lib/python3.8/site-packages/torch/ao/pruning/sparsifier/base_sparsifier.pyr   8   s
    


zBaseSparsifier.__init__)returnc                 C   s   | j | j| jdS )Nr   r   r   r$   )r   r!   r!   r"   __getstate__@   s    zBaseSparsifier.__getstate__)r   r#   c                 C   s   | j | d S N)__dict__update)r   r   r!   r!   r"   __setstate__G   s    zBaseSparsifier.__setstate__c                 C   s   | j jd }t| jD ]n\}}|d }|d7 }|d| d7 }|d| d7 }t| D ](}|dkrhqZ|d| d||  d7 }qZq|d7 }|S )	Nz (r   
z	Group z	    module: z	    z: ))r    __name__	enumerater   sortedkeys)r   format_stringiZsparse_argsr   keyr!   r!   r"   __repr__J   s    zBaseSparsifier.__repr__c                 C   s   dd | j D }| j|dS )a  Returns the state of the optimizer as a :class:`dict`.

        It contains:
        * state - current state of the sparsification.
        * groups - a list containing all sparsity configuration groups
            with the key 'tensor_fqn' specifying the path to the sparsified tensor within a model

        TODO: Need a clean way of loading the state of the "prepared" module
        c                 S   s"   g | ]}t td d | qS )c                 S   s   | d t kS )Nr   )KEYS_NOT_IN_STATE_DICT)	key_valuer!   r!   r"   <lambda>f       z6BaseSparsifier.state_dict.<locals>.<listcomp>.<lambda>)r   filteritems).0mgr!   r!   r"   
<listcomp>c   s   z-BaseSparsifier.state_dict.<locals>.<listcomp>r   r   )r   r   )r   r   r!   r!   r"   
state_dictX   s    zBaseSparsifier.state_dictT)r>   strictc                 C   s  t |d }|d }| D ]\}}t| j|}|d }|d }	|r^|d kr^td| dd}
|j|	 D ]}t|trld}
 qql|
stt	
t||	j}t||	| |d	d d k	r|d	}||_|D ]}|d
 |kr|| qq| ||d d S )Nr   r   r   r   zError loading z into the modelFTmask
tensor_fqnr=   )copydeepcopyr9   r   modelRuntimeErrorZparametrizations
isinstancer   torchZonesgetattrshaper   register_parametrizationgetpopr@   r(   r)   )r   r>   r?   r   ZstatesrA   sZarg_infor   r   foundpr@   r;   r!   r!   r"   load_state_dictr   s.    

zBaseSparsifier.load_state_dict)rD   SUPPORTED_MODULESr#   c                 C   st   g | _ |g}|rp| }| D ]L\}}t||krbt||}t|tsLt| j d|d i q || q qd S )NrA   z.weight)	configrL   named_childrentyper   rF   strAssertionErrorappend)r   rD   rQ   stackr   namechildr   r!   r!   r"   make_config_from_model   s    
z%BaseSparsifier.make_config_from_modelc                 C   s   || _ || _| jdkr | | | jD ]}t|ts<tdt| jtsLtt	| j}|
| |dd}|dk	s~tdt||}| D ]H}||kr|| || ks|dkrd||  || kstd| dq|
| | j| q&|   dS )zPrepares a model, by adding the parametrizations.

        Note::

            The model is modified inplace. If you need to preserve the original
            model, use copy.deepcopy.
        Nznconfig elements should be dicts not modules i.e.:[{`tensor_fqn`: `foo.bar.weight`}, {`tensor_fqn`: ... }, ...]rA   zttensor_fqn is a required argument in the sparsity config whichreplaces previous `module` and [module]`fqn` arguments.zGiven both `z?` and `tensor_fqn` in the config, it is expected them to agree!)rD   rR   r[   rF   r   rV   r   r   rB   rC   r(   rK   r   r/   r   rW   _prepare)r   rD   rR   Zmodule_configZ
local_argsrA   Zinfo_from_tensor_fqnr2   r!   r!   r"   prepare   s:    








zBaseSparsifier.preparec              	   O   sh   | j D ]\}|d }|d }|dt}|dtt||}|| j|d  d< t|||| qdS )z-Adds mask parametrization to the layer weightr   r   parametrizationr@   rA   N)	r   rK   r   rG   Z	ones_likerH   r   r   rJ   )r   argskwargsrR   r   r   r_   r@   r!   r!   r"   r]      s    
  zBaseSparsifier._prepare.)params_to_keepparams_to_keep_per_layerc                    s   | j D ]  d } d }tj||dd i }|dk	rR fdd|D }|| |dk	r| d d}	|	dk	r fd	d|	D }
||
 |r||_qdS )
a=	  Squashes the sparse masks into the appropriate tensors.

        If either the `params_to_keep` or `params_to_keep_per_layer` is set,
        the module will have a `sparse_params` dict attached to it.

        Args:
            params_to_keep: List of keys to save in the module or a dict
                            representing the modules and keys that will have
                            sparsity parameters saved
            params_to_keep_per_layer: Dict to specify the params that should be
                            saved for specific layers. The keys in the dict
                            should be the module fqn, while the values should
                            be a list of strings with the names of the variables
                            to save in the `sparse_params`

        Examples:
            >>> # xdoctest: +SKIP("locals are undefined")
            >>> # Don't save any sparse params
            >>> sparsifier.squash_mask()
            >>> hasattr(model.submodule1, 'sparse_params')
            False

            >>> # Keep sparse params per layer
            >>> sparsifier.squash_mask(
            ...     params_to_keep_per_layer={
            ...         'submodule1.linear1': ('foo', 'bar'),
            ...         'submodule2.linear42': ('baz',)
            ...     })
            >>> print(model.submodule1.linear1.sparse_params)
            {'foo': 42, 'bar': 24}
            >>> print(model.submodule2.linear42.sparse_params)
            {'baz': 0.1}

            >>> # Keep sparse params for all layers
            >>> sparsifier.squash_mask(params_to_keep=('foo', 'bar'))
            >>> print(model.submodule1.linear1.sparse_params)
            {'foo': 42, 'bar': 24}
            >>> print(model.submodule2.linear42.sparse_params)
            {'foo': 42, 'bar': 24}

            >>> # Keep some sparse params for all layers, and specific ones for
            >>> # some other layers
            >>> sparsifier.squash_mask(
            ...     params_to_keep=('foo', 'bar'),
            ...     params_to_keep_per_layer={
            ...         'submodule2.linear42': ('baz',)
            ...     })
            >>> print(model.submodule1.linear1.sparse_params)
            {'foo': 42, 'bar': 24}
            >>> print(model.submodule2.linear42.sparse_params)
            {'foo': 42, 'bar': 24, 'baz': 0.1}
        r   r   T)Zleave_parametrizedNc                    s   i | ]}| | qS r!   r!   r:   krR   r!   r"   
<dictcomp>"  s      z.BaseSparsifier.squash_mask.<locals>.<dictcomp>r   c                    s   i | ]}| | qS r!   r!   rd   rf   r!   r"   rg   '  s      )r   r   Zremove_parametrizationsr(   rK   sparse_params)r   rb   rc   r`   ra   r   r   rh   Zglobal_paramsparamsZper_layer_paramsr!   rf   r"   squash_mask   s&    ;
  

zBaseSparsifier.squash_maskF)r   mappinginplaceparameterizationc           
      C   s   |dkrt d|st|}i }| D ]D\}}t||rXt||krXt||||< q*| j||d|d||< q*| D ]\}}	|	|j	|< qx|S )a  Converts submodules in input module to a different module according to `mapping`
        by calling `from_dense` method on the target module class
        Args:
            module: input module
            mapping: a dictionary that maps from source module type to target
                module type, can be overwritten to allow swapping user defined
                Modules
            inplace: carry out model transformations in-place, the original module
                is mutated
        NzNeed to auto generate mapping T)rk   rl   rm   )
NotImplementedErrorrB   rC   rS   r   r   r   convertr9   Z_modules)
r   r   rk   rl   rm   ZreassignrY   modr2   valuer!   r!   r"   ro   -  s(    

zBaseSparsifier.convert)use_pathr#   c              	   C   s:   | j s
d S t  | jD ]}| jf | qW 5 Q R X d S r&   )r   rG   Zno_gradr   update_mask)r   rr   rR   r!   r!   r"   stepY  s
    

zBaseSparsifier.step)r   r   c                 K   s   d S r&   r!   )r   r   r   ra   r!   r!   r"   rs   `  s    zBaseSparsifier.update_mask)N)T)NN)T) r,   
__module____qualname____doc__r   r   rU   r   r   r%   r)   r3   r>   boolrP   rQ   r
   Moduler   r	   r[   r^   r]   r   rj   r   ro   rt   abcabstractmethodrs   __classcell__r!   r!   r   r"   r      sB     5  Q,)rz   rB   collectionsr   typingr   r   r   r   r   r   r	   rG   r
   Ztorch.nn.utilsr   Ztorch.nn.utils.parametrizer   utilsr   r   r   r   r   __all__ZLinearrQ   r4   ABCr   r!   r!   r!   r"   <module>   s   $
