U h @sddlZddlmZddlmZddlmZmZmZm Z m Z m Z m Z ddl Z ddlZ ddlmZddlmZmZddlmZddlmZd d lmZmZmZd d lmZd d lmZm Z d ddddgZ!eGdddZ"e e#e#dddZ$e j%e#e#e e j%e#fdddZ&e j%e#e#e#e j%dddZ'e j()de j()dGdddej*Z+e j%e#e j%dd d!Z,e j%e j%e e#e#e#fe e#e#e#fe j%e j%e j%e j%d"d#d$Z-e j%e j%e.d%d&d'Z/e j()d$e j()d'Gd(d)d)ej*Z0Gd*d+d+ej*Z1Gd,d-d-ej*Z2Gd.d d ej*Z3e e"e4e ee.ee3d/d0d1Z5Gd2ddeZ6Gd3ddeZ7ee d4e6j8fd5dd6d7e e6e.ee3d8d9dZ9ee d4e7j8fd5dd6d7e e7e.ee3d8d:dZ:dS);N) dataclass)partial)AnyCallableDictListOptionalSequenceTuple)MLPStochasticDepth)VideoClassification)_log_api_usage_once)register_modelWeights WeightsEnum)_KINETICS400_CATEGORIES)_ovewrite_named_paramhandle_legacy_interfaceMViTMViT_V1_B_Weights mvit_v1_bMViT_V2_S_Weights mvit_v2_sc@sVeZdZUeed<eed<eed<eeed<eeed<eeed<eeed<dS) MSBlockConfig num_headsinput_channelsoutput_channelskernel_q kernel_kvstride_q stride_kvN)__name__ __module__ __qualname__int__annotations__rr)r)O/var/www/html/venv/lib/python3.8/site-packages/torchvision/models/video/mvit.pyrs    r)sreturncCsd}|D] }||9}q|SNr))r+productvr)r)r*_prod&s r1)x target_dim expand_dimr,cCs@|}||dkr ||}n||kr8td|j||fS)Nr.zUnsupported input dimension )dim unsqueeze ValueErrorshaper2r3r4 tensor_dimr)r)r* _unsqueeze-s   r;)r2r3r4r:r,cCs||dkr||}|Sr-)squeezer9r)r)r*_squeeze6s  r=csleZdZd ejeejeejeddfdd Zej e e e e fe ej e e e e ffdddZ Z S) PoolNF)poolnorm activationnorm_before_poolr,csVt||_g}|dk r&|||dk r8|||rFtj|nd|_||_dS)N)super__init__r?appendnn Sequentialnorm_actrB)selfr?r@rArBlayers __class__r)r*rDAs   z Pool.__init__r2thwr,c Cst|dd\}}tj|ddd\}}|dd}|jdd\}}}||||f|}|jrx|jdk rx||}| |}|jdd\}} } ||||ddd}tj ||fdd}|js|jdk r||}t |dd|}||| | ffS) Nr.r.r)indicesr5r r5) r;torchZ tensor_split transposer8reshape contiguousrBrHr?catr=) rIr2rNr: class_tokenBNCTHWr)r)r*forwardRs    z Pool.forward)NF)r$r%r&rFModulerboolrDrTTensorr r'r` __classcell__r)r)rKr*r>@sr>) embeddingdr,cCs@|jd|kr|Stjj|ddd|dddddS)Nrr.Zlinear)sizemode)r8rFZ functionalZ interpolatepermuter6r<)rerfr)r)r* _interpolatelsrj)attnqq_thwk_thw rel_pos_h rel_pos_w rel_pos_tr,c% Cs6|\}}} |\} } } tdt|| d} tdt| | d}tdt|| d}t| |d}t|| d}t|dddf|t| dddfd| |}t| | d}t| | d}t| dddf|t| dddfd| |}t| |d}t|| d}t|dddf|t| dddfd| |}t|| }t||}t||}||}||}||}|j\}}}}|ddddddf||||| |} td| |}!td| |}"| dddddd ||||| |} t | | dd dd}#|# |||| ||  dddddd }#|!ddddddddddddddf|"ddddddddddddddf|#ddddddddddddddf||||| | | | }$|ddddddddf|$7<|S) Nrr.?zbythwc,hkc->bythwkzbythwc,wkc->bythwkrr rO) r'maxrTZarangerjlongr8rVZeinsumrimatmulrUview)%rkrlrmrnrorprqZq_tZq_hZq_wZk_tZk_hZk_wZdhZdwdtZ q_h_ratioZ k_h_ratioZdist_hZ q_w_ratioZ k_w_ratioZdist_wZ q_t_ratioZ k_t_ratioZdist_tZRhZRwZRtrZZn_head_r5Zr_qZrel_h_qZrel_w_qZrel_q_tZrel_posr)r)r* _add_rel_pos{sP  <<<      **$... (rzr2Zshortcutresidual_with_cls_embedc CsX|r||nD|ddddddddf|ddddddddf7<|Sr-)add_r{r)r)r* _add_shortcuts Dr~cseZdZdejfeeeeeeeeeeeeeeeeee dej fddfdd Z e j eeeefee j eeeeffddd ZZS) MultiscaleAttention.N) input_size embed_dim output_dimrr r!r"r# residual_poolr| rel_pos_embeddropout norm_layerr,c stt||_||_||_|||_dt|j|_| |_ | |_ t |d||_ t ||g}| dkr|t j| ddt j||_d|_t|dkst|dkrdd|D}tt j|j|j||||jd d | |j|_d|_d|_t|dks t|dkrrd d|D}tt j|j|j||||jd d | |j|_tt j|j|j||||jd d | |j|_d|_d|_d|_| rpt|dd}t|d kr||dn|}t|d kr||dn|}d t||d}d |d d}t t||j|_t t||j|_t t||j|_t j j!|jddt j j!|jddt j j!|jdddS)Nrrr rTinplacer.cSsg|]}t|dqSrr').0rlr)r)r* sz0MultiscaleAttention.__init__..F)stridepaddinggroupsbiascSsg|]}t|dqSrr)rkvr)r)r*rsrr{Gz?std)"rCrDrrrhead_dimmathsqrtscalerrr|rFLinearqkvrEDropoutrGprojectpool_qr1r>Conv3dpool_kpool_vrorprqrtlen ParameterrTzerosinit trunc_normal_)rIrrrrr r!r"r#rr|rrrrJZ padding_qZ padding_kvrgZq_sizeZkv_sizeZ spatial_dimZ temporal_dimrKr)r*rDs         zMultiscaleAttention.__init__rMc Cs<|j\}}}||||d|j|jddjdd\}}}|jdk rZ|||\}} n|} |jdk rx|||d}|j dk r| ||\}}t |j ||dd} |j dk r|jdk r|jdk rt| ||| |j |j|j} | jdd} t | |}|jrt|||j|dd|d|j}||}||fS)Nr r.rrSrrR)r8rrVrrrUZunbindrrrrTrvrrorprqrzZsoftmaxrr~r|rr) rIr2rNrZr[r\rlkr0rnrkr)r)r*r` s6 2      zMultiscaleAttention.forward)r$r%r&rF LayerNormrr'rbfloatrrarDrTrcr r`rdr)r)rKr*rs& \rc s~eZdZddejfeeeeeeee e e dej fdd fdd Z e jeeeefee jeeeeffddd ZZS) MultiscaleBlockr.N) rcnfrr|rproj_after_attnrstochastic_depth_probrr,c st||_d|_t|jdkr\dd|jD} dd| D} ttj| |j| dd|_|rf|j n|j } | |j |_ | | |_ t |j tj|_t||j | |j|j|j|j|j||||| d |_t| d| |j gtj|dd|_t|d |_d|_|j |j krt|j |j |_dS) Nr.cSs g|]}|dkr|dn|qSrPr))rr+r)r)r*rUsz,MultiscaleBlock.__init__..cSsg|]}t|dqSrr)rrr)r)r*rVs)rr) r r!r"r#rrr|rrrO)Zactivation_layerrrrow)rCrDr pool_skipr1r"r>rFZ MaxPool3drrnorm1norm2 isinstanceZ BatchNorm1dneeds_transposalrrr r!r#rkr ZGELUmlpr stochastic_depthrr) rIrrrr|rrrrrZ kernel_skipZ padding_skipZattn_dimrKr)r*rDDsP     zMultiscaleBlock.__init__rMc Cs|jr ||ddddn||}|||\}}|jdksJ|jsN|n||}|jdkrf|n|||d}|||}|jr||ddddn||}|jdks|jr|n||}||| ||fS)Nr.rr) rrrUrkrrrrrr) rIr2rNZx_norm1Zx_attnZthw_newZx_skipZx_norm2Zx_projr)r)r*r`~s**zMultiscaleBlock.forward)r$r%r&rFrrr'rrbrrrarDrTrcr r`rdr)r)rKr*rCs   :rcsFeZdZeeeefeeddfdd ZejejdddZ Z S)PositionalEncodingN) embed_size spatial_size temporal_sizerr,cst||_||_tt||_d|_ d|_ d|_ |stt|jd|jd||_ tt|j||_ tt||_ dS)Nrr.) rCrDrrrFrrTrrY spatial_pos temporal_pos class_pos)rIrrrrrKr)r*rDs $zPositionalEncoding.__init__r2r,cCs|j|ddd}tj||fdd}|jdk r|jdk r|jdk r|jj \}}tj |j|dd}| |jd|j dd d|tj|jd|fddd}| ||S)NrrRr.rS)rYexpandrgr6rTrXrrrr8Zrepeat_interleaver}rrV)rIr2rYZhw_sizerZ pos_embeddingr)r)r*r`s &  zPositionalEncoding.forward) r$r%r&r'r rbrDrTrcr`rdr)r)rKr*rs"rcseZdZdeeefeeeeeeeeeeee e de j fe e de j feeeefeeeefeeeefdd fd d Z ejejd d dZZS)r?rNr rrrOrOr.r r .)rr block_settingrr|rrrattention_dropoutr num_classesblockrpatch_embed_kernelpatch_embed_stridepatch_embed_paddingr,cs*tt|t|}|dkr*td| dkr6t} | dkrLttjdd} tj d|dj |||d|_ dd t |f||j j D}t|dj |d |d f|d|d |_t|_t|D]^\}}| ||d }|j| ||||||| || d t|jdkrdd t ||jD}q| |dj|_ttj|ddt|dj| |_|D]}t|tjrtjj|jddt|tjr"|j dk r"tj!|j dntt|tjr|jdk rtj!|jd |j dk r"tj!|j dn,t|tr^|"D]}tjj|ddq q^dS)a MViT main class. Args: spatial_size (tuple of ints): The spacial size of the input as ``(H, W)``. temporal_size (int): The temporal size ``T`` of the input. block_setting (sequence of MSBlockConfig): The Network structure. residual_pool (bool): If True, use MViTv2 pooling residual connection. residual_with_cls_embed (bool): If True, the addition on the residual connection will include the class embedding. rel_pos_embed (bool): If True, use MViTv2's relative positional embeddings. proj_after_attn (bool): If True, apply the projection after the attention. dropout (float): Dropout rate. Default: 0.0. attention_dropout (float): Attention dropout rate. Default: 0.0. stochastic_depth_prob: (float): Stochastic depth rate. Default: 0.0. num_classes (int): The number of classes. block (callable, optional): Module specifying the layer which consists of the attention and mlp. norm_layer (callable, optional): Module specifying the normalization layer to use. patch_embed_kernel (tuple of ints): The kernel of the convolution that patchifies the input. patch_embed_stride (tuple of ints): The stride of the convolution that patchifies the input. patch_embed_padding (tuple of ints): The padding of the convolution that patchifies the input. rz+The configuration parameter can't be empty.Ngư>)epsr )Z in_channelsZ out_channelsZ kernel_sizerrcSsg|]\}}||qSr)r)rrgrr)r)r*rsz!MViT.__init__..r.r)rrrrrr) rrrr|rrrrrcSsg|]\}}||qSr)r)rr)r)r*r srRTrrrr)#rCrDrrr7rrrFrrr conv_projziprr pos_encodingZ ModuleListblocks enumeraterEr"rr@rGrrheadmodulesrrrweightrZ constant_ parameters)rIrrrrr|rrrrrrrrrrrZtotal_stage_blocksrZstage_block_idrZsd_probmweightsrKr)r*rDsv)        z MViT.__init__rcCst|ddd}||}|ddd}||}|jjf|jj}|jD]}|||\}}qN||}|dddf}| |}|S)Nrsrrr.) r;rflattenrUrrrrr@r)rIr2rNrr)r)r*r`!s     z MViT.forward) rrrrNNrrr)r$r%r&r r'r rrbrrrrFrarDrTrcr`rdr)r)rKr*rs:     x)rrrprogresskwargsr,c Ks|dk rbt|dt|jd|jdd|jddks>tt|d|jdt|d|jd|dd }|dd }tf||||d d |d d|dd |dd |d|}|dk r||j|dd|S)Nr categoriesmin_sizerr.rrmin_temporal_sizerrFr|Trr)rrrrr|rrr)rZ check_hash)rrmetaAssertionErrorpoprZload_state_dictZget_state_dict)rrrrrrrmodelr)r)r*_mvit8s,         rc @sJeZdZedeedddddddedd d d d d diddd dZeZdS)rz:https://download.pytorch.org/models/mvit_v1_b-dbeb1030.pthr?rr?rrZ crop_sizeZ resize_sizeZmeanrrzShttps://github.com/facebookresearch/pytorchvideo/blob/main/docs/source/model_zoo.mdThe weights were ported from the paper. The accuracies are estimated on video-level with parameters `frame_rate=7.5`, `clips_per_video=5`, and `clip_len=16`ip. Kinetics-400gJ +S@gh|?eW@zacc@1zacc@5guVQ@g rxa@ rrrZrecipeZ_docsZ num_paramsZ_metricsZ_ops _file_sizeurlZ transformsrN r$r%r&rrrrKINETICS400_V1DEFAULTr)r)r)r*rYs2c @sJeZdZedeedddddddedd d d d d diddd dZeZdS)rz:https://download.pytorch.org/models/mvit_v2_s-ae3be167.pthrrrrrrzChttps://github.com/facebookresearch/SlowFast/blob/main/MODEL_ZOO.mdrirg r0T@g(\W@rguVP@g?5^I|`@rrNrr)r)r)r*rzs2Z pretrained)rT)rr)rrrr,cKszt|}ddddddddddddddddgddddddddddddddddgddddddddddddddddggd d d ggd d d gggggggggggd d d gggd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gggdddggdddgggggggggggdddgggdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddggd }g}tt|d D]Z}|t|d ||d ||d ||d||d||d||d|d qtfdd|dd|dd||d|S)a Constructs a base MViTV1 architecture from `Multiscale Vision Transformers `__. .. betastatus:: video module Args: weights (:class:`~torchvision.models.video.MViT_V1_B_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.video.MViT_V1_B_Weights` below for more details, and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.video.MViT`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.video.MViT_V1_B_Weights :members: r.rrO`r rrrr r!r"r#rrrr r!r"r#rrFr皙?)rrrrr|rrr)rverifyrangerrErrrrrrconfigrir)r)r*rs """44,          cKst|}ddddddddddddddddgddddddddddddddddgddddddddddddddddgd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d ggd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d gd d d ggdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddggdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddgdddggd }g}tt|d D]Z}|t|d ||d ||d ||d||d||d||d|d qtfdd|dddd|dd||d |S)aCConstructs a small MViTV2 architecture from `Multiscale Vision Transformers `__ and `MViTv2: Improved Multiscale Vision Transformers for Classification and Detection `__. .. betastatus:: video module Args: weights (:class:`~torchvision.models.video.MViT_V2_S_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.video.MViT_V2_S_Weights` below for more details, and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.video.MViT`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.video.MViT_V2_S_Weights :members: r.rrOrrrrrr rrrrr r!r"r#rrTFrr) rrrrr|rrrrr)rrrrrErrrrr)r)r*rs """N          );r dataclassesr functoolsrtypingrrrrrr r rTZtorch.fxZtorch.nnrFZopsr r Ztransforms._presetsrutilsrZ_apirrr_metarZ_utilsrr__all__rr'r1rcr;r=Zfxwraprar>rjrzrbr~rrrrrrrrrrrr)r)r)r*sx  $         ,   <  H !!!$`