U Mhb@sUddlZddlmZmZmZddlmZddlmZm Z m Z m Z m Z m Z mZddlmZddlmZddlmZdd lmZdd lmZddlZd d gZejjZd dZiZe e e fed<ddZ dMdd Z!e!ej"dde#dddZ$e!ej%dNe#dddZ&e!ej'dOe#dddZ(e!ej)dPe#dddZ*dQee#ee#ee#e+e#ddd Z,e!ej-ej.gdde#dd!d"Z/e!ej0e#dd#d$Z1d%d&Z2e!ej3ej4gdde#dd'd(Z5dd)ee e e#d*fe e#d*fe e#d*fe e e#d*ffdd+d,Z6dd)ee e e#d*fe e#d*fe e#d*fe e e#d*ffdd-d.Z7e!ej8d/d0dde#dd1d2Z9e!ej:d/d0e#dd3d4Z;d5d6Zgdde#dd7d8Z?e!ej@d/d0e#dd9d:ZAe!ejBd/d0e#dd;d<ZCej"e$ej%e&ej'e(ej)e*ej-e/ej.e/ej0e1ej3e5ej4e5ej=e?ej>e?ej8e9ej:e;ej@eAejBeCiZd=d>ZDd?d@dAdBdCgZEdDdEZFdFdGZGdHdIZHdJdKZIGdLd d eZJdS)RN)tree_map tree_flattentree_unflatten) ModuleTracker)ListAnyDictOptionalUnionTupleIterator) defaultdict)TorchDispatchMode)register_decompositionprodwrapsFlopCounterModeregister_flop_formulacCst|tjr|jS|SN) isinstancetorchZTensorshape)irJ/var/www/html/venv/lib/python3.8/site-packages/torch/utils/flop_counter.py get_shapes r flop_registrycstddfdd }|S)N)out_valcs(tt|||f\}}}|d|i|S)N out_shape)rr)r argskwargsr!frrnfszshape_wrapper..nfrr%r&rr$r shape_wrappersr(Fcsfdd}|S)Ncs"s t|}ttdd||S)NT)registryunsafe)r(rr)Z flop_formulaget_rawtargetsrr register_fun"sz+register_flop_formula..register_funr)r-r,r.rr+rr!s)r!returnc Os,|\}}|\}}||kst||d|S)zCount flops for matmul.AssertionError) a_shapeb_shaper!r"r#mkk2nrrrmm_flop*s r:cKs t||S)zCount flops for addmm.)r:Z self_shaper4r5r!r#rrr addmm_flop5sr<c KsD|\}}}|\}}} ||ks t||ks,t||| d|} | S)z"Count flops for the bmm operation.r1r2) r4r5r!r#br6r7b2r8r9floprrrbmm_flop:s     r@cKs t||S)z&Count flops for the baddbmm operation.)r@r;rrr baddbmm_flopGsrA)x_shapew_shaper! transposedr0c CsJ|d}|r|n|dd}|^}}}t|t||||d} | S)aCount flops for convolution. Note only multiplication is counted. Computation for bias are ignored. Flops for a transposed convolution are calculated as flops = (x_shape[2:] * prod(w_shape) * batch_size). Args: x_shape (list(int)): The input shape before convolution. w_shape (list(int)): The filter shape. out_shape (list(int)): The output shape after convolution. transposed (bool): is the convolution transposed Returns: int: the number of flops rr1Nr) rBrCr!rDZ batch_sizeZ conv_shapeZc_outZc_inZ filter_sizer?rrrconv_flop_countOs  rEc Ost||||dS)zCount flops for convolution.rD)rE) rBrC_bias_stride_padding _dilationrDr!r"r#rrr conv_flopvsrKc Csdd} d} | dr4t| d}| t|||| 7} | drt| d}|rn| t| || || |dd7} n | t| || || |dd7} | S)NcSs |d|dgt|ddS)Nrrr1)list)rrrrtszconv_backward_flop..trrFrF)rrE)grad_out_shaperBrCrGrHrIrJrDZ_output_paddingZ_groupsZ output_maskr!rM flop_countZgrad_input_shapeZgrad_weight_shaperrrconv_backward_flop|sH  " rPcCs|\}}}}|\}}} } |\} } } }||kr8| krlnn0||krP| krlnn|| krl| | krl|| ksptd}|t||||f|||| f7}|t|||| f||| |f7}|S)z^ Count flops for self-attention. NB: We can assume that value_shape == key_shape rr3r@) query_shape key_shape value_shaper=hs_qd_q_b2_h2s_k_d2_b3_h3_s3d_v total_flopsrrrsdpa_flop_counts   L""racOs t|||S)Count flops for self-attention.ra)rRrSrTr!r"r#rrr sdpa_flopsrd)grad_out.ccsN|dk r&t|jdkstt|jdks.t|dksF|j|jksFt|j\}} } |j\}} } |j\}} }|dk svt|dk st|j|jkst|dd|dd}|dd|dd}t||D]L\}}d| || f}d| || f}d| ||f}|dk r|nd}||||fVqdS|j|j|j|dk rB|jndfVdS)a; Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrlenrr3tolistzip)querykeyvaluere cum_seq_q cum_seq_kmax_qmax_k_h_qrWh_kd_kh_vr_Z seq_q_lengthsZ seq_k_lengthsZ seq_q_lenZ seq_k_lennew_query_shape new_key_shapenew_value_shapenew_grad_out_shaperrr%_unpack_flash_attention_nested_shapess(         r|ccsT|dk r,t|jdkstt|jdks.t|dksF|j|jksFt|j\}}} } |j\}}} } |j\}}} }|dk s|t|dk st|j|jkst|dd|dd}|dd|dd}t||D]L\}}d| || f}d| || f}d| ||f}|dk r|nd}||||fVqdS|j|j|j|dk rH|jndfVdS)a? Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for each batch element. In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. Nrrgrh)rlrmrnre cu_seqlens_q cu_seqlens_k max_seqlen_q max_seqlen_krsrtrWrurvrwr_Z seqlens_qZ seqlens_klen_qZlen_krxryrzr{rrr)_unpack_efficient_attention_nested_shapes+s(      rT)r,c Os(t|||||||d} tdd| DS)rb)rlrmrnrorprqrrcss"|]\}}}}t|||VqdSrrc.0rRrSrTrsrrr us z0_flash_attention_forward_flop..r|sum) rlrmrnrorprqrrr!r"r#sizesrrr_flash_attention_forward_flop[s rc Os(t|||||||d} tdd| DS)rb)rlrmrnr~rrrcss"|]\}}}}t|||VqdSrrcrrrrrs z4_efficient_attention_forward_flop..rr) rlrmrnbiasr~rrrr"r#rrrr!_efficient_attention_forward_flop{s rcCsRd}|\}}}}|\} } } } |\} }}}|\}}}}|| krR| krR|krnn*|| krt|krt|krnn|| kst||kr| |kr||kstd}|t||||f|||| f7}|t||||f|||| f7}|t||| |f||||f7}|t|||| f||| |f7}|t||||f|||| f7}|S)NrrQ)rNrRrSrTr`r=rUrVrWrXrYrZr[r\r]r^r_Z_b4Z_h4Z_s4Z_d4rrrsdpa_backward_flop_counts    P"""""rcOst||||S)z(Count flops for self-attention backward.r)rNrRrSrTr!r"r#rrrsdpa_backward_flopsrc Os*t|||||||| d} tdd| DS)N)rlrmrnrerorprqrrcss$|]\}}}}t||||VqdSrrrrRrSrTrNrrrrs z1_flash_attention_backward_flop..r) rerlrmrnoutZ logsumexprorprqrrr"r#shapesrrr_flash_attention_backward_flops rc Os*t|||||||| d} tdd| DS)N)rlrmrnrer~rrrcss$|]\}}}}t||||VqdSrrrrrrrs z5_efficient_attention_backward_flop..r) rerlrmrnrrr~rrrr"r#rrrr"_efficient_attention_backward_flops rcCst|ts|fS|Sr)rtuple)xrrrnormalize_tuples rKMBTcCs0tdtttdtt|dd}t|S)Nrrr1rf)maxminrisuffixesstr)numberindexrrrget_suffix_strs(rcCs&t|}|d|d}|t|S)Niz.3f)rr)rsuffixrrnrrrconvert_num_with_suffixs rcCs|dkr dS||dS)Nr0%z.2%r)numdenomrrrconvert_to_percent_str&srcstfdd}|S)Ncst|\}}|}t||Sr)rr)r"Z flat_argsspecrr$rrr&,s z)_pytreeify_preserve_structure..nfrr'rr$r_pytreeify_preserve_structure+srcseZdZdZdeeejje ejjfe e ee e e fdddZe dd d Ze ee e e ffdd d Zdd dZfddZfddZdddZddZZS)ra ``FlopCounterMode`` is a context manager that counts the number of flops within its context. It does this using a ``TorchDispatchMode``. It also supports hierarchical output by passing a module (or list of modules) to FlopCounterMode on construction. If you do not need hierarchical output, you do not need to use it with a module. Example usage .. code-block:: python mod = ... with FlopCounterMode(mod) as flop_counter: mod.sum().backward() Nr1T)modsdepthdisplaycustom_mappingcCs`tdd|_||_||_|dkr&i}|dk rOz*FlopCounterMode.__init__..zXsz,FlopCounterMode.__init__..) r flop_countsrrwarningswarnritemsr mod_tracker)selfrrrrrrr__init__IszFlopCounterMode.__init__r/cCst|jdS)NGlobal)rrvaluesrrrrget_total_flops\szFlopCounterMode.get_total_flopscCsdd|jDS)aReturn the flop counts as a dictionary of dictionaries. The outer dictionary is keyed by module name, and the inner dictionary is keyed by operation name. Returns: Dict[str, Dict[Any, int]]: The flop counts as a dictionary. cSsi|]\}}|t|qSr)dictrrrrrisz3FlopCounterMode.get_flop_counts..)rrrrrrget_flop_counts_s zFlopCounterMode.get_flop_countsc s|dkrj}|dkrd}ddl}d|_dddg}g}tdfdd }tjD]>}|d krxqj|d d }||krqj|||d }| |qjd jkrst |D] \} } d || d|| d<q|d d|}t |dkr d ddgg}|j||ddS)Ni?BrTModuleZFLOPz% TotalFcstj|}|kOd|}g}|||t|t|gj|D]0\}}||dt|t|t|gqX|S)N z - )rrrappendrrrr)mod_namerr`paddingrr7rZ global_flopsZ global_suffixZis_global_subsumedrrr process_modys  z.FlopCounterMode.get_table..process_modr.rr0r)leftrightr)headersZcolalign) rtabulateZPRESERVE_WHITESPACErrsortedrkeyscountextend enumerateri) rrrheaderrrmodZ mod_depthZ cur_valuesidxrnrrr get_tableks6   zFlopCounterMode.get_tablecs"|j|jt|Sr)rclearr __enter__superr __class__rrrs   zFlopCounterMode.__enter__cs0tj||j|jr,t||jdSr)r__exit__rrprintrr)rr"rrrrs  zFlopCounterMode.__exit__rcCs(|r|ni}|||}||j|||Sr) _count_flopsZ_overloadpacket)rfunctypesr"r#rrrr__torch_dispatch__s  z"FlopCounterMode.__torch_dispatch__cCsR||jkrN|j|}|||d|i}t|jjD]}|j|||7<q2|S)Nr )rsetrparentsr)rZ func_packetrr"r#Zflop_count_funcrOZparrrrrs   zFlopCounterMode._count_flops)Nr1TN)N)rN)__name__ __module__ __qualname____doc__r r rnnrrrboolr rrrrrrrrrr __classcell__rrrrr5s$  <   )F)N)N)N)F)KrZtorch.utils._pytreerrrZmodule_trackerrtypingrrr r r r r collectionsrZtorch.utils._python_dispatchrZ torch._decomprmathr functoolsrr__all__ZopsZatenrr__annotations__r(rmmrr:Zaddmmr<Zbmmr@ZbaddbmmrArrEZ convolutionZ _convolutionrKZconvolution_backwardrPraZ'_scaled_dot_product_efficient_attentionZ#_scaled_dot_product_flash_attentionrdr|rZ_flash_attention_forwardrZ_efficient_attention_forwardrrZ0_scaled_dot_product_efficient_attention_backwardZ,_scaled_dot_product_flash_attention_backwardrZ_flash_attention_backwardrZ_efficient_attention_backwardrrrrrrrrrrrrs $          ' g  6 3 6 0     !