U zh6@s8ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z ddl Z ddl Z ddlmZddlmZddlmmZddl mZddlmZddlmZdd lmZmZmZdd lm Z dd l!m"Z"m#Z#m$Z$e%e&Z'd d Z(eddZ)eej*e dddZ+d0ddZ,ddZ-eej*e dddZ.Gdddej/Z0eej*e dddZ1eddZ2d d!Z3e j4j5Z5e5j6e5j7e5j8e5j9e5j:e5j;e5je5j?e5j@e5jAe5jBe5jCjDe5jCjEe5jFe5jGe5jHe5jIe5jJe5jKe5jLhZMeeMZMed"d#ZNe e ejOfd$d%d&ZPd'd(ZQdaRd)d*ZSd+d,ZTd1d.d/ZUdS)2N)contextmanager)partial)CallableUnion)SymInt)get_decompositions) bind_symbols) aot_function aot_modulemake_boxed_compiler)strip_overloads)default_partition draw_graph#min_cut_rematerialization_partitioncCs4|jjdtjjjdD]}tjjj|_q||S)N call_functionoptarget) graph find_nodestorchopsaten_to_copytor recompile)fx_gnoderL/var/www/html/venv/lib/python3.8/site-packages/torch/_functorch/compilers.py _canonicalize$s r!c cs*tjd}z dVW5tj|XdS)NF)r_CZ_jit_set_autocast_mode)Zold_jit_autocast_flagrrr _disable_jit_autocast-s  r#)rreturnc Cstt||jjdtjjjdD]6}t|j dkr(t|j dkr(d|j kr(tjjj |_ q(|jj D]<}i}|j D]"\}}t|tjr|j}|||<qz||_ qh|j|tj|}tj|jtj|}tj|}tdd|Ds ||W5QRX|S)a Compiles the :attr:`fx_g` with Torchscript compiler. .. warning:: This API is experimental and likely to change. Args: fx_g(fx.GraphModule): The input Fx graph module to be compiled. Returns: Torch scripted model. rrr dtypecss|]}t|tjjVqdSN) isinstancerZ _subclassesZ FakeTensor).0trrr `szts_compile..)r#r rrrrrrlenargskwargsrrnodesitemsr'devicetypeZlintrjitscriptr"Z_jit_pass_remove_mutationfreezeevalZoptimize_for_inferenceany)rinpsrZ new_kwargskvfrrr ts_compile6s0  &      r;TcCst|jt|||d|S)N) clear_meta)printcoder)r_namer<rrr _draw_graph_compilees rAcCsttt|dS)Nr@)r rrArBrrr draw_graph_compileksrCcCs|S)z Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler and can be used to check accuracy. .. warning:: This API is experimental and likely to change. rrr?rrr nopos rEcs(eZdZfddZfddZZS)DebugInterpretercs"t|jf||_tj|dSr&)rmodulesymbol_mappingsuperrun)selfr, __class__rr rJ}szDebugInterpreter.runc sfddfddfddfdd}t|}d |jkrt|jd \}}t|\}}t|t|kstt|d t|ttt|||D].\}} t | t j sq||| fd d q|S) Ncs:t|ts|St|jjj}|js2t |t |Sr&) r'rsympyexpandrexprZxreplacerHZ is_numberAssertionErrorint)nir)rKrr subst_symints  z/DebugInterpreter.run_node..subst_symintcstfdd|DS)Nc3s|]}|VqdSr&r)r(rSrUrr r*szHDebugInterpreter.run_node..subst_symint_tuple..)tuple)ZnisrVrr subst_symint_tuplesz5DebugInterpreter.run_node..subst_symint_tuplecsT|dkrPt|jD]4}||||kr||dkrdSqdS)Nrr FT)Znumelrangendimstridesize)abidxrVrr check_significant_stridessz.check_significant_stridesc st|s t|j|jks6t|d|jd|j||kszt|d|d|d|||}|st|d|d|d|dS)Nz:  != z aka )callablerQr%r\r[)nvrvdescZ same_strides)r`rXrr checks ** *z(DebugInterpreter.run_node..checkvalracsddjS)Nzoutput z where )rHr)irKrr z+DebugInterpreter.run_node..) rIrun_nodemetapytreeZ tree_flattenr+rQziprYr'rTensor) rKnrfrTZn_valsZn_specZr_valsZr_specrcrdrL)r`rhrKrUrXr rks     * zDebugInterpreter.run_node)__name__ __module__ __qualname__rJrk __classcell__rrrLr rF|s rFcCs t|jS)z Returns a (slow) interpreter over the FX graph module that also checks various debugging properties (e.g., that tracing strides matched real strides.) )rFrJrDrrr debug_nopsrucCs(t|tj|}tj|}|Sr&)r rr2r3r4r5)rr?r:rrr simple_ts_compiles rvcCs t|tSr&)r rv)r:rrr nnc_jitsrwcCst|j|Sr&)r=r>rDrrr print_compiles rx)fncKsBttttd}||t|tjjr2t|f|St |f|SdS)a Wrapper function over :func:`aot_function` and :func:`aot_module` to perform memory efficient fusion. It uses the :func:`min_cut_rematerialization_partition` partitioner to perform efficient recomputation. It uses NVFuser to compile the generated forward and backward graphs. .. warning:: This API is experimental and likely to change. Args: fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` that takes one ore more arguments. Must return one or more Tensors. **kwargs: Any other overrides you want to make to the settings Returns: Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior of the original :attr:`fn`, but whose forward and backward graphs have gone through recomputation optimizations, and the graphs have been compiled with nvfuser. Z fw_compilerZ bw_compilerZ partition_fnZdecompositionsN) r;rdefault_decompositionsupdater'rnnModuler r )ryr-configrrr memory_efficient_fusions  rcCsH|dtddd|Ddddlm}||t||S)NfooaQ ############################################################## # To minimize FX graph, copy and paste the below and run it # ############################################################## import torch import torch.fx as fx from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess inps = cSsg|]}|j|jfqSr)shaper%)r(rhrrr sz!debug_compile..a? inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] from foo import FxModule mod = FxModule().cuda() with torch.jit.fuser("fuser2"): # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) r)FxModule) to_folderr=rrcudar;)rr7rrrr debug_compiles   rc Csg}t|d}t|}g}|D]}t|dkrD|}|t}nX|\}}}} } | tjtjtj tj tjtj tt hkrtj dd|| | d}ntj|| | d}||q"W5QRX|S)zZ Return a random input for the given inputs meta generated from _save_fx_default. rbr r)r%r0)openpickleloadr+randomZrandrrRZint32Zint64boolZuint8floatrandintappend) Zinput_data_pathinputsr:Z inputs_metarlr1inputrr[r%r0rrr get_inputs3s.    rc sbddlm}fddfddfdd}fd d }fd d }||||||td S)aO The forward, backward, and joint computation graph will be stored in {folder_name}/{current_name}/{current_name}_forward_{graph_index}, {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. The input shape of the graphs will be stored in the .input files. These files can be loaded with pickle, and is a list of format (type, shape, stride, dtype, device). In the case of type = int or float, it is just (type,). For joint graph input, it is a nested list [[],[]] where the two inner lists have the same format. If dump_example_input is True, example_inputs will be stored in .pt file. Since each function might produce multiple graphs, the graph_index is used to distinguish difference graphs r)aot_module_simplifiedcsg}t|dkrBt|dtrB||d7}||d7}|S|D]P}t|tksbt|tkrt|t|fqF|t||j||j |j fqF|S)Nrr ) r+r'rWr1rRrrrr[r%r0)r, input_metaarg)get_input_metarr rdsz(_save_fx_default..get_input_metacst|jjdkr(ttjd|tdSt|}|j t j j | |}tjddd|ddd|dt t|tddd|dtdd|dtddrt |ddd|dtdd|dtd dS) Nrz!No nodes in graph {%s}_{%s}_{%s}./T)exist_okr?z.inputwbz.pt)r+rr.logloggingWARNING graph_indexcopydeepcopyZ set_codegenrfxZCodeGenrosmakedirsrrdumprsave)Z gm_to_saver, type_namegmr) current_namedump_example_input folder_namerrr graph_saver_helperss: 22z,_save_fx_default..graph_saver_helpercs||d|S)Nforwardr)rZfw_argsrrr graph_saver_forwards z-_save_fx_default..graph_saver_forwardcs||dtd7a|S)NZbackwardr )r)rZbw_argsrrr graph_saver_backwards z._save_fx_default..graph_saver_backwardcs||dt||S)NZjoint)r)rZ joint_argsrrr graph_saver_joints z+_save_fx_default..graph_saver_jointrz)Zfunctorch.compilerr{) rrrrZexample_inputsrrrrr)rrrrrr _save_fx_defaultRs  #   rFcCsdatt|||S)as Dump the forward, backward, and joint computation graph. Example Usage: save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) optimize_ctx = torchdynamo.optimize( save_fx_func ) with torch.enable_grad(): with optimize_ctx: result = forward_and_backward_pass(model, example_inputs) r)rrr)rrrrrr graph_dumper_aots r)T)F)Vrrrrr contextlibr functoolsrtypingrrrNrZtorch.fxrZtorch.nnr}Ztorch.utils._pytreeutilsZ_pytreermrZ torch._decomprZ%torch.fx.experimental.symbolic_shapesrZ aot_autogradr r r Z compile_utilsr Z partitionersrrr getLoggerrqrr!r#Z GraphModuler;rArCrEZ InterpreterrFrurvrwrrdetachZ gelu_backwardZleaky_relu_backwardZsigmoid_backwardZthreshold_backwardZhardtanh_backwardZhardsigmoid_backwardZhardswish_backwardZ tanh_backwardZ silu_backwardZ elu_backwardZcudnn_batch_normZcudnn_batch_norm_backwardZ masked_fillZScalarroZeluZ leaky_reluZhardtanhZ hardswishZ hardsigmoidZ conj_physicalZ is_same_sizer{rxr~rrrrrrrrrr s           .  7    ']