U zhg@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZddlmZmZmZmZmZmZddlZddlmZddlZddlmZddlmZddlmZddlm Z dd l!m"Z"m#Z#d d l$m%Z%d d lm&Z&m'Z'e(e)Z*ed Z+edZ,e,-Z.e.r8ddl/Z0gZ1dZ2e.r|ddddgZ1e0j3j4j567ddZ8d9dde1DZ2dddgZ:GdddZ;ddZe?dd#d$Z@d%d&d'd(ZAd)d*ZBd+d,ZCGd-d.d.eDZEd/d0ZFdQd1d2ZGdRd%d%d3d4d5ZHd6d7ZId8d9ZJd:d;ZKdSd%d%d3dd?d>d@dAdBZMe+eee+ge+fdCdDdEZNeNejOZPeNeQdFZReNdZSeNd%ZTeNd%ZUGdGdHdHZVGdIdJdJZWGdKdLdLZXdTeeegeefeYeeeYeZfeeZeeYefdNdOdPZ[dS)UN)Counter) import_module)AnyCallableDictListOptionalTypeVar)Tensor) rand_strided)is_float_dtype)StorageWeakRef)ContentStoreReaderContentStoreWriter)config) clone_inputs get_debug_dirTztorch._inductor.configz1//caffe2/torch/fb/sparsenn:sparsenn_operators_gpuz-//caffe2/torch/fb/sparsenn:sparsenn_operatorsz///deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpuz+//deeplearning/fbgemm/fbgemm_gpu:sparse_opszfbcode:// cCsg|]}d|dqS)ztorch.ops.load_library("z").0xrrK/var/www/html/venv/lib/python3.8/site-packages/torch/_dynamo/debug_utils.py 6srZbuck2runz@mode/dev-nosanc@s&eZdZddZddZd ddZdS) BuckTargetWritercCstjtj|\|_|_|jdd|_|jddd|j|_|j|jdd|_|jdd|_|j}||dddd}d|d |j|_ dS) Nz.pyr/.zfbcode.zfbcode/r:) ospathsplitabspathsubdirpy_filereplacetargetfind cmd_line_path)selffilenametmprrr__init__=szBuckTargetWriter.__init__c CsDdddtD}td|jd|jdtd|d|jd S) NrcSsg|]}d|dqS)z "z",rrrrrrLsz*BuckTargetWriter.build..za load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") python_binary( name="z", srcs = ["z"], compile = False, deps = [ "//caffe2:torch", "//caffe2/functorch:functorch", "//triton:triton", "z", ], cpp_deps = [ z ], main_module = "z", par_style = "xar", ) )join extra_depstextwrapdedentr+r) cur_targetr%)r.Zextra_cpp_depsrrrbuildKs zBuckTargetWriter.buildTc CsZtj|jd}t|d}||W5QRXt|jg}|rVt dd||S)NZTARGETSwzFFound an example that reproduces the error. Run this cmd to repro - %s ) r$r%r2r(openwriter7BUCK_CMD_PREFIXr-logwarning)r.Z print_msgZ target_filefdZ cmd_splitrrrr;ds  zBuckTargetWriter.writeN)T)__name__ __module__ __qualname__r1r7r;rrrrr<srcCsLtjtd}|dkr.tdt}tj|sHtj |dd|S)NZminifierz /minifier_T)exist_ok) r$r%r2rtempfile gettempdirgetpassgetuserexistsmakedirs)r%rrr minifier_dirrs  rJc@seZdZejjejjejjejjejj ejj ejj ejj ejj ejjejjejjejjejjejjejjejjejjejjejjejjgZeddZeddZdS)NNModuleToStringcCsLt}|D] \}}t|tjkr||qt|dkrHtd|dS)Nrz-We have not tested reprs of some modules - %sT) setnamed_childrentyperL safe_reprsaddlenr=r>)gmZ cant_convert_modulerrrcan_convert_to_strings   z&NNModuleToString.can_convert_to_stringcCsddlm}d}td}|D]V\}}|}t|d}|dk rZ|jrZ|d}||dd|d|d 7}q"|j D]\}} | dkrq| t krdd l m} | jt kstt| } n@t| rd t| jd | jd } ndt| jd | jd } | jr| d} ||dd|d| d7}q|j D]d\} } | dkrXqBd}| jrhd}dt| jd | j|d} ||dd| d| d 7}qB|||jdd 7}|S)Nr) _addindent z from torch.nn import * class Repro(torch.nn.Module): def __init__(self): super().__init__() z.cuda()zself.z = r) PRINT_OPTSz torch.randn(z, dtype=)ztorch.randint(1, size=zself.register_buffer('z', z) rz, device="cuda"ztorch.nn.Parameter(torch.randn(z))rK)Ztorch.nn.modules.modulerWr4r5rN__repr__next parametersZis_cuda_buffersitemsZnumelMAX_CONSTANT_NUMEL_INLINEZtorch._tensor_strrZ thresholdAssertionErrorreprtorchis_floating_pointlistshapedtype _parameterscode)rSrWtab model_str module_namerUZ module_strZ example_paramZ buffer_namebufferrZZ tensor_str param_nameparam maybe_devicerrrconvertsF            " zNNModuleToString.convertN)r@rArBrennZLinearZConv1dZConv2dZConv3dZ BatchNorm1dZ BatchNorm2dZ BatchNorm3dZ LayerNormZDropoutZSoftmaxZReLUZGELUZIdentityZ MaxPool2dZ EmbeddingZTanhZConvTranspose1dZGLUZLSTMZFlattenZAdaptiveAvgPool2drP staticmethodrVrsrrrrrL~s4 rLc CstjsdSd}zBtddg}|d}ddd|D}||d7}Wn"ttj fk rv|d 7}YnXt d d t tj D}|d 7}| D]\}}|d |d|d7}q|d7}|S)Nz:# torch.cuda.is_available()==False, no GPU info collected z# CUDA Info: Znvccz --versionrrcSs g|]}|dkrd|dqS))r#  r)rsrrrrsz-_cuda_system_info_comment..z# nvcc not found css|]}tj|VqdSN)recudaZget_device_name)rirrr sz,_cuda_system_info_comment..z# GPU Hardware Info: rvz : rw)rerzZ is_available subprocess check_outputdecoder&r2FileNotFoundErrorCalledProcessErrorrrangeZ device_countr`)rmZcuda_version_outZcuda_version_linescommentZ gpu_namesnamecountrrr_cuda_system_info_comments$  rF) stable_outputc Cs\ddl}ddl}|rdS|jjj}d|jjd|jjd|j jd|d S)Nrz*# config omitted due to stable_output=Truez~import torch._dynamo.config import torch._inductor.config import torch._functorch.config import torch.fx.experimental._config r) Ztorch._functorch.configZtorch._inductor.configZfxZ experimental_configZcodegen_config_dynamorZ _inductorZ _functorch)rreZexperimental_configrrrgenerate_config_strings   rcCstjtdS)Nzminifier_launcher.py)r$r%r2rJrrrrget_minifier_repro_pathsrc Cst}td|tr"t|z$t|d}||W5QRXWn6tk r|}ztdt d|W5d}~XYnXdS)NzWriting minified repro to: %sr8rz(Could not write to {minified_repro_path}) rr=r>use_buckrr;r:OSError exceptionNotImplementedError)contentsZminified_repro_pathr?errrhelper_for_dump_minifys    rc@s eZdZdS) AccuracyErrorN)r@rArBrrrrr"srcCsBt|}tt|D](}t||tjr||||jq|S)z This clone inputs is different from utils clone_input. In case of minifier, all the tensors are leaf tensors while creating a new graph. So, we set the requires_grad field w/o checking the leafness of the tensor. )rrrR isinstancerer Zrequires_grad_ requires_grad)example_inputsZ cloned_inputsidxrrrclone_inputs_retaining_gradness&s rc Csddlm}m}m}t|}|s*t|}t|dr>|dt|drP||n||}|r`|S||rx||}| |||d|S)z Runs a forward and possibly backward iteration for a given mod and args. When disable_clone is True, we will use args as-is without cloning. This is higher fidelity but we may destroy the args in the process. r)collect_resultsreduce_to_scalar_lossrequires_bwd_pass zero_gradTZ _boxed_callN) ZtestingrrrcopydeepcopyrhasattrrZbackward) rSargsonly_fwdZ disable_clonerrroutZlossrrrrun_fwd_maybe_bwd3s   r require_fp64 ignore_non_fpc Csddlm}t|||}d}tjrvz(tt|t|\} } t| | |}Wn*t k rt|rft dt dYnXzt|||} Wn2t k r} zt dWYdSd} ~ XYnX||| |tjd|d} | S)aa Check two models have same accuracy. require_fp64: if True, raise an error if we unable to calculate the fp64 reference ignore_non_fp: if True, do not compare outputs which are not floating point. This is mostly useful for the minifier (which wants to avoid quantizing floating point error into integer/boolean error) r)sameNzCould not generate fp64 outputszWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graph.T)ZtolZ equal_nanr)utilsrrrZsame_two_models_use_fp64 cast_to_fp64rrr Exception RuntimeErrorr=r>rZrepro_tolerance)rSZopt_gmrrrrrrefZfp64_refZ fp64_modelZ fp64_examplesresrZpassingrrrsame_two_modelsNs<  rcCs|jjD]}|jdkrj|jtjjjjkrjt |j dks:t t |j drj|j dtj krj|j dtj f|_ |jdkr|jd}|dk rt |rt|j}tj |d<||_q|j||S)NZ call_functionrYrrri)graphZnodesopr+reZopsZprimsZconvert_element_typedefaultrRrrcr float64kwargsgetdictZlintZ recompile)modelnoderiZ new_kwargsrrrcast_dtype_args_to_fp64s"      rcsBddlm}|}tjkr(t|}|fdd|}||fS)Nr)tree_mapcs"t|tjr|r|S|Sry)rrer rftorrirrs   zcast_to..)Ztorch.utils._pytreerrrerr)ririnputsrrrrcast_tos    rcCsttj||Sry)rrer)rrrrrrsrc Csdz,|t|t|}t||||||d WStk r^}ztdWYdSd}~XYnXdS)NrzWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graphF)rrrrrr=r)rSrZ compiler_fnrrrZ compiled_gmrrrrbackend_accuracy_failss$  rztorch._prims_common.StrideTypeztorch._prims_common.ShapeType)striderhreturncCs|dk r |St|Sry)rZmake_contiguous_strides_for)rrhrrr_stride_or_defaultsr)drcs fddS)Ncs|dk r |SSryrrrrrrz_mk_defaulter..rrrrr _mk_defaultersrcpuc@s4eZdZddZdddddZddZd d ZdS) NopInputReadercCs d|_dS)Nrtotal)r.rrrr1szNopInputReader.__init__Ndevice dtype_hintcCs|jd7_dS)Nrr)r. storage_hashnbytesrrrrrstorageszNopInputReader.storagecOsdSryrr.rrrrrtensorszNopInputReader.tensorcOsdSryrrrrrsymintszNopInputReader.symintr@rArBr1rrrrrrrrsrc@sJeZdZd ddddZdddddZddddddd d Zd d ZdS) InputReaderN)pbarcCs8|dkrtd|dk r"t|nd|_g|_||_dS)Nz0no save_dir specified, will generate random data)r=r>rstorerr)r.save_dirrrrrr1s  zInputReader.__init__rcCs|jdk r|jdt|}t|}|jdk r||dk r|z|j|}Wntk r\Yn X||jkrxt d||j|St d|||j f}t d|d}t |||| S)Nrzdevice mismatch: %s != %sz1could not load %s, generating random data insteadrh)rupdate_device_or_default_dtype_or_defaultrZ read_storagerrr=r>itemsizerr untyped_storage)r.rrrrrrhrrrrr s       zInputReader.storage)storage_offsetriris_leafc Kst||d}t|}t|}t|}t|}tjg||j|d} t| ||||W5QRX|st | j tj d} W5QRXt| ||||W5QRXtj j| |ksttj| ||j| | S)Nr)rirr)Z memory_format)r_storage_offset_or_defaultr_is_leaf_or_default_requires_grad_or_defaultrerrZno_gradset_Z enable_gradcloneZpreserve_format _subclasses meta_utils safe_is_leafrc_utilsZset_tensor_metadatarappend) r.rrhrrrirrmetadatatrrrrs,     zInputReader.tensorcCs|j||Sry)rr)r.valrrrr@s zInputReader.symint)N)Nrrrrrrs  !rc@sTeZdZddddZddZddded d d Zdd d d Zdd ddZdS) InputWriterF stable_hashcCs:g|_t|_||_|dk r*t||dnd|_i|_dS)Nr)_lines itertoolsrstorage_counterrrr seen_storages)r.rrrrrr1Rs zInputWriter.__init__cCs*dg}|dd|jD|d|S)Nzdef load_args(reader):css|]}d|VqdS)rXNr)rlrrrr|bsz$InputWriter.lines..zload_args._version = 0)extendrr)r.rrrrlines^s  zInputWriter.linesNr device_hintrc Cst|}|j|}|dk r |Sdt|j}d}tdt|krNd|}d}|j}|jdkrr|dk snt|}t d|krd|}| } d} |j dk r|jjdkr|j |} |j |d| d| ||d||j|<|S) Nbufrz , dtype_hint=metaz , device=z = reader.storage(, r[)r rrr]rrrrOrcrrrZ write_storagerr) r.rrrwsvZmaybe_dtype_hintrrrrrrrrrms0        zInputWriter.storagec Cs4|j||j|jd}g}td|jd|krH|tt |t d|jkrh|d|jt d| kr|d| t j|}|r|dd|Dtd|jkr|d|jt jj|}td|kr|d||jd d |tt |jf|d |dS) Nrrzdtype=zstorage_offset=css |]\}}|d|VqdS)=Nr)rkrrrrr|sz%InputWriter.tensor..zrequires_grad=zis_leaf=zreader.tensor(r) # )rrrirrrhrrstrtuplerrrrerZget_tensor_metadatarr`rrrrrrrr2)r.rrrrZtensor_metadatarrrrrs6  zInputWriter.tensorcCs0t|tjr|jj}|jd|d|dS)Nzreader.symint(r)rreZSymIntrhintrr)r.rrrrrrs zInputWriter.symint) r@rArBr1rrrrrrrrrrQs  rrz)funcr sym_shapesdefault_sym_shapercsddlm}dd|D}d|}t|}d|d}d|d } d } Gd d d } i} pjifd dtdfdd } |j}|D]t\}}|dkrqt | |}|r| \}}t | d}||}| ||| |<t | |}|r|d| |<qdt|jkr~| }|| d<t ||D]>}| \}}}}t | d}||}t||| ||q>| S)a Takes in a function which has been printed with print_readable() and constructs kwargs to run it. Handles Tensor inputs, Symints, and a graph module which might have tensor constants. Consider a function `forward` defined as follows: def forward(self, primals_1: "f32[1001, 6]", primals_2: "f32[s0]", primals_3: "Sym(s0)",): _tensor_constant0: "i64[4190]" = self._tensor_constant0 # Further implementation kwargs = aot_graph_input_parser(forward) forward(**kwargs) r) dtype_abbrscSsi|]\}}||qSrr)rkeyvaluerrr sz*aot_graph_input_parser..|z(_tensor_constant\d+): \"(z0)\[\s*(.*?)\s*\]\" = self\.(_tensor_constant\d+)(z)\[\s*(.*?)\s*\]z Sym\((s\d+)\)c@seZdZdZdS)z/aot_graph_input_parser..TensorContainerz#Container for tensors as attributesN)r@rArB__doc__rrrrTensorContainersrcs,tkpdk fddS)Ncs dS)Nz; not in symbolic_shapes and default sym shape not passed inrrrrrrrz=aot_graph_input_parser..get_sym_int..)re_checkrr)r r rr get_sym_ints  z+aot_graph_input_parser..get_sym_intrc sg}g}t|D]D\}}|}d|krF|}||||q|t|q|jrbtjntj}|||d}|D]} tj || qz|S)Nrx)rir) enumeratestriprintrfreZrandnZzerosrZ mark_dynamic) rhriZresolved_shapeZ dynamic_dimsr{dimrx constructorrr)rrrr gen_tensors  z*aot_graph_input_parser..gen_tensorr,rr.)Ztorch.fx.graphr r`r2valuesinspect getsourcer __annotations__researchgroupsrr&group signaturer^finditersetattr)rrr r r Z dtype_mapZ dtype_patternsourceZtensor_assignment_regexZ tensor_regexZsym_shape_regexrrr annotationsrq annotationmatchZ data_typeZ shape_strrhri container attr_namerTr)r rrr raot_graph_input_parsersD       r.)FF)F)F)rzNN)\r functoolsrFrrloggingr$r!r}rDr4 collectionsr importlibrtypingrrrrrr reZtorch._prims_commonZ _prims_commonrZtorch._subclasses.meta_utilsr Ztorch._dynamo.testingr r Z torch.multiprocessing.reductionsr Ztorch.utils._content_storerrrrrr getLoggerr@r=rZinductor_configZ is_fbcoderZlibfb.py.build_infoZlibfbr3Z extra_importspyZ build_infoZ BuildInfoZget_build_ruler*r6r2r<rrJrarL lru_cacherrrrrrrrrrrrrrrZfloat32rrrrrrrrrrrr.rrrrs           6 d   8 *  T[