U U?h@sddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlm Z Gd d d eZ!dS) N)onnx_pb) BaseQuantizerQuantizationParams) TensorData) ONNXModel)TENSOR_NAME_QUANT_SUFFIXQuantizationModeQuantizedValueQuantizedValueType __producer__ __version__add_infer_metadataattribute_to_kwargcompute_scale_zpcompute_scale_zp_float8 find_by_nameget_qmin_qmax_for_qTypeget_qrange_for_qType ms_domain&save_and_reload_model_with_shape_infertensor_proto_to_array)CreateOpQuantizerc@seZdZd:ddZddZddZdd Zd d Zd d ZddZ ddZ d;ddZ ddZ ddZ ddZddZdd$d%Zd&d'Zd?d(d)Zd@d+d,ZdAd.d/ZdBd0d1ZdCd2d3Zd4d5Zd6d7Zd8d9ZdS)D ONNXQuantizerNc  Cs~t|||||||| | | | |s|jt|jj}dd|jjD|_|jdd|jj D|jdd|jj Dt ||_||_ ||_ |jdk|_d|jko|jd|_g|_d|_i|_|jdd|jj D|jd d|jj D|jjjjD]} |jd d| j Dq |j tkrFtd |j ||_d |_d |_d|_d|_i|_|j|_ dS)NcSsi|] }|j|qSname).0virrY/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/quantization/onnx_quantizer.py Gsz*ONNXQuantizer.__init__..cSsi|] }|j|qSrrrZotrrrr HscSsi|] }|j|qSrrritrrrr Is ZMatMulConstBOnly/cSsi|] }|jdqSrrr!rrrr UscSsi|] }|jdqSr&rr"rrrr VscSsi|] }|dqSr&r)r output_namerrrr Xszunsupported quantization mode Zfixed_quantization_range_uint8Zfixed_quantization_range_int8Z fixed_zeroZ fixed_zero_zp)!r__init__modelZreplace_gemm_with_matmulrgraph value_info value_infosupdateoutputinputrmodestaticZ opset_versionfuse_dynamic_quant extra_optionsZq_matmul_const_b_only new_nodes graph_scope tensor_namesnoder ValueErrorcalculate_quantization_paramsquantization_paramsfixed_qrange_uint8_namefixed_qrange_int8_namefixed_zero_namefixed_zero_zp_namequantized_value_mapZget_non_initializer_inputsgenerated_value_names)selfr) per_channel reduce_ranger0r1 weight_qTypeactivation_qType tensors_rangenodes_to_quantizenodes_to_excludeop_types_to_quantizer3r7rrrr(&sP      zONNXQuantizer.__init__c Cs~tjj|d|jjjd}t|t||j|j|j |j |j |j |j |j|j|j|j }||_|j|d|_||jjjS)z generate submodel for the subgraph, so that we re-utilize current quantization implementation. quantize the submodel update subgraph and set it back to node onnx-quantizer) producer_nameZ opset_importsr%)onnxhelperZ make_modelr) opset_importrrrBrCr0r1rDrErFrGrHrIr3parentr5quantize_modelr*)rAsubgraphZ graph_keyZ warped_modelZ sub_quantizerrrrquantize_subgraphos0zONNXQuantizer.quantize_subgraphc Cs dd|jD}t|dkr |S|jr,|jn|jdt|j}i}|jD]}|jtjjkr|j| |j |d|ji}n\|jtjj krg}|j D].}| | ||d|jdt|gq|j|i}nt|}||qLtjj|j|j|jfd|ji|S)z| Check subgraph, if any, quantize it and replace it. return new_nodes added for quantizing subgraph cSs,g|]$}|jtjjks$|jtjjkr|qSr)typerLAttributeProtoGRAPHGRAPHS)rattrrrr sz>ONNXQuantizer.quantize_node_with_sub_graph..rZ _node_count_:r) attributelenrop_typer4rSrLrTrUrRgrVZgraphsextendrr-rM make_noder/r.) rAr7Z graph_attrsZ node_namekwargsrWkvvaluerQrrrquantize_node_with_sub_graphs0 " "   z*ONNXQuantizer.quantize_node_with_sub_graphcCstdd|jDS)zQ Detect if model already has QuantizeLinear or DequantizeLinear. css"|]}|jdkp|jdkVqdS)QuantizeLinearDequantizeLinearN)r\rr7rrr sz.ONNXQuantizer.has_QDQ_nodes..)anyr)nodes)rArrr has_QDQ_nodesszONNXQuantizer.has_QDQ_nodescCs2t||jdk rdS|jdk r.|j|SdS)NTF)rr) initializerrOfind_initializer_in_path)rAZinitializer_namerrrrls   z&ONNXQuantizer.find_initializer_in_pathcCs2|j||D]}|jD]}|j|qqdSN)r4r^r.r@add)rArir7r'rrr add_new_nodess  zONNXQuantizer.add_new_nodesc CsH|rtd|jD]d}|jr0||}t|j}t ||}| t |t|jD]"}|j|j D]}|j |qlq\q||jd|jj|j|jdkr|j\}}t|dkrtdt|t|jj_t|jj_dd|jjjD}|s@dd|jD} | r@|jjj} d| _t| _|jjS) NzPlease check if the model is already quantized. Note you don't need to quantize a QAT model. OnnxRuntime support to run QAT model directly.r7rz0Invalid model with unknown initializers/tensors.cSsg|]}|jtkr|qSr)domainr)ropsetrrrrXs z0ONNXQuantizer.quantize_model..cSsg|]}|jdkr|qS)z com.microsoft)rprfrrrrXs r) rjloggingwarningr)rienable_subgraph_quantizationrcr[r4rquantizeranger.r@rn_dequantize_outputsr*Z ClearFieldr7r^rOZclean_initializers RuntimeErrorstrr rKr Zproducer_versionrNversionrrp) rAr7Znumber_of_existing_new_nodesZ op_quantizerir'_Zinitializers_not_foundZms_opsetZms_nodesrqrrrrPs<       zONNXQuantizer.quantize_modelcCs<d|jkr(td||jd|jdStd|ddS)NZDefaultTensorTypezDget_tensor_type returns DefaultTensorType for tensor name %r, use %dz)Unable to find data type for weight_name=a7. shape_inference failed to return a type probably this node is from a different domain or using an input produced by such an operator. This may happen if you quantize a model already quantized. You may use extra_options `DefaultTensorType` to indicate the default weight type, usually `onnx.TensorProto.FLOAT`.)r3rrinforxrA tensor_namerrr_get_default_tensor_types   z&ONNXQuantizer._get_default_tensor_typeFcCst||j}|dk r|jS||jkrd|j|}|jdrd|rZ|jjjdkrZ| |S|jjjS|j rt|j dkr|r| |SdS|j |}|dk r|S|j r|j r|j |}|dk r|S|r| |SdS)N tensor_typer)rr)rkZ data_typer,rSHasFieldr elem_typerrtrOis_valid_quantize_weightget_tensor_type)rAr mandatoryweightrotyperesrrrrs.          zONNXQuantizer.get_tensor_typecCs||r||S||jkrp|j|}|jdrR|jjjtjj tjj fkrRdSt d|d|jddS|j r|jr|j|St d|ddS) NrTz)rArrrrinput_scale_namereduce_min_namereduce_min_nodereduce_max_namereduce_max_nodeZreduce_min_abs_nameZreduce_min_abs_nodeZreduce_max_abs_nameZreduce_max_abs_nodeZ abs_max_nameZ abs_max_nodeZinitializer_divscale_div_namescale_div_nodeZinitializer_zprrrrDs|           z9ONNXQuantizer._get_dynamic_input_quantization_params_int8cCstjj}|d}|d}|d}tjjd|g|dg|dd}|||d} tjjd |g| dg| dd} || tj|j|gt |g} |j | tj|j |gd g} |j | |d } tjd | j d|j dg| dg| }|||d }tjd|j d|jg|g|}|||d}tjd |j |j dg|dg|}|||d}tjd|j d|g|dg|}|||d}tjd|j |dg|}|||d}tjjd|j |g||d}||||ggfS)a{ Create nodes for dynamic quantization of input to uint8 and add them to nodes_list parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter initial_type: initial weight type (FLAOT or FLOAT16) return: scale_name, zero_point_name, scale_shape, zero_point_shape. r _zero_pointrrrrrrrgZ _scale_SubSubZ _scale_DivrZ_zero_point_SubZ_zero_point_DivZ_zero_point_FloorZFloorZ_zero_point_CastZCast)to)rrrrLrMr_rrr;rr)rr=r.)rArrrrrZ input_zp_namerrrrZinitializer_qrangeZinitializer_qvalueZscale_sub_nameZscale_sub_noderrZ zp_sub_nameZ zp_sub_nodeZ zp_div_nameZ zp_div_nodeZ zp_floor_nameZ zp_floor_nodeZ zp_cast_nameZ zp_cast_noderrrrs           z:ONNXQuantizer._get_dynamic_input_quantization_params_uint8cCs |j}|dks|dkr|jdks,||jkrBtd|ddS|j|}t|tsptdt|d|d|dkst|dkrt d |d |t |d g}t |d d r|d j t jt jfkrt dt|d d|t |d g}|j t jks t|d}nRt |g}t |g}|j|}d |krT|d j }||}|j t jksftg} |d} g} |d} tj| || |} |j| |j t jkrtjj}n.|j t jkrtjj}nt d|j d|tj| || |d}|j|d| | | | fS)a\ Create initializers and inputs in the graph for zero point and scale of output. Zero point and scale values are obtained from self.quantization_params if specified. parameter param_name: Name of the quantization parameter. return: result, scale_name, zero_point_name, scale_shape, zero_point_shape. Nz$Quantization parameters for tensor:"z" not specified)FrrrUnexpected type  for rzbQuantization parameters should contain zero point, scale, quant type. Specified values for output z: zero_pointscaledtypez and param_name= quant_typerrzUnexpected dtype=z for param_name=)T)rEr:rrr} isinstancer TypeErrorrSr[r8nparrayhasattrrZfloat32Zfloat16Zfloat64AssertionErrorZastyperLrMrZraveltolistr)rrrrrZreshape)rA param_nameZ use_scaleZ use_zeropointZzero_point_typeparamsZzero_point_valuesZ scale_valuesrZzero_point_shapeZzero_point_name scale_shape scale_nameZinit_zpZ scale_typeZ init_scalerrr_get_quantization_paramssX  $            z&ONNXQuantizer._get_quantization_paramsc CsH|j|}|dkstd|t}|d} |dk rL|dk rLd||} } } n||\} } } } } g}| rtjd|| | g|g| }n|jrdS|jr|t j j kr|d} |d} tjd |g|| | g| }n\|dk std |d |d |d ||j ||||d\} } }}tjd|| | g|g| }t ||| | ||j|<||fS)a Given an input for a node (which is not a initializer), this function - add nodes to compute zero point and scale for this input if they don't exist. - add new QuantizeLinear node to quantize the input. :param node: node being quantized in NodeProto format. :param input_index: index of input in node.input. :param qType: type to quantize to. :param given_scale_name: if those inputs need to be quanitzed using this scale tensor. :param given_zp_name: if those inputs to be quantized using this zeropoint tensor. :param initial_type: type of the weight to quantize :return: List of newly created nodes in NodeProto format. rz*Cannot access undefined variable in graph._QuantizeLinearNTrdrrZDynamicQuantizeLinearzCCannot quantize input without knowing the initial type, input_name=z, input_index=z, qType=z, node=r)r/rrrrLrMr_r1r2rrrrr r?)rAr7 input_indexrZgiven_scale_nameZ given_zp_namerrr'Z ql_node_nameZ data_foundrzp_namer|ri qlinear_noderZzp_shaperrr_get_quantize_input_nodes1sV   z'ONNXQuantizer._get_quantize_input_nodescCs.||jkr|j|S|jdk r*|j|SdSrm)r?rOfind_quantized_value)rArrrrrws     z"ONNXQuantizer.find_quantized_value?c Cs||jkr|j|jS|j|j}t||j}t|}||jkrR|j|j}n0||jkrr||\} }} } } nt d|dt||j} t| } | || ||\} } }}}}||jkst t || | |t j|jdkrdnd||d}||j|<| S)z] Quantized the bias. Zero Point == 0 and Scale == Input_Scale * Weight_Scale z Expected z5 to be in quantized value map for static quantizationrrN) node_type node_qtype)r?q_namerrr)rkrr:rr8Zquantize_bias_static_implrr r Initializersize)rAZ bias_namer weight_namebetaZweight_scale_nameZweight_initializerZ weight_scalerr|Zinputscale_initializerZ input_scaleZquantized_bias_nameZquantized_bias_scale_nameZquantized_bias_zp_nameZbias_scale_datarrquantized_valuerrrquantize_bias_static~sB       z"ONNXQuantizer.quantize_bias_staticcCs||jkp||jkp||jkS)zq only check for value info and newly generated tensor names, initializers are checked separately )r,r6r@r~rrrcontains_tensors  zONNXQuantizer.contains_tensorc Cs|j||dddd|dS)NFrr7indicesinitializer_use_weight_qTyperCop_level_per_channelaxis from_subgraph_ONNXQuantizer__quantize_inputs)rAr7rrrrrquantize_activationsz!ONNXQuantizer.quantize_activationrc Cs|j||d||||dS)NTrr)rAr7rrCrrrrrrquantize_weights zONNXQuantizer.quantize_weightTc Csg}g} g} g} |D]} |j| } | |jkr^|j| }||j| |j| |jq| s| d|d| dqt| |j}|dk r |j r|r| |j |r|j n|j ||\}}}n |||r|j n|j |\}}}| || |||q|| r~|j| d|j|j}|dkr|j| }||jkr|j|}|dsztd|d|jdstd|d|jjj}n&||jkstd |d |j|}|j|| |j |d }|dkrd S|r||n | ||d }|jdkrL| |j||jd| |jdn0| |jd||jd| |jdq|jdk r|jj || g||||dd\}}}}| |d||d| |dqt!d| d|j"q| | || fS)a Given a node, this function quantizes the inputs as follows: - If input is an initializer, quantize the initializer data, replace old initializer with new initializer - Else, add QuantizeLinear nodes to perform quantization parameter node: node being quantized in NodeProto format. parameter indices: input indices to quantize. return: (List of quantized input names, List of zero point names used for input quantization, List of scale names used for input quantization, List of new QuantizeLinear nodes created) rNrrSz value_info=z has no type.rz is not a tensor.zshape inference failed for zF and attribute 'tensor_names' does not have any value for this tensor.r)NNNNrrdrrT)rrCrrrz!Invalid tensor name to quantize: z @graph scope)#r/r?rrrrrr)rkrBquantize_weight_per_channelrrDrEquantize_initializerrfind_node_by_namer4r*r,rrrSrrr6rror^r\r.rOrr8r5)rAr7rrrCrrrZ scale_namesZzero_point_namesZquantized_input_namesrirZ node_inputrrk q_weight_namerrrrr+rZquantize_input_nodesZparent_quantized_input_namesZparent_zero_point_namesZparent_scale_namesr|rrrZ__quantize_inputss                                zONNXQuantizer.__quantize_inputsc Csj|j|jkr(|j|j}|j|j|jfS|||||\}}}t|j|||tjd}||j|j<|||fS)a :param weight: TensorProto initializer :param qType: type to quantize to :param keep_float_weight: Whether to quantize the weight. In some cases, we only want to qunatize scale and zero point. If keep_float_weight is False, quantize the weight, or don't quantize the weight. :return: quantized weight name, zero point name, scale name N) rr?rrrZquantize_initializer_implr r r) rArrrCkeep_float_weightrrrrrrrr\s,    z"ONNXQuantizer.quantize_initializerc Csd||jkr$|j|}|j|j|jfS||||||\}}} t||| |tjd}||j|<||| fSrm)r?rrrZ quantize_weight_per_channel_implr r r) rArrDZ channel_axisrCrrrrrrrrr}s.    z)ONNXQuantizer.quantize_weight_per_channelcCs||jkr||jkr|j|}t|j|j}|jjjdksT|jjjdkrj|dk rjtj |j dksjt |d}|j ||j |j}|dkr|j|j|jg}tjd||g|}|S||jdkst dS)a Given a value (input/output) which is quantized, add a DequantizeLinear node to dequantize it back to float32 or float16 parameter value_name: value to dequantize parameter new_nodes_list: List of new nodes created before processing current node return: None if there is already a DequantizeLinear node that dequantizes it A DequantizeLinear node otherwise rJNrZ_DequantizeLinearrer)r?r@rrr)rkrKrLZ numpy_helperZto_arrayrrrr4r*rrrMr_r.)rAZ value_namerZ scale_initZ dqlinear_nameZ dqlinear_nodeZdqlinear_inputsdequantize_noderrr_dequantize_values2   zONNXQuantizer._dequantize_valuecCs6|jjD]$}||j}|dk r |j|q dS)z Dequantize output if it is quantized parameter new_nodes_list: List of new nodes created before processing current node return: List of new nodes created N)r)r*r.rrr4r)rAr.rrrrrws z!ONNXQuantizer._dequantize_outputscCsB|jdkrdS|i}|jD]}|j|}t|tsTtdt|d|d|jj|id}|j}d|kr||dj }d|krd|kr|d|d}}n|t j j krt ||jd\}}nh|d |jd }|d |jd} |d |j} |d d} t|| | d\} } t|| | | | |j\}}t|||d||<q |S)Nrrr)Z default_valrrrrrminrrmax symmetricrCF)rCr)rrr)rFZadjust_tensor_rangesrrrrSZtensor_quant_overridesZget_per_tensor_overridesrErrLrZ FLOAT8E4M3FNrZavg_stdgetZ range_valueZis_activation_symmetricrrZmin_real_ranger)rAr:rtdZquant_overridesrzerorrrrrCZqminZqmaxrrrr9s0       z+ONNXQuantizer.calculate_quantization_params)N)F)NN)NNN)r)F)FFrF)TFFrF)FF)TF)__name__ __module__ __qualname__r(rRrcrjrlrorPrrrrrrrrrrrrrrrrrrwr9rrrrr%sT I"- T^ < F 1    & ' r)"rrnumpyrrLZonnx.numpy_helperrrZbase_quantizerrrZ calibraterZ onnx_modelrZ quant_utilsrr r r r r rrrrrrrrrrregistryrrrrrrs   H