U U?h{@sddlmZddlZddlZddlZddlmZddlmZddl Z ddl Z ddl m Z m Z m Z ddl mZddlmZmZmZmZddlmZdd lmZmZmZzdd lmZWnek rdZYnXzdd lmZmZWnek rdZdZYnXd Z d Z!dZ"dZ#dZ$dZ%dZ&dZ'dZ(iZ)dde*e DZ+GdddeZ,GdddeZ-GdddeZ.GdddeZ/ej j0e 1dej j2e 1d ej j3e 1d!ej j4e 1d"ej j5eej j6eej j7eiZ8ej j2e j9de j:d#e j9d$e j:d#fej j0e j9d%e j;d#e j9d&e j;d#fej j4e j9de jej j0e j9d-e j;d#e j9d&e j;d#fej j3e j9d.e j=d#e j9d)e j=d#fiZ?ej j2e j9de j:d#e j9d&e j:d#fej j0e j9d/e j;d#e j9d0e j;d#fej j4e j9de jd?ZDdd@dAZEddBdCZFddDdEZGdFdFdGdHdIdJZHdKdLdMdNdOZIGdPdQdQZJGdRdSdSZKGdTdUdUZLdVdWZMdXdYZNdZd[ZOd\d]ZPd^d_d^d`dadbZQdcddZRddfdgZSddidjZTd^dkdldmZUd^d^dndodpZVdqdrdsdtZWdqdudvdwdxZXdqdrdydzZYdqdudvd{d|ZZd^dqd}d~dZ[dqdqdvddZ\dddddZ]d_d_dddZ^d_d_dddZ_d_dddZ`d_dddZad_dddZbd_dddZcdS)) annotationsN)Enum)Path) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions float8e4m3fn)int4uint4z onnx.quantizez0.1.0zai.onnxz com.microsoftQuantizeLinearZ_QuantizeLinear_InputZDequantizeLinearZ_DequantizeLinear_OutputZ _quantizedcCs(i|] }ttt|trtt||qS) isinstancegetattrrint).0krrV/var/www/html/venv/lib/python3.8/site-packages/onnxruntime/quantization/quant_utils.py 1src@s(eZdZdZdZddZeddZdS)QuantizationModercCs|jSNnameselfrrr__str__<szQuantizationMode.__str__cCs*z t|WStk r$tYnXdSr )rKeyError ValueError)moderrr from_string?s zQuantizationMode.from_stringN)__name__ __module__ __qualname__Z IntegerOpsZ QLinearOpsr% staticmethodr)rrrrr8s rc@s(eZdZdZdZddZeddZdS)QuantizedValueTyperrcCs|jSr r!r#rrrr%KszQuantizedValueType.__str__cCs*z t|WStk r$tYnXdSr )r.r&r')vrrrr)Ns zQuantizedValueType.from_stringN)r*r+r,ZInputZ Initializerr%r-r)rrrrr.Gs r.c@sHeZdZdZdZdZdZdZdZdZ dd Z e d d Z e d d ZdS) QuantTyperrcCs|jSr r!r#rrrr%_szQuantType.__str__cCs*z t|WStk r$tYnXdSr )r0r&r')trrrr)bs zQuantType.from_stringcCs|tjkrtjS|tjkr tjS|tjkr0tjS|tjkr@tj S|tj krPtj S|tj kr`tj S|tjkrptjStd|ddS)NzUnexpected value qtype=.)r0QInt8rINT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r'r#rrr tensor_typeis       zQuantType.tensor_typeN)r*r+r,r8r:r@r>r<rDrBr%r-r)propertyrFrrrrr0Vs r0c@s(eZdZdZdZddZeddZdS) QuantFormatrrcCs|jSr r!r#rrrr%szQuantFormat.__str__cCs*z t|WStk r$tYnXdSr )rHr&r')formatrrrr)s zQuantFormat.from_stringN)r*r+r,Z QOperatorZQDQr%r-r)rrrrrH|s rHint8uint8int16uint16dtypeiiiiiiii@ii@r2zero_point_indexcGsg}t|D]\}}tt|tjr8|t|n,t|tjrP||nt d|d|||kr |d}|j tj ks|j tj kr t d|j q t |dkrt|S|dS)Nzarg z is not an array: rVzzero_point cannot be rr) enumeratenumpyZ issubdtypetypenumberappendarrayrndarray TypeErrorrOfloat32float16lentuple)rXargsnew_argsiar/rrr _check_types  ric Cs|tkstd|d|tjjtjjtjjtjjfkr|dkrRtd|d|j t j krftj }n&|j t j krztj}ntd|j dtttdgdgtjd|gdgd td d d dgd ggdtd |dtd |dgtd |dg}t|}t|d||ddSt|} t|ddd\} } |dk r>t| |n| } |dk rVt| |n| } t |t j ||}t j || | |dt|| SdS)NUnexpected data type > requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not r7zUnexpected dtype Constant zero_point)valuerXscaleYZqu)rorpFT) reduce_range symmetric)out)!ONNX_TYPE_TO_NP_TYPEAssertionError onnx_protorrAZFLOAT8E4M3FNUZZ FLOAT8E5M2ZFLOAT8E5M2FNUZNotImplementedErrorrOrZraFLOATrbFLOAT16r'r r r onnxhelperZ make_tensorr r rirunget_qmin_qmax_for_qTypemaxminasarrayastyperoundZclip)qTypeZarrrprmlowhighZ onnx_typeZ onnx_modelrefrOqminqmaxZcliplowZcliphighZarr_fp32rrrquantize_nparraysV      rFc Cs|dks|dkr$td|d|t|tjd|jd}t|tjd|jd}|dk rjt|||}|rtt|t|}| }| }||kstd|d|tj||tj d}tj|tj dtj|tj d}t||} | dkstd| t |jj kr8tjd |jd} tjd|jd} n\|rjtjt ||tjd tj d|jd} ntjt ||| |jd} | |j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rNNzqmin=z > qmax=z scale isse?g@)r'rZminimumr^rOmaximumrabsrvZfloat64ZfinfoZtinyrr) rminrmaxrrrsmin_real_rangeZabsmaxZdrZdqrprmrrrcompute_scale_zps4  rc sd}|tkr~|tjkrdddlmddlm}|}fddtdD}tj dd|Dtj d }nt d |d |t|<n|tjkrddlm}|}|dkrt d |d t t|}tj d|d }tj |||jd }||gS)arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nrfloat8e4m3_to_float32rcsg|] }|qSrr)rrgrrr Asz+compute_scale_zp_float8..cSs$g|]}t|st|s|qSr)rZisnanisinf)rfrrrrCs rNzQuantization to element_type=z not implemented.zUnexpected element_type r7)FLOAT8_DISTRIBUTIONSrrAZonnx.numpy_helperr#onnx.reference.custom_element_typesrrangerZr^rar'r`stdrO) Z element_typerZzp_dtyperZ all_valuesvaluesZstd_f8zerorprrrcompute_scale_zp_float80s,        rc Cst|tjs tdt|d|dk r.|}nt|r>|nd}|dk rP|}nt|r`|nd}tj||j d}tj||j d}d} tjd|j d} |t j krD|rt dt |} t|| \} } t||| | } t| tjd @d kr0t|} t d | d | d | d | d t||| | | ddS|t jt jt jt jt jt jfkrt|rt|||d\}}t||||||\} } t||| | } t||| | | ddStd|ddS)a :param data: data to quantize :param qType: data type to quantize to. Supported types UINT8 and INT8 :param symmetric: whether symmetric quantization is used or not. This is applied to INT8. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: minimum, maximum, zero point, scale, and quantized weights To pack weights, we compute a linear transformation - when data `type == uint8` mode, from `[rmin, rmax]` -> :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point z%Weight must be given as an array not r7NgrNrrz1Unsupported option reduce_range=True for float 8.rQz+One of the quantized value is NaN data in [z, z], quantized_data in [z].r1rWrszUnexpected value for qType=)rrZr_r`r[rcrrr^rOrrA RuntimeErrorrrranyrrKZravelrrir9r;r?r=rErCr~rr')datarrsrrrZ rmin_overrideZ rmax_overriderrrmrprquantized_dataZnp_datarrrrr quantize_dataUsL    , rc Cs|tjjkrtdd}|r(t|}n |r>|tkr>t|}n t|}|s\td|d|\}}|dkst|dkrtd|d|d|j d |d |d | |S) z Return qmin and qmax, the minimum and maximum value representable by the given qType :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 :return: qmin, qmax z;This function is not implemented for float 8 as not needed.Nrjrkrrrz, dtype=z, reduce_range=z , symmetric=z, qType=) rwrrArxONNX_INT_TYPE_REDUCED_RANGEgetONNX_INT_TYPE_SYMMETRIC_RANGEONNX_INT_TYPE_RANGEr'rO)rrrrsZqrangerrrrrr~s      (r~cCst|||d\}}||S)z Helper function to get the quantization range for a type. parameter qType: quantization type. return: quantization range. r)r~)rrrrsrrrrrget_qrange_for_qTypesrrztuple[bool, int])axisrankreturncCs,|dkr||n|}|dko"||k}||fS)z Helper function that tries to return a normalized axis in the range [0, rank - 1]. :parameter axis: The axis to normalize. :parameter rank: The tensor rank (number of dimensions). :return (is_valid, axis_norm) rr)rrZ axis_normZis_validrrrnormalize_axissrbytes bytearray)src_8bitrcCst|}|dkrtS|dd}t|}d}d}||dkrt||dd@d>||d@B||<|d7}|d7}q2||kr||d@||<|S)aB Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values. Assumes that the source values are already in the appropriate int4 range. :parameter src_8bit: The 8-bit element values to pack. :return A bytearray with every two 8-bit src elements packed into a single byte. rrr1rRr3)rcr)rZ num_elemsZdst_sizedstZsrc_iZdst_irrrpack_bytes_to_4bits  $ rc@s eZdZdZggdfddZdS)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc Cs:||_||_||_||_||_||_||_||_| |_dSr ) r" initializerrminsrmaxs zero_pointsscalesrrr) r$r"rrrrrrrrrrr__init__s zQuantizedInitializer.__init__r*r+r,__doc__rrrrrrs  rc@seZdZdZdddZdS)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc Cs:||_||_||_||_||_||_||_||_| |_dSr ) original_nameZq_name scale_nameZzp_nameZ value_typer node_type node_qtype scale_type) r$r"Znew_quantized_namerZzero_point_nameZquantized_value_typerrrrrrrrs zQuantizedValue.__init__)NNNNrrrrrrs  rc@seZdZdZddZdS)BiasToQuantizez+ Represents a bias to be quantized cCs||_||_||_dSr ) bias_name input_name weight_name)r$rrrrrrr5szBiasToQuantize.__init__Nrrrrrr0srcCs|jdkrtd|jd|jdkr.|j}n|jdkr@|j}n|jdkrR|j}n|jdkrd|j}n|jdkrv|j}nt|jd kr|j}nb|jd kr|j }nP|jd kr|j }n>|jd kr|j }n,|jd kr|j }ntd|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.rr1r2r3r4r5rS z has unsupported type r7) r[r'r"rrgsr6gZfloatsZintsstringsZtensorsZgraphs) attributernrrrattribute_to_kwarg;s0           rcs*fdd|D}t|dkr&|dSdS)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. csg|]}|jkr|qSrr!)ritem item_namerrrgs z find_by_name..rN)rc)rZ item_listitemsrrr find_by_name`srcCs*d}tt|D]}|||kr|}q|S)zC Helper function to return index of an item in a node list rV)rrc)Z elem_nameZ elem_listZelem_idxrgrrrget_elem_indexks  rcCstjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. ZMul)r{r|r )inputsoutputr"rrr get_mul_nodevsrrstr)filename identifierrcCs|j|j||jS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. )parentjoinpathstemsuffix)rrrrrgenerate_identified_filenamesrcCsddl}ddlm}ddl}|j|jdtdt|tdt||j||dd|d| d| d | dS) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueZCountszTensor value V.S. Counts) sysZmatplotlib.pyplotZpyplotrZZset_printoptionsmaxsizeprintZstairsZxlabelZylabeltitleshow)histZ hist_edgesrZpltrZrrr apply_plots    rr7c Cstddl}ddl}ddlmmm}ddlmmm}t d|t t j |dd}|||W5QRX|d}g}t|D]t} || } ttt| dt| d} || } || } ||||| ||| ||}||q||t||D]}||q$|}| ||!|||"|}|#||$}t t j |dd }||W5QRXt j%&d dd kr|j'|d}|(}t)|D],}|*|}t |+t |,qt t j |d d\}t|D]H} || } | d ttt| dt| d}|||dqW5QRXdS)z> Helper function to write calibration table to files. rNzcalibration cache: zcalibration.jsonwirzcalibration.flatbufferswbZQUANTIZATION_DEBUG)r1zcalibration.cache  )-json flatbuffersZ5onnxruntime.quantization.CalTableFlatBuffers.KeyValueZ quantizationZCalTableFlatBuffersKeyValueZ5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTablelogginginfoopenospathjoinwritedumpsZBuildersortedkeysrrrZ CreateStringZ KeyValueStartZKeyValueAddKeyZKeyValueAddValueZ KeyValueEndr]ZTrtTableStartDictVectorrcZPrependUOffsetTRelativeZ EndVectorZ TrtTableStartZTrtTableAddDictZ TrtTableEndZFinishOutputenvironrZGetRootAsTrtTableZ DictLengthrDictKeyValue)Zcalibration_cachedirrrrrfilebuilderZkey_value_listkeyrrnZflat_keyZ flat_value key_valueZ main_dictZ cal_tablebufZdict_lenrgrrrrwrite_calibration_tablesV              & r -C6?cCs|dktj}|dktj}|}|j|}|s:dS|t|t|}|dkshtd|||f|tj}|||| |7}|dkdkst|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrz"n_zeros=%d, n_nonzeros=%d, eps1=%f)rrZrasumsizefloatrv)pepsZis_zerosZ is_nonzerosZn_zerosZ n_nonzerosZeps1rrrrsmooth_distributions    r) model_pathcCs4tj|dd}|jjD]}t|rdSqdS)NF)Zload_external_dataT)r{loadas_posixgraphrrZuses_external_data)rmodelZ intializerrrrmodel_has_external_datas   r)ropt_model_pathcCsFt}||_tj|_i}dg|d<t||fddgi|}dS)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ZConstantSharingZdisabled_optimizers providersZCPUExecutionProviderN)rrZoptimized_model_filepathrZORT_ENABLE_BASICZgraph_optimization_levelr)rrZ sess_optionkwargs_rrroptimize_models   rr)rcCs>ddi}|jr,|jD]}||j|jiqtj||dS)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdaterrnr{r|Zset_model_props)rr proprrradd_pre_process_metadatas  r#bool)rrcCs0|jr,|jD]}|jdkr |jdkr dSq dS)zCCheck the model whether it went through quantization pre-processingrrTFr rrn)rr"rrrmodel_has_pre_process_metadatas  r&cCs>ddi}|jr,|jD]}||j|jiqtj||dS)N onnx.inferrr)rr rrrradd_infer_metadata$s  r(cCs0|jr,|jD]}|jdkr |jdkr dSq dS)Nr'rTFr%)rrrrrmodel_has_infer_metadata,s  r))rrcCsBt|d}tjt|t|t|}t|||S)Nz -inferred) rr{Zshape_inferenceZinfer_shapes_pathrrrr(unlink)rZinferred_model_pathrrrrload_model_with_shape_infer4s  r+c CsNtjdd8}t|d}tj||ddt|W5QRSQRXdS)Nz ort.quant.)prefixz model.onnxT)Zsave_as_external_data)tempfileTemporaryDirectoryrrr{Z save_modelrr+)rZ quant_tmp_dirrrrr&save_and_reload_model_with_shape_infer=sr/rz numpy.ndarray)rrcCsB|jtjjtjjfkr"tj|Std|j dt |jdS)Nz&Only float type is supported. Weights z is ) Z data_typerwrryrzr{Z numpy_helperZto_arrayr'r" type_to_name)rrrrtensor_proto_to_arrayDs  r1) tensor_namercCs|dS)NZ_QuantizeLinearrr2rrradd_quant_suffixMsr4cCs|tSr )QUANT_INPUT_SUFFIXr3rrradd_quant_input_suffixQsr6)rcCs|dS)NZ_QuantizeLinear_Outputrr3rrradd_quant_output_suffixUsr7cCs|dS)NZ_DequantizeLinearrr3rrradd_dequant_suffixYsr8cCs|dS)NZ_DequantizeLinear_Inputrr3rrradd_dequant_input_suffix]sr9cCs|tSr )DEQUANT_OUTPUT_SUFFIXr3rrradd_dequant_output_suffixasr;)NN)FN)FNNN)FF)FF)r7)r )d __future__rrrr-enumrpathlibrrZr{rrrrrwZ onnx.helperr r r r Zonnx.referencer Z onnxruntimerrrrr ImportErrorrrZ __producer__ __version__Z onnx_domainZ ms_domainZ QUANT_OP_NAMEr5ZDEQUANT_OP_NAMEr:ZTENSOR_NAME_QUANT_SUFFIXrrr0rr.r0rHr9rOr;r?r=rArErCrur^rKrJrMrLrrrrirrrrr~rrrrrrrrrrrrr rrrr#r&r(r)r+r/r1r4r6r7r8r9r;rrrrs       &    4 ?& L   %    C