\j$ ddlmZddlZddlZddlZddlZddlmZddlm Z ddl Z ddl Z ddl m Z mZmZddl mZmZmZddl mZddlmZmZmZmZdd lmZdd lmZmZmZ dd l m!Z!d Z#d Z$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,iZ-e.eDcic]}e/e0e|e1se0e||c}Z2GddeZ3GddeZ4GddeZ5GddeZ6ej"jne jpdej"jre jpdej"jte jpd ej"jve jpd!ej"jxe ej"jzeej"j|eiZ?ej"jre jde j"e jd#e j"fej"jne jd$e j"e jd%e j"fej"jve jde j"e jd&e j"fej"jte jd'e j"e jd(e j"fej"j|e jde"e jd)e"fej"jze jd*e"e jd+e"fiZEej"jne jd,e j"e jd%e j"fej"jte jd-e j"e jd(e j"fiZFej"jre jde j"e jd%e j"fej"jne jd.e j"e jd/e j"fej"jve jde j"e jd(e j"fej"jte jd0e j"e jd1e j"fej"j|e jde"e jd+e"fej"jze jd2e"e jd3e"fiZGd4d5d6ZHdbd7ZIdcd8ZJddded9ZKd:ZL dfd;ZM dg dhd<ZN dg did=ZO dj dkd>ZPdld?ZQdld@ZRdmdAZSdndBZTGdCdDZUGdEdFZVGdGdHZWdIZXdJZYdKZZdLZ[dodMZ\dNZ]dpdOZ^dqdPZ_drdQZ`dsdRZadtdSZbdudTZcdtdUZddudVZedvdWZf dj dwdXZgdxdYZhdydZZidzd[Zjd{d\Zkd{d]Zld|d^Zmd|d_Znd|d`Zod|daZpy#e"$rdZ!YfwxYwcc}w)}) annotationsN)Enum)Path) float8_e4m3fnint4uint4) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions)to_array_extendedz onnx.quantizez0.1.0ai.onnxz com.microsoftQuantizeLinear_QuantizeLinear_InputDequantizeLinear_DequantizeLinear_Output _quantizedlc*eZdZdZdZdZedZy)QuantizationModerc|jSNnameselfs W/root/.hermes/venv/lib/python3.12/site-packages/onnxruntime/quantization/quant_utils.py__str__zQuantizationMode.__str__8 yycD t|S#t$r twxYwr )rKeyError ValueError)modes r% from_stringzQuantizationMode.from_string;s) #D) ) ,   N)__name__ __module__ __qualname__ IntegerOps QLinearOpsr& staticmethodr-r(r%rr4s%JJr(rc*eZdZdZdZdZedZy)QuantizedValueTyperrc|jSr r!r#s r%r&zQuantizedValueType.__str__Gr'r(cD t|S#t$r twxYwr )r7r*r+)vs r%r-zQuantizedValueType.from_stringJs) %a( ( ,  r.N)r/r0r1Input Initializerr&r4r-r5r(r%r7r7Cs% EKr(r7cNeZdZdZdZdZdZdZdZdZ dZ e d Z e d Zy ) QuantTyperrc|jSr r!r#s r%r&zQuantType.__str__[r'r(cD t|S#t$r twxYwr )r>r*r+)ts r%r-zQuantType.from_string^s( Q<  ,  r.c |tjk(rtjS|tjk(rtj S|tj k(rtjS|tjk(rtjS|tjk(rtjS|tjk(rtjS|tjk(rtjSt!d|d)NzUnexpected value qtype=.)r>QInt8r INT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r+r#s r% tensor_typezQuantType.tensor_typees 9?? "## # 9## #$$ $ 9$$ $%% % 9## #$$ $ 9** *++ + 9## #$$ $ 9?? "## #24(!<==r(N)r/r0r1rIrKrQrOrMrUrSr&r4r-propertyrWr5r(r%r>r>RsR E FM FG E F >>r(r>c*eZdZdZdZdZedZy) QuantFormatrrc|jSr r!r#s r%r&zQuantFormat.__str__|r'r(cD t|S#t$r twxYwr )rZr*r+)formats r%r-zQuantFormat.from_strings) v& & ,  r.N)r/r0r1 QOperatorQDQr&r4r-r5r(r%rZrZxs%I Cr(rZint8uint8int16uint16dtypeiiiiiiii@ii@r@zero_point_indexc@g}t|D]\}}tjt|tjr%|j tj |n=t|tjr|j |ntd|d|||k(s|d}|jtjk(s|jtjk(std|jt|dkDr t|S|dS)Nzarg z is not an array: rlzzero_point cannot be rr) enumeratenumpy issubdtypetypenumberappendarray isinstancendarray TypeErrorrefloat32float16lentuple)rnargsnew_argsiar:s r% _check_typersH$ C1   DGU\\ 2 OOEKKN + 5== ) OOA d1#%7s;< <   Aww%--'177emm+C"7y ABB C"(ma/5?@Xa[@r(c|tvs Jd|d|tjjtjjtjj tjj fvr/|dk7rtd|d|jtjk(rtj}nG|jtjk(rtj}ntd|jdtt!t#dgdgt$j&j)d|gdg t#d gd d ggd t+d|dt+d|dgt+d |dg}t-|}t/|j1d||ddSt|} t3|dd\} } | t5| |n| } | t7| |n| } tj8|j;tj|z j=|z}tj>|| | |t/|j;| S)NUnexpected data type > requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not rHzUnexpected dtype Constant zero_point)valuer)XscalerYqurr)rrF reduce_range symmetric)out) ONNX_TYPE_TO_NP_TYPE onnx_protor rRFLOAT8E4M3FNUZ FLOAT8E5M2FLOAT8E5M2FNUZNotImplementedErrorrerqrzFLOATr{FLOAT16r+rr ronnxhelper make_tensorrrrrunget_qmin_qmax_for_qTypemaxminasarrayastyperoundclip)qTypearrrrlowhigh onnx_type onnx_modelrefreqminqmaxcliplowcliphigharr_fp32s r%quantize_nparrayrs- ( ( w&de ( ++--))--  ?%(Z[eZhhi&jk k 99 %#))I YY%-- '#++I0 1=> > "Bdkk>U>UVbdikmpqor>s.0LseT  *3 4@*7ItD(UD9:  !,3774sU)CDQGHH%U+,URWX d$'O#dC.&*&63tT?D==#**U]]";e"C!J!J!Lz!YZ 8WhH=8??5122r(c |dkDs|dkrtd|d|tj|tjd|j}tj |tjd|j}|.t ||tj||jz}|rBtj tj|tj|}| }|}||ks Jd|d|tj||z tj}tj|tjtj|tjz }tj||z } | dk\sJd| tj|jjkrFtjd|j} tjd|j} | | gS|r^tjtj||ztjd tjz |j} n:tjtj||| z z |j} | j|j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rdzqmin=z > qmax=z scale issue?@)r+rqminimumrvremaximumrrabsfloat64finfotinyrr) rminrmaxrrrmin_real_rangeabsmaxdrdqrrs r%compute_scale_zprs ( ax4!8]^b]ccklpkqrss ==u{{1DJJ? @D ==u{{1DJJ? @D!4 nDJJ OOPuyy $@ww 4<55htf55< TD[ 6B T /%++d%--2X XB KKR E A:$}$: u{{4::&+++ Ctzz2[[$**5      TD[EKK5==,QQRZ^ZdZdJU[[u 1D%ETZZXJ TZZ(  r(ct|}t|}||zdzdz}ttj|}ttj|}t |d}t |d}||dkDrt ||t|z}||kr|dk\r|n|}t t |t |} ||k(r||z nt d||z dz} | dkDr| ndt d| z } || |||z z kr|||z z } tj|tjtj| tjfS|dk\rQtj|tj} tj|||z z tj} netj|tj} | ||z z }|||z z }tjt ||tj} |?t| |||z z kr+tj|||z z tj} | | fS)a6Snap a uint8 activation zero-point to qmin (when rmin >= 0) or mid (when rmin < 0). Used by the ActivationRestrictedAsymmetric quantization option. Recomputes scale so the dequantized range still covers [rmin, rmax] without clipping. :parameter rmin: calibrated minimum activation value (numpy scalar) :parameter rmax: calibrated maximum activation value (numpy scalar) :parameter qmin: minimum quantized value (int, default 0) :parameter qmax: maximum quantized value (int, default 255) :parameter min_real_range: minimum floating-point range to enforce (same semantics as compute_scale_zp). When not None and > 0, rmax is adjusted to max(rmax, rmin + min_real_range) before scale computation. :return: (zero_point, scale) with zero_point dtype uint8 and scale dtype float32 rr?rrrd) intfloatrqsqueezerrrrvrarz)rrrrrqmin_valqmax_valmid degenerate_zpabs_maxdenom scale_valrr scale_neg scale_poss r%snap_zero_point_to_uint8r,s4yH4yH h  "q (C t$ %D t$ %D tS>D tS>D!nq&84n 556 t|%)CKS c$iT+)6()BH$APX[cPchiOiHj '! WAu E  %)nS[H[6\*\&(X*=>I{{= iii s{[[=  DHx$78 N[[EKK8 ES8^, HsN+  C 95U]]K!eEl^xRZGZ5[&[ Nh.AB%--X u r(cd}|tvr|tjk(rddlm}|}t dDcgc] }t |}}tj|Dcgc]0}tj|rtj|r/|2c}tj}ntd|d|t|<n|tjk(rddlm}|}|td|d tjt|}tjd|} tj||z |j} | | gScc}wcc}w) arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nr)rrdzQuantization to element_type=z not implemented.zUnexpected element_type rH)FLOAT8_DISTRIBUTIONSr rR ml_dtypesrrangerrqrvisnanisinfrzr+rystdre) element_typerzp_dtyperr all_valuesfvaluesstd_f8zerors r%compute_scale_zp_float8rgsH// ;33 3 /$H,1#J7q%(7J7[[&Tqekk!nU[[QR^T\a\i\iF<\NJ[\] ]-3\* 11 1+ 2<.BCC YY+L9 :F ;;q )D KKf CII 6E %=#8TsEE 5E  E c |jdk7r&td|jd|jd|j|}||zdz |z}tt j t |jDcgc] \}}||k7s |c}}} t j||d} | j|| } t|d|\} } t|dj} ||z|z }|dkDr=t j|| f| j }t j| |gd }n| }|j||| }|jd }|jd }t j |t j"|}t j$|t j"|}|rAt j$t j&|t j&|}| }|}t j(| }t j(| }||z j+tj(}||z }||z j+|j}t j,|jj.}||k}t j0|t j2||}|r?tt j4||zd z }t j6|| f|| }nt j4||j+tj(|j+tj(z z }t j8|||}|j+| }d||<t j|d|}t j|d|}||fScc}}w) aCompute per-block scale and zero-point for a weight tensor. The weight is sliced along *axis* into blocks of *block_size* elements. Per the ONNX opset-21 spec, QuantizeLinear/DequantizeLinear require the scale and zero_point tensors to have the **same rank** as the input tensor. Only rank-2 weight tensors are supported; rank > 2 is explicitly rejected. Returns arrays with the same rank and dimensions as *weight*, except ``shape[axis] == ceil(weight.shape[axis] / block_size)``. This matches the ONNX opset-21 QuantizeLinear/DequantizeLinear blocked-quantization spec. :param weight: Float32/float16 weight array (must be rank-2). :param quant_type: ONNX tensor data type for quantization. :param axis: Axis along which to apply block-wise quantization. :param block_size: Number of elements per block along *axis*. :param symmetric: Whether to use symmetric quantization per block. :return: Tuple of (zero_point, scale), each with shape matching *weight* except ``shape[axis] == n_blocks``, where ``n_blocks == ceil(weight.shape[axis] / block_size)``. :raises NotImplementedError: If weight rank is not 2 (opset-21 constraint). r?zXPer-block (opset-21) quantization is only supported for rank-2 weight tensors. Got rank-z tensor with shape zY. For rank > 2 tensors, reshape to 2-D before quantizing or use per-channel quantization.rrFrrd)axisr)ndimrshaperrqprodrpmoveaxisreshaperONNX_INT_TYPE_RANGErezeros concatenaterrr zeros_likerrrrrrwhere ones_likerfullr) weight quant_typer block_sizerkn_blocksrdothermovedrrrpad_lenpad moved_paddedblocksrrrrrrr raw_scaler degeneratescaleszp_val zero_pointsraw_zps r%compute_scale_zp_blockedrs/6{{a! }$7 ~Ff f   TAJ"z1H  )FLL*AO$!QQ$YAOP QE NN64 +E MM!U #E(%S\]JD$":.q177H#a'G{kk7E*%++>((%A>   ! !(J >F ::1: D ::1: D ==u//5 6D ==u//5 6Duyy $@w}}T"H}}T"H +  emm ,B H Bb  .I ;;v|| $ ) )DT!J [[U__Y%? KFU[[(X"5!<=>jj(E!2F(K X EMM(BV]]SXS`S`Ea(aabFHh7mmH- "# J ^^FAt ,F..a6K  uPs < N> N>ct|tjstdt |d||}nt |r|j nd}||}nt |r|jnd}tj||j}tj||j}tjd|j} |tjk(r?|r tdtj|} t|| \} } t| | dS|tj tj"tj$tj&tj(tj*fvr_t-||| \} } t |rt/||| | ||\} } n!tjd| j} t| | dSt1d |d) a Returns the zero_point and scale for the given data. :param data: The data for which to compute quantization parameters. :param quant_type: The quantization data type. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: zero point and scale z%Weight must be given as an array not rHrrdrz1Unsupported option reduce_range=True for float 8.rrmrz Unexpected value for quant_type=)rwrqrxryrsr|rrrvrer rR RuntimeErrorrrrrJrLrPrNrVrTrrr+)datarrrr rmin_override rmax_overriderrrrrrrs r%compute_data_quant_paramsr s* dEMM *?T |1MNN  YtxxzC  YtxxzC ;;t4:: .D ;;t4:: .D KK4:: .E[--- RS Siio3JD E:uqAA -ZQZ[ d t9 0tT4Tb c JQdjj9J:uqAA 7 |1E FFr(c t|||||||\}}|tjk(rt||||} t | j t jjdzdk(ret j|} td| jd| jd| jd| jd ||| fS|tjtjtjtj tj"tj$fvrt||||} ||| fSt'd|d)al :param data: data to quantize :param qType: data type to quantize to. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: minimum, maximum, zero point, scale, and quantized weights To pack weights, we compute a linear transformation - when data `type == uint8` mode, from `[rmin, rmax]` -> :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point rgz+One of the quantized value is NaN data in [z, z], quantized_data in [z].zUnexpected value for qType=rH)r r rRranyviewrqraravelrrrrrJrLrPrNrVrTr+) rrrrrrrrrquantized_datanp_datas r% quantize_datar,sZ82  J ((()%ujI ##EKK06683>3F GmmD)G=gkkm_Bw{{}o^&&4&8&8&:%;2n>P>P>R=SSUW 5.00  *%ujI5.00 25'; <>*dA6$$Q'==jI""5?? C*$Eej(!,CU| *:eCin 5 ; ; ={3PS8?TV^_bdg_gVh+c 3'   OOQC;;L;L1M"[AQRVZQZ1"[[ \^_ae  ([5F5F5H%Q[\ #))$/ K--.  T&(#}% lA&++At4L!!HM!+A %5L..0-AS& " ( . .u}}=S/T/\/\]i/j k l))*EtL ):%6;;-PhOi@jMT%%222#//1)3&!!((5$1!(5(=(=(?(D(D(F(N(N(P%  (&&:;E{{k///5==?mF[F[F]3]"/ 0A0A/BBc$,,.s34F5==?3B;O:PP[\b\h\h[iS!56t<=Q@(  ((--t/?/?/E/EF F   tUm 3!7 Ftuv v.}/D/D/FGH  ${{66}jRXR]R]_jpt6u  ==jI m>JRRSYS^S^_ #00;;M=Y Q Y"#\s T T& T"4T"cj|tjjk(r tdd}|rtj |}n)|r|t vr t |}ntj |}|std|d|\}}|dkDs|dkr'td|d|d|jd |d |d | |S) z Return qmin and qmax, the minimum and maximum value representable by the given qType :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 :return: qmin, qmax z;This function is not implemented for float 8 as not needed.Nrrrrrz, dtype=z, reduce_range=z , symmetric=z, qType=) rr rRrONNX_INT_TYPE_REDUCED_RANGEgetONNX_INT_TYPE_SYMMETRIC_RANGErr+re)rrrqrangerrs r%rrs  &&333!"_`` F,007 u ==.u5$((/ 07uvwwJD$ ax4!86$x |?<.Y" 8E7 4  Mr(c.t|||\}}||z S)z Helper function to get the quantization range for a type. parameter qType: quantization type. return: quantization range. r)r)rrrrrs r%get_qrange_for_qTyperDs ) RJD$ $;r(c:|dkr||zn|}|dk\xr||k}||fS)z Helper function that tries to return a normalized axis in the range [0, rank - 1]. :parameter axis: The axis to normalize. :parameter rank: The tensor rank (number of dimensions). :return (is_valid, axis_norm) rr5)rrank axis_normis_valids r%normalize_axisrIs3 $axt TIA~2)d"2H Y r(ct|}|dk(r tS|dzdz}t|}d}d}||dz kr-||dzdzdz||dzz||<|dz }|dz }||dz kr-||kr ||dz||<|S)aB Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values. Assumes that the source values are already in the appropriate int4 range. :parameter src_8bit: The 8-bit element values to pack. :return A bytearray with every two 8-bit src elements packed into a single byte. rrr?rhrA)r| bytearray)src_8bit num_elemsdst_sizedstsrc_idst_is r%r$r$sH IA~{A !#H H CEE )a-  *S0Q68E?S;PQE      )a-   ye_s*E Jr(ceZdZdZggdfdZy)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc ||_||_||_||_||_||_||_||_| |_yr ) r" initializerrminsrmaxsrrrrr) r$r"rUrVrWrrrrrs r%__init__zQuantizedInitializer.__init__(sF &  &  , r(r/r0r1__doc__rXr5r(r%rSrS#s r(rSc eZdZdZ ddZy)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc ||_||_||_||_||_||_||_||_| |_yr ) original_nameq_name scale_namezp_name value_typer node_type node_qtype scale_type) r$r"new_quantized_namer`zero_point_namequantized_value_typerrcrdres r%rXzQuantizedValue.__init__GsD"( $& . "$$r()NNNNrYr5r(r%r\r\Bs%r(r\ceZdZdZdZy)BiasToQuantizez+ Represents a bias to be quantized c.||_||_||_yr ) bias_name input_name weight_name)r$rlrmrns r%rXzBiasToQuantize.__init__cs"$&r(NrYr5r(r%rjrj^s 'r(rjc|jdk(rtd|jd|jdk(r|j}n#|jdk(r|j}n|jdk(r |j }n|jdk(r |j }n|jdk(r |j}n|jd k(r |j}n|jd k(r |j}nz|jd k(r |j}n^|jd k(r |j}nB|jd k(r |j}n&td|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.rr?r@rArBrCri rz has unsupported type rH) rsr+r"rrsrFgfloatsintsstringstensorsgraphs) attributers r%attribute_to_kwargrzis; ~~:inn%55STUU~~  1   1   1   1   1    1  1 !! 1 !! 2   :inn%55KINNK[[\]^^ NNE ""r(ct|Dcgc]}|j|k(s|}}t|dkDr|dSdScc}w)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. rN)r"r|) item_name item_listitemitemss r% find_by_namersA( Bd499 +AT BE B5zA~58/4/ Cs55cRd}tt|D] }|||k(s |}|S)zC Helper function to return index of an item in a node list rl)rr|) elem_name elem_listelem_idxrs r%get_elem_indexrs9H 3y> " Q<9 $H Or(cHtjjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. Mul)rrr)inputsoutputr"s r% get_mul_noders! ;; $ ??r(cl|jj|j|z|jzS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. )parentjoinpathstemsuffix)filename identifiers r%generate_identified_filenamers+ ?? # #HMMJ$>$P QQr(c`ddl}ddlm}ddl}|j|j t dt |t dt ||j||d|jd|jd|jd |jy) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueCountszTensor value V.S. Counts) sysmatplotlib.pyplotpyplotrqset_printoptionsmaxsizeprintstairsxlabelylabeltitleshow)hist hist_edgesrpltrqs r% apply_plotrs#ES[[1 , $K  *JJtZdJ+JJ~JJxII()HHJr(c ddl}ddl}ddl}ddlmcmcm}ddlmcmcm}ddl m }tjd||j||}ttj j#|dd5} | j%|ddd|j'd} |j)d} g} t+|j-D]} || }|j/}t1|j3d | j5t1|j3d | j5g}t7t9|}| j;| }| j;|}|j=| |j?| ||jA| ||jC| }| jE||jG| tI| | D]}| jK|| jM}|jO| |jQ| ||jS| }| jU|| jW}ttj j#|d d 5} | j%|dddtjXj3d ddvr|jj[|d}|j]}t_|D]Y}|ja|}tj|jctj|je[ttj j#|dd5} t+|j-D]} || }|j/}t1|j3d | j5t1|j3d | j5g}| dzt7t9|z}| j%|| j%d dddy#1swYxYw#1swYxYw#1swYyxYw)z> Helper function to write calibration table to files. rN)CalibrationCacheEncoderzcalibration cache: )clszcalibration.jsonwihighestlowestzcalibration.flatbufferswbQUANTIZATION_DEBUG0)r1zcalibration.cache  )3json flatbuffersrq5onnxruntime.quantization.CalTableFlatBuffers.KeyValue quantizationCalTableFlatBuffersKeyValue5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTable"onnxruntime.quantization.calibraterlogginginfodumpsopenospathjoinwritervBuildersortedkeysto_dictrr@r~r"r CreateString KeyValueStartKeyValueAddKeyKeyValueAddValue KeyValueEndruTrtTableStartDictVectorr|PrependUOffsetTRelative EndVector TrtTableStartTrtTableAddDict TrtTableEndFinishOutputenvironGetRootAsTrtTable DictLengthrDictKeyValue)calibration_cachedirrrnprrr json_datafilerbuilderkey_value_listkeyrd_valuesrtrflat_key flat_value key_value main_dict cal_tablebufdict_lenrs r%write_calibration_tablers LLLLK LL&'8&9:; ,2I JI bggll3 23S 9T 9 88A;D!!$'GN',,./)"3'>># (,,y$/446 7 (,,x.335 6 CK '',))%0 w'2!!':6((1 i(#)& $$Wc..AB#3 '' 23!!#I 7# Wi0$$W-I NN9 .. C bggll3 9:D AT 3 zz~~*C0H<%%77Q? '')x ,A!q)I LL ) LL* + , bggll3 34c : d+0023 C&s+F~~'Hhll9d388:;hll8T2779:F#ICK 00E JJu  JJt    gL  s%QQ$ CQ1Q!$Q.1Q:c|dk(jtj}|dk7jtj}|j}|j|z }|sy|t |zt |z }|dksJd|d|d||jtj}|||z| |zzz }|dkjdk(sJ|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrzn_zeros=z , n_nonzeros=z, eps1=)rrqrzsumsizer)pepsis_zeros is_nonzerosn_zeros n_nonzeroseps1rs r%smooth_distributionrsQu}}-H6//%--0KllnG'!J  w % "3 3D #:Q'- |74&QQ: 88EMM "DC(Nte{2 22D AI??  !! ! Kr(ctj|jd}td|jj DS)NF)load_external_datac3FK|]}tj|ywr )r uses_external_data).0 intializers r% z*model_has_external_data..8smz#66zBms!)rloadas_posixr graphrU) model_pathmodels r%model_has_external_datar6s9 IIj))+ FE mUZU`U`UlUlm mmr(ct}|j|_tj|_i}dg|d<t |j|fddgi|}y)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ConstantSharingdisabled_optimizers providersCPUExecutionProviderN)rroptimized_model_filepathrORT_ENABLE_BASICgraph_optimization_levelr)ropt_model_path sess_optionkwargs_s r%optimize_modelr;sb!"K+9+B+B+DK(+A+R+RK( F%6$7F !,,. jH^G_jcijAr(cddi}|jr8|jD])}|j|j|ji+tj j ||y)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdaterrrrset_model_props)rrprops r%add_pre_process_metadatarKsZ.0CDN (( :D  ! !488TZZ"8 9 :KK~6r(c|jr2|jD]#}|jdk(s|jdk(s#yy)zCCheck the model whether it went through quantization pre-processingrrTFrrr)rrs r%model_has_pre_process_metadatarTsA (( Dxx33 FY8Y  r(cddi}|jr8|jD])}|j|j|ji+tj j ||y)N onnx.inferrr)rrrs r%add_infer_metadatar]sZ"$78N %% 4A  ! !155!''"2 3 4KK~6r(c|jr2|jD]#}|jdk(s|jdk(s#yy)NrrTFr)rrs r%model_has_infer_metadatar es@ %% Auu $4G)G  r(c|jDcgc] }|jr|jdk(s|"}}t|dk7r td|dj}|Scc}w)Nrrz$Failed to find proper ai.onnx domainr) opset_importdomainr|r+version)ropsetai_onnx_domain opset_versions r%get_opset_versionr(mse).););m5<a?@@"1%--M  ns A A c0t|}|}t|d|}| t|d|nd}tjjtjj f} || vxs|| v} | s|rt jt jh} |jD]S} | D]H} | jd}|| vrd} n/| jd}|0|jd}|| vsFd} n| sSn|dkr!|dkDrtjd|d d}n|d kr9|tjj k(rtjd|d d }nb|dkr| rtjd|d d}n?|d k(rtjd|dn |d krtjd|dd}||k7r+tj"j%||}t'|}|S#ttf$rtjdYwxYw)NrWrTconvertzYSkipping 16-bit opset bump heuristic for TensorQuantOverrides: structure not as expected.rz$The original model opset version is z, which does not support block-wise quantization natively. Please update the model to opset >= 21. Automatically updating the model to opset 21. Please verify the quantized model.z, which does not support quantization to float 8. Please update the model to opset >= 19. Automatically update the model to opset 19. Please verify the quantized model.z, which does not support 16-bit integer quantization natively. Please update the model to opset >= 21. Automatically update the model to opset 21. Please verify the quantized model.rze, which does not support node fusions. Please update the model to opset >= 11 for better performance.z, which does not support quantization. Please update the model to opset >= 11. Automatically update the model to opset 11. Please verify the quantized model. )r(getattrrr rNrPr>rOrMrr@AttributeErrorryrdebugwarningrRversion_converterconvert_version&save_and_reload_model_with_shape_infer)r weight_typeactivation_typetensor_quant_overridesrr'target_opset_versionweight_quant_typeactivation_quant_type _int16_typesneeds_opset21_for_16bit_int16_quant_typesoverrides_listoverrideqtr* convert_qts r%update_opset_versionrBvsz&e,M( ]KHDSD_@ei$$++T-=-=-C-CDL/<?hCX\hCh #'='.. 0A0AB w"8"?"?"A  . "H!l3B//26/&ll95G*%,[[%> %);;6:3! "+ &rj1n2=/B1 1 "   1T5E5E5R5R R2=/B1 1 "   72=/B1 1 " " 2=/BM M  2=/B1 1 "},&&66u>RS7u= Lg * w MMu v ws%AG-G-*G-2G--$HHct|d}tjjt |t |tj |j }t||j|S)Nz -inferred) rrshape_inferenceinfer_shapes_pathr"rrrunlink)rinferred_model_pathrs r%load_model_with_shape_inferrHs`6z;O**3z?C@Sr6zQuantType | Noner7z dict | Nonerrrlr )rrrlr )rr rlr )rUr rlrj)rXr"rlr")rlr")q __future__rrrrrLenumrpathlibrrqrrrrrr r r r r onnx.helperr rrronnx.referencer onnxruntimerrronnx.reference.op_runr ImportError __producer__ __version__ onnx_domain ms_domain QUANT_OP_NAMEr[DEQUANT_OP_NAMErgrMODEL_SIZE_THRESHOLDrrrwr.rrTrr7r>rZrJrerLrPrNrRrVrTrrvrar`rcrbrrAr?rrrrrrr rr=rrDrIr$rSr\rjrzrrrrrrrrrrrrr r(rBrHr4rrYr\r_rbrerh)rs0r%rs> #   00>>&QQ-PP7      ,$2'!47 4Dqq SZ[fhiSjloHp Q'*q  t   #>#>L $  V!4  +%++g"6  +%++g"6!!;5;;x#8''  %  ;5;;q #DkekkRU]b]h]hFi"j+%++d%**"E{u{{SV^c^h^hGi!j!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q  ;5;;q#>  BV[@\"]+%++b"={u{{1TX?Y!Z +%++d%**"E{u{{SV^c^h^hGi!j  ;5;;vU[[#I;5;;W\didodoKp"q!   ;5;;q #DkekkRU]b]h]hFi"j+%++c"DkekkRT\a\f\fFg!h!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q  ;5;;q#>  AUZ@["\+%++b"={u{{1TX?Y!Z )+ A 13h<~8v!H` `` ` `  ` ) `N#'"&"&;G ;G);G;G ;G ! ;G  ;G ;G);G~hl:=7:=D$(c c )c c   c  c " c c c L@ <>%%8''"#J0@R$Rj2n k 77)-*. W WW&W( W  W  Wt7+,2-3/G'$rs"[[ [[[