U yhVC@sddlZddlmZmZddlZddlZddlmZddgZGdddejj Z Gdddejj Z Gd d d ejj Z Gd ddejj Z dS) N)OptionalTuple)TensorLSTMCellLSTMcseZdZdZejjZdeee ddfdd Z de e e e e fe e e fddd Zdee e e e fd d d ZddZedddZedddZZS)raA quantizable long short-term memory (LSTM) cell. For the description and the argument types, please, refer to :class:`~torch.nn.LSTMCell` Examples:: >>> import torch.ao.nn.quantizable as nnqa >>> rnn = nnqa.LSTMCell(10, 20) >>> input = torch.randn(6, 10) >>> hx = torch.randn(3, 20) >>> cx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx, cx = rnn(input[i], (hx, cx)) ... output.append(hx) TN input_dim hidden_dimbiasreturncs||d}t||_||_||_tjj|d|fd|i||_tjj|d|fd|i||_ tj jj |_ tj|_tj|_tj|_tj|_tj jj |_tj jj |_tj jj |_tj jj |_d|_d|_tj|_tj|_dS)Ndevicedtyper )?r)super__init__ input_size hidden_sizer torchnnZLinearigateshgatesaoZ quantizedZFloatFunctionalgatesZSigmoid input_gate forget_gateZTanh cell_gate output_gatefgate_cx igate_cgatefgate_cx_igate_cgateogate_cyinitial_hidden_state_qparamsinitial_cell_state_qparamsZquint8hidden_state_dtypecell_state_dtypeselfrr r r rfactory_kwargs __class__U/var/www/html/venv/lib/python3.8/site-packages/torch/ao/nn/quantizable/modules/rnn.pyr&s(        zLSTMCell.__init__)xhiddenr cCs|dks |ddks |ddkr4||jd|j}|\}}||}||}|j||}|dd\}} } } ||}| | } | | } | | } |j | |} |j || } |j| | }|}t|}|j | |}||fS)Nrr)initialize_hiddenshape is_quantizedrrraddchunkrrrrrmulr r!rtanhr")r(r.r/hxcxrrrrrrZout_gaterr r!cyZtanh_cyhyr,r,r-forwardBs$        zLSTMCell.forwardF) batch_sizer3r c Csjt||jft||jf}}|rb|j\}}|j\}}tj||||jd}tj||||jd}||fS)NscaleZ zero_pointr)rzerosrr#r$quantize_per_tensorr%r&) r(r=r3hcZh_scaleZh_zpZc_scaleZc_zpr,r,r-r1\s"  zLSTMCell.initialize_hiddencCsdS)NZQuantizableLSTMCellr,r(r,r,r- _get_nameeszLSTMCell._get_namecCs|dk|dkkst|jd}|jd}||||dk d}tj||j_|dk rbtj||j_tj||j_|dk rtj||j_|S)zUses the weights and biases to create a new LSTM cell. Args: wi, wh: Weights for the input and hidden layers bi, bh: Biases for the input and hidden layers Nr0)rr r ) AssertionErrorr2rr Parameterrweightr r)clswiwhbibhrrcellr,r,r- from_paramshs  zLSTMCell.from_paramscCs\t||jkstt|ds$td||j|j|j|j}|j |_ |j |j _ |j |j _ |S)Nqconfigz$The float module must have 'qconfig') type _FLOAT_MODULErFhasattrrOZ weight_ihZ weight_hhZbias_ihZbias_hhrPrr)rIotherZuse_precomputed_fake_quantobservedr,r,r- from_float}s   zLSTMCell.from_float)TNN)N)F)NN)F)__name__ __module__ __qualname____doc__rrrrRintboolrrrrr<r1rE classmethodrOrV __classcell__r,r,r*r-rs(  csXeZdZdZd eeeddfdd Zd eee eefddd Z e d d Z Z S)_LSTMSingleLayerzA single one-directional LSTM layer. The difference between a layer and a cell is that the layer can process a sequence, while the cell only expects an instantaneous value. TNrcs0||d}tt||fd|i||_dSNr r )rrrrNr'r*r,r-rs  z_LSTMSingleLayer.__init__r.r/cCsNg}|jd}t|D]"}||||}||dqt|d}||fS)Nr)r2rangerNappendrstack)r(r.r/resultZseq_leniZ result_tensorr,r,r-r<s   z_LSTMSingleLayer.forwardcOs(tj||}||j|j|j}||_|S)N)rrOrrr rN)rIargskwargsrNlayerr,r,r-rOs z_LSTMSingleLayer.from_params)TNN)N)rWrXrYrZr[r\rrrrr<r]rOr^r,r,r*r-r_s r_cs^eZdZdZdeeeeeddfdd Zdeee eefdd d Z e dd d Z Z S) _LSTMLayerz#A single bi-directional LSTM layer.TFN)rr r batch_first bidirectionalr c sZ||d}t||_||_t||fd|i||_|jrVt||fd|i||_dSr`)rrrkrlr_layer_fwlayer_bw) r(rr r rkrlr rr)r*r,r-rs  z_LSTMLayer.__init__racCs|jr|dd}|dkr$d\}}n|\}}d}|jr|dkrDd}n|d}|d}|dkrbd}n|d}|d}|dk r|dk r||f}|dkr|dkrd}ntj|tj|f}|||\} }t|dr|jr|d} | | |\} }| d} t | | g| d} |dkr6|dkr6d} d}nh|dkrRtj|\} }nL|dkrntj|\} }n0t |d|dgd} t |d|dgd}n| } tj|\} }|jr| dd| | |ffS)Nrr0)NNrn)rk transposerlrjit_unwrap_optionalrmrSfliprncatdimrdZ transpose_)r(r.r/Zhx_fwZcx_fwZ hidden_bwZhx_bwZcx_bwZ hidden_fwZ result_fwZ x_reversedZ result_bwrerBrCr,r,r-r<sP       z_LSTMLayer.forwardrcKs:t|ds|dk st|d|j}|d|j}|d|j}|d|j}|d|j} |||||| } t|d|| _ t|d|} t|d |} t|d |d} t|d |d}t | | | || _ |jr6t|d|d } t|d |d } t|d |d d} t|d |d d}t | | | || _ | S) z There is no FP equivalent of this class. This function is here just to mimic the behavior of the `prepare` within the `torch.ao.quantization` flow. rPNrrr rkrlZ weight_ih_lZ weight_hh_lZ bias_ih_lZ bias_hh_lZ_reverse)rSrFgetrrr rkrlgetattrrPr_rOrmrn)rIrTZ layer_idxrPrhrrr rkrlrirJrKrLrMr,r,r-rVs(z_LSTMLayer.from_float)TFFNN)N)rN)rWrXrYrZr[r\rrrrr<r]rVr^r,r,r*r-rjs 4rjc s~eZdZdZejjZdeeee e e e ddfdd Z de e ee e fd d d Zd dZedddZeddZZS)raXA quantizable long short-term memory (LSTM). For the description and the argument types, please, refer to :class:`~torch.nn.LSTM` Attributes: layers : instances of the `_LSTMLayer` .. note:: To access the weights and biases, you need to access them per layer. See examples below. Examples:: >>> import torch.ao.nn.quantizable as nnqa >>> rnn = nnqa.LSTM(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> c0 = torch.randn(2, 3, 20) >>> output, (hn, cn) = rnn(input, (h0, c0)) >>> # To get the weights: >>> # xdoctest: +SKIP >>> print(rnn.layers[0].weight_ih) tensor([[...]]) >>> print(rnn.layers[0].weight_hh) AssertionError: There is no reverse path in the non-bidirectional layer r0TFN)rr num_layersr rkdropoutrlr c s(|| d} t||_||_||_||_||_t||_||_ d|_ |rPdnd} t |t j rd|krtdkrnn t |trtd|dkrtd|dkrtd|d |t|j|j|jfd|j d | g} td|D],} | t|j|j|jfd|j d | qtj| |_dS) Nr Fr0rzbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz|dropout option for quantizable LSTM is ignored. If you are training, please, use nn.LSTM version followed by `prepare` step.zdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=z and num_layers=)rkrl)rrrrrxr rkfloatryrltraining isinstancenumbersNumberr\ ValueErrorwarningswarnrjrbrcrrZ ModuleListlayers)r(rrrxr rkryrlr rr)num_directionsrrir*r,r-r%sJ    $   z LSTM.__init__racs|jr|dd}|d}|jr&dnd}|dkrtj|||jtj|jd d|j rptj dd|j dfddt |jD}njtj|}t|dtr|d|j|||j|d|j|||jfd dt |jD}n|}g}g}t|jD]D\} } | ||| \}\} } |tj| |tj| qt|} t|}| d | jd | jd } |d |jd |jd }|jr|dd}|| |ffS) Nrr0rz)rr rr>csg|] }fqSr,r,).0_)r@r,r- ]sz LSTM.forward..cs(g|] }|d|dfqS)r)Zsqueeze)ridx)r9r8r,r-rgs)rkrosizerlrr@rr{r Zsqueeze_r3rArrbrxrprqr}rZreshape enumeraterrcrdr2)r(r.r/Zmax_batch_sizerZhxcxZhidden_non_optZhx_listZcx_listrrirBrCZ hx_tensorZ cx_tensorr,)r9r8r@r-r<OsT       z LSTM.forwardcCsdS)NZQuantizableLSTMr,rDr,r,r-rE~szLSTM._get_namecCst||jstt|ds"|s"t||j|j|j|j|j|j |j }t |d||_ t |jD]}tj|||dd|j|<q\|jr|tjjj|dd}n|tjjj|dd}|S)NrPF)rkT)Zinplace)r}rRrFrSrrrxr rkryrlrvrPrbrjrVrr|trainrrZ quantizationZ prepare_qatevalprepare)rIrTrPrUrr,r,r-rVs& zLSTM.from_floatcCs tddS)NzuIt looks like you are trying to convert a non-quantizable LSTM module. Please, see the examples on quantizable LSTMs.)NotImplementedError)rIrTr,r,r- from_observedszLSTM.from_observed)r0TFrwFNN)N)N)rWrXrYrZrrrrRr[r\r{rrrrr<rEr]rVrr^r,r,r*r-rs.*/ )r~typingrrrrr__all__rModulerr_rjrr,r,r,r-s vb