\chapter{Bilinear Products for ABIPNN}
\label{appendix:bilinear_product_induction}

% Open and edit \href{run:./AppendixA.tex}{AppendixA.tex}

In Chapter~\ref{chp:abpinn_method}, we use circular convolution as the bilinear product in ABIPNN and briefly derive the results of matrices $\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}$ and $\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}$ used in the backpropagation process. In this section, we will derive some more bilinear products in detail. The feedforward connection is still described between layer $l-1$ and layer $l$.


\section{Vector product}
Here $\mathbf{w}_{ij}^l\bullet\mathbf{a}_{j}^{l-1}$ stands for the usual vector product, $\mathbf{w}_{ij}^l=\left[w_{ij1}^l\textrm{,}~w_{ij2}^l\textrm{,}~w_{ij3}^l\right]^T\in \mathbb{R}^3$, and the input $\mathbf{a}_{j}^{l-1}=\left[a_{j1}^{l-1}\textrm{,}~a_{j2}^{l-1}\textrm{,}~a_{j3}^{l-1}\right]^T\in \mathbb{R}^3$ are all three-dimensional vectors $(N=3)$. The matrix representation of $\mathbf{w}_{ij}^l\bullet\mathbf{a}_{j}^{l-1}$ is:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\left[\mathbf{w}_{ij}^{l}\right]_{\bullet}=
\begin{bmatrix}
0             & -w_{ij3}^{l}    & w_{ij2}^{l}   \\ 
w_{ij3}^{l}   & 0               & -w_{ij1}^{l}  \\
-w_{ij2}^{l}  & w_{ij1}^{l}     & 0             \\
\end{bmatrix} \textrm{,}
\end{split}
\end{equation}
where the weight vector is formulated as a $3 \times 3$ square matrix with zeros on the main diagonal. When calculating $\frac{\partial C}{\partial \mathbf{w}_{ij}^l}$,
the matrix $\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}$ in 
Eq.~\ref{eq:partial_C_over_partial_wij_result} can be extended as follows:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}
=
\begin{bmatrix}
& 0            & a_{j3}^{l-1} &-a_{j2}^{l-1} \\
&-a_{j3}^{l-1} & 0            & a_{j1}^{l-1} \\
& a_{j2}^{l-1} &-a_{j1}^{l-1} & 0            \\
\end{bmatrix} \textrm{.} 
\end{split}
\end{equation}
Then, we calculate $\frac{\partial \mathbf{z}_{k}^{l+1}}{\partial \mathbf{a}_{i}^l}$ in Eq.~\ref{eq:partial_zk_over_partial_ai_result} using the following matrix:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}
=
\begin{bmatrix}
& 0             &-w_{ki3}^{l+1} & w_{ki2}^{l+1} \\
& w_{ki3}^{l+1} & 0             &-w_{ki1}^{l+1} \\
&-w_{ki2}^{l+1} & w_{ki1}^{l+1} & 0            \\
\end{bmatrix} \textrm{.}
\end{split}
\end{equation}

ABIPNN with this product is identical to the three-dimensional vector product neural network \cite{nitta93ijcnn}.

\section{Quaternion multiplication}
Quaternions are four-dimensional numbers $a+bi+cj+dk$ with the multiplication rule $i^2=j^2=k^2=ijk=-1$. Here $\mathbf{w}_{ij}^l\bullet \mathbf{a}_{j}^{l-1}$ stands for quaternion multiplication, the weight $\mathbf{w}_{ij}^l=\left[w_{ij1}^l\textrm{,}~w_{ij2}^l\textrm{,}~w_{ij3}^l\textrm{,}~w_{ij4}^l\right]^T\in \mathbb{R}^4$ and the input $\mathbf{a}_{j}^{l-1}=\left[a_{j1}^{l-1}\textrm{,}~a_{j2}^{l-1}\textrm{,}~a_{j3}^{l-1}\textrm{,}~a_{j4}^{l-1}\right]^T\in \mathbb{R}^4$ are all four-dimensional vectors $(N=4)$. The matrix representation of $\mathbf{w}_{ij}^l\bullet \mathbf{a}_{j}^{l-1}$ is:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\left[\mathbf{w}_{ij}^{l}\right]_{\bullet}=
\begin{bmatrix}
w_{ij1}^{l}   &-w_{ij2}^{l}     &-w_{ij3}^{l}   &-w_{ij4}^{l}  \\ 
w_{ij2}^{l}   & w_{ij1}^{l}     &-w_{ij4}^{l}   & w_{ij3}^{l}  \\
w_{ij3}^{l}   & w_{ij4}^{l}     & w_{ij1}^{l}   &-w_{ij2}^{l}  \\
w_{ij4}^{l}   &-w_{ij3}^{l}     & w_{ij2}^{l}   & w_{ij1}^{l}  \\
\end{bmatrix} \textrm{,}
\end{split}
\end{equation}
where the weight vector is formulated as a $4 \times 4$ square matrix with $w_{ij1}^l$ on the main diagonal. The matrix $\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}$ for the calculation of $\frac{\partial C}{\partial \mathbf{w}_{ij}^l}$ in Eq.~\ref{eq:partial_C_over_partial_wij_result} can be extended as follows:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}
=
\begin{bmatrix}
a_{j1}^{l-1}   &-a_{j2}^{l-1}     &-a_{j3}^{l-1}   &-a_{j4}^{l-1}  \\ 
a_{j2}^{l-1}   & a_{j1}^{l-1}     & a_{j4}^{l-1}   &-a_{j3}^{l-1}  \\
a_{j3}^{l-1}   &-a_{j4}^{l-1}     & a_{j1}^{l-1}   & a_{j2}^{l-1}  \\
a_{j4}^{l-1}   & a_{j3}^{l-1}     &-a_{j2}^{l-1}   & a_{j1}^{l-1}  \\
\end{bmatrix} \textrm{.}
\end{split}
\end{equation}
After that, we can use the following matrix $\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}$ when computing $\frac{\partial \mathbf{z}_{k}^{l+1}}{\partial \mathbf{a}_{i}^l}$ in Eq.~\ref{eq:partial_zk_over_partial_ai_result}:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}=
\begin{bmatrix}
w_{ki1}^{l+1}   &-w_{ki2}^{l+1}     &-w_{ki3}^{l+1}   &-w_{ki4}^{l+1}  \\ 
w_{ki2}^{l+1}   & w_{ki1}^{l+1}     &-w_{ki4}^{l+1}   & w_{ki3}^{l+1}  \\
w_{ki3}^{l+1}   & w_{ki4}^{l+1}     & w_{ki1}^{l+1}   &-w_{ki2}^{l+1}  \\
w_{ki4}^{l+1}   &-w_{ki3}^{l+1}     & w_{ki2}^{l+1}   & w_{ki1}^{l+1}  \\
\end{bmatrix} \textrm{.}
\end{split}
\end{equation}

With quaternion multiplication, the ABIPNN is equivalent to the quaternion-valued neural network \cite{arena94iscas}.

\section{Seven-Dimensional Vector Product}
The seven-dimensional vector product is defined as per \cite{lounesto01}. Here $\mathbf{w}_{ij}^{l}\bullet\mathbf{a}_{j}^{l-1}$ stands for the seven-dimensional vector product, $\mathbf{w}_{ij}^l=\left[w_{ij1}^l\textrm{,}~w_{ij2}^l\textrm{,}\dots\textrm{,}~w_{ij7}^l\right]^T\in \mathbb{R}^7$ and $\mathbf{a}_{j}^{l-1}=\left[a_{j1}^{l-1}\textrm{,}~a_{j2}^{l-1}\textrm{,}\dots\textrm{,}~a_{j7}^{l-1}\right]^T\in \mathbb{R}^7$ are both seven-dimensional vectors $(N=7)$, and the matrix representation of $\mathbf{w}_{ij}^{l}\bullet\mathbf{a}_{j}^{l}$ is:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\setlength{\arraycolsep}{5pt}
\begin{split}
&\left[\mathbf{w}_{ij}^{l}\right]_{\bullet}=
\begin{bmatrix}
 0    		   &-w_{ij4}^{l}    &-w_{ij7}^{l}   & w_{ij2}^{l}   &-w_{ij6}^{l}  & w_{ij5}^{l}   & w_{ij3}^{l} \\ 
 w_{ij4}^{l}   & 0     			&-w_{ij5}^{l}   &-w_{ij1}^{l}   & w_{ij3}^{l}  &-w_{ij7}^{l}   & w_{ij6}^{l} \\
 w_{ij7}^{l}   & w_{ij5}^{l}    & 0   			&-w_{ij6}^{l}   &-w_{ij2}^{l}  & w_{ij4}^{l}   &-w_{ij1}^{l} \\
-w_{ij2}^{l}   & w_{ij1}^{l}    & w_{ij6}^{l}   & 0			    &-w_{ij7}^{l}  &-w_{ij3}^{l}   & w_{ij5}^{l} \\
 w_{ij6}^{l}   &-w_{ij3}^{l}    & w_{ij2}^{l}   & w_{ij7}^{l}   & 0   		   &-w_{ij1}^{l}   &-w_{ij4}^{l} \\ 
-w_{ij5}^{l}   & w_{ij7}^{l}    &-w_{ij4}^{l}   & w_{ij3}^{l}   & w_{ij1}^{l}  & 0   		   &-w_{ij2}^{l} \\ 
-w_{ij3}^{l}   &-w_{ij6}^{l}    & w_{ij1}^{l}   &-w_{ij5}^{l}   & w_{ij4}^{l}  & w_{ij2}^{l}   & 0           \\ 
\end{bmatrix}\textrm{,}
\end{split}
\end{equation}
where the weight vector is formulated as a $7 \times 7$ square matrix in a similar way as the vector product one. The matrix $\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}$ in Eq.~\ref{eq:partial_C_over_partial_wij_result} becomes:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\setlength{\arraycolsep}{5pt}
\begin{split}
&\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}=
\begin{bmatrix}
 0    		    & a_{j4}^{l-1}    & a_{j7}^{l-1}   &-a_{j2}^{l-1}   & a_{j6}^{l-1}  &-a_{j5}^{l-1}   &-a_{j3}^{l-1} \\ 
-a_{j4}^{l-1}   & 0     		  & a_{i5}^{l-1}   & a_{j1}^{l-1}   &-a_{j3}^{l-1}  & a_{j7}^{l-1}   &-a_{j6}^{l-1} \\
-a_{j7}^{l-1}   &-a_{j5}^{l-1}    & 0   	       & a_{j6}^{l-1}   & a_{j2}^{l-1}  &-a_{j4}^{l-1}   & a_{j1}^{l-1} \\
 a_{j2}^{l-1}   &-a_{j1}^{l-1}    &-a_{j6}^{l-1}   & 0			    & a_{j7}^{l-1}  & a_{j3}^{l-1}   &-a_{j5}^{l-1} \\
-a_{j6}^{l-1}   & a_{j3}^{l-1}    &-a_{j2}^{l-1}   &-a_{j7}^{l-1}   & 0   		    & a_{j1}^{l-1}   & a_{j4}^{l-1} \\ 
 a_{j5}^{l-1}   &-a_{j7}^{l-1}    & a_{j4}^{l-1}   &-a_{j3}^{l-1}   &-a_{j1}^{l-1}  & 0   		     & a_{j2}^{l-1} \\ 
 a_{j3}^{l-1}   & a_{j6}^{l-1}    &-a_{j1}^{l-1}   & a_{j5}^{l-1}   &-a_{j4}^{l-1}  &-a_{j2}^{l-1}   & 0            \\ 
\end{bmatrix} \textrm{,}
\end{split}
\end{equation}
where the signs are flipped when compared to the matrix $\left[\mathbf{w}_{ij}^{l}\right]_{\bullet}$ above. The matrix $\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}$ for the seven-dimensional vector product is then extended as:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\setlength{\arraycolsep}{5pt}
\begin{split}
&\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}=
\begin{bmatrix}
 0    		   &-w_{ki4}^{l+1}  &-w_{ki7}^{l+1} & w_{ki2}^{l+1} &-w_{ki6}^{l+1}  & w_{ki5}^{l+1} & w_{ki3}^{l+1} \\ 
 w_{ki4}^{l+1} & 0     			&-w_{ki5}^{l+1} &-w_{ki1}^{l+1} & w_{ki3}^{l+1}  &-w_{ki7}^{l+1} & w_{ki6}^{l+1} \\
 w_{ki7}^{l+1} & w_{ki5}^{l+1}  & 0   			&-w_{ki6}^{l+1} &-w_{ki2}^{l+1}  & w_{ki4}^{l+1} &-w_{ki1}^{l+1} \\
-w_{ki2}^{l+1} & w_{ki1}^{l+1}  & w_{ki6}^{l+1} & 0			    &-w_{ki7}^{l+1}  &-w_{ki3}^{l+1} & w_{ki5}^{l+1} \\
 w_{ki6}^{l+1} &-w_{ki3}^{l+1}  & w_{ki2}^{l+1} & w_{ki7}^{l+1} & 0   		     &-w_{ki1}^{l+1} &-w_{ki4}^{l+1} \\ 
-w_{ki5}^{l+1} & w_{ki7}^{l+1}  &-w_{ki4}^{l+1} & w_{ki3}^{l+1} & w_{ki1}^{l+1}  & 0   		     &-w_{ki2}^{l+1} \\ 
-w_{ki3}^{l+1} &-w_{ki6}^{l+1}  & w_{ki1}^{l+1} &-w_{ki5}^{l+1} & w_{ki4}^{l+1}  & w_{ki2}^{l+1} & 0           \\ 
\end{bmatrix}\textrm{.}
\end{split}
\end{equation}


\section{Circular convolution}
Here $\mathbf{w}_{ij}^{l}\bullet\mathbf{a}_{ij}^{l-1}$ stands for the usual circular convolution, $\mathbf{w}_{ij}^{l}=\left[w_{ij1}^l\textrm{,}~w_{ij2}^l\textrm{,}\dots\textrm{,}~w_{ijN}^l\right]^T\in \mathbb{R}^N$ and $\mathbf{a}_{j}^{l-1}=\left[a_{j1}^{l-1}\textrm{,}~a_{j2}^{l-1}\textrm{,}\dots\textrm{,}~a_{jN}^{l-1}\right]^T\in \mathbb{R}^N$ are both $N$-dimensional vectors, and the matrix representation of $\mathbf{w}_{ij}^{l}\bullet\mathbf{a}_{ij}^{l-1}$ is:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\label{eq:circular_convolution}
\begin{split}
\left[\mathbf{w}_{ij}^{l}\right]_{\bullet}=
\begin{bmatrix}
w_{ij1}^l  & w_{ijN}^l      & w_{ij(N-1)}^l  & \dots     & w_{ij2}^l   \\ 
w_{ij2}^l  & w_{ij1}^l      & w_{ijN}^l      & \dots     & w_{ij3}^l   \\
w_{ij3}^l  & w_{ij2}^l      & w_{ij1}^l      & \dots     & w_{ij4}^l   \\
\vdots     & \vdots         & \vdots         & \ddots    & \vdots      \\
w_{ijN}^l  & w_{ij(N-1)}^l  & w_{ij(N-2)}^l  & \dots     & w_{ij1}^l   \\
\end{bmatrix} \textrm{,}
\end{split}
\end{equation}
where the weight vector is formulated as an $N \times N$ square matrix with $w_{ij1}^l$ on the main diagonal. The matrix $\mathbf{A}_{j}^{l-1}$ for Eq.~\ref{eq:partial_C_over_partial_wij_result} is extended as follows:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\mathbf{A}_{j}^{l-1} =
\begin{bmatrix}
a_{j1}^{l-1}  & a_{jN}^{l-1}      & a_{j(N-1)}^{l-1}  & \dots     & a_{j2}^{l-1}  \\ 
a_{j2}^{l-1}  & a_{j1}^{l-1}      & a_{jN}^{l-1}      & \dots     & a_{j3}^{l-1}  \\
a_{j3}^{l-1}  & a_{j2}^{l-1}      & a_{j1}^{l-1}      & \dots     & a_{j4}^{l-1}  \\
\vdots        & \vdots            & \vdots            & \ddots    & \vdots        \\
a_{jN}^{l-1}  & a_{j(N-1)}^{l-1}  & a_{j(N-2)}^{l-1}  & \dots     & a_{j1}^{l-1}  \\
\end{bmatrix} \textrm{,}
\end{split}
\end{equation}
where the permutation of elements in the matrix $\mathbf{A}_{j}^{l-1}$ is identical to the matrix $\left[\mathbf{w}_{ij}^{l}\right]_{\bullet}$. The matrix $\mathbf{W}_{ki}^{l+1}$ for circular convolution is extended as:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\mathbf{W}_{ki}^{l+1} =
\begin{bmatrix}
w_{ki1}^{l+1}  & w_{kiN}^{l+1}      & w_{ki(N-1)}^{l+1}  & \dots     & w_{ki2}^{l+1}   \\ 
w_{ki2}^{l+1}  & w_{ki1}^{l+1}      & w_{kiN}^{l+1}      & \dots     & w_{ki3}^{l+1}   \\
w_{ki3}^{l+1}  & w_{ki2}^{l+1}      & w_{ki1}^{l+1}      & \dots     & w_{ki4}^{l+1}   \\
\vdots         & \vdots             & \vdots             & \ddots    & \vdots          \\
w_{kiN}^{l+1}  & w_{ki(N-1)}^{l+1}  & w_{ki(N-2)}^{l+1}  & \dots     & w_{ki1}^{l+1}   \\
\end{bmatrix} \textrm{.}
\end{split}
\end{equation}


\section{Skew circular convolution}
The skew-circular convolution is obtained by replacing the circulant matrix in Eq.~\ref{eq:circular_convolution} by a skew-circulant one \cite{davis12circulant}. The matrix representation of $\mathbf{w}_{ij}^{l}\bullet\mathbf{a}_{j}^{l-1}$ then becomes:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\label{eq:skew_circular_convolution}
\begin{split}
\left[\mathbf{w}_{ij}^{l}\right]_{\bullet}=
\begin{bmatrix}
w_{ij1}^l  &-w_{ijN}^l      &-w_{ij(N-1)}^l  & \dots     &-w_{ij2}^l   \\ 
w_{ij2}^l  & w_{ij1}^l      &-w_{ijN}^l      & \dots     &-w_{ij3}^l   \\
w_{ij3}^l  & w_{ij2}^l      & w_{ij1}^l      & \dots     &-w_{ij4}^l   \\
\vdots     & \vdots         & \vdots         & \ddots    & \vdots      \\
w_{ijN}^l  & w_{ij(N-1)}^l  & w_{ij(N-2)}^l  & \dots     & w_{ij1}^l   \\
\end{bmatrix}\textrm{,}
\end{split}
\end{equation}
where the weight vector is formulated as an $N \times N$ square matrix with $w_{ij1}^l$ on the main diagonal and the upper triangular part of the weight matrix is multiplied by minus one. 
When we compute $\frac{\partial C}{\partial \mathbf{w}_{ij}^l}$ in Eq.~\ref{eq:partial_C_over_partial_wij_result}, $\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}$ is extended as follows:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}=
\begin{bmatrix}
a_{j1}^{l-1}  &-a_{jN}^{l-1}      &-a_{j(N-1)}^{l-1}  & \dots     &-a_{j2}^{l-1}  \\ 
a_{j2}^{l-1}  & a_{j1}^{l-1}      &-a_{jN}^{l-1}      & \dots     &-a_{j3}^{l-1}  \\
a_{j3}^{l-1}  & a_{j2}^{l-1}      & a_{j1}^{l-1}      & \dots     &-a_{j4}^{l-1}  \\
\vdots        & \vdots            & \vdots            & \ddots    & \vdots        \\
a_{jN}^{l-1}  & a_{j(N-1)}^{l-1}  & a_{j(N-2)}^{l-1}  & \dots     & a_{j1}^{l-1}  \\
\end{bmatrix} \textrm{.}
\end{split}
\end{equation}
When $\frac{\partial \mathbf{z}_{k}^{l+1}}{\partial \mathbf{a}_{i}^l}$ is computed, the matrix $\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}$ in Eq.~\ref{eq:partial_zk_over_partial_ai_result} is extended for skew circular convolution as follows:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}=
\begin{bmatrix}
w_{ki1}^{l+1}  &-w_{kiN}^{l+1}      &-w_{ki(N-1)}^{l+1}  & \dots     &-w_{ki2}^{l+1}   \\ 
w_{ki2}^{l+1}  & w_{ki1}^{l+1}      &-w_{kiN}^{l+1}      & \dots     &-w_{ki3}^{l+1}   \\
w_{ki3}^{l+1}  & w_{ki2}^{l+1}      & w_{ki1}^{l+1}      & \dots     &-w_{ki4}^{l+1}   \\
\vdots         & \vdots             & \vdots             & \ddots    & \vdots          \\
w_{kiN}^{l+1}  & w_{ki(N-1)}^{l+1}  & w_{ki(N-2)}^{l+1}  & \dots     &-w_{ki1}^{l+1}   \\
\end{bmatrix} \textrm{.}
\end{split}
\end{equation}
If $N=2$, then ABIPNN becomes a complex-valued neural network \cite{nitta97nn}. 
For even $N\geq 2$, the skew circular convolution is also known as planar complex multiplication \cite{olariu02elsevier}.


\section{Reverse-time circular convolution}
The reversed-time circular convolution is obtained by flipping the circulant matrix upside down (see also \cite{powell90iscs}). The matrix representation of $\mathbf{w}_{ij}^{l}\bullet\mathbf{a}_{j}^{l-1}$ becomes:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\label{eq:reverse_time_circular_convolution}
\begin{split}
&\left[\mathbf{w}_{ij}^{l}\right]_{\bullet}= 
\begin{bmatrix}
w_{ijN}^l      & w_{ij(N-1)}^l    & w_{ij(N-2)}^l   & \dots     & w_{ij1}^l       \\ 
w_{ij(N-1)}^l  & w_{ij(N-2)}^l    & w_{ij(N-3)}^l   & \dots     & w_{ijN}^l       \\
w_{ij(N-2)}^l  & w_{ij(N-3)}^l    & w_{ij(N-4)}^l   & \dots     & w_{ij(N-1)}^l   \\
\vdots         & \vdots           & \vdots          & \ddots    & \vdots          \\
w_{ij1}^l  	   & w_{ijN}^l      & w_{ij(N-1)}^l   & \dots     & w_{ij2}^l       \\
\end{bmatrix}\textrm{,}
\end{split}
\end{equation}
where the weight vector is formulated as an $N \times N$ square matrix which is the same as the weight matrix for circular convolution rotated by 90 degrees. The matrix $\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}$ in Eq.~\ref{eq:partial_C_over_partial_wij_result} is extended as follows:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
&\left[\mathbf{a}_{j}^{l-1}\right]_{\bullet}^{\dagger}= 
\begin{bmatrix}
a_{jN}^{l-1}      & a_{j(N-1)}^{l-1}  & a_{j(N-2)}^{l-1}  & \dots     & a_{j1}^{l-1}      \\ 
a_{j(N-1)}^{l-1}  & a_{j(N-2)}^{l-1}  & a_{j(N-3)}^{l-1}  & \dots     & a_{jN}^{l-1}      \\
a_{j(N-2)}^{l-1}  & a_{2(N-3)}^{l-1}  & a_{j(N-4)}^{l-1}  & \dots     & a_{j(N-1)}^{l-1}  \\
\vdots            & \vdots            & \vdots            & \ddots    & \vdots            \\
a_{j1}^{l-1}      & a_{jN}^{l-1}      & a_{j(N-1)}^{l-1}  & \dots     & a_{j2}^{l-1}      \\
\end{bmatrix} \textrm{,}
\end{split}
\end{equation}
Then, the derivative $\frac{\partial \mathbf{z}_{k}^{l+1}}{\partial \mathbf{a}_{i}^l}$ in Eq.~\ref{eq:partial_zk_over_partial_ai_result} is extended for reverse-time circular convolution by the following:
\renewcommand{\arraystretch}{0.95}
\begin{equation}
\begin{split}
&\left[\mathbf{w}_{ki}^{l+1}\right]_{\bullet}=
\begin{bmatrix}
w_{kiN}^{l-1}      & w_{ki(N-1)}^{l-1}  & w_{ki(N-2)}^{l-1}  & \dots     & w_{ki1}^{l-1}      \\ 
w_{ki(N-1)}^{l-1}  & w_{ki(N-2)}^{l-1}  & w_{ki(N-3)}^{l-1}  & \dots     & w_{kiN}^{l-1}      \\
w_{ki(N-2)}^{l-1}  & w_{ki(N-3)}^{l-1}  & w_{ki(N-4)}^{l-1}  & \dots     & w_{ki(N-1)}^{l-1}  \\
\vdots             & \vdots             & \vdots             & \ddots    & \vdots             \\
w_{ki1}^{l-1}      & w_{kiN}^{l-1}      & w_{ki(N-1)}^{l-1}  & \dots     & w_{ki2}^{l-1}      \\
\end{bmatrix}  \textrm{.}
\end{split}
\end{equation}
