cgemm_kernel_loongson3b_2x2.S

#define ASSEMBLER
#include "common.h"

#define FETCH	ld
#define gsLQC1(base,fq,ft,offset) .word(0x32<<26|base<<21|ft<<16|0x1<<15|offset<<6|0x1<<5|fq)
#define gsSQC1(base,fq,ft,offset) .word(0x3A<<26|base<<21|ft<<16|0x1<<15|offset<<6|0x1<<5|fq)


#define STACKSIZE	160
#define M	$4
#define	N	$5
#define	K	$6
#define A	$9
#define B	$10
#define C	$11
#define LDC	$8

#define AO	$12
#define BO	$13

#define	R12	12
#define	R13	13

#define I	$2
#define J	$3
#define L	$7

#define CO1	$14
#define CO2	$15
#define PREA	$16
#define PREB	$17

#if defined(TRMMKERNEL)
#define OFFSET	$18
#define KK	$19
#define TEMP	$20
#endif

#define a1	$f0
#define a2	$f1
#define a3	$f2
#define a4	$f3

#define b1	$f4
#define b2	$f5
#define b3	$f6
#define b4	$f7

#define a5	$f8
#define a6	$f9
#define a7	$f10
#define a8	$f11

#define b5	$f12
#define b6	$f13
#define b7	$f15
#define b8	$f16

#define c11	$f14
#define c12	$f17
#define c13	$f18
#define c14	$f19
#define c21	$f20
#define c22	$f21
#define c23	$f22
#define c24	$f23
#define c31	$f24
#define c32	$f25
#define c33	$f26
#define c34	$f27
#define c41	$f28
#define c42	$f29
#define c43	$f30
#define c44	$f31

#define F0	0
#define F1	1
#define F2	2
#define F3	3
#define F4	4
#define F5	5
#define	F6	6
#define	F7	7
#define	F8	8
#define	F9	9
#define	F10	10
#define	F11	11
#define	F12	12
#define	F13	13
#define	F14	14
#define	F15	15
#define	F16	16
#define	F17	17
#define	F18	18
#define	F19	19
#define	F20	20
#define	F21	21
#define	F22	22
#define	F23	23
#define	F24	24
#define	F25	25
#define	F26	26
#define	F27	27
#define	F28	28
#define	F29	29
#define	F30	30
#define	F31	31

#define ALPHA_R	$f15
#define ALPHA_I	$f16

#################################
##	MADD1	a*c
##	MADD2	b*c
##	MADD3	a*d
##	MADD4	d*b
##################################
#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
#define MADD1	  MADD
#define MADD2	  MADD
#define MADD3	  MADD
#define MADD4	  NMSUB
#endif

#if   defined(NR) || defined(NC) || defined(TR) || defined(TC)
#define MADD1	  MADD
#define MADD2	  MADD
#define MADD3	  NMSUB
#define MADD4	  MADD
#endif

#if   defined(RN) || defined(RT) || defined(CN) || defined(CT)
#define MADD1	  MADD
#define MADD2	  NMSUB
#define MADD3	  MADD
#define MADD4	  MADD
#endif

#if   defined(RR) || defined(RC) || defined(CR) || defined(CC)
#define MADD1	  MADD
#define MADD2	  NMSUB
#define MADD3	  NMSUB
#define MADD4	  NMSUB
#endif

	PROLOGUE

	LDARG	LDC,   0($sp)
	daddiu	$sp, $sp, -STACKSIZE

	SDARG	$16,   0($sp)
	SDARG	$17,   8($sp)
	sdc1	$f24, 16($sp)
	sdc1	$f25, 24($sp)
	sdc1	$f26, 32($sp)
	sdc1	$f27, 40($sp)
	sdc1	$f28, 48($sp)
	sdc1	$f29, 56($sp)

#if defined(TRMMKERNEL)
	SDARG	$18,  64($sp)
	SDARG	$19,  72($sp)
	SDARG	$20,  80($sp)

	LDARG	OFFSET, STACKSIZE + 8($sp)
#endif

#ifndef __64BIT__
	sdc1	$f20, 88($sp)
	sdc1	$f21, 96($sp)
	sdc1	$f22,104($sp)
	sdc1	$f23,112($sp)
#endif

	dsra	J,  N, 1							#	J=N/2
	ST	ALPHA_R, 128($sp)						#	store alpha_r & alpha_i

#if defined(TRMMKERNEL) && !defined(LEFT)
	neg	KK, OFFSET
#endif

	dsll	LDC, LDC, ZBASE_SHIFT				#	LDC*SIZE*COMPSIZE
	blez	J, .L20
	ST	ALPHA_I, 136($sp)


	.align	5
.L10:
#if defined(TRMMKERNEL) &&  defined(LEFT)
	move	KK, OFFSET
#endif

	daddiu	J, J, -1
	dsra	I,  M, 1							#	I=M/2

	dsll	PREB, K, 1 + ZBASE_SHIFT				#	PREA=K*2*2^4
	dsll	PREA, K, 1 + ZBASE_SHIFT				#	PREA=K*2*2^4

	move	CO1, C								#	Fix pointer Cx
	daddu	CO2, C,   LDC

	move	AO, A								#	Reset AO
	blez	I, .L30
	daddu	PREA, PREA, A						#	PREA=A+panel size

.L11:
#if defined(TRMMKERNEL)
#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
	move	BO,  B
#else
	dsll	TEMP, KK, 1 + ZBASE_SHIFT

	daddu	AO, AO, TEMP
	daddu	BO, B,  TEMP
#endif

	MTC	$0,  c11								#	Clear results regs
	LD	a1,	0 * SIZE(AO)
	MOV	c12, c11
	LD	a2,	1 * SIZE(AO)

	MOV	c13, c11
	LD	b1, 0 * SIZE(BO)
	MOV	c14, c11
	LD	b2, 1 * SIZE(BO)

	MOV	c21, c11
	LD	a3,	2 * SIZE(AO)
	MOV	c22, c11
	LD	a4, 3 * SIZE(AO)

	MOV	c23, c11
	LD	b3, 2 * SIZE(BO)
	MOV	c24, c11
	LD	b4,	3 * SIZE(BO)

	FETCH	$0, 0 * SIZE(CO2)
	MOV	c31, c11
	MOV	c32, c11

	FETCH	$0, 0 * SIZE(CO1)
	MOV	c33, c11
	MOV	c34, c11

	FETCH	$0, 4 * SIZE(CO2)
	MOV	c41, c11
	MOV	c42, c11

	FETCH	$0, 4 * SIZE(CO1)
	MOV	c43, c11
	MOV	c44, c11

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
	dsubu	TEMP, K, KK
#elif defined(LEFT)
	daddiu	TEMP, KK, 2
#else
	daddiu	TEMP, KK, 2
#endif
	dsra	L,  TEMP, 2
	daddu	PREB, PREB, B						#	PREA=A+panel size
	blez	L, .L15
	NOP

#else

	dsra	L,  K, 2							#	Unroll K 4 times
	move	BO,  B

	MTC	$0,  c11								#	Clear results regs
	LD	a1,	0 * SIZE(AO)
	MOV	c12, c11
	LD	a2,	1 * SIZE(AO)

	MOV	c13, c11
	LD	b1,	0 * SIZE(BO)
	MOV	c14, c11
	LD	b2,	1 * SIZE(BO)

	MOV	c21, c11
	LD	a3,	2 * SIZE(AO)
	MOV	c22, c11
	LD	a4, 3 * SIZE(AO)

	MOV	c23, c11
	LD	b3, 2 * SIZE(BO)
	MOV	c24, c11
	LD	b4,	3 * SIZE(BO)

	MOV	c31, c11
	MOV	c32, c11
	FETCH	$0, 0 * SIZE(CO2)

	MOV	c33, c11
	MOV	c34, c11
	FETCH	$0, 0 * SIZE(CO1)

	MOV	c41, c11
	MOV	c42, c11
	FETCH	$0, 4 * SIZE(CO2)

	MOV	c43, c11
	NOP
	FETCH	$0, 4 * SIZE(CO1)

	daddu	PREB, PREB, B						#	PREA=A+panel size
	blez	L, .L15
	MOV	c44, c11
#endif

	.align	5

.L12:
	LD	a5,	4 * SIZE(AO)
	LD	a6,	5 * SIZE(AO)
	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd

	LD	b5,	4 * SIZE(BO)
	LD	b6,	5 * SIZE(BO)
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

	LD	a7,	6 * SIZE(AO)
	LD	a8,	7 * SIZE(AO)
	MADD1	c21, c21, a3, b1 					#	A2xB1
	MADD3	c23, c23, a3, b2

	LD	b7,	6 * SIZE(BO)
	LD	b8, 7 * SIZE(BO)
	MADD2	c22, c22, a4, b1
	MADD4	c24, c24, a4, b2

	FETCH	$0, 4 * SIZE(PREA)
	FETCH	$0, 4 * SIZE(PREB)
	MADD1	c31, c31, a1, b3 					#	A1xB2
	MADD3	c33, c33, a1, b4

	MADD2	c32, c32, a2, b3
	MADD4	c34, c34, a2, b4

	MADD1	c41, c41, a3, b3 					#	A2xB2
	MADD3	c43, c43, a3, b4
	MADD2	c42, c42, a4, b3
	MADD4	c44, c44, a4, b4

	LD	a1,	8 * SIZE(AO)
	LD	a2, 9 * SIZE(AO)
	MADD1	c11, c11, a5, b5 					#	axc		A1xB1
	MADD3	c13, c13, a5, b6					#	axd

	LD	b1,	8 * SIZE(BO)
	LD	b2, 9 * SIZE(BO)
	MADD2	c12, c12, a6, b5					#	bxc
	MADD4	c14, c14, a6, b6					#	bxd

	LD	a3,	10 * SIZE(AO)
	LD	a4,	11 * SIZE(AO)
	MADD1	c21, c21, a7, b5 					#	A2xB1
	MADD3	c23, c23, a7, b6

	LD	b3,	10 * SIZE(BO)
	LD	b4,	11 * SIZE(BO)
	MADD2	c22, c22, a8, b5
	MADD4	c24, c24, a8, b6

	FETCH	$0, 8 * SIZE(PREA)
	FETCH	$0, 8 * SIZE(PREB)
	MADD1	c31, c31, a5, b7 					#	A1xB2
	MADD3	c33, c33, a5, b8

	MADD2	c32, c32, a6, b7
	MADD4	c34, c34, a6, b8

	MADD1	c41, c41, a7, b7 					#	A2xB2
	MADD3	c43, c43, a7, b8
	MADD2	c42, c42, a8, b7
	MADD4	c44, c44, a8, b8

	LD	a5,	12 * SIZE(AO)
	LD	a6,	13 * SIZE(AO)
	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd

	LD	b5,	12 * SIZE(BO)
	LD	b6,	13 * SIZE(BO)
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

	LD	a7,	14 * SIZE(AO)
	LD	a8,	15 * SIZE(AO)
	MADD1	c21, c21, a3, b1 					#	A2xB1
	MADD3	c23, c23, a3, b2

	LD	b7, 14 * SIZE(BO)
	LD	b8,	15 * SIZE(BO)
	MADD2	c22, c22, a4, b1
	MADD4	c24, c24, a4, b2

	FETCH	$0, 12 * SIZE(PREA)
	MADD1	c31, c31, a1, b3 					#	A1xB2
	MADD3	c33, c33, a1, b4
	daddiu	L, L, -1

	FETCH	$0, 12 * SIZE(PREB)
	MADD2	c32, c32, a2, b3
	MADD4	c34, c34, a2, b4
	daddiu	AO, AO, 16 * SIZE

	daddiu	BO, BO, 16 * SIZE  					#	2nr*4kr*cmpx
	MADD1	c41, c41, a3, b3 					#	A2xB2
	MADD3	c43, c43, a3, b4
	daddu	PREA, PREA, 16 * SIZE

	MADD2	c42, c42, a4, b3
	MADD4	c44, c44, a4, b4
	daddu	PREB, PREB, 16 * SIZE

	LD	a1,	0 * SIZE(AO)
	LD	a2,	1 * SIZE(AO)
	MADD1	c11, c11, a5, b5 					#	axc		A1xB1
	MADD3	c13, c13, a5, b6					#	axd

	LD	b1, 0 * SIZE(BO)
	LD	b2,	1 * SIZE(BO)
	MADD2	c12, c12, a6, b5					#	bxc
	MADD4	c14, c14, a6, b6					#	bxd

	LD	a3,	2 * SIZE(AO)
	LD	a4,	3 * SIZE(AO)
	MADD1	c21, c21, a7, b5 					#	A2xB1
	MADD3	c23, c23, a7, b6

	LD	b3, 2 * SIZE(BO)
	LD	b4,	3 * SIZE(BO)
	MADD2	c22, c22, a8, b5
	MADD4	c24, c24, a8, b6

	FETCH	$0, 0 * SIZE(PREA)
	FETCH	$0, 0 * SIZE(PREB)
	MADD1	c31, c31, a5, b7 					#	A1xB2
	MADD3	c33, c33, a5, b8

	MADD2	c32, c32, a6, b7
	MADD4	c34, c34, a6, b8

	MADD1	c41, c41, a7, b7 					#	A2xB2
	MADD3	c43, c43, a7, b8

	MADD2	c42, c42, a8, b7
	bgtz	L, .L12
	MADD4	c44, c44, a8, b8

	.align	5

.L15:
#ifndef	TRMMKERNEL
	andi	L,  K, 3
	LD	ALPHA_R, 128($sp)
#else
	andi	L,  TEMP, 3
	LD	ALPHA_R, 128($sp)
#endif
	blez	L, .L18
	LD	ALPHA_I, 136($sp)

	.align	5

.L16:
	daddiu	BO, BO, 4 * SIZE					#	2nr*1kr*cmpx
	daddiu	AO, AO, 4 * SIZE					#	2mr*1kr*cmpx
	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd

	daddiu	PREA, PREA, 4 * SIZE
	daddiu	PREB, PREB, 4 * SIZE
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

	MADD1	c21, c21, a3, b1 					#	A2xB1
	MADD3	c23, c23, a3, b2

	MADD2	c22, c22, a4, b1
	MADD4	c24, c24, a4, b2

	FETCH	$0, 0 * SIZE(PREA)
	MADD1	c31, c31, a1, b3 					#	A1xB2
	MADD3	c33, c33, a1, b4
	daddiu	L, L, -1

	MADD2	c32, c32, a2, b3
	MADD4	c34, c34, a2, b4

	FETCH	$0, 0 * SIZE(PREB)
	MADD1	c41, c41, a3, b3 					#	A2xB2
	MADD3	c43, c43, a3, b4

	MADD2	c42, c42, a4, b3
	MADD4	c44, c44, a4, b4

	LD	a1,	0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	LD	a3, 2 * SIZE(AO)
	LD	a4, 3 * SIZE(AO)
	LD	b3, 2 * SIZE(BO)
	LD	b4, 3 * SIZE(BO)
	bgtz	L, .L16
	NOP

.L18:
#ifndef TRMMKERNEL
 	ADD	c11, c14, c11
	LD	a1,  0 * SIZE(CO1)
	ADD	c12, c13, c12
	LD	a2,  1 * SIZE(CO1)
	ADD	c21, c24, c21
	LD	b1,  2 * SIZE(CO1)
	ADD	c22, c23, c22
	LD	b2,  3 * SIZE(CO1)

	ADD	c31, c34, c31
	LD	a3,  0 * SIZE(CO2)
	ADD	c32, c33, c32
	LD	a4,  1 * SIZE(CO2)
	ADD	c41, c44, c41
	LD	b3,  2 * SIZE(CO2)
	ADD	c42, c43, c42
	LD	b4,  3 * SIZE(CO2)

	daddiu	I, I, -1
	MADD	a1, a1, ALPHA_R, c11
	MADD	a2, a2, ALPHA_R, c12
	MADD	b1, b1, ALPHA_R, c21
	MADD	b2, b2, ALPHA_R, c22

	NMSUB	a1, a1, ALPHA_I, c12
	MADD	a2, a2, ALPHA_I, c11
	NMSUB	b1, b1, ALPHA_I, c22
	MADD	b2, b2, ALPHA_I, c21

	MADD	a3, a3, ALPHA_R, c31
	MADD	a4, a4, ALPHA_R, c32
	ST	a1, 0 * SIZE(CO1)
	MADD	b3, b3, ALPHA_R, c41
	MADD	b4, b4, ALPHA_R, c42
	ST	a2, 1 * SIZE(CO1)

	NMSUB	a3, a3, ALPHA_I, c32
	MADD	a4, a4, ALPHA_I, c31
	ST	b1, 2 * SIZE(CO1)

	NMSUB	b3, b3, ALPHA_I, c42
	MADD	b4, b4, ALPHA_I, c41
	ST	b2, 3 * SIZE(CO1)

	ST	a3, 0 * SIZE(CO2)
	ST	a4, 1 * SIZE(CO2)
	ST	b3, 2 * SIZE(CO2)
	ST	b4, 3 * SIZE(CO2)

#else
 	ADD	c11, c14, c11
	ADD	c12, c13, c12
	ADD	c21, c24, c21
	ADD	c22, c23, c22

	ADD	c31, c34, c31
	ADD	c32, c33, c32
	ADD	c41, c44, c41
	ADD	c42, c43, c42

	daddiu	I, I, -1
	MUL	a1, ALPHA_R, c11
	MUL	a2, ALPHA_R, c12
	MUL	b1, ALPHA_R, c21
	MUL	b2, ALPHA_R, c22

	NMSUB	a1, a1, ALPHA_I, c12
	MADD	a2, a2, ALPHA_I, c11
	NMSUB	b1, b1, ALPHA_I, c22
	MADD	b2, b2, ALPHA_I, c21

	MUL	a3, ALPHA_R, c31
	MUL	a4, ALPHA_R, c32
	MUL	b3, ALPHA_R, c41
	MUL	b4, ALPHA_R, c42

	NMSUB	a3, a3, ALPHA_I, c32
	MADD	a4, a4, ALPHA_I, c31
	NMSUB	b3, b3, ALPHA_I, c42
	MADD	b4, b4, ALPHA_I, c41

	ST	a1, 0 * SIZE(CO1)
	ST	a2, 1 * SIZE(CO1)
	ST	b1, 2 * SIZE(CO1)
	ST	b2, 3 * SIZE(CO1)

	ST	a3, 0 * SIZE(CO2)
	ST	a4, 1 * SIZE(CO2)
	ST	b3, 2 * SIZE(CO2)
	ST	b4, 3 * SIZE(CO2)


#if ( defined(LEFT) &&  defined(TRANSA)) || \
    (!defined(LEFT) && !defined(TRANSA))
	dsubu	TEMP, K, KK
#ifdef LEFT
	daddiu	TEMP, TEMP, -2
#else
	daddiu	TEMP, TEMP, -2
#endif

	dsll	TEMP, TEMP, 1 + ZBASE_SHIFT

	daddu	AO, AO, TEMP
	daddu	BO, BO, TEMP
#endif

#ifdef LEFT
	daddiu	KK, KK, 2
#endif
#endif

	dsll	PREB, K, 1 + ZBASE_SHIFT				#	PREA=K*2*2^4
	daddiu	CO1,CO1, 4 * SIZE
	bgtz	I, .L11
	daddiu	CO2,CO2, 4 * SIZE

	.align	5
.L30:
	andi	I, M, 1
	daddu	C,   C, LDC						#	Change C to next panel

	daddu	PREB, PREB, B						#	PREA=A+panel size
	blez	I, .L19
	daddu	C,   C, LDC						#	Change C to next panel

#if defined(TRMMKERNEL)
#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
	move	BO,  B
#else
	dsll	L,    KK, ZBASE_SHIFT			#	MR=1
	dsll	TEMP, KK, 1 + ZBASE_SHIFT		#	NR=2

	daddu	AO, AO, L
	daddu	BO, B,  TEMP
#endif

	LD	a1, 0 * SIZE(AO)
	LD	a2,	1 * SIZE(AO)
	MTC	$0,  c11								#	Clear results regs
	MOV	c12, c11

	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	MOV	c13, c11
	MOV	c14, c11

	LD	b3, 2 * SIZE(BO)
	LD	b4, 3 * SIZE(BO)
	MOV	c31, c11
	MOV	c32, c11

	FETCH	$0, 0 * SIZE(PREB)
	MOV	c33, c11
	MOV	c34, c11

	FETCH	$0, 0 * SIZE(CO1)
	FETCH	$0, 0 * SIZE(CO2)
	FETCH	$0, 4 * SIZE(CO1)
	FETCH	$0, 4 * SIZE(CO2)

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
	dsubu	TEMP, K, KK
#elif defined(LEFT)
	daddiu	TEMP, KK, 1							#	MR=1
#else
	daddiu	TEMP, KK, 2							#	NR=2
#endif
	dsra	L,  TEMP, 2
	blez	L, .L35
	NOP

#else

	LD	a1, 0 * SIZE(AO)
	LD	a2,	1 * SIZE(AO)
	dsra	L,  K, 2							#	Unroll K 4 times
	move	BO,  B

	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	MTC	$0,  c11								#	Clear results regs
	MOV	c12, c11

	LD	b3, 2 * SIZE(BO)
	LD	b4, 3 * SIZE(BO)
	MOV	c13, c11
	MOV	c14, c11

	FETCH	$0, 0 * SIZE(PREB)
	MOV	c31, c11
	MOV	c32, c11

	FETCH	$0, 0 * SIZE(CO1)
	FETCH	$0, 0 * SIZE(CO2)
	FETCH	$0, 4 * SIZE(CO1)
	FETCH	$0, 4 * SIZE(CO2)

	MOV	c33, c11
	blez	L, .L35
	MOV	c34, c11
#endif

	.align	5

.L32:
	LD	a3,	2 * SIZE(AO)
	LD	a4, 3 * SIZE(AO)
	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd

	LD	b5, 4 * SIZE(BO)
	LD	b6, 5 * SIZE(BO)
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

	LD	b7, 6 * SIZE(BO)
	LD	b8, 7 * SIZE(BO)
	MADD1	c31, c31, a1, b3 					#	A1xB2
	MADD3	c33, c33, a1, b4

	FETCH	$0, 4 * SIZE(PREB)
	MADD2	c32, c32, a2, b3
	MADD4	c34, c34, a2, b4
	NOP

	LD	a5,	4 * SIZE(AO)
	LD	a6, 5 * SIZE(AO)
	MADD1	c11, c11, a3, b5 					#	axc		A1xB1
	MADD3	c13, c13, a3, b6					#	axd

	LD	b1, 8 * SIZE(BO)
	LD	b2, 9 * SIZE(BO)
	MADD2	c12, c12, a4, b5					#	bxc
	MADD4	c14, c14, a4, b6					#	bxd

	LD	b3, 10 * SIZE(BO)
	LD	b4, 11 * SIZE(BO)
	MADD1	c31, c31, a3, b7 					#	A1xB2
	MADD3	c33, c33, a3, b8

	FETCH	$0, 8 * SIZE(PREB)
	MADD2	c32, c32, a4, b7
	MADD4	c34, c34, a4, b8
	daddiu	L, L, -1

	LD	a7, 6 * SIZE(AO)
	LD	a8, 7 * SIZE(AO)
	MADD1	c11, c11, a5, b1 					#	axc		A1xB1
	MADD3	c13, c13, a5, b2					#	axd

	LD	b5, 12 * SIZE(BO)
	LD	b6, 13 * SIZE(BO)
	MADD2	c12, c12, a6, b1					#	bxc
	MADD4	c14, c14, a6, b2					#	bxd

	LD	b7, 14 * SIZE(BO)
	LD	b8, 15 * SIZE(BO)
	MADD1	c31, c31, a5, b3 					#	A1xB2
	MADD3	c33, c33, a5, b4

	daddiu	AO, AO, 8 * SIZE 					#	2mr*4kr*cmpx
	daddiu	BO, BO, 16 * SIZE  					#	2nr*4kr*cmpx

	FETCH	$0, 12 * SIZE(PREB)
	MADD2	c32, c32, a6, b3
	MADD4	c34, c34, a6, b4
	NOP

	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	MADD1	c11, c11, a7, b5 					#	axc		A1xB1
	MADD3	c13, c13, a7, b6					#	axd

	LD	b1,	0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	MADD2	c12, c12, a8, b5					#	bxc
	MADD4	c14, c14, a8, b6					#	bxd

	LD	b3, 2 * SIZE(BO)
	LD	b4, 3 * SIZE(BO)
	MADD1	c31, c31, a7, b7 					#	A1xB2
	NOP

	MADD3	c33, c33, a7, b8
	daddiu	PREB, PREB, 16 * SIZE

	FETCH	$0, 0 * SIZE(PREB)
	MADD2	c32, c32, a8, b7
	bgtz	L, .L32
	MADD4	c34, c34, a8, b8


.L35:
#ifndef TRMMKERNEL
	andi	L,  K, 3
	LD	ALPHA_R, 128($sp)
#else
	andi	L,  TEMP, 3
	LD	ALPHA_R, 128($sp)
#endif
	blez	L, .L38
	LD	ALPHA_I, 136($sp)
	.align	5

.L36:
	daddiu	L, L, -1
	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd

	daddiu	BO, BO, 4 * SIZE 					#	2nr*1kr*cmpx
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

	daddiu	AO, AO, 2 * SIZE					#	2mr*1kr*cmpx
	MADD1	c31, c31, a1, b3 					#	A1xB2
	MADD3	c33, c33, a1, b4

	daddiu	PREB, PREB, 4 * SIZE
	MADD2	c32, c32, a2, b3
	MADD4	c34, c34, a2, b4

	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)

	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	LD	b3, 2 * SIZE(BO)
	LD	b4, 3 * SIZE(BO)
	bgtz	L, .L36
	NOP

.L38:
#ifndef	TRMMKERNEL
 	ADD	c11, c14, c11
	LD	a1,  0 * SIZE(CO1)
	ADD	c12, c13, c12
	LD	a2,  1 * SIZE(CO1)

	ADD	c31, c34, c31
	LD	a3,  0 * SIZE(CO2)
	ADD	c32, c33, c32
	LD	a4,  1 * SIZE(CO2)

	MADD	a1, a1, ALPHA_R, c11
	MADD	a2, a2, ALPHA_R, c12

	MADD	a3, a3, ALPHA_R, c31
	MADD	a4, a4, ALPHA_R, c32

	NMSUB	a1, a1, ALPHA_I, c12
	MADD	a2, a2, ALPHA_I, c11

	NMSUB	a3, a3, ALPHA_I, c32
	MADD	a4, a4, ALPHA_I, c31

	ST	a1, 0 * SIZE(CO1)
	ST	a2, 1 * SIZE(CO1)

	ST	a3, 0 * SIZE(CO2)
	ST	a4, 1 * SIZE(CO2)

	daddiu	CO1,CO1, 2 * SIZE
	daddiu	CO2,CO2, 2 * SIZE

#else
	ADD	c11, c14, c11
	ADD	c12, c13, c12

	ADD	c31, c34, c31
	ADD	c32, c33, c32

	MUL	a1, ALPHA_R, c11
	MUL	a2, ALPHA_R, c12

	MUL	a3, ALPHA_R, c31
	MUL	a4, ALPHA_R, c32

	NMSUB	a1, a1, ALPHA_I, c12
	MADD	a2, a2, ALPHA_I, c11

	NMSUB	a3, a3, ALPHA_I, c32
	MADD	a4, a4, ALPHA_I, c31

	ST	a1, 0 * SIZE(CO1)
	ST	a2, 1 * SIZE(CO1)

	ST	a3, 0 * SIZE(CO2)
	ST	a4, 1 * SIZE(CO2)

	daddiu	CO1,CO1, 2 * SIZE
	daddiu	CO2,CO2, 2 * SIZE

#if ( defined(LEFT) &&  defined(TRANSA)) || \
    (!defined(LEFT) && !defined(TRANSA))
	dsubu	TEMP, K, KK
#ifdef LEFT
	daddiu	TEMP, TEMP, -1
#else
	daddiu	TEMP, TEMP, -2
#endif
	dsll	L,    TEMP, ZBASE_SHIFT
	dsll	TEMP, TEMP, 1 + ZBASE_SHIFT

	daddu	AO, AO, L
	daddu	BO, BO, TEMP
#endif

#ifdef LEFT
	daddiu	KK, KK, 1
#endif
#endif

	.align 5

.L19:
#if defined(TRMMKERNEL) && !defined(LEFT)
	daddiu	KK, KK, 2
#endif

	bgtz	J, .L10
	move	B, BO

	.align 5

.L20:
	andi	J,  N, 1
	blez	J, .L999
	dsll	PREA, K, 1+ZBASE_SHIFT				#	PREA=K*2*2^4

	dsra	I,  M, 1							#	I=M/2
	move	CO1, C

#if defined(TRMMKERNEL) &&  defined(LEFT)
	move	KK, OFFSET
#endif

	move	AO, A								#	Reset AO
	blez	I, .L29
	daddu	PREA, PREA, A

.L21:
#if defined(TRMMKERNEL)
#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
	move	BO,  B
#else
	dsll	L,    KK, 1 + ZBASE_SHIFT
	dsll	TEMP, KK, ZBASE_SHIFT

	daddu	AO, AO, L
	daddu	BO, B,  TEMP
#endif

	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	MTC	$0,  c11								#	Clear results regs
	MOV	c12, c11

	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	MOV	c13, c11
	MOV	c14, c11

	LD	a3, 2 * SIZE(AO)
	LD	a4, 3 * SIZE(AO)
	MOV	c21, c11
	MOV	c22, c11

	FETCH	$0, 0 * SIZE(PREA)
	MOV	c23, c11
	MOV	c24, c11

	FETCH	$0, 0 * SIZE(CO1)
	FETCH	$0, 4 * SIZE(CO1)

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
	dsubu	TEMP, K, KK
#elif defined(LEFT)
	daddiu	TEMP, KK, 2						#	define Mr=2
#else
	daddiu	TEMP, KK, 1						#	define	NR=1
#endif
	dsra	L,  TEMP, 2
	blez	L, .L25
	NOP

#else
	dsra	L,  K, 2							#	Unroll K 4 times
	move	BO,  B

	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	MTC	$0,  c11								#	Clear results regs
	MOV	c12, c11

	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	MOV	c13, c11
	MOV	c14, c11

	LD	a3, 2 * SIZE(AO)
	LD 	a4, 3 * SIZE(AO)
	MOV	c21, c11
	MOV	c22, c11

	FETCH	$0, 0 * SIZE(PREA)
	MOV	c23, c11
	MOV	c24, c11

	FETCH	$0, 0 * SIZE(CO1)
	FETCH	$0, 4 * SIZE(CO1)

	blez	L, .L25
	NOP
#endif

	.align	5

.L22:
	LD	 a5, 4 * SIZE(AO)
	LD	 a6, 5 * SIZE(AO)
	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd

	LD	 b3, 2 * SIZE(BO)
	LD	 b4, 3 * SIZE(BO)
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

	LD 	 a7, 6 * SIZE(AO)
	LD 	 a8, 7 * SIZE(AO)
	MADD1	c21, c21, a3, b1 					#	A2xB1
	MADD3	c23, c23, a3, b2

	FETCH	$0, 4 * SIZE(PREA)
	MADD2	c22, c22, a4, b1
	MADD4	c24, c24, a4, b2

	LD 	a1, 8 * SIZE(AO)
	LD	a2, 9 * SIZE(AO)
	MADD1	c11, c11, a5, b3 					#	axc		A1xB1
	MADD3	c13, c13, a5, b4					#	axd

	LD	b5, 4 * SIZE(BO)
	LD	b6, 5 * SIZE(BO)
	MADD2	c12, c12, a6, b3					#	bxc
	MADD4	c14, c14, a6, b4					#	bxd

	LD	a3, 10 * SIZE(AO)
	LD	a4, 11 * SIZE(AO)
	MADD1	c21, c21, a7, b3 					#	A2xB1
	MADD3	c23, c23, a7, b4

	FETCH	$0, 8 * SIZE(PREA)
	MADD2	c22, c22, a8, b3
	MADD4	c24, c24, a8, b4
	daddiu	L, L, -1

	LD	a5, 12 * SIZE(AO)
	LD	a6, 13 * SIZE(AO)
	MADD1	c11, c11, a1, b5 					#	axc		A1xB1
	MADD3	c13, c13, a1, b6					#	axd

	LD	b7, 6 * SIZE(BO)
	LD	b8, 7 * SIZE(BO)
	MADD2	c12, c12, a2, b5					#	bxc
	MADD4	c14, c14, a2, b6					#	bxd

	LD	a7, 14 * SIZE(AO)
	LD	a8, 15 * SIZE(AO)
	MADD1	c21, c21, a3, b5 					#	A2xB1
	MADD3	c23, c23, a3, b6

	daddiu	BO, BO,  8 * SIZE  					#	1nr*4kr*cmpx
	daddiu	AO, AO, 16 * SIZE 					#	2mr*4kr*cmpx

	FETCH	$0, 12 * SIZE(PREA)
	MADD2	c22, c22, a4, b5
	MADD4	c24, c24, a4, b6
	daddiu	PREA, PREA, 16 * SIZE

	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	MADD1	c11, c11, a5, b7 					#	axc		A1xB1
	MADD3	c13, c13, a5, b8					#	axd

	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	MADD2	c12, c12, a6, b7					#	bxc
	MADD4	c14, c14, a6, b8					#	bxd

	LD	a3, 2 * SIZE(AO)
	LD	a4, 3 * SIZE(AO)
	MADD1	c21, c21, a7, b7 					#	A2xB1
	MADD3	c23, c23, a7, b8

	FETCH	$0, 0 * SIZE(PREA)
	MADD2	c22, c22, a8, b7
	bgtz	L, .L22
	MADD4	c24, c24, a8, b8


.L25:
#ifndef TRMMKERNEL
	andi	L,  K, 3
	LD	ALPHA_R, 128($sp)
#else
	andi	L, TEMP, 3
	LD	ALPHA_R, 128($sp)
#endif
	blez	L, .L28
	LD	ALPHA_I, 136($sp)
	.align	3

.L26:
	daddiu	L, L, -1
	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd

	daddiu	BO, BO, 2 * SIZE 					#	2nr*1kr*cmpx
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

	daddiu	AO, AO, 4 * SIZE 					#	2mr*1kr*cmpx
	MADD1	c21, c21, a3, b1 					#	A2xB1
	MADD3	c23, c23, a3, b2

	daddiu	PREA, PREA, 4 * SIZE 				#	2mr*1kr*cmpx
	MADD2	c22, c22, a4, b1
	MADD4	c24, c24, a4, b2

#	gsLQC1(R12, F1, F0, 0)						#	R:a1	I:a2	Unroll K=4
#	gsLQC1(R12, F3, F2, 1)						#	R:a3	I:a4
#	gsLQC1(R13, F5, F4, 0)						#	R:b1	I:b2
	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	LD	a3, 2 * SIZE(AO)
	LD	a4, 3 * SIZE(AO)

	bgtz	L, .L26
	FETCH	$0, 0 * SIZE(PREA)

.L28:
#ifndef	TRMMKERNEL
 	ADD	c11, c14, c11
	LD	a1,  0 * SIZE(CO1)
	ADD	c12, c13, c12
	LD	a2,  1 * SIZE(CO1)
	ADD	c21, c24, c21
	LD	b1,  2 * SIZE(CO1)
	ADD	c22, c23, c22
	LD	b2,  3 * SIZE(CO1)

	daddiu	I, I, -1
	MADD	a1, a1, ALPHA_R, c11
	MADD	a2, a2, ALPHA_R, c12
	MADD	b1, b1, ALPHA_R, c21
	MADD	b2, b2, ALPHA_R, c22

	NMSUB	a1, a1, ALPHA_I, c12
	MADD	a2, a2, ALPHA_I, c11
	NMSUB	b1, b1, ALPHA_I, c22
	MADD	b2, b2, ALPHA_I, c21

	ST	a1, 0 * SIZE(CO1)
	ST	a2, 1 * SIZE(CO1)
	ST	b1, 2 * SIZE(CO1)
	ST	b2, 3 * SIZE(CO1)

#else
 	ADD	c11, c14, c11
	ADD	c12, c13, c12
	ADD	c21, c24, c21
	ADD	c22, c23, c22

	daddiu	I, I, -1
	MUL	a1, ALPHA_R, c11
	MUL	a2, ALPHA_R, c12
	MUL	b1, ALPHA_R, c21
	MUL	b2, ALPHA_R, c22

	NMSUB	a1, a1, ALPHA_I, c12
	MADD	a2, a2, ALPHA_I, c11
	NMSUB	b1, b1, ALPHA_I, c22
	MADD	b2, b2, ALPHA_I, c21

	ST	a1, 0 * SIZE(CO1)
	ST	a2, 1 * SIZE(CO1)
	ST	b1, 2 * SIZE(CO1)
	ST	b2, 3 * SIZE(CO1)

#if ( defined(LEFT) &&  defined(TRANSA)) || \
    (!defined(LEFT) && !defined(TRANSA))
	dsubu	TEMP, K, KK
#ifdef LEFT
	daddiu	TEMP, TEMP, -2
#else
	daddiu	TEMP, TEMP, -1
#endif

	dsll	L,    TEMP, 1 + ZBASE_SHIFT
	dsll	TEMP, TEMP, ZBASE_SHIFT

	daddu	AO, AO, L
	daddu	BO, BO, TEMP
#endif

#ifdef LEFT
	daddiu	KK, KK, 2
#endif
#endif
	daddiu	CO1,CO1, 4 * SIZE
	bgtz	I, .L21
	NOP

.L29:
	andi	I, M, 1
	blez	I, .L999
	NOP

#if defined(TRMMKERNEL)
#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
	move	BO,  B
#else
	dsll	TEMP, KK,  ZBASE_SHIFT

	daddu	AO, AO, TEMP
	daddu	BO, B,  TEMP
#endif

#	gsLQC1(R12, F1, F0, 0)						#	R:a1	I:a2
	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	MTC	$0,  c11								#	Clear results regs
	MOV	c12, c11

#	gsLQC1(R13, F5, F4, 0)						#	R:b1	I:b2
	LD	b1, 0 * SIZE(BO)
	LD 	b2, 1 * SIZE(BO)
	MOV	c13, c11
	MOV	c14, c11

	FETCH	$0, 0 * SIZE(PREA)
	FETCH	$0, 4 * SIZE(PREA)

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
	dsubu	TEMP, K, KK
#elif defined(LEFT)
	daddiu	TEMP, KK, 1
#else
	daddiu	TEMP, KK, 1
#endif
	dsra	L,  TEMP, 2
	blez	L, .L45
	NOP

#else
	dsra	L,  K, 2							#	Unroll K 4 times
	move	BO,  B

#	gsLQC1(R12, F1, F0, 0)						#	R:a1	I:a2
	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	MTC	$0,  c11								#	Clear results regs
	MOV	c12, c11

#	gsLQC1(R13, F5, F4, 0)						#	R:b1	I:b2
	LD	b1, 0 * SIZE(BO)
	LD 	b2, 1 * SIZE(BO)
	MOV	c13, c11
	MOV	c14, c11

	FETCH	$0, 0 * SIZE(PREA)
	FETCH	$0, 4 * SIZE(PREA)
	blez	L, .L45
	NOP
#endif

	.align	3

.L42:
#	gsLQC1(R12, F3, F2, 1)						#	R:a3	I:a4
	LD 	a3, 2 * SIZE(AO)
	LD	a4, 3 * SIZE(AO)
	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd

#	gsLQC1(R13, F7, F6, 1)						#	R:b2	I:b3
	LD	b3, 2 * SIZE(BO)
	LD	b4, 3 * SIZE(BO)
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

#	gsLQC1(R12, F9, F8, 2)						#	Unroll K=1
	LD	a5, 4 * SIZE(AO)
	LD	a6, 5 * SIZE(AO)
	MADD1	c11, c11, a3, b3 					#	axc		A1xB1
	MADD3	c13, c13, a3, b4					#	axd

#	gsLQC1(R13, F13, F12, 2)
	LD	b5, 4 * SIZE(BO)
	LD 	b6, 5 * SIZE(BO)
	MADD2	c12, c12, a4, b3					#	bxc
	MADD4	c14, c14, a4, b4					#	bxd

#	gsLQC1(R12, F11, F10, 3)
	LD 	a7, 6 * SIZE(AO)
	LD	a8, 7 * SIZE(AO)
	MADD1	c11, c11, a5, b5 					#	axc		A1xB1
	MADD3	c13, c13, a5, b6					#	axd

	daddiu	L, L, -1

#	gsLQC1(R13, F16, F15, 3)
	LD	b7, 6 * SIZE(BO)
	LD	b8, 7 * SIZE(BO)
	MADD2	c12, c12, a6, b5					#	bxc
	MADD4	c14, c14, a6, b6					#	bxd

	daddiu	AO, AO, 8 * SIZE 					#	2mr*4kr*cmpx
	daddiu	BO, BO, 8 * SIZE  					#	2nr*4kr*cmpx

#	gsLQC1(R12, F1, F0, 0)						#	R:a1	I:a2	Unroll K=4
	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	MADD1	c11, c11, a7, b7 					#	axc		A1xB1
	MADD3	c13, c13, a7, b8					#	axd

#	gsLQC1(R13, F5, F4, 0)						#	R:b1	I:b2
	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	MADD2	c12, c12, a8, b7					#	bxc
	MADD4	c14, c14, a8, b8					#	bxd

	bgtz	L, .L42
	NOP


	.align 5

.L45:
#ifndef	TRMMKERNEL
	andi	L,  K, 3
	LD	ALPHA_R, 128($sp)
#else
	andi	L,  TEMP, 3
	LD	ALPHA_R, 128($sp)
#endif
	blez	L, .L48
	LD	ALPHA_I, 136($sp)

.L46:
	daddiu	L, L, -1
	daddiu	BO, BO, 1 * SIZE * COMPSIZE			#	2nr*1kr*cmpx
	daddiu	AO, AO, 1 * SIZE * COMPSIZE			#	2mr*1kr*cmpx

	MADD1	c11, c11, a1, b1 					#	axc		A1xB1
	MADD3	c13, c13, a1, b2					#	axd
	MADD2	c12, c12, a2, b1					#	bxc
	MADD4	c14, c14, a2, b2					#	bxd

#	gsLQC1(R12, F1, F0, 0)						#	R:a1	I:a2	Unroll K=4
#	gsLQC1(R13, F5, F4, 0)						#	R:b1	I:b2
	LD	a1, 0 * SIZE(AO)
	LD	a2, 1 * SIZE(AO)
	LD	b1, 0 * SIZE(BO)
	LD	b2, 1 * SIZE(BO)
	bgtz	L, .L46
	NOP

.L48:
#ifndef	TRMMKERNEL
 	ADD	c11, c14, c11
	ADD	c12, c13, c12

	LD	a1,  0 * SIZE(CO1)
	LD	a2,  1 * SIZE(CO1)

	MADD	a1, a1, ALPHA_R, c11
	MADD	a2, a2, ALPHA_R, c12

	NMSUB	a1, a1, ALPHA_I, c12
	MADD	a2, a2, ALPHA_I, c11

	ST	a1, 0 * SIZE(CO1)
	ST	a2, 1 * SIZE(CO1)

#else
	ADD	c11, c14, c11
	ADD	c12, c13, c12

	MUL	a1, ALPHA_R, c11
	MUL	a2, ALPHA_R, c12

	NMSUB	a1, a1, ALPHA_I, c12
	MADD	a2, a2, ALPHA_I, c11

	ST	a1, 0 * SIZE(CO1)
	ST	a2, 1 * SIZE(CO1)

#if ( defined(LEFT) &&  defined(TRANSA)) || \
    (!defined(LEFT) && !defined(TRANSA))
	dsubu	TEMP, K, KK
#ifdef LEFT
	daddiu	TEMP, TEMP, -1
#else
	daddiu	TEMP, TEMP, -1
#endif

	dsll	TEMP, TEMP, ZBASE_SHIFT

	daddu	AO, AO, TEMP
	daddu	BO, BO, TEMP
#endif

#ifdef LEFT
	daddiu	KK, KK, 1
#endif

	daddiu	CO1,CO1, 2 * SIZE
#endif



	.align 5

.L999:
	LDARG	$16,   0($sp)
	LDARG	$17,   8($sp)
	ldc1	$f24, 16($sp)
	ldc1	$f25, 24($sp)
	ldc1	$f26, 32($sp)
	ldc1	$f27, 40($sp)
	ldc1	$f28, 48($sp)
	ldc1	$f29, 56($sp)

#if defined(TRMMKERNEL)
	LDARG	$18,  64($sp)
	LDARG	$19,  72($sp)
	LDARG	$20,  80($sp)
#endif

#ifndef __64BIT__
	ldc1	$f20, 88($sp)
	ldc1	$f21, 96($sp)
	ldc1	$f22,104($sp)
	ldc1	$f23,112($sp)
#endif

	j	$31
	daddiu	$sp, $sp, STACKSIZE

	EPILOGUE