;**************************************************************************
;* Copyright (C) 2007 Loren Merritt <lorenm@u.washington.edu>
;*
;* This program is free software; you can redistribute it and/or modify
;* it under the terms of the GNU General Public License as published by
;* the Free Software Foundation; either version 2 of the License, or
;* (at your option) any later version.
;*
;* This program is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;* GNU General Public License for more details.
;*
;* You should have received a copy of the GNU General Public License
;* along with this program; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
;**************************************************************************

%include "amd64inc.asm"

section .rodata align=16

pw_8: times 8 dw 8
pw_808: times 8 dw 0x808

pw_c1: times 8 dw 64277
pw_c2: times 8 dw 60547
pw_c3: times 8 dw 54491
pw_c4: times 8 dw 46341
pw_c5: times 8 dw 36410
pw_c6: times 8 dw 25080
pw_c7: times 8 dw 12785

pw_c1b: times 8 dw 64277/2
pw_c2b: times 8 dw 60547/2
pw_c3b: times 8 dw 54491/2
pw_c4b: times 8 dw 46341/2
pw_c5b: times 8 dw 36410/2
pw_c6b: times 8 dw 25080/2
pw_c7b: times 8 dw 12785/2

%define mm_c1 [pw_c1 GLOBAL]
%define mm_c2 [pw_c2 GLOBAL]
%define mm_c3 [pw_c3 GLOBAL]
%define mm_c4 xmm15 ; most frequently used constant
%define mm_c5 [pw_c5 GLOBAL]
%define mm_c6 [pw_c6 GLOBAL]
%define mm_c7 [pw_c7 GLOBAL]

cextern xlsc_quant_tab

section .text align=16

%macro SUMSUB 2
    paddw %1, %2
    paddw %2, %2
    psubw %2, %1
%endmacro

; compensates for the constants that overflow int16
%macro MULSHIFT 3
%ifidn %2, c6
    pmulhw   %1, mm_%2
%elifidn %2, c7
    pmulhw   %1, mm_%2
%else
    movdqa   %3, %1
    pmulhw   %1, mm_%2
    paddw    %1, %3
%endif
%endmacro

%macro MULROUND_SSE2 3
    movdqa   %3, %1
    pmulhw   %1, mm_%2
%ifnidn %2, c6
%ifnidn %2, c7
    paddw    %1, %3
%endif
%endif
    pmullw   %3, mm_%2
    psrlw    %3, 15
    paddw    %1, %3
%endmacro

%macro MULROUND_SSSE3 3
    pmulhrsw %1, mm_%2
%endmacro

%define MULROUND MULROUND_SSE2

; %1 = (%3*%2 >> 16) + (%4*%1 >> 16);
; %2 = (%4*%2 >> 16) - (%3*%1 >> 16);
; clobber %5..%8
%macro MULSHIFTSUMSUB 8
    movdqa   %5, %1
    movdqa   %6, %2
    MULSHIFT %1, %4, %7
    MULSHIFT %2, %4, %8
    MULSHIFT %5, %3, %7
    MULSHIFT %6, %3, %8
    psubw    %2, %5
    paddw    %1, %6
%endmacro

; %1 = (%3*%2 + 0x8000 >> 16) + (%4*%1 + 0x8000 >> 16);
; %2 = (%4*%2 + 0x8000 >> 16) - (%3*%1 + 0x8000 >> 16);
; clobber %5..%8
%macro MULROUNDSUMSUB 8
    movdqa   %5, %1
    movdqa   %6, %2
    MULROUND %1, %4, %7
    MULROUND %2, %4, %8
    MULROUND %5, %3, %7
    MULROUND %6, %3, %8
    psubw    %2, %5
    paddw    %1, %6
%endmacro

; 01234567 -> 51604372
%macro DCT_1D 12
    SUMSUB   %8, %1 ; %8=t0, %1=t7
    SUMSUB   %7, %2 ; %7=t1, %2=t6
    SUMSUB   %6, %3 ; %6=t2, %3=t5
    SUMSUB   %5, %4 ; %5=t3, %4=t4
    SUMSUB   %5, %8 ; %5=t0, %8=t3
    SUMSUB   %6, %7 ; %6=t1, %7=t2
    SUMSUB   %3, %2 ; %3=t6, %2=t5
    MULROUND %2, c4, %9
    MULROUND %3, c4, %10
    SUMSUB   %2, %4 ; %2=t4, %4=t5
    SUMSUB   %3, %1 ; %3=t7, %1=t6
    SUMSUB   %6, %5 ; %6=t0, %5=t1
    MULROUND %6, c4, %9  ; %6=x0
    MULROUND %5, c4, %10 ; %5=x4
    MULROUNDSUMSUB %7, %8, c2, c6, %9, %10, %11, %12 ; %7=x2, %8=x6
    MULROUNDSUMSUB %2, %3, c1, c7, %9, %10, %11, %12 ; %2=x1, %3=x7
    MULROUNDSUMSUB %4, %1, c5, c3, %9, %10, %11, %12 ; %4=x5, %1=x3
%endmacro

; 01234567 -> 31754026
%macro IDCT_1D 13
    SUMSUB   %5, %1
    MULSHIFT %1, c4, %9  ; %1=t1
    MULSHIFT %5, c4, %10 ; %5=t0
%ifnidn %13, 0
    paddw    %1, %13
    paddw    %5, %13
%endif
    MULSHIFTSUMSUB %7, %3, c2, c6, %9, %10, %11, %12 ; %7=t3, %3=t2
    MULSHIFTSUMSUB %8, %2, c1, c7, %9, %10, %11, %12 ; %8=t7, %2=t4
    MULSHIFTSUMSUB %4, %6, c5, c3, %9, %10, %11, %12 ; %4=t6, %6=t5
    SUMSUB   %6, %2 ; %6=t4, %2=t5
    SUMSUB   %4, %8 ; %4=t7, %8=t6
    MULSHIFT %2, c4, %9
    MULSHIFT %8, c4, %10
    SUMSUB   %7, %5 ; %7=t0, %5=t3
    SUMSUB   %3, %1 ; %3=t1, %1=t2
    SUMSUB   %2, %8 ; %2=t6, %8=t5
    SUMSUB   %4, %7 ; %4=x0, %7=x7
    SUMSUB   %2, %3 ; %2=x1, %3=x6
    SUMSUB   %8, %1 ; %8=x2, %1=x5
    SUMSUB   %6, %5 ; %6=x3, %5=x4
%endmacro

%macro SBUTTERFLY 4
    movdqa    %4, %2
    punpckl%1 %2, %3
    punpckh%1 %4, %3
%endmacro

; 01234567 -> 05738421
%macro TRANSPOSE_8x8W 9
    SBUTTERFLY wd,  %1, %2, %9
    SBUTTERFLY wd,  %3, %4, %2
    SBUTTERFLY wd,  %5, %6, %4
    SBUTTERFLY wd,  %7, %8, %6
    SBUTTERFLY dq,  %1, %3, %8
    SBUTTERFLY dq,  %9, %2, %3
    SBUTTERFLY dq,  %5, %7, %2
    SBUTTERFLY dq,  %4, %6, %7
    SBUTTERFLY qdq, %1, %5, %6
    SBUTTERFLY qdq, %9, %4, %5
    SBUTTERFLY qdq, %8, %2, %4
    SBUTTERFLY qdq, %3, %7, %2
%endmacro

%macro OP4 6
    %1 xmm%2, %6
    %1 xmm%3, %6
    %1 xmm%4, %6
    %1 xmm%5, %6
%endmacro

%macro OP8 10
    OP4 %1, %2, %3, %4, %5, %10
    OP4 %1, %6, %7, %8, %9, %10
%endmacro

%macro LOAD4 11
    mov%1 xmm%8,  [%2+%3*%4]
    mov%1 xmm%9,  [%2+%3*%5]
    mov%1 xmm%10, [%2+%3*%6]
    mov%1 xmm%11, [%2+%3*%7]
%endmacro

%macro STORE4 11
    mov%1 [%2+%3*%4], xmm%8
    mov%1 [%2+%3*%5], xmm%9
    mov%1 [%2+%3*%6], xmm%10
    mov%1 [%2+%3*%7], xmm%11
%endmacro

; 01234567 -> 50148367
cglobal xlsc_idct_sse2
    movdqa mm_c4, [pw_c4 GLOBAL]
    IDCT_1D        xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, 0
    TRANSPOSE_8x8W xmm3, xmm1, xmm7, xmm5, xmm4, xmm0, xmm2, xmm6, xmm8
    IDCT_1D        xmm3, xmm0, xmm6, xmm5, xmm8, xmm4, xmm7, xmm1, xmm2, xmm9, xmm10, xmm11, [rax]
    OP8 psraw, 5,0,1,4,8,3,6,7, 4
    ret

; 01234567 -> 43658012
cglobal xlsc_fdct_sse2
    movdqa mm_c4, [pw_c4 GLOBAL]
    DCT_1D         xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11
    TRANSPOSE_8x8W xmm5, xmm1, xmm6, xmm0, xmm4, xmm3, xmm7, xmm2, xmm8
    DCT_1D         xmm5, xmm3, xmm2, xmm0, xmm8, xmm4, xmm6, xmm1, xmm7, xmm9, xmm10, xmm11
    ret

%define mm_c1 [pw_c1b GLOBAL]
%define mm_c2 [pw_c2b GLOBAL]
%define mm_c3 [pw_c3b GLOBAL]
%define mm_c4 xmm15
%define mm_c5 [pw_c5b GLOBAL]
%define mm_c6 [pw_c6b GLOBAL]
%define mm_c7 [pw_c7b GLOBAL]
%define MULROUND MULROUND_SSSE3

; 01234567 -> 43658012
cglobal xlsc_fdct_ssse3
    movdqa mm_c4, [pw_c4b GLOBAL]
    DCT_1D         xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11
    TRANSPOSE_8x8W xmm5, xmm1, xmm6, xmm0, xmm4, xmm3, xmm7, xmm2, xmm8
    DCT_1D         xmm5, xmm3, xmm2, xmm0, xmm8, xmm4, xmm6, xmm1, xmm7, xmm9, xmm10, xmm11
    ret

%macro FDCT_FUNCS 1
; void xlsc_dct_inter_sse2( int16_t *dct, uint8_t *fenc, uint8_t *fdec );
cglobal xlsc_dct_inter_%1
    pxor xmm8, xmm8
    LOAD4 q, parm2q, FENC_STRIDE, 0,1,2,3, 0,1,2,3
    LOAD4 q, parm3q, FENC_STRIDE, 0,1,2,3, 9,10,11,12
    OP8 punpcklbw, 0,1,2,3,9,10,11,12, xmm8
    psubw xmm0, xmm9
    psubw xmm1, xmm10
    psubw xmm2, xmm11
    psubw xmm3, xmm12
    LOAD4 q, parm2q, FENC_STRIDE, 4,5,6,7, 4,5,6,7
    LOAD4 q, parm3q, FENC_STRIDE, 4,5,6,7, 9,10,11,12
    OP8 punpcklbw, 4,5,6,7,9,10,11,12, xmm8
    psubw xmm4, xmm9
    psubw xmm5, xmm10
    psubw xmm6, xmm11
    psubw xmm7, xmm12
    call xlsc_fdct_%1
    STORE4 dqa, parm1q, 16, 0,1,2,3, 4,3,6,5
    STORE4 dqa, parm1q, 16, 4,5,6,7, 8,0,1,2
    ret

; void xlsc_dct_intra_sse2( int16_t *dct, uint8_t *fenc );
cglobal xlsc_dct_intra_%1
    pxor xmm8, xmm8
    LOAD4 q, parm2q, FENC_STRIDE, 0,1,2,3, 0,1,2,3
    LOAD4 q, parm2q, FENC_STRIDE, 4,5,6,7, 4,5,6,7
    OP8 punpcklbw, 0,1,2,3,4,5,6,7, xmm8
    call xlsc_fdct_%1
    STORE4 dqa, parm1q, 16, 0,1,2,3, 4,3,6,5
    STORE4 dqa, parm1q, 16, 4,5,6,7, 8,0,1,2
    sub word[parm1q], 0x1000
    ret
%endmacro

FDCT_FUNCS sse2
FDCT_FUNCS ssse3

; void xlsc_idct_inter_sse2( uint8_t *fdec, int16_t *dct )
cglobal xlsc_idct_inter_sse2
    LOAD4 dqa, parm2q, 16, 0,1,2,3, 0,1,2,3
    LOAD4 dqa, parm2q, 16, 4,5,6,7, 4,5,6,7
    lea  rax, [pw_8 GLOBAL]
    call xlsc_idct_sse2
    pxor xmm2, xmm2
    LOAD4 q, parm1q, FENC_STRIDE, 0,1,2,3, 9,10,11,12
    OP4 punpcklbw, 9,10,11,12, xmm2
    paddw xmm5, xmm9
    paddw xmm0, xmm10
    paddw xmm1, xmm11
    paddw xmm4, xmm12
    LOAD4 q, parm1q, FENC_STRIDE, 4,5,6,7, 9,10,11,12
    OP4 punpcklbw, 9,10,11,12, xmm2
    paddw xmm8, xmm9
    paddw xmm3, xmm10
    paddw xmm6, xmm11
    paddw xmm7, xmm12
    OP8 packuswb, 5,0,1,4,8,3,6,7, xmm2
    STORE4 q, parm1q, FENC_STRIDE, 0,1,2,3, 5,0,1,4
    STORE4 q, parm1q, FENC_STRIDE, 4,5,6,7, 8,3,6,7
    ret

; void xlsc_idct_intra_sse2( uint8_t *fdec, int16_t *dct )
cglobal xlsc_idct_intra_sse2
    LOAD4 dqa, parm2q, 16, 0,1,2,3, 0,1,2,3
    LOAD4 dqa, parm2q, 16, 4,5,6,7, 4,5,6,7
    lea  rax, [pw_808 GLOBAL]
    call xlsc_idct_sse2
    OP8 packuswb, 5,0,1,4,8,3,6,7, xmm2
    STORE4 q, parm1q, FENC_STRIDE, 0,1,2,3, 5,0,1,4
    STORE4 q, parm1q, FENC_STRIDE, 4,5,6,7, 8,3,6,7
    ret


%macro QUANT_SSE2 0
    movdqa    xmm0, [parm1q]
    movdqa    xmm2, [parm1q+16]
    movdqa    xmm1, xmm0
    movdqa    xmm3, xmm2
    psraw     xmm1, 15
    psraw     xmm3, 15
    pxor      xmm0, xmm1
    pxor      xmm2, xmm3
    psubw     xmm0, xmm1
    psubw     xmm2, xmm3
    paddw     xmm0, xmm6
    paddw     xmm2, xmm6
    pmulhuw   xmm0, xmm7
    pmulhuw   xmm2, xmm7
    paddw     xmm0, xmm1
    paddw     xmm2, xmm3
    pxor      xmm0, xmm1
    pxor      xmm2, xmm3
    movdqa [parm1q], xmm0
    movdqa [parm1q+16], xmm2
%endmacro

%macro QUANT_SSSE3 0
    movdqa    xmm1, [parm1q]
    movdqa    xmm3, [parm1q+16]
    pabsw     xmm0, xmm1
    pabsw     xmm2, xmm3
    paddw     xmm0, xmm6
    paddw     xmm2, xmm6
    pmulhuw   xmm0, xmm7
    pmulhuw   xmm2, xmm7
    psignw    xmm0, xmm1
    psignw    xmm2, xmm3
    movdqa [parm1q], xmm0
    movdqa [parm1q+16], xmm2
%endmacro

%macro QUANT_FUNC 1
; void xlsc_quant_sse2( int16_t *dct, int qp, int bias )
cglobal xlsc_quant_%1
    movd      xmm6, parm3d
    movd      xmm7, [xlsc_quant_tab + 2*parm2q]
    pshuflw   xmm6, xmm6, 0
    pshuflw   xmm7, xmm7, 0
    punpcklwd xmm6, xmm6
    punpcklwd xmm7, xmm7

    mov        eax, 4
.loop:
    QUANT
    add     parm1q, 32
    sub        eax, byte 1
    jg .loop
    rep ret
%endmacro

%define QUANT QUANT_SSE2
QUANT_FUNC sse2
%define QUANT QUANT_SSSE3
QUANT_FUNC ssse3

; void xlsc_dequant_sse2( int16_t *dct, int qp )
cglobal xlsc_dequant_sse2
    shl     parm2d, 2
    movd      xmm7, parm2d
    pshuflw   xmm7, xmm7, 0
    punpcklwd xmm7, xmm7
    LOAD4  dqa, parm1q, 16, 0,1,2,3, 0,1,2,3
    pmullw    xmm0, xmm7
    pmullw    xmm1, xmm7
    pmullw    xmm2, xmm7
    pmullw    xmm3, xmm7
    STORE4 dqa, parm1q, 16, 0,1,2,3, 0,1,2,3
    LOAD4  dqa, parm1q, 16, 4,5,6,7, 0,1,2,3
    pmullw    xmm0, xmm7
    pmullw    xmm1, xmm7
    pmullw    xmm2, xmm7
    pmullw    xmm3, xmm7
    STORE4 dqa, parm1q, 16, 4,5,6,7, 0,1,2,3
    ret

