falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
37120 Commits
32 Branches
696MB
Tree: 34e6af9e20
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '34e6af9e20'
${ noResults }
FFmpeg/libavcodec/x86/h264_idct.asm

1074 lines
27KB
Raw Blame History 

  1. ;*****************************************************************************

  2. ;* MMX/SSE2-optimized H.264 iDCT

  3. ;*****************************************************************************

  4. ;* Copyright (C) 2004-2005 Michael Niedermayer, Loren Merritt

  5. ;* Copyright (C) 2003-2008 x264 project

  6. ;*

  7. ;* Authors: Laurent Aimar <fenrir@via.ecp.fr>

  8. ;*          Loren Merritt <lorenm@u.washington.edu>

  9. ;*          Holger Lubitz <hal@duncan.ol.sub.de>

  10. ;*          Min Chen <chenm001.163.com>

  11. ;*

  12. ;* This file is part of Libav.

  13. ;*

  14. ;* Libav is free software; you can redistribute it and/or

  15. ;* modify it under the terms of the GNU Lesser General Public

  16. ;* License as published by the Free Software Foundation; either

  17. ;* version 2.1 of the License, or (at your option) any later version.

  18. ;*

  19. ;* Libav is distributed in the hope that it will be useful,

  20. ;* but WITHOUT ANY WARRANTY; without even the implied warranty of

  21. ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  22. ;* Lesser General Public License for more details.

  23. ;*

  24. ;* You should have received a copy of the GNU Lesser General Public

  25. ;* License along with Libav; if not, write to the Free Software

  26. ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  27. ;*****************************************************************************

  28. 

  29. %include "libavutil/x86/x86util.asm"

  30. 

  31. SECTION_RODATA

  32. 

  33. ; FIXME this table is a duplicate from h264data.h, and will be removed once the tables from, h264 have been split

  34. scan8_mem: db  4+ 1*8, 5+ 1*8, 4+ 2*8, 5+ 2*8

  35.            db  6+ 1*8, 7+ 1*8, 6+ 2*8, 7+ 2*8

  36.            db  4+ 3*8, 5+ 3*8, 4+ 4*8, 5+ 4*8

  37.            db  6+ 3*8, 7+ 3*8, 6+ 4*8, 7+ 4*8

  38.            db  4+ 6*8, 5+ 6*8, 4+ 7*8, 5+ 7*8

  39.            db  6+ 6*8, 7+ 6*8, 6+ 7*8, 7+ 7*8

  40.            db  4+ 8*8, 5+ 8*8, 4+ 9*8, 5+ 9*8

  41.            db  6+ 8*8, 7+ 8*8, 6+ 9*8, 7+ 9*8

  42.            db  4+11*8, 5+11*8, 4+12*8, 5+12*8

  43.            db  6+11*8, 7+11*8, 6+12*8, 7+12*8

  44.            db  4+13*8, 5+13*8, 4+14*8, 5+14*8

  45.            db  6+13*8, 7+13*8, 6+14*8, 7+14*8

  46. %ifdef PIC

  47. %define npicregs 1

  48. %define scan8 picregq

  49. %else

  50. %define npicregs 0

  51. %define scan8 scan8_mem

  52. %endif

  53. 

  54. cextern pw_32

  55. cextern pw_1

  56. 

  57. SECTION .text

  58. 

  59. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  60. %macro IDCT4_ADD 3

  61.     ; Load dct coeffs

  62.     movq         m0, [%2]

  63.     movq         m1, [%2+8]

  64.     movq         m2, [%2+16]

  65.     movq         m3, [%2+24]

  66. 

  67.     IDCT4_1D      w, 0, 1, 2, 3, 4, 5

  68.     mova         m6, [pw_32]

  69.     TRANSPOSE4x4W 0, 1, 2, 3, 4

  70.     paddw        m0, m6

  71.     IDCT4_1D      w, 0, 1, 2, 3, 4, 5

  72.     pxor         m7, m7

  73.     movq    [%2+ 0], m7

  74.     movq    [%2+ 8], m7

  75.     movq    [%2+16], m7

  76.     movq    [%2+24], m7

  77. 

  78.     STORE_DIFFx2 m0, m1, m4, m5, m7, 6, %1, %3

  79.     lea          %1, [%1+%3*2]

  80.     STORE_DIFFx2 m2, m3, m4, m5, m7, 6, %1, %3

  81. %endmacro

  82. 

  83. INIT_MMX mmx

  84. ; ff_h264_idct_add_mmx(uint8_t *dst, int16_t *block, int stride)

  85. cglobal h264_idct_add_8, 3, 3, 0

  86.     IDCT4_ADD    r0, r1, r2

  87.     RET

  88. 

  89. %macro IDCT8_1D 2

  90.     mova         m0, m1

  91.     psraw        m1, 1

  92.     mova         m4, m5

  93.     psraw        m4, 1

  94.     paddw        m4, m5

  95.     paddw        m1, m0

  96.     paddw        m4, m7

  97.     paddw        m1, m5

  98.     psubw        m4, m0

  99.     paddw        m1, m3

  100. 

  101.     psubw        m0, m3

  102.     psubw        m5, m3

  103.     psraw        m3, 1

  104.     paddw        m0, m7

  105.     psubw        m5, m7

  106.     psraw        m7, 1

  107.     psubw        m0, m3

  108.     psubw        m5, m7

  109. 

  110.     mova         m7, m1

  111.     psraw        m1, 2

  112.     mova         m3, m4

  113.     psraw        m3, 2

  114.     paddw        m3, m0

  115.     psraw        m0, 2

  116.     paddw        m1, m5

  117.     psraw        m5, 2

  118.     psubw        m0, m4

  119.     psubw        m7, m5

  120. 

  121.     mova         m5, m6

  122.     psraw        m6, 1

  123.     mova         m4, m2

  124.     psraw        m4, 1

  125.     paddw        m6, m2

  126.     psubw        m4, m5

  127. 

  128.     mova         m2, %1

  129.     mova         m5, %2

  130.     SUMSUB_BA    w, 5, 2

  131.     SUMSUB_BA    w, 6, 5

  132.     SUMSUB_BA    w, 4, 2

  133.     SUMSUB_BA    w, 7, 6

  134.     SUMSUB_BA    w, 0, 4

  135.     SUMSUB_BA    w, 3, 2

  136.     SUMSUB_BA    w, 1, 5

  137.     SWAP         7, 6, 4, 5, 2, 3, 1, 0 ; 70315246 -> 01234567

  138. %endmacro

  139. 

  140. %macro IDCT8_1D_FULL 1

  141.     mova         m7, [%1+112]

  142.     mova         m6, [%1+ 96]

  143.     mova         m5, [%1+ 80]

  144.     mova         m3, [%1+ 48]

  145.     mova         m2, [%1+ 32]

  146.     mova         m1, [%1+ 16]

  147.     IDCT8_1D   [%1], [%1+ 64]

  148. %endmacro

  149. 

  150. ; %1=int16_t *block, %2=int16_t *dstblock

  151. %macro IDCT8_ADD_MMX_START 2

  152.     IDCT8_1D_FULL %1

  153.     mova       [%1], m7

  154.     TRANSPOSE4x4W 0, 1, 2, 3, 7

  155.     mova         m7, [%1]

  156.     mova    [%2   ], m0

  157.     mova    [%2+16], m1

  158.     mova    [%2+32], m2

  159.     mova    [%2+48], m3

  160.     TRANSPOSE4x4W 4, 5, 6, 7, 3

  161.     mova    [%2+ 8], m4

  162.     mova    [%2+24], m5

  163.     mova    [%2+40], m6

  164.     mova    [%2+56], m7

  165. %endmacro

  166. 

  167. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  168. %macro IDCT8_ADD_MMX_END 3-4

  169.     IDCT8_1D_FULL %2

  170.     mova    [%2   ], m5

  171.     mova    [%2+16], m6

  172.     mova    [%2+32], m7

  173. 

  174.     pxor         m7, m7

  175. %if %0 == 4

  176.     movq   [%4+  0], m7

  177.     movq   [%4+  8], m7

  178.     movq   [%4+ 16], m7

  179.     movq   [%4+ 24], m7

  180.     movq   [%4+ 32], m7

  181.     movq   [%4+ 40], m7

  182.     movq   [%4+ 48], m7

  183.     movq   [%4+ 56], m7

  184.     movq   [%4+ 64], m7

  185.     movq   [%4+ 72], m7

  186.     movq   [%4+ 80], m7

  187.     movq   [%4+ 88], m7

  188.     movq   [%4+ 96], m7

  189.     movq   [%4+104], m7

  190.     movq   [%4+112], m7

  191.     movq   [%4+120], m7

  192. %endif

  193.     STORE_DIFFx2 m0, m1, m5, m6, m7, 6, %1, %3

  194.     lea          %1, [%1+%3*2]

  195.     STORE_DIFFx2 m2, m3, m5, m6, m7, 6, %1, %3

  196.     mova         m0, [%2   ]

  197.     mova         m1, [%2+16]

  198.     mova         m2, [%2+32]

  199.     lea          %1, [%1+%3*2]

  200.     STORE_DIFFx2 m4, m0, m5, m6, m7, 6, %1, %3

  201.     lea          %1, [%1+%3*2]

  202.     STORE_DIFFx2 m1, m2, m5, m6, m7, 6, %1, %3

  203. %endmacro

  204. 

  205. INIT_MMX mmx

  206. ; ff_h264_idct8_add_mmx(uint8_t *dst, int16_t *block, int stride)

  207. cglobal h264_idct8_add_8, 3, 4, 0

  208.     %assign pad 128+4-(stack_offset&7)

  209.     SUB         rsp, pad

  210. 

  211.     add   word [r1], 32

  212.     IDCT8_ADD_MMX_START r1  , rsp

  213.     IDCT8_ADD_MMX_START r1+8, rsp+64

  214.     lea          r3, [r0+4]

  215.     IDCT8_ADD_MMX_END   r0  , rsp,   r2, r1

  216.     IDCT8_ADD_MMX_END   r3  , rsp+8, r2

  217. 

  218.     ADD         rsp, pad

  219.     RET

  220. 

  221. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  222. %macro IDCT8_ADD_SSE 4

  223.     IDCT8_1D_FULL %2

  224. %if ARCH_X86_64

  225.     TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, 8

  226. %else

  227.     TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, [%2], [%2+16]

  228. %endif

  229.     paddw        m0, [pw_32]

  230. 

  231. %if ARCH_X86_64 == 0

  232.     mova    [%2   ], m0

  233.     mova    [%2+16], m4

  234.     IDCT8_1D   [%2], [%2+ 16]

  235.     mova    [%2   ], m6

  236.     mova    [%2+16], m7

  237. %else

  238.     SWAP          0, 8

  239.     SWAP          4, 9

  240.     IDCT8_1D     m8, m9

  241.     SWAP          6, 8

  242.     SWAP          7, 9

  243. %endif

  244. 

  245.     pxor         m7, m7

  246.     lea          %4, [%3*3]

  247.     STORE_DIFF   m0, m6, m7, [%1     ]

  248.     STORE_DIFF   m1, m6, m7, [%1+%3  ]

  249.     STORE_DIFF   m2, m6, m7, [%1+%3*2]

  250.     STORE_DIFF   m3, m6, m7, [%1+%4  ]

  251. %if ARCH_X86_64 == 0

  252.     mova         m0, [%2   ]

  253.     mova         m1, [%2+16]

  254. %else

  255.     SWAP          0, 8

  256.     SWAP          1, 9

  257. %endif

  258.     mova   [%2+  0], m7

  259.     mova   [%2+ 16], m7

  260.     mova   [%2+ 32], m7

  261.     mova   [%2+ 48], m7

  262.     mova   [%2+ 64], m7

  263.     mova   [%2+ 80], m7

  264.     mova   [%2+ 96], m7

  265.     mova   [%2+112], m7

  266.     lea          %1, [%1+%3*4]

  267.     STORE_DIFF   m4, m6, m7, [%1     ]

  268.     STORE_DIFF   m5, m6, m7, [%1+%3  ]

  269.     STORE_DIFF   m0, m6, m7, [%1+%3*2]

  270.     STORE_DIFF   m1, m6, m7, [%1+%4  ]

  271. %endmacro

  272. 

  273. INIT_XMM sse2

  274. ; ff_h264_idct8_add_sse2(uint8_t *dst, int16_t *block, int stride)

  275. cglobal h264_idct8_add_8, 3, 4, 10

  276.     IDCT8_ADD_SSE r0, r1, r2, r3

  277.     RET

  278. 

  279. %macro DC_ADD_MMXEXT_INIT 2

  280.     add          %1, 32

  281.     sar          %1, 6

  282.     movd         m0, %1d

  283.     lea          %1, [%2*3]

  284.     pshufw       m0, m0, 0

  285.     pxor         m1, m1

  286.     psubw        m1, m0

  287.     packuswb     m0, m0

  288.     packuswb     m1, m1

  289. %endmacro

  290. 

  291. %macro DC_ADD_MMXEXT_OP 4

  292.     %1           m2, [%2     ]

  293.     %1           m3, [%2+%3  ]

  294.     %1           m4, [%2+%3*2]

  295.     %1           m5, [%2+%4  ]

  296.     paddusb      m2, m0

  297.     paddusb      m3, m0

  298.     paddusb      m4, m0

  299.     paddusb      m5, m0

  300.     psubusb      m2, m1

  301.     psubusb      m3, m1

  302.     psubusb      m4, m1

  303.     psubusb      m5, m1

  304.     %1    [%2     ], m2

  305.     %1    [%2+%3  ], m3

  306.     %1    [%2+%3*2], m4

  307.     %1    [%2+%4  ], m5

  308. %endmacro

  309. 

  310. INIT_MMX mmxext

  311. ; ff_h264_idct_dc_add_mmxext(uint8_t *dst, int16_t *block, int stride)

  312. %if ARCH_X86_64

  313. cglobal h264_idct_dc_add_8, 3, 4, 0

  314.     movsx        r3, word [r1]

  315.     mov  dword [r1], 0

  316.     DC_ADD_MMXEXT_INIT r3, r2

  317.     DC_ADD_MMXEXT_OP movh, r0, r2, r3

  318.     RET

  319. 

  320. ; ff_h264_idct8_dc_add_mmxext(uint8_t *dst, int16_t *block, int stride)

  321. cglobal h264_idct8_dc_add_8, 3, 4, 0

  322.     movsx        r3, word [r1]

  323.     mov  dword [r1], 0

  324.     DC_ADD_MMXEXT_INIT r3, r2

  325.     DC_ADD_MMXEXT_OP mova, r0, r2, r3

  326.     lea          r0, [r0+r2*4]

  327.     DC_ADD_MMXEXT_OP mova, r0, r2, r3

  328.     RET

  329. %else

  330. cglobal h264_idct_dc_add_8, 2, 3, 0

  331.     movsx        r2, word [r1]

  332.     mov  dword [r1], 0

  333.     mov          r1, r2m

  334.     DC_ADD_MMXEXT_INIT r2, r1

  335.     DC_ADD_MMXEXT_OP movh, r0, r1, r2

  336.     RET

  337. 

  338. ; ff_h264_idct8_dc_add_mmxext(uint8_t *dst, int16_t *block, int stride)

  339. cglobal h264_idct8_dc_add_8, 2, 3, 0

  340.     movsx        r2, word [r1]

  341.     mov  dword [r1], 0

  342.     mov          r1, r2m

  343.     DC_ADD_MMXEXT_INIT r2, r1

  344.     DC_ADD_MMXEXT_OP mova, r0, r1, r2

  345.     lea          r0, [r0+r1*4]

  346.     DC_ADD_MMXEXT_OP mova, r0, r1, r2

  347.     RET

  348. %endif

  349. 

  350. INIT_MMX mmx

  351. ; ff_h264_idct_add16_mmx(uint8_t *dst, const int *block_offset,

  352. ;             int16_t *block, int stride, const uint8_t nnzc[6*8])

  353. cglobal h264_idct_add16_8, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  354.     xor          r5, r5

  355. %ifdef PIC

  356.     lea     picregq, [scan8_mem]

  357. %endif

  358. .nextblock:

  359.     movzx        r6, byte [scan8+r5]

  360.     movzx        r6, byte [r4+r6]

  361.     test         r6, r6

  362.     jz .skipblock

  363.     mov         r6d, dword [r1+r5*4]

  364.     lea          r6, [r0+r6]

  365.     IDCT4_ADD    r6, r2, r3

  366. .skipblock:

  367.     inc          r5

  368.     add          r2, 32

  369.     cmp          r5, 16

  370.     jl .nextblock

  371.     REP_RET

  372. 

  373. ; ff_h264_idct8_add4_mmx(uint8_t *dst, const int *block_offset,

  374. ;                        int16_t *block, int stride, const uint8_t nnzc[6*8])

  375. cglobal h264_idct8_add4_8, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  376.     %assign pad 128+4-(stack_offset&7)

  377.     SUB         rsp, pad

  378. 

  379.     xor          r5, r5

  380. %ifdef PIC

  381.     lea     picregq, [scan8_mem]

  382. %endif

  383. .nextblock:

  384.     movzx        r6, byte [scan8+r5]

  385.     movzx        r6, byte [r4+r6]

  386.     test         r6, r6

  387.     jz .skipblock

  388.     mov         r6d, dword [r1+r5*4]

  389.     add          r6, r0

  390.     add   word [r2], 32

  391.     IDCT8_ADD_MMX_START r2  , rsp

  392.     IDCT8_ADD_MMX_START r2+8, rsp+64

  393.     IDCT8_ADD_MMX_END   r6  , rsp,   r3, r2

  394.     mov         r6d, dword [r1+r5*4]

  395.     lea          r6, [r0+r6+4]

  396.     IDCT8_ADD_MMX_END   r6  , rsp+8, r3

  397. .skipblock:

  398.     add          r5, 4

  399.     add          r2, 128

  400.     cmp          r5, 16

  401.     jl .nextblock

  402.     ADD         rsp, pad

  403.     RET

  404. 

  405. INIT_MMX mmxext

  406. ; ff_h264_idct_add16_mmxext(uint8_t *dst, const int *block_offset,

  407. ;                           int16_t *block, int stride, const uint8_t nnzc[6*8])

  408. cglobal h264_idct_add16_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  409.     xor          r5, r5

  410. %ifdef PIC

  411.     lea     picregq, [scan8_mem]

  412. %endif

  413. .nextblock:

  414.     movzx        r6, byte [scan8+r5]

  415.     movzx        r6, byte [r4+r6]

  416.     test         r6, r6

  417.     jz .skipblock

  418.     cmp          r6, 1

  419.     jnz .no_dc

  420.     movsx        r6, word [r2]

  421.     test         r6, r6

  422.     jz .no_dc

  423.     mov   word [r2], 0

  424.     DC_ADD_MMXEXT_INIT r6, r3

  425. %if ARCH_X86_64 == 0

  426. %define dst2q r1

  427. %define dst2d r1d

  428. %endif

  429.     mov       dst2d, dword [r1+r5*4]

  430.     lea       dst2q, [r0+dst2q]

  431.     DC_ADD_MMXEXT_OP movh, dst2q, r3, r6

  432. %if ARCH_X86_64 == 0

  433.     mov          r1, r1m

  434. %endif

  435.     inc          r5

  436.     add          r2, 32

  437.     cmp          r5, 16

  438.     jl .nextblock

  439.     REP_RET

  440. .no_dc:

  441.     mov         r6d, dword [r1+r5*4]

  442.     add          r6, r0

  443.     IDCT4_ADD    r6, r2, r3

  444. .skipblock:

  445.     inc          r5

  446.     add          r2, 32

  447.     cmp          r5, 16

  448.     jl .nextblock

  449.     REP_RET

  450. 

  451. INIT_MMX mmx

  452. ; ff_h264_idct_add16intra_mmx(uint8_t *dst, const int *block_offset,

  453. ;                             int16_t *block, int stride, const uint8_t nnzc[6*8])

  454. cglobal h264_idct_add16intra_8, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  455.     xor          r5, r5

  456. %ifdef PIC

  457.     lea     picregq, [scan8_mem]

  458. %endif

  459. .nextblock:

  460.     movzx        r6, byte [scan8+r5]

  461.     movzx        r6, byte [r4+r6]

  462.     or          r6w, word [r2]

  463.     test         r6, r6

  464.     jz .skipblock

  465.     mov         r6d, dword [r1+r5*4]

  466.     add          r6, r0

  467.     IDCT4_ADD    r6, r2, r3

  468. .skipblock:

  469.     inc          r5

  470.     add          r2, 32

  471.     cmp          r5, 16

  472.     jl .nextblock

  473.     REP_RET

  474. 

  475. INIT_MMX mmxext

  476. ; ff_h264_idct_add16intra_mmxext(uint8_t *dst, const int *block_offset,

  477. ;                                int16_t *block, int stride,

  478. ;                                const uint8_t nnzc[6*8])

  479. cglobal h264_idct_add16intra_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  480.     xor          r5, r5

  481. %ifdef PIC

  482.     lea     picregq, [scan8_mem]

  483. %endif

  484. .nextblock:

  485.     movzx        r6, byte [scan8+r5]

  486.     movzx        r6, byte [r4+r6]

  487.     test         r6, r6

  488.     jz .try_dc

  489.     mov         r6d, dword [r1+r5*4]

  490.     lea          r6, [r0+r6]

  491.     IDCT4_ADD    r6, r2, r3

  492.     inc          r5

  493.     add          r2, 32

  494.     cmp          r5, 16

  495.     jl .nextblock

  496.     REP_RET

  497. .try_dc:

  498.     movsx        r6, word [r2]

  499.     test         r6, r6

  500.     jz .skipblock

  501.     mov   word [r2], 0

  502.     DC_ADD_MMXEXT_INIT r6, r3

  503. %if ARCH_X86_64 == 0

  504. %define dst2q r1

  505. %define dst2d r1d

  506. %endif

  507.     mov       dst2d, dword [r1+r5*4]

  508.     add       dst2q, r0

  509.     DC_ADD_MMXEXT_OP movh, dst2q, r3, r6

  510. %if ARCH_X86_64 == 0

  511.     mov          r1, r1m

  512. %endif

  513. .skipblock:

  514.     inc          r5

  515.     add          r2, 32

  516.     cmp          r5, 16

  517.     jl .nextblock

  518.     REP_RET

  519. 

  520. ; ff_h264_idct8_add4_mmxext(uint8_t *dst, const int *block_offset,

  521. ;                           int16_t *block, int stride,

  522. ;                           const uint8_t nnzc[6*8])

  523. cglobal h264_idct8_add4_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  524.     %assign pad 128+4-(stack_offset&7)

  525.     SUB         rsp, pad

  526. 

  527.     xor          r5, r5

  528. %ifdef PIC

  529.     lea     picregq, [scan8_mem]

  530. %endif

  531. .nextblock:

  532.     movzx        r6, byte [scan8+r5]

  533.     movzx        r6, byte [r4+r6]

  534.     test         r6, r6

  535.     jz .skipblock

  536.     cmp          r6, 1

  537.     jnz .no_dc

  538.     movsx        r6, word [r2]

  539.     test         r6, r6

  540.     jz .no_dc

  541.     mov   word [r2], 0

  542.     DC_ADD_MMXEXT_INIT r6, r3

  543. %if ARCH_X86_64 == 0

  544. %define dst2q r1

  545. %define dst2d r1d

  546. %endif

  547.     mov       dst2d, dword [r1+r5*4]

  548.     lea       dst2q, [r0+dst2q]

  549.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  550.     lea       dst2q, [dst2q+r3*4]

  551.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  552. %if ARCH_X86_64 == 0

  553.     mov          r1, r1m

  554. %endif

  555.     add          r5, 4

  556.     add          r2, 128

  557.     cmp          r5, 16

  558.     jl .nextblock

  559. 

  560.     ADD         rsp, pad

  561.     RET

  562. .no_dc:

  563.     mov         r6d, dword [r1+r5*4]

  564.     add          r6, r0

  565.     add   word [r2], 32

  566.     IDCT8_ADD_MMX_START r2  , rsp

  567.     IDCT8_ADD_MMX_START r2+8, rsp+64

  568.     IDCT8_ADD_MMX_END   r6  , rsp,   r3, r2

  569.     mov         r6d, dword [r1+r5*4]

  570.     lea          r6, [r0+r6+4]

  571.     IDCT8_ADD_MMX_END   r6  , rsp+8, r3

  572. .skipblock:

  573.     add          r5, 4

  574.     add          r2, 128

  575.     cmp          r5, 16

  576.     jl .nextblock

  577. 

  578.     ADD         rsp, pad

  579.     RET

  580. 

  581. INIT_XMM sse2

  582. ; ff_h264_idct8_add4_sse2(uint8_t *dst, const int *block_offset,

  583. ;                         int16_t *block, int stride, const uint8_t nnzc[6*8])

  584. cglobal h264_idct8_add4_8, 5, 8 + npicregs, 10, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  585.     xor          r5, r5

  586. %ifdef PIC

  587.     lea     picregq, [scan8_mem]

  588. %endif

  589. .nextblock:

  590.     movzx        r6, byte [scan8+r5]

  591.     movzx        r6, byte [r4+r6]

  592.     test         r6, r6

  593.     jz .skipblock

  594.     cmp          r6, 1

  595.     jnz .no_dc

  596.     movsx        r6, word [r2]

  597.     test         r6, r6

  598.     jz .no_dc

  599. INIT_MMX cpuname

  600.     mov   word [r2], 0

  601.     DC_ADD_MMXEXT_INIT r6, r3

  602. %if ARCH_X86_64 == 0

  603. %define dst2q r1

  604. %define dst2d r1d

  605. %endif

  606.     mov       dst2d, dword [r1+r5*4]

  607.     add       dst2q, r0

  608.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  609.     lea       dst2q, [dst2q+r3*4]

  610.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  611. %if ARCH_X86_64 == 0

  612.     mov          r1, r1m

  613. %endif

  614.     add          r5, 4

  615.     add          r2, 128

  616.     cmp          r5, 16

  617.     jl .nextblock

  618.     REP_RET

  619. .no_dc:

  620. INIT_XMM cpuname

  621.     mov       dst2d, dword [r1+r5*4]

  622.     add       dst2q, r0

  623.     IDCT8_ADD_SSE dst2q, r2, r3, r6

  624. %if ARCH_X86_64 == 0

  625.     mov          r1, r1m

  626. %endif

  627. .skipblock:

  628.     add          r5, 4

  629.     add          r2, 128

  630.     cmp          r5, 16

  631.     jl .nextblock

  632.     REP_RET

  633. 

  634. INIT_MMX mmx

  635. h264_idct_add8_mmx_plane:

  636. .nextblock:

  637.     movzx        r6, byte [scan8+r5]

  638.     movzx        r6, byte [r4+r6]

  639.     or          r6w, word [r2]

  640.     test         r6, r6

  641.     jz .skipblock

  642. %if ARCH_X86_64

  643.     mov         r0d, dword [r1+r5*4]

  644.     add          r0, [dst2q]

  645. %else

  646.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  647.     mov          r0, [r0]

  648.     add          r0, dword [r1+r5*4]

  649. %endif

  650.     IDCT4_ADD    r0, r2, r3

  651. .skipblock:

  652.     inc          r5

  653.     add          r2, 32

  654.     test         r5, 3

  655.     jnz .nextblock

  656.     rep ret

  657. 

  658. ; ff_h264_idct_add8_mmx(uint8_t **dest, const int *block_offset,

  659. ;                       int16_t *block, int stride, const uint8_t nnzc[6*8])

  660. cglobal h264_idct_add8_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  661.     mov          r5, 16

  662.     add          r2, 512

  663. %ifdef PIC

  664.     lea     picregq, [scan8_mem]

  665. %endif

  666. %if ARCH_X86_64

  667.     mov       dst2q, r0

  668. %endif

  669.     call         h264_idct_add8_mmx_plane

  670.     mov          r5, 32

  671.     add          r2, 384

  672. %if ARCH_X86_64

  673.     add       dst2q, gprsize

  674. %else

  675.     add        r0mp, gprsize

  676. %endif

  677.     call         h264_idct_add8_mmx_plane

  678.     RET

  679. 

  680. h264_idct_add8_mmxext_plane:

  681. .nextblock:

  682.     movzx        r6, byte [scan8+r5]

  683.     movzx        r6, byte [r4+r6]

  684.     test         r6, r6

  685.     jz .try_dc

  686. %if ARCH_X86_64

  687.     mov         r0d, dword [r1+r5*4]

  688.     add          r0, [dst2q]

  689. %else

  690.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  691.     mov          r0, [r0]

  692.     add          r0, dword [r1+r5*4]

  693. %endif

  694.     IDCT4_ADD    r0, r2, r3

  695.     inc          r5

  696.     add          r2, 32

  697.     test         r5, 3

  698.     jnz .nextblock

  699.     rep ret

  700. .try_dc:

  701.     movsx        r6, word [r2]

  702.     test         r6, r6

  703.     jz .skipblock

  704.     mov   word [r2], 0

  705.     DC_ADD_MMXEXT_INIT r6, r3

  706. %if ARCH_X86_64

  707.     mov         r0d, dword [r1+r5*4]

  708.     add          r0, [dst2q]

  709. %else

  710.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  711.     mov          r0, [r0]

  712.     add          r0, dword [r1+r5*4]

  713. %endif

  714.     DC_ADD_MMXEXT_OP movh, r0, r3, r6

  715. .skipblock:

  716.     inc          r5

  717.     add          r2, 32

  718.     test         r5, 3

  719.     jnz .nextblock

  720.     rep ret

  721. 

  722. INIT_MMX mmxext

  723. ; ff_h264_idct_add8_mmxext(uint8_t **dest, const int *block_offset,

  724. ;                          int16_t *block, int stride, const uint8_t nnzc[6*8])

  725. cglobal h264_idct_add8_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  726.     mov          r5, 16

  727.     add          r2, 512

  728. %if ARCH_X86_64

  729.     mov       dst2q, r0

  730. %endif

  731. %ifdef PIC

  732.     lea     picregq, [scan8_mem]

  733. %endif

  734.     call h264_idct_add8_mmxext_plane

  735.     mov          r5, 32

  736.     add          r2, 384

  737. %if ARCH_X86_64

  738.     add       dst2q, gprsize

  739. %else

  740.     add        r0mp, gprsize

  741. %endif

  742.     call h264_idct_add8_mmxext_plane

  743.     RET

  744. 

  745. ; r0 = uint8_t *dst, r2 = int16_t *block, r3 = int stride, r6=clobbered

  746. h264_idct_dc_add8_mmxext:

  747.     movd         m0, [r2   ]          ;  0 0 X D

  748.     mov word [r2+ 0], 0

  749.     punpcklwd    m0, [r2+32]          ;  x X d D

  750.     mov word [r2+32], 0

  751.     paddsw       m0, [pw_32]

  752.     psraw        m0, 6

  753.     punpcklwd    m0, m0               ;  d d D D

  754.     pxor         m1, m1               ;  0 0 0 0

  755.     psubw        m1, m0               ; -d-d-D-D

  756.     packuswb     m0, m1               ; -d-d-D-D d d D D

  757.     pshufw       m1, m0, 0xFA         ; -d-d-d-d-D-D-D-D

  758.     punpcklwd    m0, m0               ;  d d d d D D D D

  759.     lea          r6, [r3*3]

  760.     DC_ADD_MMXEXT_OP movq, r0, r3, r6

  761.     ret

  762. 

  763. ALIGN 16

  764. INIT_XMM sse2

  765. ; r0 = uint8_t *dst (clobbered), r2 = int16_t *block, r3 = int stride

  766. h264_add8x4_idct_sse2:

  767.     movq   m0, [r2+ 0]

  768.     movq   m1, [r2+ 8]

  769.     movq   m2, [r2+16]

  770.     movq   m3, [r2+24]

  771.     movhps m0, [r2+32]

  772.     movhps m1, [r2+40]

  773.     movhps m2, [r2+48]

  774.     movhps m3, [r2+56]

  775.     IDCT4_1D w,0,1,2,3,4,5

  776.     TRANSPOSE2x4x4W 0,1,2,3,4

  777.     paddw m0, [pw_32]

  778.     IDCT4_1D w,0,1,2,3,4,5

  779.     pxor  m7, m7

  780.     mova [r2+ 0], m7

  781.     mova [r2+16], m7

  782.     mova [r2+32], m7

  783.     mova [r2+48], m7

  784.     STORE_DIFFx2 m0, m1, m4, m5, m7, 6, r0, r3

  785.     lea   r0, [r0+r3*2]

  786.     STORE_DIFFx2 m2, m3, m4, m5, m7, 6, r0, r3

  787.     ret

  788. 

  789. %macro add16_sse2_cycle 2

  790.     movzx       r0, word [r4+%2]

  791.     test        r0, r0

  792.     jz .cycle%1end

  793.     mov        r0d, dword [r1+%1*8]

  794. %if ARCH_X86_64

  795.     add         r0, r5

  796. %else

  797.     add         r0, r0m

  798. %endif

  799.     call        h264_add8x4_idct_sse2

  800. .cycle%1end:

  801. %if %1 < 7

  802.     add         r2, 64

  803. %endif

  804. %endmacro

  805. 

  806. ; ff_h264_idct_add16_sse2(uint8_t *dst, const int *block_offset,

  807. ;                         int16_t *block, int stride, const uint8_t nnzc[6*8])

  808. cglobal h264_idct_add16_8, 5, 5 + ARCH_X86_64, 8

  809. %if ARCH_X86_64

  810.     mov         r5, r0

  811. %endif

  812.     ; unrolling of the loop leads to an average performance gain of

  813.     ; 20-25%

  814.     add16_sse2_cycle 0, 0xc

  815.     add16_sse2_cycle 1, 0x14

  816.     add16_sse2_cycle 2, 0xe

  817.     add16_sse2_cycle 3, 0x16

  818.     add16_sse2_cycle 4, 0x1c

  819.     add16_sse2_cycle 5, 0x24

  820.     add16_sse2_cycle 6, 0x1e

  821.     add16_sse2_cycle 7, 0x26

  822.     RET

  823. 

  824. %macro add16intra_sse2_cycle 2

  825.     movzx       r0, word [r4+%2]

  826.     test        r0, r0

  827.     jz .try%1dc

  828.     mov        r0d, dword [r1+%1*8]

  829. %if ARCH_X86_64

  830.     add         r0, r7

  831. %else

  832.     add         r0, r0m

  833. %endif

  834.     call        h264_add8x4_idct_sse2

  835.     jmp .cycle%1end

  836. .try%1dc:

  837.     movsx       r0, word [r2   ]

  838.     or         r0w, word [r2+32]

  839.     jz .cycle%1end

  840.     mov        r0d, dword [r1+%1*8]

  841. %if ARCH_X86_64

  842.     add         r0, r7

  843. %else

  844.     add         r0, r0m

  845. %endif

  846.     call        h264_idct_dc_add8_mmxext

  847. .cycle%1end:

  848. %if %1 < 7

  849.     add         r2, 64

  850. %endif

  851. %endmacro

  852. 

  853. ; ff_h264_idct_add16intra_sse2(uint8_t *dst, const int *block_offset,

  854. ;                              int16_t *block, int stride, const uint8_t nnzc[6*8])

  855. cglobal h264_idct_add16intra_8, 5, 7 + ARCH_X86_64, 8

  856. %if ARCH_X86_64

  857.     mov         r7, r0

  858. %endif

  859.     add16intra_sse2_cycle 0, 0xc

  860.     add16intra_sse2_cycle 1, 0x14

  861.     add16intra_sse2_cycle 2, 0xe

  862.     add16intra_sse2_cycle 3, 0x16

  863.     add16intra_sse2_cycle 4, 0x1c

  864.     add16intra_sse2_cycle 5, 0x24

  865.     add16intra_sse2_cycle 6, 0x1e

  866.     add16intra_sse2_cycle 7, 0x26

  867.     RET

  868. 

  869. %macro add8_sse2_cycle 2

  870.     movzx       r0, word [r4+%2]

  871.     test        r0, r0

  872.     jz .try%1dc

  873. %if ARCH_X86_64

  874.     mov        r0d, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  875.     add         r0, [r7]

  876. %else

  877.     mov         r0, r0m

  878.     mov         r0, [r0]

  879.     add         r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  880. %endif

  881.     call        h264_add8x4_idct_sse2

  882.     jmp .cycle%1end

  883. .try%1dc:

  884.     movsx       r0, word [r2   ]

  885.     or         r0w, word [r2+32]

  886.     jz .cycle%1end

  887. %if ARCH_X86_64

  888.     mov        r0d, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  889.     add         r0, [r7]

  890. %else

  891.     mov         r0, r0m

  892.     mov         r0, [r0]

  893.     add         r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  894. %endif

  895.     call        h264_idct_dc_add8_mmxext

  896. .cycle%1end:

  897. %if %1 == 1

  898.     add         r2, 384+64

  899. %elif %1 < 3

  900.     add         r2, 64

  901. %endif

  902. %endmacro

  903. 

  904. ; ff_h264_idct_add8_sse2(uint8_t **dest, const int *block_offset,

  905. ;                        int16_t *block, int stride, const uint8_t nnzc[6*8])

  906. cglobal h264_idct_add8_8, 5, 7 + ARCH_X86_64, 8

  907.     add          r2, 512

  908. %if ARCH_X86_64

  909.     mov          r7, r0

  910. %endif

  911.     add8_sse2_cycle 0, 0x34

  912.     add8_sse2_cycle 1, 0x3c

  913. %if ARCH_X86_64

  914.     add          r7, gprsize

  915. %else

  916.     add        r0mp, gprsize

  917. %endif

  918.     add8_sse2_cycle 2, 0x5c

  919.     add8_sse2_cycle 3, 0x64

  920.     RET

  921. 

  922. ;void ff_h264_luma_dc_dequant_idct_mmx(int16_t *output, int16_t *input, int qmul)

  923. 

  924. %macro WALSH4_1D 5

  925.     SUMSUB_BADC w, %4, %3, %2, %1, %5

  926.     SUMSUB_BADC w, %4, %2, %3, %1, %5

  927.     SWAP %1, %4, %3

  928. %endmacro

  929. 

  930. %macro DEQUANT_MMX 3

  931.     mova        m7, [pw_1]

  932.     mova        m4, %1

  933.     punpcklwd   %1, m7

  934.     punpckhwd   m4, m7

  935.     mova        m5, %2

  936.     punpcklwd   %2, m7

  937.     punpckhwd   m5, m7

  938.     movd        m7, t3d

  939.     punpckldq   m7, m7

  940.     pmaddwd     %1, m7

  941.     pmaddwd     %2, m7

  942.     pmaddwd     m4, m7

  943.     pmaddwd     m5, m7

  944.     psrad       %1, %3

  945.     psrad       %2, %3

  946.     psrad       m4, %3

  947.     psrad       m5, %3

  948.     packssdw    %1, m4

  949.     packssdw    %2, m5

  950. %endmacro

  951. 

  952. %macro STORE_WORDS 5-9

  953. %if cpuflag(sse)

  954.     movd  t0d, %1

  955.     psrldq  %1, 4

  956.     movd  t1d, %1

  957.     psrldq  %1, 4

  958.     mov [t2+%2*32], t0w

  959.     mov [t2+%4*32], t1w

  960.     shr   t0d, 16

  961.     shr   t1d, 16

  962.     mov [t2+%3*32], t0w

  963.     mov [t2+%5*32], t1w

  964.     movd  t0d, %1

  965.     psrldq  %1, 4

  966.     movd  t1d, %1

  967.     mov [t2+%6*32], t0w

  968.     mov [t2+%8*32], t1w

  969.     shr   t0d, 16

  970.     shr   t1d, 16

  971.     mov [t2+%7*32], t0w

  972.     mov [t2+%9*32], t1w

  973. %else

  974.     movd  t0d, %1

  975.     psrlq  %1, 32

  976.     movd  t1d, %1

  977.     mov [t2+%2*32], t0w

  978.     mov [t2+%4*32], t1w

  979.     shr   t0d, 16

  980.     shr   t1d, 16

  981.     mov [t2+%3*32], t0w

  982.     mov [t2+%5*32], t1w

  983. %endif

  984. %endmacro

  985. 

  986. %macro DEQUANT_STORE 1

  987. %if cpuflag(sse2)

  988.     movd      xmm4, t3d

  989.     movq      xmm5, [pw_1]

  990.     pshufd    xmm4, xmm4, 0

  991.     movq2dq   xmm0, m0

  992.     movq2dq   xmm1, m1

  993.     movq2dq   xmm2, m2

  994.     movq2dq   xmm3, m3

  995.     punpcklwd xmm0, xmm5

  996.     punpcklwd xmm1, xmm5

  997.     punpcklwd xmm2, xmm5

  998.     punpcklwd xmm3, xmm5

  999.     pmaddwd   xmm0, xmm4

  1000.     pmaddwd   xmm1, xmm4

  1001.     pmaddwd   xmm2, xmm4

  1002.     pmaddwd   xmm3, xmm4

  1003.     psrad     xmm0, %1

  1004.     psrad     xmm1, %1

  1005.     psrad     xmm2, %1

  1006.     psrad     xmm3, %1

  1007.     packssdw  xmm0, xmm1

  1008.     packssdw  xmm2, xmm3

  1009.     STORE_WORDS xmm0,  0,  1,  4,  5,  2,  3,  6,  7

  1010.     STORE_WORDS xmm2,  8,  9, 12, 13, 10, 11, 14, 15

  1011. %else

  1012.     DEQUANT_MMX m0, m1, %1

  1013.     STORE_WORDS m0,  0,  1,  4,  5

  1014.     STORE_WORDS m1,  2,  3,  6,  7

  1015. 

  1016.     DEQUANT_MMX m2, m3, %1

  1017.     STORE_WORDS m2,  8,  9, 12, 13

  1018.     STORE_WORDS m3, 10, 11, 14, 15

  1019. %endif

  1020. %endmacro

  1021. 

  1022. %macro IDCT_DC_DEQUANT 1

  1023. cglobal h264_luma_dc_dequant_idct, 3, 4, %1

  1024.     ; manually spill XMM registers for Win64 because

  1025.     ; the code here is initialized with INIT_MMX

  1026.     WIN64_SPILL_XMM %1

  1027.     movq        m3, [r1+24]

  1028.     movq        m2, [r1+16]

  1029.     movq        m1, [r1+ 8]

  1030.     movq        m0, [r1+ 0]

  1031.     WALSH4_1D    0,1,2,3,4

  1032.     TRANSPOSE4x4W 0,1,2,3,4

  1033.     WALSH4_1D    0,1,2,3,4

  1034. 

  1035. ; shift, tmp, output, qmul

  1036. %if WIN64

  1037.     DECLARE_REG_TMP 0,3,1,2

  1038.     ; we can't avoid this, because r0 is the shift register (ecx) on win64

  1039.     xchg        r0, t2

  1040. %elif ARCH_X86_64

  1041.     DECLARE_REG_TMP 3,1,0,2

  1042. %else

  1043.     DECLARE_REG_TMP 1,3,0,2

  1044. %endif

  1045. 

  1046.     cmp        t3d, 32767

  1047.     jg .big_qmul

  1048.     add        t3d, 128 << 16

  1049.     DEQUANT_STORE 8

  1050.     RET

  1051. .big_qmul:

  1052.     bsr        t0d, t3d

  1053.     add        t3d, 128 << 16

  1054.     mov        t1d, 7

  1055.     cmp        t0d, t1d

  1056.     cmovg      t0d, t1d

  1057.     inc        t1d

  1058.     shr        t3d, t0b

  1059.     sub        t1d, t0d

  1060. %if cpuflag(sse2)

  1061.     movd      xmm6, t1d

  1062.     DEQUANT_STORE xmm6

  1063. %else

  1064.     movd        m6, t1d

  1065.     DEQUANT_STORE m6

  1066. %endif

  1067.     RET

  1068. %endmacro

  1069. 

  1070. INIT_MMX mmx

  1071. IDCT_DC_DEQUANT 0

  1072. INIT_MMX sse2

  1073. IDCT_DC_DEQUANT 7