falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
47082 Commits
32 Branches
283MB
Tree: bf2f93cdbf
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bf2f93cdbf'
${ noResults }
FFmpeg/libavcodec/x86/h264_idct.asm

1015 lines
26KB
Raw Blame History 

  1. ;*****************************************************************************

  2. ;* MMX/SSE2-optimized H.264 iDCT

  3. ;*****************************************************************************

  4. ;* Copyright (C) 2004-2005 Michael Niedermayer, Loren Merritt

  5. ;* Copyright (C) 2003-2008 x264 project

  6. ;*

  7. ;* Authors: Laurent Aimar <fenrir@via.ecp.fr>

  8. ;*          Loren Merritt <lorenm@u.washington.edu>

  9. ;*          Holger Lubitz <hal@duncan.ol.sub.de>

  10. ;*          Min Chen <chenm001.163.com>

  11. ;*

  12. ;* This file is part of FFmpeg.

  13. ;*

  14. ;* FFmpeg is free software; you can redistribute it and/or

  15. ;* modify it under the terms of the GNU Lesser General Public

  16. ;* License as published by the Free Software Foundation; either

  17. ;* version 2.1 of the License, or (at your option) any later version.

  18. ;*

  19. ;* FFmpeg is distributed in the hope that it will be useful,

  20. ;* but WITHOUT ANY WARRANTY; without even the implied warranty of

  21. ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  22. ;* Lesser General Public License for more details.

  23. ;*

  24. ;* You should have received a copy of the GNU Lesser General Public

  25. ;* License along with FFmpeg; if not, write to the Free Software

  26. ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  27. ;*****************************************************************************

  28. 

  29. %include "libavutil/x86/x86util.asm"

  30. 

  31. SECTION_RODATA

  32. 

  33. ; FIXME this table is a duplicate from h264data.h, and will be removed once the tables from, h264 have been split

  34. scan8_mem: db  4+ 1*8, 5+ 1*8, 4+ 2*8, 5+ 2*8

  35.            db  6+ 1*8, 7+ 1*8, 6+ 2*8, 7+ 2*8

  36.            db  4+ 3*8, 5+ 3*8, 4+ 4*8, 5+ 4*8

  37.            db  6+ 3*8, 7+ 3*8, 6+ 4*8, 7+ 4*8

  38.            db  4+ 6*8, 5+ 6*8, 4+ 7*8, 5+ 7*8

  39.            db  6+ 6*8, 7+ 6*8, 6+ 7*8, 7+ 7*8

  40.            db  4+ 8*8, 5+ 8*8, 4+ 9*8, 5+ 9*8

  41.            db  6+ 8*8, 7+ 8*8, 6+ 9*8, 7+ 9*8

  42.            db  4+11*8, 5+11*8, 4+12*8, 5+12*8

  43.            db  6+11*8, 7+11*8, 6+12*8, 7+12*8

  44.            db  4+13*8, 5+13*8, 4+14*8, 5+14*8

  45.            db  6+13*8, 7+13*8, 6+14*8, 7+14*8

  46. %ifdef PIC

  47. %define npicregs 1

  48. %define scan8 picregq

  49. %else

  50. %define npicregs 0

  51. %define scan8 scan8_mem

  52. %endif

  53. 

  54. cextern pw_32

  55. cextern pw_1

  56. 

  57. SECTION .text

  58. 

  59. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  60. %macro IDCT4_ADD 3

  61.     ; Load dct coeffs

  62.     movq         m0, [%2]

  63.     movq         m1, [%2+8]

  64.     movq         m2, [%2+16]

  65.     movq         m3, [%2+24]

  66. 

  67.     IDCT4_1D      w, 0, 1, 2, 3, 4, 5

  68.     mova         m6, [pw_32]

  69.     TRANSPOSE4x4W 0, 1, 2, 3, 4

  70.     paddw        m0, m6

  71.     IDCT4_1D      w, 0, 1, 2, 3, 4, 5

  72.     pxor         m7, m7

  73. 

  74.     STORE_DIFFx2 m0, m1, m4, m5, m7, 6, %1, %3

  75.     lea          %1, [%1+%3*2]

  76.     STORE_DIFFx2 m2, m3, m4, m5, m7, 6, %1, %3

  77. %endmacro

  78. 

  79. INIT_MMX

  80. ; ff_h264_idct_add_mmx(uint8_t *dst, int16_t *block, int stride)

  81. cglobal h264_idct_add_8_mmx, 3, 3, 0

  82.     IDCT4_ADD    r0, r1, r2

  83.     RET

  84. 

  85. %macro IDCT8_1D 2

  86.     mova         m0, m1

  87.     psraw        m1, 1

  88.     mova         m4, m5

  89.     psraw        m4, 1

  90.     paddw        m4, m5

  91.     paddw        m1, m0

  92.     paddw        m4, m7

  93.     paddw        m1, m5

  94.     psubw        m4, m0

  95.     paddw        m1, m3

  96. 

  97.     psubw        m0, m3

  98.     psubw        m5, m3

  99.     psraw        m3, 1

  100.     paddw        m0, m7

  101.     psubw        m5, m7

  102.     psraw        m7, 1

  103.     psubw        m0, m3

  104.     psubw        m5, m7

  105. 

  106.     mova         m7, m1

  107.     psraw        m1, 2

  108.     mova         m3, m4

  109.     psraw        m3, 2

  110.     paddw        m3, m0

  111.     psraw        m0, 2

  112.     paddw        m1, m5

  113.     psraw        m5, 2

  114.     psubw        m0, m4

  115.     psubw        m7, m5

  116. 

  117.     mova         m5, m6

  118.     psraw        m6, 1

  119.     mova         m4, m2

  120.     psraw        m4, 1

  121.     paddw        m6, m2

  122.     psubw        m4, m5

  123. 

  124.     mova         m2, %1

  125.     mova         m5, %2

  126.     SUMSUB_BA    w, 5, 2

  127.     SUMSUB_BA    w, 6, 5

  128.     SUMSUB_BA    w, 4, 2

  129.     SUMSUB_BA    w, 7, 6

  130.     SUMSUB_BA    w, 0, 4

  131.     SUMSUB_BA    w, 3, 2

  132.     SUMSUB_BA    w, 1, 5

  133.     SWAP         7, 6, 4, 5, 2, 3, 1, 0 ; 70315246 -> 01234567

  134. %endmacro

  135. 

  136. %macro IDCT8_1D_FULL 1

  137.     mova         m7, [%1+112]

  138.     mova         m6, [%1+ 96]

  139.     mova         m5, [%1+ 80]

  140.     mova         m3, [%1+ 48]

  141.     mova         m2, [%1+ 32]

  142.     mova         m1, [%1+ 16]

  143.     IDCT8_1D   [%1], [%1+ 64]

  144. %endmacro

  145. 

  146. ; %1=int16_t *block, %2=int16_t *dstblock

  147. %macro IDCT8_ADD_MMX_START 2

  148.     IDCT8_1D_FULL %1

  149.     mova       [%1], m7

  150.     TRANSPOSE4x4W 0, 1, 2, 3, 7

  151.     mova         m7, [%1]

  152.     mova    [%2   ], m0

  153.     mova    [%2+16], m1

  154.     mova    [%2+32], m2

  155.     mova    [%2+48], m3

  156.     TRANSPOSE4x4W 4, 5, 6, 7, 3

  157.     mova    [%2+ 8], m4

  158.     mova    [%2+24], m5

  159.     mova    [%2+40], m6

  160.     mova    [%2+56], m7

  161. %endmacro

  162. 

  163. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  164. %macro IDCT8_ADD_MMX_END 3

  165.     IDCT8_1D_FULL %2

  166.     mova    [%2   ], m5

  167.     mova    [%2+16], m6

  168.     mova    [%2+32], m7

  169. 

  170.     pxor         m7, m7

  171.     STORE_DIFFx2 m0, m1, m5, m6, m7, 6, %1, %3

  172.     lea          %1, [%1+%3*2]

  173.     STORE_DIFFx2 m2, m3, m5, m6, m7, 6, %1, %3

  174.     mova         m0, [%2   ]

  175.     mova         m1, [%2+16]

  176.     mova         m2, [%2+32]

  177.     lea          %1, [%1+%3*2]

  178.     STORE_DIFFx2 m4, m0, m5, m6, m7, 6, %1, %3

  179.     lea          %1, [%1+%3*2]

  180.     STORE_DIFFx2 m1, m2, m5, m6, m7, 6, %1, %3

  181. %endmacro

  182. 

  183. INIT_MMX

  184. ; ff_h264_idct8_add_mmx(uint8_t *dst, int16_t *block, int stride)

  185. cglobal h264_idct8_add_8_mmx, 3, 4, 0

  186.     %assign pad 128+4-(stack_offset&7)

  187.     SUB         rsp, pad

  188. 

  189.     add   word [r1], 32

  190.     IDCT8_ADD_MMX_START r1  , rsp

  191.     IDCT8_ADD_MMX_START r1+8, rsp+64

  192.     lea          r3, [r0+4]

  193.     IDCT8_ADD_MMX_END   r0  , rsp,   r2

  194.     IDCT8_ADD_MMX_END   r3  , rsp+8, r2

  195. 

  196.     ADD         rsp, pad

  197.     RET

  198. 

  199. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  200. %macro IDCT8_ADD_SSE 4

  201.     IDCT8_1D_FULL %2

  202. %if ARCH_X86_64

  203.     TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, 8

  204. %else

  205.     TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, [%2], [%2+16]

  206. %endif

  207.     paddw        m0, [pw_32]

  208. 

  209. %if ARCH_X86_64 == 0

  210.     mova    [%2   ], m0

  211.     mova    [%2+16], m4

  212.     IDCT8_1D   [%2], [%2+ 16]

  213.     mova    [%2   ], m6

  214.     mova    [%2+16], m7

  215. %else

  216.     SWAP          0, 8

  217.     SWAP          4, 9

  218.     IDCT8_1D     m8, m9

  219.     SWAP          6, 8

  220.     SWAP          7, 9

  221. %endif

  222. 

  223.     pxor         m7, m7

  224.     lea          %4, [%3*3]

  225.     STORE_DIFF   m0, m6, m7, [%1     ]

  226.     STORE_DIFF   m1, m6, m7, [%1+%3  ]

  227.     STORE_DIFF   m2, m6, m7, [%1+%3*2]

  228.     STORE_DIFF   m3, m6, m7, [%1+%4  ]

  229. %if ARCH_X86_64 == 0

  230.     mova         m0, [%2   ]

  231.     mova         m1, [%2+16]

  232. %else

  233.     SWAP          0, 8

  234.     SWAP          1, 9

  235. %endif

  236.     lea          %1, [%1+%3*4]

  237.     STORE_DIFF   m4, m6, m7, [%1     ]

  238.     STORE_DIFF   m5, m6, m7, [%1+%3  ]

  239.     STORE_DIFF   m0, m6, m7, [%1+%3*2]

  240.     STORE_DIFF   m1, m6, m7, [%1+%4  ]

  241. %endmacro

  242. 

  243. INIT_XMM

  244. ; ff_h264_idct8_add_sse2(uint8_t *dst, int16_t *block, int stride)

  245. cglobal h264_idct8_add_8_sse2, 3, 4, 10

  246.     IDCT8_ADD_SSE r0, r1, r2, r3

  247.     RET

  248. 

  249. %macro DC_ADD_MMXEXT_INIT 2-3

  250. %if %0 == 2

  251.     movsx        %1, word [%1]

  252.     add          %1, 32

  253.     sar          %1, 6

  254.     movd         m0, %1d

  255.     lea          %1, [%2*3]

  256. %else

  257.     add          %3, 32

  258.     sar          %3, 6

  259.     movd         m0, %3d

  260.     lea          %3, [%2*3]

  261. %endif

  262.     pshufw       m0, m0, 0

  263.     pxor         m1, m1

  264.     psubw        m1, m0

  265.     packuswb     m0, m0

  266.     packuswb     m1, m1

  267. %endmacro

  268. 

  269. %macro DC_ADD_MMXEXT_OP 4

  270.     %1           m2, [%2     ]

  271.     %1           m3, [%2+%3  ]

  272.     %1           m4, [%2+%3*2]

  273.     %1           m5, [%2+%4  ]

  274.     paddusb      m2, m0

  275.     paddusb      m3, m0

  276.     paddusb      m4, m0

  277.     paddusb      m5, m0

  278.     psubusb      m2, m1

  279.     psubusb      m3, m1

  280.     psubusb      m4, m1

  281.     psubusb      m5, m1

  282.     %1    [%2     ], m2

  283.     %1    [%2+%3  ], m3

  284.     %1    [%2+%3*2], m4

  285.     %1    [%2+%4  ], m5

  286. %endmacro

  287. 

  288. INIT_MMX

  289. ; ff_h264_idct_dc_add_mmxext(uint8_t *dst, int16_t *block, int stride)

  290. cglobal h264_idct_dc_add_8_mmxext, 3, 3, 0

  291.     DC_ADD_MMXEXT_INIT r1, r2

  292.     DC_ADD_MMXEXT_OP movh, r0, r2, r1

  293.     RET

  294. 

  295. ; ff_h264_idct8_dc_add_mmxext(uint8_t *dst, int16_t *block, int stride)

  296. cglobal h264_idct8_dc_add_8_mmxext, 3, 3, 0

  297.     DC_ADD_MMXEXT_INIT r1, r2

  298.     DC_ADD_MMXEXT_OP mova, r0, r2, r1

  299.     lea          r0, [r0+r2*4]

  300.     DC_ADD_MMXEXT_OP mova, r0, r2, r1

  301.     RET

  302. 

  303. ; ff_h264_idct_add16_mmx(uint8_t *dst, const int *block_offset,

  304. ;             DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  305. cglobal h264_idct_add16_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  306.     xor          r5, r5

  307. %ifdef PIC

  308.     lea     picregq, [scan8_mem]

  309. %endif

  310. .nextblock:

  311.     movzx        r6, byte [scan8+r5]

  312.     movzx        r6, byte [r4+r6]

  313.     test         r6, r6

  314.     jz .skipblock

  315.     mov         r6d, dword [r1+r5*4]

  316.     lea          r6, [r0+r6]

  317.     IDCT4_ADD    r6, r2, r3

  318. .skipblock:

  319.     inc          r5

  320.     add          r2, 32

  321.     cmp          r5, 16

  322.     jl .nextblock

  323.     REP_RET

  324. 

  325. ; ff_h264_idct8_add4_mmx(uint8_t *dst, const int *block_offset,

  326. ;                        DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  327. cglobal h264_idct8_add4_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  328.     %assign pad 128+4-(stack_offset&7)

  329.     SUB         rsp, pad

  330. 

  331.     xor          r5, r5

  332. %ifdef PIC

  333.     lea     picregq, [scan8_mem]

  334. %endif

  335. .nextblock:

  336.     movzx        r6, byte [scan8+r5]

  337.     movzx        r6, byte [r4+r6]

  338.     test         r6, r6

  339.     jz .skipblock

  340.     mov         r6d, dword [r1+r5*4]

  341.     add          r6, r0

  342.     add   word [r2], 32

  343.     IDCT8_ADD_MMX_START r2  , rsp

  344.     IDCT8_ADD_MMX_START r2+8, rsp+64

  345.     IDCT8_ADD_MMX_END   r6  , rsp,   r3

  346.     mov         r6d, dword [r1+r5*4]

  347.     lea          r6, [r0+r6+4]

  348.     IDCT8_ADD_MMX_END   r6  , rsp+8, r3

  349. .skipblock:

  350.     add          r5, 4

  351.     add          r2, 128

  352.     cmp          r5, 16

  353.     jl .nextblock

  354.     ADD         rsp, pad

  355.     RET

  356. 

  357. ; ff_h264_idct_add16_mmxext(uint8_t *dst, const int *block_offset,

  358. ;                           DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  359. cglobal h264_idct_add16_8_mmxext, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  360.     xor          r5, r5

  361. %ifdef PIC

  362.     lea     picregq, [scan8_mem]

  363. %endif

  364. .nextblock:

  365.     movzx        r6, byte [scan8+r5]

  366.     movzx        r6, byte [r4+r6]

  367.     test         r6, r6

  368.     jz .skipblock

  369.     cmp          r6, 1

  370.     jnz .no_dc

  371.     movsx        r6, word [r2]

  372.     test         r6, r6

  373.     jz .no_dc

  374.     DC_ADD_MMXEXT_INIT r2, r3, r6

  375. %if ARCH_X86_64 == 0

  376. %define dst2q r1

  377. %define dst2d r1d

  378. %endif

  379.     mov       dst2d, dword [r1+r5*4]

  380.     lea       dst2q, [r0+dst2q]

  381.     DC_ADD_MMXEXT_OP movh, dst2q, r3, r6

  382. %if ARCH_X86_64 == 0

  383.     mov          r1, r1m

  384. %endif

  385.     inc          r5

  386.     add          r2, 32

  387.     cmp          r5, 16

  388.     jl .nextblock

  389.     REP_RET

  390. .no_dc:

  391.     mov         r6d, dword [r1+r5*4]

  392.     add          r6, r0

  393.     IDCT4_ADD    r6, r2, r3

  394. .skipblock:

  395.     inc          r5

  396.     add          r2, 32

  397.     cmp          r5, 16

  398.     jl .nextblock

  399.     REP_RET

  400. 

  401. ; ff_h264_idct_add16intra_mmx(uint8_t *dst, const int *block_offset,

  402. ;                             DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  403. cglobal h264_idct_add16intra_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  404.     xor          r5, r5

  405. %ifdef PIC

  406.     lea     picregq, [scan8_mem]

  407. %endif

  408. .nextblock:

  409.     movzx        r6, byte [scan8+r5]

  410.     movzx        r6, byte [r4+r6]

  411.     or          r6w, word [r2]

  412.     test         r6, r6

  413.     jz .skipblock

  414.     mov         r6d, dword [r1+r5*4]

  415.     add          r6, r0

  416.     IDCT4_ADD    r6, r2, r3

  417. .skipblock:

  418.     inc          r5

  419.     add          r2, 32

  420.     cmp          r5, 16

  421.     jl .nextblock

  422.     REP_RET

  423. 

  424. ; ff_h264_idct_add16intra_mmxext(uint8_t *dst, const int *block_offset,

  425. ;                                DCTELEM *block, int stride,

  426. ;                                const uint8_t nnzc[6*8])

  427. cglobal h264_idct_add16intra_8_mmxext, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  428.     xor          r5, r5

  429. %ifdef PIC

  430.     lea     picregq, [scan8_mem]

  431. %endif

  432. .nextblock:

  433.     movzx        r6, byte [scan8+r5]

  434.     movzx        r6, byte [r4+r6]

  435.     test         r6, r6

  436.     jz .try_dc

  437.     mov         r6d, dword [r1+r5*4]

  438.     lea          r6, [r0+r6]

  439.     IDCT4_ADD    r6, r2, r3

  440.     inc          r5

  441.     add          r2, 32

  442.     cmp          r5, 16

  443.     jl .nextblock

  444.     REP_RET

  445. .try_dc:

  446.     movsx        r6, word [r2]

  447.     test         r6, r6

  448.     jz .skipblock

  449.     DC_ADD_MMXEXT_INIT r2, r3, r6

  450. %if ARCH_X86_64 == 0

  451. %define dst2q r1

  452. %define dst2d r1d

  453. %endif

  454.     mov       dst2d, dword [r1+r5*4]

  455.     add       dst2q, r0

  456.     DC_ADD_MMXEXT_OP movh, dst2q, r3, r6

  457. %if ARCH_X86_64 == 0

  458.     mov          r1, r1m

  459. %endif

  460. .skipblock:

  461.     inc          r5

  462.     add          r2, 32

  463.     cmp          r5, 16

  464.     jl .nextblock

  465.     REP_RET

  466. 

  467. ; ff_h264_idct8_add4_mmxext(uint8_t *dst, const int *block_offset,

  468. ;                           DCTELEM *block, int stride,

  469. ;                           const uint8_t nnzc[6*8])

  470. cglobal h264_idct8_add4_8_mmxext, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  471.     %assign pad 128+4-(stack_offset&7)

  472.     SUB         rsp, pad

  473. 

  474.     xor          r5, r5

  475. %ifdef PIC

  476.     lea     picregq, [scan8_mem]

  477. %endif

  478. .nextblock:

  479.     movzx        r6, byte [scan8+r5]

  480.     movzx        r6, byte [r4+r6]

  481.     test         r6, r6

  482.     jz .skipblock

  483.     cmp          r6, 1

  484.     jnz .no_dc

  485.     movsx        r6, word [r2]

  486.     test         r6, r6

  487.     jz .no_dc

  488.     DC_ADD_MMXEXT_INIT r2, r3, r6

  489. %if ARCH_X86_64 == 0

  490. %define dst2q r1

  491. %define dst2d r1d

  492. %endif

  493.     mov       dst2d, dword [r1+r5*4]

  494.     lea       dst2q, [r0+dst2q]

  495.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  496.     lea       dst2q, [dst2q+r3*4]

  497.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  498. %if ARCH_X86_64 == 0

  499.     mov          r1, r1m

  500. %endif

  501.     add          r5, 4

  502.     add          r2, 128

  503.     cmp          r5, 16

  504.     jl .nextblock

  505. 

  506.     ADD         rsp, pad

  507.     RET

  508. .no_dc:

  509.     mov         r6d, dword [r1+r5*4]

  510.     add          r6, r0

  511.     add   word [r2], 32

  512.     IDCT8_ADD_MMX_START r2  , rsp

  513.     IDCT8_ADD_MMX_START r2+8, rsp+64

  514.     IDCT8_ADD_MMX_END   r6  , rsp,   r3

  515.     mov         r6d, dword [r1+r5*4]

  516.     lea          r6, [r0+r6+4]

  517.     IDCT8_ADD_MMX_END   r6  , rsp+8, r3

  518. .skipblock:

  519.     add          r5, 4

  520.     add          r2, 128

  521.     cmp          r5, 16

  522.     jl .nextblock

  523. 

  524.     ADD         rsp, pad

  525.     RET

  526. 

  527. INIT_XMM

  528. ; ff_h264_idct8_add4_sse2(uint8_t *dst, const int *block_offset,

  529. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  530. cglobal h264_idct8_add4_8_sse2, 5, 8 + npicregs, 10, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  531.     xor          r5, r5

  532. %ifdef PIC

  533.     lea     picregq, [scan8_mem]

  534. %endif

  535. .nextblock:

  536.     movzx        r6, byte [scan8+r5]

  537.     movzx        r6, byte [r4+r6]

  538.     test         r6, r6

  539.     jz .skipblock

  540.     cmp          r6, 1

  541.     jnz .no_dc

  542.     movsx        r6, word [r2]

  543.     test         r6, r6

  544.     jz .no_dc

  545. INIT_MMX

  546.     DC_ADD_MMXEXT_INIT r2, r3, r6

  547. %if ARCH_X86_64 == 0

  548. %define dst2q r1

  549. %define dst2d r1d

  550. %endif

  551.     mov       dst2d, dword [r1+r5*4]

  552.     add       dst2q, r0

  553.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  554.     lea       dst2q, [dst2q+r3*4]

  555.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  556. %if ARCH_X86_64 == 0

  557.     mov          r1, r1m

  558. %endif

  559.     add          r5, 4

  560.     add          r2, 128

  561.     cmp          r5, 16

  562.     jl .nextblock

  563.     REP_RET

  564. .no_dc:

  565. INIT_XMM

  566.     mov       dst2d, dword [r1+r5*4]

  567.     add       dst2q, r0

  568.     IDCT8_ADD_SSE dst2q, r2, r3, r6

  569. %if ARCH_X86_64 == 0

  570.     mov          r1, r1m

  571. %endif

  572. .skipblock:

  573.     add          r5, 4

  574.     add          r2, 128

  575.     cmp          r5, 16

  576.     jl .nextblock

  577.     REP_RET

  578. 

  579. INIT_MMX

  580. h264_idct_add8_mmx_plane:

  581. .nextblock:

  582.     movzx        r6, byte [scan8+r5]

  583.     movzx        r6, byte [r4+r6]

  584.     or          r6w, word [r2]

  585.     test         r6, r6

  586.     jz .skipblock

  587. %if ARCH_X86_64

  588.     mov         r0d, dword [r1+r5*4]

  589.     add          r0, [dst2q]

  590. %else

  591.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  592.     mov          r0, [r0]

  593.     add          r0, dword [r1+r5*4]

  594. %endif

  595.     IDCT4_ADD    r0, r2, r3

  596. .skipblock:

  597.     inc          r5

  598.     add          r2, 32

  599.     test         r5, 3

  600.     jnz .nextblock

  601.     rep ret

  602. 

  603. ; ff_h264_idct_add8_mmx(uint8_t **dest, const int *block_offset,

  604. ;                       DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  605. cglobal h264_idct_add8_8_mmx, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  606.     mov          r5, 16

  607.     add          r2, 512

  608. %ifdef PIC

  609.     lea     picregq, [scan8_mem]

  610. %endif

  611. %if ARCH_X86_64

  612.     mov       dst2q, r0

  613. %endif

  614.     call         h264_idct_add8_mmx_plane

  615.     mov          r5, 32

  616.     add          r2, 384

  617. %if ARCH_X86_64

  618.     add       dst2q, gprsize

  619. %else

  620.     add        r0mp, gprsize

  621. %endif

  622.     call         h264_idct_add8_mmx_plane

  623.     RET

  624. 

  625. h264_idct_add8_mmxext_plane:

  626. .nextblock:

  627.     movzx        r6, byte [scan8+r5]

  628.     movzx        r6, byte [r4+r6]

  629.     test         r6, r6

  630.     jz .try_dc

  631. %if ARCH_X86_64

  632.     mov         r0d, dword [r1+r5*4]

  633.     add          r0, [dst2q]

  634. %else

  635.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  636.     mov          r0, [r0]

  637.     add          r0, dword [r1+r5*4]

  638. %endif

  639.     IDCT4_ADD    r0, r2, r3

  640.     inc          r5

  641.     add          r2, 32

  642.     test         r5, 3

  643.     jnz .nextblock

  644.     rep ret

  645. .try_dc:

  646.     movsx        r6, word [r2]

  647.     test         r6, r6

  648.     jz .skipblock

  649.     DC_ADD_MMXEXT_INIT r2, r3, r6

  650. %if ARCH_X86_64

  651.     mov         r0d, dword [r1+r5*4]

  652.     add          r0, [dst2q]

  653. %else

  654.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  655.     mov          r0, [r0]

  656.     add          r0, dword [r1+r5*4]

  657. %endif

  658.     DC_ADD_MMXEXT_OP movh, r0, r3, r6

  659. .skipblock:

  660.     inc          r5

  661.     add          r2, 32

  662.     test         r5, 3

  663.     jnz .nextblock

  664.     rep ret

  665. 

  666. ; ff_h264_idct_add8_mmxext(uint8_t **dest, const int *block_offset,

  667. ;                          DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  668. cglobal h264_idct_add8_8_mmxext, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  669.     mov          r5, 16

  670.     add          r2, 512

  671. %if ARCH_X86_64

  672.     mov       dst2q, r0

  673. %endif

  674. %ifdef PIC

  675.     lea     picregq, [scan8_mem]

  676. %endif

  677.     call h264_idct_add8_mmxext_plane

  678.     mov          r5, 32

  679.     add          r2, 384

  680. %if ARCH_X86_64

  681.     add       dst2q, gprsize

  682. %else

  683.     add        r0mp, gprsize

  684. %endif

  685.     call h264_idct_add8_mmxext_plane

  686.     RET

  687. 

  688. INIT_MMX

  689. ; r0 = uint8_t *dst, r2 = int16_t *block, r3 = int stride, r6=clobbered

  690. h264_idct_dc_add8_mmxext:

  691.     movd         m0, [r2   ]          ;  0 0 X D

  692.     punpcklwd    m0, [r2+32]          ;  x X d D

  693.     paddsw       m0, [pw_32]

  694.     psraw        m0, 6

  695.     punpcklwd    m0, m0               ;  d d D D

  696.     pxor         m1, m1               ;  0 0 0 0

  697.     psubw        m1, m0               ; -d-d-D-D

  698.     packuswb     m0, m1               ; -d-d-D-D d d D D

  699.     pshufw       m1, m0, 0xFA         ; -d-d-d-d-D-D-D-D

  700.     punpcklwd    m0, m0               ;  d d d d D D D D

  701.     lea          r6, [r3*3]

  702.     DC_ADD_MMXEXT_OP movq, r0, r3, r6

  703.     ret

  704. 

  705. ALIGN 16

  706. INIT_XMM

  707. ; r0 = uint8_t *dst (clobbered), r2 = int16_t *block, r3 = int stride

  708. h264_add8x4_idct_sse2:

  709.     movq   m0, [r2+ 0]

  710.     movq   m1, [r2+ 8]

  711.     movq   m2, [r2+16]

  712.     movq   m3, [r2+24]

  713.     movhps m0, [r2+32]

  714.     movhps m1, [r2+40]

  715.     movhps m2, [r2+48]

  716.     movhps m3, [r2+56]

  717.     IDCT4_1D w,0,1,2,3,4,5

  718.     TRANSPOSE2x4x4W 0,1,2,3,4

  719.     paddw m0, [pw_32]

  720.     IDCT4_1D w,0,1,2,3,4,5

  721.     pxor  m7, m7

  722.     STORE_DIFFx2 m0, m1, m4, m5, m7, 6, r0, r3

  723.     lea   r0, [r0+r3*2]

  724.     STORE_DIFFx2 m2, m3, m4, m5, m7, 6, r0, r3

  725.     ret

  726. 

  727. %macro add16_sse2_cycle 2

  728.     movzx       r0, word [r4+%2]

  729.     test        r0, r0

  730.     jz .cycle%1end

  731.     mov        r0d, dword [r1+%1*8]

  732. %if ARCH_X86_64

  733.     add         r0, r5

  734. %else

  735.     add         r0, r0m

  736. %endif

  737.     call        h264_add8x4_idct_sse2

  738. .cycle%1end:

  739. %if %1 < 7

  740.     add         r2, 64

  741. %endif

  742. %endmacro

  743. 

  744. ; ff_h264_idct_add16_sse2(uint8_t *dst, const int *block_offset,

  745. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  746. cglobal h264_idct_add16_8_sse2, 5, 5 + ARCH_X86_64, 8

  747. %if ARCH_X86_64

  748.     mov         r5, r0

  749. %endif

  750.     ; unrolling of the loop leads to an average performance gain of

  751.     ; 20-25%

  752.     add16_sse2_cycle 0, 0xc

  753.     add16_sse2_cycle 1, 0x14

  754.     add16_sse2_cycle 2, 0xe

  755.     add16_sse2_cycle 3, 0x16

  756.     add16_sse2_cycle 4, 0x1c

  757.     add16_sse2_cycle 5, 0x24

  758.     add16_sse2_cycle 6, 0x1e

  759.     add16_sse2_cycle 7, 0x26

  760.     RET

  761. 

  762. %macro add16intra_sse2_cycle 2

  763.     movzx       r0, word [r4+%2]

  764.     test        r0, r0

  765.     jz .try%1dc

  766.     mov        r0d, dword [r1+%1*8]

  767. %if ARCH_X86_64

  768.     add         r0, r7

  769. %else

  770.     add         r0, r0m

  771. %endif

  772.     call        h264_add8x4_idct_sse2

  773.     jmp .cycle%1end

  774. .try%1dc:

  775.     movsx       r0, word [r2   ]

  776.     or         r0w, word [r2+32]

  777.     jz .cycle%1end

  778.     mov        r0d, dword [r1+%1*8]

  779. %if ARCH_X86_64

  780.     add         r0, r7

  781. %else

  782.     add         r0, r0m

  783. %endif

  784.     call        h264_idct_dc_add8_mmxext

  785. .cycle%1end:

  786. %if %1 < 7

  787.     add         r2, 64

  788. %endif

  789. %endmacro

  790. 

  791. ; ff_h264_idct_add16intra_sse2(uint8_t *dst, const int *block_offset,

  792. ;                              DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  793. cglobal h264_idct_add16intra_8_sse2, 5, 7 + ARCH_X86_64, 8

  794. %if ARCH_X86_64

  795.     mov         r7, r0

  796. %endif

  797.     add16intra_sse2_cycle 0, 0xc

  798.     add16intra_sse2_cycle 1, 0x14

  799.     add16intra_sse2_cycle 2, 0xe

  800.     add16intra_sse2_cycle 3, 0x16

  801.     add16intra_sse2_cycle 4, 0x1c

  802.     add16intra_sse2_cycle 5, 0x24

  803.     add16intra_sse2_cycle 6, 0x1e

  804.     add16intra_sse2_cycle 7, 0x26

  805.     RET

  806. 

  807. %macro add8_sse2_cycle 2

  808.     movzx       r0, word [r4+%2]

  809.     test        r0, r0

  810.     jz .try%1dc

  811. %if ARCH_X86_64

  812.     mov        r0d, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  813.     add         r0, [r7]

  814. %else

  815.     mov         r0, r0m

  816.     mov         r0, [r0]

  817.     add         r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  818. %endif

  819.     call        h264_add8x4_idct_sse2

  820.     jmp .cycle%1end

  821. .try%1dc:

  822.     movsx       r0, word [r2   ]

  823.     or         r0w, word [r2+32]

  824.     jz .cycle%1end

  825. %if ARCH_X86_64

  826.     mov        r0d, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  827.     add         r0, [r7]

  828. %else

  829.     mov         r0, r0m

  830.     mov         r0, [r0]

  831.     add         r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  832. %endif

  833.     call        h264_idct_dc_add8_mmxext

  834. .cycle%1end:

  835. %if %1 == 1

  836.     add         r2, 384+64

  837. %elif %1 < 3

  838.     add         r2, 64

  839. %endif

  840. %endmacro

  841. 

  842. ; ff_h264_idct_add8_sse2(uint8_t **dest, const int *block_offset,

  843. ;                        DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  844. cglobal h264_idct_add8_8_sse2, 5, 7 + ARCH_X86_64, 8

  845.     add          r2, 512

  846. %if ARCH_X86_64

  847.     mov          r7, r0

  848. %endif

  849.     add8_sse2_cycle 0, 0x34

  850.     add8_sse2_cycle 1, 0x3c

  851. %if ARCH_X86_64

  852.     add          r7, gprsize

  853. %else

  854.     add        r0mp, gprsize

  855. %endif

  856.     add8_sse2_cycle 2, 0x5c

  857.     add8_sse2_cycle 3, 0x64

  858.     RET

  859. 

  860. ;void ff_h264_luma_dc_dequant_idct_mmx(DCTELEM *output, DCTELEM *input, int qmul)

  861. 

  862. %macro WALSH4_1D 5

  863.     SUMSUB_BADC w, %4, %3, %2, %1, %5

  864.     SUMSUB_BADC w, %4, %2, %3, %1, %5

  865.     SWAP %1, %4, %3

  866. %endmacro

  867. 

  868. %macro DEQUANT_MMX 3

  869.     mova        m7, [pw_1]

  870.     mova        m4, %1

  871.     punpcklwd   %1, m7

  872.     punpckhwd   m4, m7

  873.     mova        m5, %2

  874.     punpcklwd   %2, m7

  875.     punpckhwd   m5, m7

  876.     movd        m7, t3d

  877.     punpckldq   m7, m7

  878.     pmaddwd     %1, m7

  879.     pmaddwd     %2, m7

  880.     pmaddwd     m4, m7

  881.     pmaddwd     m5, m7

  882.     psrad       %1, %3

  883.     psrad       %2, %3

  884.     psrad       m4, %3

  885.     psrad       m5, %3

  886.     packssdw    %1, m4

  887.     packssdw    %2, m5

  888. %endmacro

  889. 

  890. %macro STORE_WORDS_MMX 5

  891.     movd  t0d, %1

  892.     psrlq  %1, 32

  893.     movd  t1d, %1

  894.     mov [t2+%2*32], t0w

  895.     mov [t2+%4*32], t1w

  896.     shr   t0d, 16

  897.     shr   t1d, 16

  898.     mov [t2+%3*32], t0w

  899.     mov [t2+%5*32], t1w

  900. %endmacro

  901. 

  902. %macro DEQUANT_STORE_MMX 1

  903.     DEQUANT_MMX m0, m1, %1

  904.     STORE_WORDS_MMX m0,  0,  1,  4,  5

  905.     STORE_WORDS_MMX m1,  2,  3,  6,  7

  906. 

  907.     DEQUANT_MMX m2, m3, %1

  908.     STORE_WORDS_MMX m2,  8,  9, 12, 13

  909.     STORE_WORDS_MMX m3, 10, 11, 14, 15

  910. %endmacro

  911. 

  912. %macro STORE_WORDS_SSE 9

  913.     movd  t0d, %1

  914.     psrldq  %1, 4

  915.     movd  t1d, %1

  916.     psrldq  %1, 4

  917.     mov [t2+%2*32], t0w

  918.     mov [t2+%4*32], t1w

  919.     shr   t0d, 16

  920.     shr   t1d, 16

  921.     mov [t2+%3*32], t0w

  922.     mov [t2+%5*32], t1w

  923.     movd  t0d, %1

  924.     psrldq  %1, 4

  925.     movd  t1d, %1

  926.     mov [t2+%6*32], t0w

  927.     mov [t2+%8*32], t1w

  928.     shr   t0d, 16

  929.     shr   t1d, 16

  930.     mov [t2+%7*32], t0w

  931.     mov [t2+%9*32], t1w

  932. %endmacro

  933. 

  934. %macro DEQUANT_STORE_SSE2 1

  935.     movd      xmm4, t3d

  936.     movq      xmm5, [pw_1]

  937.     pshufd    xmm4, xmm4, 0

  938.     movq2dq   xmm0, m0

  939.     movq2dq   xmm1, m1

  940.     movq2dq   xmm2, m2

  941.     movq2dq   xmm3, m3

  942.     punpcklwd xmm0, xmm5

  943.     punpcklwd xmm1, xmm5

  944.     punpcklwd xmm2, xmm5

  945.     punpcklwd xmm3, xmm5

  946.     pmaddwd   xmm0, xmm4

  947.     pmaddwd   xmm1, xmm4

  948.     pmaddwd   xmm2, xmm4

  949.     pmaddwd   xmm3, xmm4

  950.     psrad     xmm0, %1

  951.     psrad     xmm1, %1

  952.     psrad     xmm2, %1

  953.     psrad     xmm3, %1

  954.     packssdw  xmm0, xmm1

  955.     packssdw  xmm2, xmm3

  956.     STORE_WORDS_SSE xmm0,  0,  1,  4,  5,  2,  3,  6,  7

  957.     STORE_WORDS_SSE xmm2,  8,  9, 12, 13, 10, 11, 14, 15

  958. %endmacro

  959. 

  960. %macro IDCT_DC_DEQUANT 2

  961. cglobal h264_luma_dc_dequant_idct_%1, 3,4,%2

  962.     ; manually spill XMM registers for Win64 because

  963.     ; the code here is initialized with INIT_MMX

  964.     WIN64_SPILL_XMM %2

  965.     movq        m3, [r1+24]

  966.     movq        m2, [r1+16]

  967.     movq        m1, [r1+ 8]

  968.     movq        m0, [r1+ 0]

  969.     WALSH4_1D    0,1,2,3,4

  970.     TRANSPOSE4x4W 0,1,2,3,4

  971.     WALSH4_1D    0,1,2,3,4

  972. 

  973. ; shift, tmp, output, qmul

  974. %if WIN64

  975.     DECLARE_REG_TMP 0,3,1,2

  976.     ; we can't avoid this, because r0 is the shift register (ecx) on win64

  977.     xchg        r0, t2

  978. %elif ARCH_X86_64

  979.     DECLARE_REG_TMP 3,1,0,2

  980. %else

  981.     DECLARE_REG_TMP 1,3,0,2

  982. %endif

  983. 

  984.     cmp        t3d, 32767

  985.     jg .big_qmul

  986.     add        t3d, 128 << 16

  987. %ifidn %1,mmx

  988.     DEQUANT_STORE_MMX 8

  989. %else

  990.     DEQUANT_STORE_SSE2 8

  991. %endif

  992.     RET

  993. .big_qmul:

  994.     bsr        t0d, t3d

  995.     add        t3d, 128 << 16

  996.     mov        t1d, 7

  997.     cmp        t0d, t1d

  998.     cmovg      t0d, t1d

  999.     inc        t1d

  1000.     shr        t3d, t0b

  1001.     sub        t1d, t0d

  1002. %ifidn %1,mmx

  1003.     movd        m6, t1d

  1004.     DEQUANT_STORE_MMX m6

  1005. %else

  1006.     movd      xmm6, t1d

  1007.     DEQUANT_STORE_SSE2 xmm6

  1008. %endif

  1009.     RET

  1010. %endmacro

  1011. 

  1012. INIT_MMX

  1013. IDCT_DC_DEQUANT mmx, 0

  1014. IDCT_DC_DEQUANT sse2, 7