falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
46555 Commits
32 Branches
291MB
Tree: 2fbb37b51b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '2fbb37b51b'
${ noResults }
FFmpeg/libavcodec/x86/h264_idct.asm

1013 lines
26KB
Raw Blame History 

  1. ;*****************************************************************************

  2. ;* MMX/SSE2-optimized H.264 iDCT

  3. ;*****************************************************************************

  4. ;* Copyright (C) 2004-2005 Michael Niedermayer, Loren Merritt

  5. ;* Copyright (C) 2003-2008 x264 project

  6. ;*

  7. ;* Authors: Laurent Aimar <fenrir@via.ecp.fr>

  8. ;*          Loren Merritt <lorenm@u.washington.edu>

  9. ;*          Holger Lubitz <hal@duncan.ol.sub.de>

  10. ;*          Min Chen <chenm001.163.com>

  11. ;*

  12. ;* This file is part of FFmpeg.

  13. ;*

  14. ;* FFmpeg is free software; you can redistribute it and/or

  15. ;* modify it under the terms of the GNU Lesser General Public

  16. ;* License as published by the Free Software Foundation; either

  17. ;* version 2.1 of the License, or (at your option) any later version.

  18. ;*

  19. ;* FFmpeg is distributed in the hope that it will be useful,

  20. ;* but WITHOUT ANY WARRANTY; without even the implied warranty of

  21. ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  22. ;* Lesser General Public License for more details.

  23. ;*

  24. ;* You should have received a copy of the GNU Lesser General Public

  25. ;* License along with FFmpeg; if not, write to the Free Software

  26. ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  27. ;*****************************************************************************

  28. 

  29. %include "libavutil/x86/x86util.asm"

  30. 

  31. SECTION_RODATA

  32. 

  33. ; FIXME this table is a duplicate from h264data.h, and will be removed once the tables from, h264 have been split

  34. scan8_mem: db  4+ 1*8, 5+ 1*8, 4+ 2*8, 5+ 2*8

  35.            db  6+ 1*8, 7+ 1*8, 6+ 2*8, 7+ 2*8

  36.            db  4+ 3*8, 5+ 3*8, 4+ 4*8, 5+ 4*8

  37.            db  6+ 3*8, 7+ 3*8, 6+ 4*8, 7+ 4*8

  38.            db  4+ 6*8, 5+ 6*8, 4+ 7*8, 5+ 7*8

  39.            db  6+ 6*8, 7+ 6*8, 6+ 7*8, 7+ 7*8

  40.            db  4+ 8*8, 5+ 8*8, 4+ 9*8, 5+ 9*8

  41.            db  6+ 8*8, 7+ 8*8, 6+ 9*8, 7+ 9*8

  42.            db  4+11*8, 5+11*8, 4+12*8, 5+12*8

  43.            db  6+11*8, 7+11*8, 6+12*8, 7+12*8

  44.            db  4+13*8, 5+13*8, 4+14*8, 5+14*8

  45.            db  6+13*8, 7+13*8, 6+14*8, 7+14*8

  46. %ifdef PIC

  47. %define npicregs 1

  48. %define scan8 picregq

  49. %else

  50. %define npicregs 0

  51. %define scan8 scan8_mem

  52. %endif

  53. 

  54. cextern pw_32

  55. cextern pw_1

  56. 

  57. SECTION .text

  58. 

  59. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  60. %macro IDCT4_ADD 3

  61.     ; Load dct coeffs

  62.     movq         m0, [%2]

  63.     movq         m1, [%2+8]

  64.     movq         m2, [%2+16]

  65.     movq         m3, [%2+24]

  66. 

  67.     IDCT4_1D      w, 0, 1, 2, 3, 4, 5

  68.     mova         m6, [pw_32]

  69.     TRANSPOSE4x4W 0, 1, 2, 3, 4

  70.     paddw        m0, m6

  71.     IDCT4_1D      w, 0, 1, 2, 3, 4, 5

  72.     pxor         m7, m7

  73. 

  74.     STORE_DIFFx2 m0, m1, m4, m5, m7, 6, %1, %3

  75.     lea          %1, [%1+%3*2]

  76.     STORE_DIFFx2 m2, m3, m4, m5, m7, 6, %1, %3

  77. %endmacro

  78. 

  79. INIT_MMX

  80. ; ff_h264_idct_add_mmx(uint8_t *dst, int16_t *block, int stride)

  81. cglobal h264_idct_add_8_mmx, 3, 3, 0

  82.     IDCT4_ADD    r0, r1, r2

  83.     RET

  84. 

  85. %macro IDCT8_1D 2

  86.     mova         m0, m1

  87.     psraw        m1, 1

  88.     mova         m4, m5

  89.     psraw        m4, 1

  90.     paddw        m4, m5

  91.     paddw        m1, m0

  92.     paddw        m4, m7

  93.     paddw        m1, m5

  94.     psubw        m4, m0

  95.     paddw        m1, m3

  96. 

  97.     psubw        m0, m3

  98.     psubw        m5, m3

  99.     psraw        m3, 1

  100.     paddw        m0, m7

  101.     psubw        m5, m7

  102.     psraw        m7, 1

  103.     psubw        m0, m3

  104.     psubw        m5, m7

  105. 

  106.     mova         m7, m1

  107.     psraw        m1, 2

  108.     mova         m3, m4

  109.     psraw        m3, 2

  110.     paddw        m3, m0

  111.     psraw        m0, 2

  112.     paddw        m1, m5

  113.     psraw        m5, 2

  114.     psubw        m0, m4

  115.     psubw        m7, m5

  116. 

  117.     mova         m5, m6

  118.     psraw        m6, 1

  119.     mova         m4, m2

  120.     psraw        m4, 1

  121.     paddw        m6, m2

  122.     psubw        m4, m5

  123. 

  124.     mova         m2, %1

  125.     mova         m5, %2

  126.     SUMSUB_BA    w, 5, 2

  127.     SUMSUB_BA    w, 6, 5

  128.     SUMSUB_BA    w, 4, 2

  129.     SUMSUB_BA    w, 7, 6

  130.     SUMSUB_BA    w, 0, 4

  131.     SUMSUB_BA    w, 3, 2

  132.     SUMSUB_BA    w, 1, 5

  133.     SWAP         7, 6, 4, 5, 2, 3, 1, 0 ; 70315246 -> 01234567

  134. %endmacro

  135. 

  136. %macro IDCT8_1D_FULL 1

  137.     mova         m7, [%1+112]

  138.     mova         m6, [%1+ 96]

  139.     mova         m5, [%1+ 80]

  140.     mova         m3, [%1+ 48]

  141.     mova         m2, [%1+ 32]

  142.     mova         m1, [%1+ 16]

  143.     IDCT8_1D   [%1], [%1+ 64]

  144. %endmacro

  145. 

  146. ; %1=int16_t *block, %2=int16_t *dstblock

  147. %macro IDCT8_ADD_MMX_START 2

  148.     IDCT8_1D_FULL %1

  149.     mova       [%1], m7

  150.     TRANSPOSE4x4W 0, 1, 2, 3, 7

  151.     mova         m7, [%1]

  152.     mova    [%2   ], m0

  153.     mova    [%2+16], m1

  154.     mova    [%2+32], m2

  155.     mova    [%2+48], m3

  156.     TRANSPOSE4x4W 4, 5, 6, 7, 3

  157.     mova    [%2+ 8], m4

  158.     mova    [%2+24], m5

  159.     mova    [%2+40], m6

  160.     mova    [%2+56], m7

  161. %endmacro

  162. 

  163. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  164. %macro IDCT8_ADD_MMX_END 3

  165.     IDCT8_1D_FULL %2

  166.     mova    [%2   ], m5

  167.     mova    [%2+16], m6

  168.     mova    [%2+32], m7

  169. 

  170.     pxor         m7, m7

  171.     STORE_DIFFx2 m0, m1, m5, m6, m7, 6, %1, %3

  172.     lea          %1, [%1+%3*2]

  173.     STORE_DIFFx2 m2, m3, m5, m6, m7, 6, %1, %3

  174.     mova         m0, [%2   ]

  175.     mova         m1, [%2+16]

  176.     mova         m2, [%2+32]

  177.     lea          %1, [%1+%3*2]

  178.     STORE_DIFFx2 m4, m0, m5, m6, m7, 6, %1, %3

  179.     lea          %1, [%1+%3*2]

  180.     STORE_DIFFx2 m1, m2, m5, m6, m7, 6, %1, %3

  181. %endmacro

  182. 

  183. INIT_MMX

  184. ; ff_h264_idct8_add_mmx(uint8_t *dst, int16_t *block, int stride)

  185. cglobal h264_idct8_add_8_mmx, 3, 4, 0

  186.     %assign pad 128+4-(stack_offset&7)

  187.     SUB         rsp, pad

  188. 

  189.     add   word [r1], 32

  190.     IDCT8_ADD_MMX_START r1  , rsp

  191.     IDCT8_ADD_MMX_START r1+8, rsp+64

  192.     lea          r3, [r0+4]

  193.     IDCT8_ADD_MMX_END   r0  , rsp,   r2

  194.     IDCT8_ADD_MMX_END   r3  , rsp+8, r2

  195. 

  196.     ADD         rsp, pad

  197.     RET

  198. 

  199. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  200. %macro IDCT8_ADD_SSE 4

  201.     IDCT8_1D_FULL %2

  202. %if ARCH_X86_64

  203.     TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, 8

  204. %else

  205.     TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, [%2], [%2+16]

  206. %endif

  207.     paddw        m0, [pw_32]

  208. 

  209. %if ARCH_X86_64 == 0

  210.     mova    [%2   ], m0

  211.     mova    [%2+16], m4

  212.     IDCT8_1D   [%2], [%2+ 16]

  213.     mova    [%2   ], m6

  214.     mova    [%2+16], m7

  215. %else

  216.     SWAP          0, 8

  217.     SWAP          4, 9

  218.     IDCT8_1D     m8, m9

  219.     SWAP          6, 8

  220.     SWAP          7, 9

  221. %endif

  222. 

  223.     pxor         m7, m7

  224.     lea          %4, [%3*3]

  225.     STORE_DIFF   m0, m6, m7, [%1     ]

  226.     STORE_DIFF   m1, m6, m7, [%1+%3  ]

  227.     STORE_DIFF   m2, m6, m7, [%1+%3*2]

  228.     STORE_DIFF   m3, m6, m7, [%1+%4  ]

  229. %if ARCH_X86_64 == 0

  230.     mova         m0, [%2   ]

  231.     mova         m1, [%2+16]

  232. %else

  233.     SWAP          0, 8

  234.     SWAP          1, 9

  235. %endif

  236.     lea          %1, [%1+%3*4]

  237.     STORE_DIFF   m4, m6, m7, [%1     ]

  238.     STORE_DIFF   m5, m6, m7, [%1+%3  ]

  239.     STORE_DIFF   m0, m6, m7, [%1+%3*2]

  240.     STORE_DIFF   m1, m6, m7, [%1+%4  ]

  241. %endmacro

  242. 

  243. INIT_XMM

  244. ; ff_h264_idct8_add_sse2(uint8_t *dst, int16_t *block, int stride)

  245. cglobal h264_idct8_add_8_sse2, 3, 4, 10

  246.     IDCT8_ADD_SSE r0, r1, r2, r3

  247.     RET

  248. 

  249. %macro DC_ADD_MMXEXT_INIT 2-3

  250. %if %0 == 2

  251.     movsx        %1, word [%1]

  252.     add          %1, 32

  253.     sar          %1, 6

  254.     movd         m0, %1d

  255.     lea          %1, [%2*3]

  256. %else

  257.     add          %3, 32

  258.     sar          %3, 6

  259.     movd         m0, %3d

  260.     lea          %3, [%2*3]

  261. %endif

  262.     pshufw       m0, m0, 0

  263.     pxor         m1, m1

  264.     psubw        m1, m0

  265.     packuswb     m0, m0

  266.     packuswb     m1, m1

  267. %endmacro

  268. 

  269. %macro DC_ADD_MMXEXT_OP 4

  270.     %1           m2, [%2     ]

  271.     %1           m3, [%2+%3  ]

  272.     %1           m4, [%2+%3*2]

  273.     %1           m5, [%2+%4  ]

  274.     paddusb      m2, m0

  275.     paddusb      m3, m0

  276.     paddusb      m4, m0

  277.     paddusb      m5, m0

  278.     psubusb      m2, m1

  279.     psubusb      m3, m1

  280.     psubusb      m4, m1

  281.     psubusb      m5, m1

  282.     %1    [%2     ], m2

  283.     %1    [%2+%3  ], m3

  284.     %1    [%2+%3*2], m4

  285.     %1    [%2+%4  ], m5

  286. %endmacro

  287. 

  288. INIT_MMX

  289. ; ff_h264_idct_dc_add_mmx2(uint8_t *dst, int16_t *block, int stride)

  290. cglobal h264_idct_dc_add_8_mmx2, 3, 3, 0

  291.     DC_ADD_MMXEXT_INIT r1, r2

  292.     DC_ADD_MMXEXT_OP movh, r0, r2, r1

  293.     RET

  294. 

  295. ; ff_h264_idct8_dc_add_mmx2(uint8_t *dst, int16_t *block, int stride)

  296. cglobal h264_idct8_dc_add_8_mmx2, 3, 3, 0

  297.     DC_ADD_MMXEXT_INIT r1, r2

  298.     DC_ADD_MMXEXT_OP mova, r0, r2, r1

  299.     lea          r0, [r0+r2*4]

  300.     DC_ADD_MMXEXT_OP mova, r0, r2, r1

  301.     RET

  302. 

  303. ; ff_h264_idct_add16_mmx(uint8_t *dst, const int *block_offset,

  304. ;             DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  305. cglobal h264_idct_add16_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  306.     xor          r5, r5

  307. %ifdef PIC

  308.     lea     picregq, [scan8_mem]

  309. %endif

  310. .nextblock:

  311.     movzx        r6, byte [scan8+r5]

  312.     movzx        r6, byte [r4+r6]

  313.     test         r6, r6

  314.     jz .skipblock

  315.     mov         r6d, dword [r1+r5*4]

  316.     lea          r6, [r0+r6]

  317.     IDCT4_ADD    r6, r2, r3

  318. .skipblock:

  319.     inc          r5

  320.     add          r2, 32

  321.     cmp          r5, 16

  322.     jl .nextblock

  323.     REP_RET

  324. 

  325. ; ff_h264_idct8_add4_mmx(uint8_t *dst, const int *block_offset,

  326. ;                        DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  327. cglobal h264_idct8_add4_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  328.     %assign pad 128+4-(stack_offset&7)

  329.     SUB         rsp, pad

  330. 

  331.     xor          r5, r5

  332. %ifdef PIC

  333.     lea     picregq, [scan8_mem]

  334. %endif

  335. .nextblock:

  336.     movzx        r6, byte [scan8+r5]

  337.     movzx        r6, byte [r4+r6]

  338.     test         r6, r6

  339.     jz .skipblock

  340.     mov         r6d, dword [r1+r5*4]

  341.     add          r6, r0

  342.     add   word [r2], 32

  343.     IDCT8_ADD_MMX_START r2  , rsp

  344.     IDCT8_ADD_MMX_START r2+8, rsp+64

  345.     IDCT8_ADD_MMX_END   r6  , rsp,   r3

  346.     mov         r6d, dword [r1+r5*4]

  347.     lea          r6, [r0+r6+4]

  348.     IDCT8_ADD_MMX_END   r6  , rsp+8, r3

  349. .skipblock:

  350.     add          r5, 4

  351.     add          r2, 128

  352.     cmp          r5, 16

  353.     jl .nextblock

  354.     ADD         rsp, pad

  355.     RET

  356. 

  357. ; ff_h264_idct_add16_mmx2(uint8_t *dst, const int *block_offset,

  358. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  359. cglobal h264_idct_add16_8_mmx2, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  360.     xor          r5, r5

  361. %ifdef PIC

  362.     lea     picregq, [scan8_mem]

  363. %endif

  364. .nextblock:

  365.     movzx        r6, byte [scan8+r5]

  366.     movzx        r6, byte [r4+r6]

  367.     test         r6, r6

  368.     jz .skipblock

  369.     cmp          r6, 1

  370.     jnz .no_dc

  371.     movsx        r6, word [r2]

  372.     test         r6, r6

  373.     jz .no_dc

  374.     DC_ADD_MMXEXT_INIT r2, r3, r6

  375. %if ARCH_X86_64 == 0

  376. %define dst2q r1

  377. %define dst2d r1d

  378. %endif

  379.     mov       dst2d, dword [r1+r5*4]

  380.     lea       dst2q, [r0+dst2q]

  381.     DC_ADD_MMXEXT_OP movh, dst2q, r3, r6

  382. %if ARCH_X86_64 == 0

  383.     mov          r1, r1m

  384. %endif

  385.     inc          r5

  386.     add          r2, 32

  387.     cmp          r5, 16

  388.     jl .nextblock

  389.     REP_RET

  390. .no_dc:

  391.     mov         r6d, dword [r1+r5*4]

  392.     add          r6, r0

  393.     IDCT4_ADD    r6, r2, r3

  394. .skipblock:

  395.     inc          r5

  396.     add          r2, 32

  397.     cmp          r5, 16

  398.     jl .nextblock

  399.     REP_RET

  400. 

  401. ; ff_h264_idct_add16intra_mmx(uint8_t *dst, const int *block_offset,

  402. ;                             DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  403. cglobal h264_idct_add16intra_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  404.     xor          r5, r5

  405. %ifdef PIC

  406.     lea     picregq, [scan8_mem]

  407. %endif

  408. .nextblock:

  409.     movzx        r6, byte [scan8+r5]

  410.     movzx        r6, byte [r4+r6]

  411.     or          r6w, word [r2]

  412.     test         r6, r6

  413.     jz .skipblock

  414.     mov         r6d, dword [r1+r5*4]

  415.     add          r6, r0

  416.     IDCT4_ADD    r6, r2, r3

  417. .skipblock:

  418.     inc          r5

  419.     add          r2, 32

  420.     cmp          r5, 16

  421.     jl .nextblock

  422.     REP_RET

  423. 

  424. ; ff_h264_idct_add16intra_mmx2(uint8_t *dst, const int *block_offset,

  425. ;                              DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  426. cglobal h264_idct_add16intra_8_mmx2, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  427.     xor          r5, r5

  428. %ifdef PIC

  429.     lea     picregq, [scan8_mem]

  430. %endif

  431. .nextblock:

  432.     movzx        r6, byte [scan8+r5]

  433.     movzx        r6, byte [r4+r6]

  434.     test         r6, r6

  435.     jz .try_dc

  436.     mov         r6d, dword [r1+r5*4]

  437.     lea          r6, [r0+r6]

  438.     IDCT4_ADD    r6, r2, r3

  439.     inc          r5

  440.     add          r2, 32

  441.     cmp          r5, 16

  442.     jl .nextblock

  443.     REP_RET

  444. .try_dc:

  445.     movsx        r6, word [r2]

  446.     test         r6, r6

  447.     jz .skipblock

  448.     DC_ADD_MMXEXT_INIT r2, r3, r6

  449. %if ARCH_X86_64 == 0

  450. %define dst2q r1

  451. %define dst2d r1d

  452. %endif

  453.     mov       dst2d, dword [r1+r5*4]

  454.     add       dst2q, r0

  455.     DC_ADD_MMXEXT_OP movh, dst2q, r3, r6

  456. %if ARCH_X86_64 == 0

  457.     mov          r1, r1m

  458. %endif

  459. .skipblock:

  460.     inc          r5

  461.     add          r2, 32

  462.     cmp          r5, 16

  463.     jl .nextblock

  464.     REP_RET

  465. 

  466. ; ff_h264_idct8_add4_mmx2(uint8_t *dst, const int *block_offset,

  467. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  468. cglobal h264_idct8_add4_8_mmx2, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  469.     %assign pad 128+4-(stack_offset&7)

  470.     SUB         rsp, pad

  471. 

  472.     xor          r5, r5

  473. %ifdef PIC

  474.     lea     picregq, [scan8_mem]

  475. %endif

  476. .nextblock:

  477.     movzx        r6, byte [scan8+r5]

  478.     movzx        r6, byte [r4+r6]

  479.     test         r6, r6

  480.     jz .skipblock

  481.     cmp          r6, 1

  482.     jnz .no_dc

  483.     movsx        r6, word [r2]

  484.     test         r6, r6

  485.     jz .no_dc

  486.     DC_ADD_MMXEXT_INIT r2, r3, r6

  487. %if ARCH_X86_64 == 0

  488. %define dst2q r1

  489. %define dst2d r1d

  490. %endif

  491.     mov       dst2d, dword [r1+r5*4]

  492.     lea       dst2q, [r0+dst2q]

  493.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  494.     lea       dst2q, [dst2q+r3*4]

  495.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  496. %if ARCH_X86_64 == 0

  497.     mov          r1, r1m

  498. %endif

  499.     add          r5, 4

  500.     add          r2, 128

  501.     cmp          r5, 16

  502.     jl .nextblock

  503. 

  504.     ADD         rsp, pad

  505.     RET

  506. .no_dc:

  507.     mov         r6d, dword [r1+r5*4]

  508.     add          r6, r0

  509.     add   word [r2], 32

  510.     IDCT8_ADD_MMX_START r2  , rsp

  511.     IDCT8_ADD_MMX_START r2+8, rsp+64

  512.     IDCT8_ADD_MMX_END   r6  , rsp,   r3

  513.     mov         r6d, dword [r1+r5*4]

  514.     lea          r6, [r0+r6+4]

  515.     IDCT8_ADD_MMX_END   r6  , rsp+8, r3

  516. .skipblock:

  517.     add          r5, 4

  518.     add          r2, 128

  519.     cmp          r5, 16

  520.     jl .nextblock

  521. 

  522.     ADD         rsp, pad

  523.     RET

  524. 

  525. INIT_XMM

  526. ; ff_h264_idct8_add4_sse2(uint8_t *dst, const int *block_offset,

  527. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  528. cglobal h264_idct8_add4_8_sse2, 5, 8 + npicregs, 10, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  529.     xor          r5, r5

  530. %ifdef PIC

  531.     lea     picregq, [scan8_mem]

  532. %endif

  533. .nextblock:

  534.     movzx        r6, byte [scan8+r5]

  535.     movzx        r6, byte [r4+r6]

  536.     test         r6, r6

  537.     jz .skipblock

  538.     cmp          r6, 1

  539.     jnz .no_dc

  540.     movsx        r6, word [r2]

  541.     test         r6, r6

  542.     jz .no_dc

  543. INIT_MMX

  544.     DC_ADD_MMXEXT_INIT r2, r3, r6

  545. %if ARCH_X86_64 == 0

  546. %define dst2q r1

  547. %define dst2d r1d

  548. %endif

  549.     mov       dst2d, dword [r1+r5*4]

  550.     add       dst2q, r0

  551.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  552.     lea       dst2q, [dst2q+r3*4]

  553.     DC_ADD_MMXEXT_OP mova, dst2q, r3, r6

  554. %if ARCH_X86_64 == 0

  555.     mov          r1, r1m

  556. %endif

  557.     add          r5, 4

  558.     add          r2, 128

  559.     cmp          r5, 16

  560.     jl .nextblock

  561.     REP_RET

  562. .no_dc:

  563. INIT_XMM

  564.     mov       dst2d, dword [r1+r5*4]

  565.     add       dst2q, r0

  566.     IDCT8_ADD_SSE dst2q, r2, r3, r6

  567. %if ARCH_X86_64 == 0

  568.     mov          r1, r1m

  569. %endif

  570. .skipblock:

  571.     add          r5, 4

  572.     add          r2, 128

  573.     cmp          r5, 16

  574.     jl .nextblock

  575.     REP_RET

  576. 

  577. INIT_MMX

  578. h264_idct_add8_mmx_plane:

  579. .nextblock:

  580.     movzx        r6, byte [scan8+r5]

  581.     movzx        r6, byte [r4+r6]

  582.     or          r6w, word [r2]

  583.     test         r6, r6

  584.     jz .skipblock

  585. %if ARCH_X86_64

  586.     mov         r0d, dword [r1+r5*4]

  587.     add          r0, [dst2q]

  588. %else

  589.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  590.     mov          r0, [r0]

  591.     add          r0, dword [r1+r5*4]

  592. %endif

  593.     IDCT4_ADD    r0, r2, r3

  594. .skipblock:

  595.     inc          r5

  596.     add          r2, 32

  597.     test         r5, 3

  598.     jnz .nextblock

  599.     rep ret

  600. 

  601. ; ff_h264_idct_add8_mmx(uint8_t **dest, const int *block_offset,

  602. ;                       DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  603. cglobal h264_idct_add8_8_mmx, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  604.     mov          r5, 16

  605.     add          r2, 512

  606. %ifdef PIC

  607.     lea     picregq, [scan8_mem]

  608. %endif

  609. %if ARCH_X86_64

  610.     mov       dst2q, r0

  611. %endif

  612.     call         h264_idct_add8_mmx_plane

  613.     mov          r5, 32

  614.     add          r2, 384

  615. %if ARCH_X86_64

  616.     add       dst2q, gprsize

  617. %else

  618.     add        r0mp, gprsize

  619. %endif

  620.     call         h264_idct_add8_mmx_plane

  621.     RET

  622. 

  623. h264_idct_add8_mmx2_plane:

  624. .nextblock:

  625.     movzx        r6, byte [scan8+r5]

  626.     movzx        r6, byte [r4+r6]

  627.     test         r6, r6

  628.     jz .try_dc

  629. %if ARCH_X86_64

  630.     mov         r0d, dword [r1+r5*4]

  631.     add          r0, [dst2q]

  632. %else

  633.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  634.     mov          r0, [r0]

  635.     add          r0, dword [r1+r5*4]

  636. %endif

  637.     IDCT4_ADD    r0, r2, r3

  638.     inc          r5

  639.     add          r2, 32

  640.     test         r5, 3

  641.     jnz .nextblock

  642.     rep ret

  643. .try_dc:

  644.     movsx        r6, word [r2]

  645.     test         r6, r6

  646.     jz .skipblock

  647.     DC_ADD_MMXEXT_INIT r2, r3, r6

  648. %if ARCH_X86_64

  649.     mov         r0d, dword [r1+r5*4]

  650.     add          r0, [dst2q]

  651. %else

  652.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  653.     mov          r0, [r0]

  654.     add          r0, dword [r1+r5*4]

  655. %endif

  656.     DC_ADD_MMXEXT_OP movh, r0, r3, r6

  657. .skipblock:

  658.     inc          r5

  659.     add          r2, 32

  660.     test         r5, 3

  661.     jnz .nextblock

  662.     rep ret

  663. 

  664. ; ff_h264_idct_add8_mmx2(uint8_t **dest, const int *block_offset,

  665. ;                        DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  666. cglobal h264_idct_add8_8_mmx2, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  667.     mov          r5, 16

  668.     add          r2, 512

  669. %if ARCH_X86_64

  670.     mov       dst2q, r0

  671. %endif

  672. %ifdef PIC

  673.     lea     picregq, [scan8_mem]

  674. %endif

  675.     call h264_idct_add8_mmx2_plane

  676.     mov          r5, 32

  677.     add          r2, 384

  678. %if ARCH_X86_64

  679.     add       dst2q, gprsize

  680. %else

  681.     add        r0mp, gprsize

  682. %endif

  683.     call h264_idct_add8_mmx2_plane

  684.     RET

  685. 

  686. INIT_MMX

  687. ; r0 = uint8_t *dst, r2 = int16_t *block, r3 = int stride, r6=clobbered

  688. h264_idct_dc_add8_mmx2:

  689.     movd         m0, [r2   ]          ;  0 0 X D

  690.     punpcklwd    m0, [r2+32]          ;  x X d D

  691.     paddsw       m0, [pw_32]

  692.     psraw        m0, 6

  693.     punpcklwd    m0, m0               ;  d d D D

  694.     pxor         m1, m1               ;  0 0 0 0

  695.     psubw        m1, m0               ; -d-d-D-D

  696.     packuswb     m0, m1               ; -d-d-D-D d d D D

  697.     pshufw       m1, m0, 0xFA         ; -d-d-d-d-D-D-D-D

  698.     punpcklwd    m0, m0               ;  d d d d D D D D

  699.     lea          r6, [r3*3]

  700.     DC_ADD_MMXEXT_OP movq, r0, r3, r6

  701.     ret

  702. 

  703. ALIGN 16

  704. INIT_XMM

  705. ; r0 = uint8_t *dst (clobbered), r2 = int16_t *block, r3 = int stride

  706. h264_add8x4_idct_sse2:

  707.     movq   m0, [r2+ 0]

  708.     movq   m1, [r2+ 8]

  709.     movq   m2, [r2+16]

  710.     movq   m3, [r2+24]

  711.     movhps m0, [r2+32]

  712.     movhps m1, [r2+40]

  713.     movhps m2, [r2+48]

  714.     movhps m3, [r2+56]

  715.     IDCT4_1D w,0,1,2,3,4,5

  716.     TRANSPOSE2x4x4W 0,1,2,3,4

  717.     paddw m0, [pw_32]

  718.     IDCT4_1D w,0,1,2,3,4,5

  719.     pxor  m7, m7

  720.     STORE_DIFFx2 m0, m1, m4, m5, m7, 6, r0, r3

  721.     lea   r0, [r0+r3*2]

  722.     STORE_DIFFx2 m2, m3, m4, m5, m7, 6, r0, r3

  723.     ret

  724. 

  725. %macro add16_sse2_cycle 2

  726.     movzx       r0, word [r4+%2]

  727.     test        r0, r0

  728.     jz .cycle%1end

  729.     mov        r0d, dword [r1+%1*8]

  730. %if ARCH_X86_64

  731.     add         r0, r5

  732. %else

  733.     add         r0, r0m

  734. %endif

  735.     call        h264_add8x4_idct_sse2

  736. .cycle%1end:

  737. %if %1 < 7

  738.     add         r2, 64

  739. %endif

  740. %endmacro

  741. 

  742. ; ff_h264_idct_add16_sse2(uint8_t *dst, const int *block_offset,

  743. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  744. cglobal h264_idct_add16_8_sse2, 5, 5 + ARCH_X86_64, 8

  745. %if ARCH_X86_64

  746.     mov         r5, r0

  747. %endif

  748.     ; unrolling of the loop leads to an average performance gain of

  749.     ; 20-25%

  750.     add16_sse2_cycle 0, 0xc

  751.     add16_sse2_cycle 1, 0x14

  752.     add16_sse2_cycle 2, 0xe

  753.     add16_sse2_cycle 3, 0x16

  754.     add16_sse2_cycle 4, 0x1c

  755.     add16_sse2_cycle 5, 0x24

  756.     add16_sse2_cycle 6, 0x1e

  757.     add16_sse2_cycle 7, 0x26

  758.     RET

  759. 

  760. %macro add16intra_sse2_cycle 2

  761.     movzx       r0, word [r4+%2]

  762.     test        r0, r0

  763.     jz .try%1dc

  764.     mov        r0d, dword [r1+%1*8]

  765. %if ARCH_X86_64

  766.     add         r0, r7

  767. %else

  768.     add         r0, r0m

  769. %endif

  770.     call        h264_add8x4_idct_sse2

  771.     jmp .cycle%1end

  772. .try%1dc:

  773.     movsx       r0, word [r2   ]

  774.     or         r0w, word [r2+32]

  775.     jz .cycle%1end

  776.     mov        r0d, dword [r1+%1*8]

  777. %if ARCH_X86_64

  778.     add         r0, r7

  779. %else

  780.     add         r0, r0m

  781. %endif

  782.     call        h264_idct_dc_add8_mmx2

  783. .cycle%1end:

  784. %if %1 < 7

  785.     add         r2, 64

  786. %endif

  787. %endmacro

  788. 

  789. ; ff_h264_idct_add16intra_sse2(uint8_t *dst, const int *block_offset,

  790. ;                              DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  791. cglobal h264_idct_add16intra_8_sse2, 5, 7 + ARCH_X86_64, 8

  792. %if ARCH_X86_64

  793.     mov         r7, r0

  794. %endif

  795.     add16intra_sse2_cycle 0, 0xc

  796.     add16intra_sse2_cycle 1, 0x14

  797.     add16intra_sse2_cycle 2, 0xe

  798.     add16intra_sse2_cycle 3, 0x16

  799.     add16intra_sse2_cycle 4, 0x1c

  800.     add16intra_sse2_cycle 5, 0x24

  801.     add16intra_sse2_cycle 6, 0x1e

  802.     add16intra_sse2_cycle 7, 0x26

  803.     RET

  804. 

  805. %macro add8_sse2_cycle 2

  806.     movzx       r0, word [r4+%2]

  807.     test        r0, r0

  808.     jz .try%1dc

  809. %if ARCH_X86_64

  810.     mov        r0d, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  811.     add         r0, [r7]

  812. %else

  813.     mov         r0, r0m

  814.     mov         r0, [r0]

  815.     add         r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  816. %endif

  817.     call        h264_add8x4_idct_sse2

  818.     jmp .cycle%1end

  819. .try%1dc:

  820.     movsx       r0, word [r2   ]

  821.     or         r0w, word [r2+32]

  822.     jz .cycle%1end

  823. %if ARCH_X86_64

  824.     mov        r0d, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  825.     add         r0, [r7]

  826. %else

  827.     mov         r0, r0m

  828.     mov         r0, [r0]

  829.     add         r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  830. %endif

  831.     call        h264_idct_dc_add8_mmx2

  832. .cycle%1end:

  833. %if %1 == 1

  834.     add         r2, 384+64

  835. %elif %1 < 3

  836.     add         r2, 64

  837. %endif

  838. %endmacro

  839. 

  840. ; ff_h264_idct_add8_sse2(uint8_t **dest, const int *block_offset,

  841. ;                        DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  842. cglobal h264_idct_add8_8_sse2, 5, 7 + ARCH_X86_64, 8

  843.     add          r2, 512

  844. %if ARCH_X86_64

  845.     mov          r7, r0

  846. %endif

  847.     add8_sse2_cycle 0, 0x34

  848.     add8_sse2_cycle 1, 0x3c

  849. %if ARCH_X86_64

  850.     add          r7, gprsize

  851. %else

  852.     add        r0mp, gprsize

  853. %endif

  854.     add8_sse2_cycle 2, 0x5c

  855.     add8_sse2_cycle 3, 0x64

  856.     RET

  857. 

  858. ;void ff_h264_luma_dc_dequant_idct_mmx(DCTELEM *output, DCTELEM *input, int qmul)

  859. 

  860. %macro WALSH4_1D 5

  861.     SUMSUB_BADC w, %4, %3, %2, %1, %5

  862.     SUMSUB_BADC w, %4, %2, %3, %1, %5

  863.     SWAP %1, %4, %3

  864. %endmacro

  865. 

  866. %macro DEQUANT_MMX 3

  867.     mova        m7, [pw_1]

  868.     mova        m4, %1

  869.     punpcklwd   %1, m7

  870.     punpckhwd   m4, m7

  871.     mova        m5, %2

  872.     punpcklwd   %2, m7

  873.     punpckhwd   m5, m7

  874.     movd        m7, t3d

  875.     punpckldq   m7, m7

  876.     pmaddwd     %1, m7

  877.     pmaddwd     %2, m7

  878.     pmaddwd     m4, m7

  879.     pmaddwd     m5, m7

  880.     psrad       %1, %3

  881.     psrad       %2, %3

  882.     psrad       m4, %3

  883.     psrad       m5, %3

  884.     packssdw    %1, m4

  885.     packssdw    %2, m5

  886. %endmacro

  887. 

  888. %macro STORE_WORDS_MMX 5

  889.     movd  t0d, %1

  890.     psrlq  %1, 32

  891.     movd  t1d, %1

  892.     mov [t2+%2*32], t0w

  893.     mov [t2+%4*32], t1w

  894.     shr   t0d, 16

  895.     shr   t1d, 16

  896.     mov [t2+%3*32], t0w

  897.     mov [t2+%5*32], t1w

  898. %endmacro

  899. 

  900. %macro DEQUANT_STORE_MMX 1

  901.     DEQUANT_MMX m0, m1, %1

  902.     STORE_WORDS_MMX m0,  0,  1,  4,  5

  903.     STORE_WORDS_MMX m1,  2,  3,  6,  7

  904. 

  905.     DEQUANT_MMX m2, m3, %1

  906.     STORE_WORDS_MMX m2,  8,  9, 12, 13

  907.     STORE_WORDS_MMX m3, 10, 11, 14, 15

  908. %endmacro

  909. 

  910. %macro STORE_WORDS_SSE 9

  911.     movd  t0d, %1

  912.     psrldq  %1, 4

  913.     movd  t1d, %1

  914.     psrldq  %1, 4

  915.     mov [t2+%2*32], t0w

  916.     mov [t2+%4*32], t1w

  917.     shr   t0d, 16

  918.     shr   t1d, 16

  919.     mov [t2+%3*32], t0w

  920.     mov [t2+%5*32], t1w

  921.     movd  t0d, %1

  922.     psrldq  %1, 4

  923.     movd  t1d, %1

  924.     mov [t2+%6*32], t0w

  925.     mov [t2+%8*32], t1w

  926.     shr   t0d, 16

  927.     shr   t1d, 16

  928.     mov [t2+%7*32], t0w

  929.     mov [t2+%9*32], t1w

  930. %endmacro

  931. 

  932. %macro DEQUANT_STORE_SSE2 1

  933.     movd      xmm4, t3d

  934.     movq      xmm5, [pw_1]

  935.     pshufd    xmm4, xmm4, 0

  936.     movq2dq   xmm0, m0

  937.     movq2dq   xmm1, m1

  938.     movq2dq   xmm2, m2

  939.     movq2dq   xmm3, m3

  940.     punpcklwd xmm0, xmm5

  941.     punpcklwd xmm1, xmm5

  942.     punpcklwd xmm2, xmm5

  943.     punpcklwd xmm3, xmm5

  944.     pmaddwd   xmm0, xmm4

  945.     pmaddwd   xmm1, xmm4

  946.     pmaddwd   xmm2, xmm4

  947.     pmaddwd   xmm3, xmm4

  948.     psrad     xmm0, %1

  949.     psrad     xmm1, %1

  950.     psrad     xmm2, %1

  951.     psrad     xmm3, %1

  952.     packssdw  xmm0, xmm1

  953.     packssdw  xmm2, xmm3

  954.     STORE_WORDS_SSE xmm0,  0,  1,  4,  5,  2,  3,  6,  7

  955.     STORE_WORDS_SSE xmm2,  8,  9, 12, 13, 10, 11, 14, 15

  956. %endmacro

  957. 

  958. %macro IDCT_DC_DEQUANT 2

  959. cglobal h264_luma_dc_dequant_idct_%1, 3,4,%2

  960.     ; manually spill XMM registers for Win64 because

  961.     ; the code here is initialized with INIT_MMX

  962.     WIN64_SPILL_XMM %2

  963.     movq        m3, [r1+24]

  964.     movq        m2, [r1+16]

  965.     movq        m1, [r1+ 8]

  966.     movq        m0, [r1+ 0]

  967.     WALSH4_1D    0,1,2,3,4

  968.     TRANSPOSE4x4W 0,1,2,3,4

  969.     WALSH4_1D    0,1,2,3,4

  970. 

  971. ; shift, tmp, output, qmul

  972. %if WIN64

  973.     DECLARE_REG_TMP 0,3,1,2

  974.     ; we can't avoid this, because r0 is the shift register (ecx) on win64

  975.     xchg        r0, t2

  976. %elif ARCH_X86_64

  977.     DECLARE_REG_TMP 3,1,0,2

  978. %else

  979.     DECLARE_REG_TMP 1,3,0,2

  980. %endif

  981. 

  982.     cmp        t3d, 32767

  983.     jg .big_qmul

  984.     add        t3d, 128 << 16

  985. %ifidn %1,mmx

  986.     DEQUANT_STORE_MMX 8

  987. %else

  988.     DEQUANT_STORE_SSE2 8

  989. %endif

  990.     RET

  991. .big_qmul:

  992.     bsr        t0d, t3d

  993.     add        t3d, 128 << 16

  994.     mov        t1d, 7

  995.     cmp        t0d, t1d

  996.     cmovg      t0d, t1d

  997.     inc        t1d

  998.     shr        t3d, t0b

  999.     sub        t1d, t0d

  1000. %ifidn %1,mmx

  1001.     movd        m6, t1d

  1002.     DEQUANT_STORE_MMX m6

  1003. %else

  1004.     movd      xmm6, t1d

  1005.     DEQUANT_STORE_SSE2 xmm6

  1006. %endif

  1007.     RET

  1008. %endmacro

  1009. 

  1010. INIT_MMX

  1011. IDCT_DC_DEQUANT mmx, 0

  1012. IDCT_DC_DEQUANT sse2, 7