falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
34093 Commits
32 Branches
283MB
Tree: 5ed032fb8b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5ed032fb8b'
${ noResults }
FFmpeg/libavcodec/x86/h264_idct.asm

1014 lines
26KB
Raw Blame History 

  1. ;*****************************************************************************

  2. ;* MMX/SSE2-optimized H.264 iDCT

  3. ;*****************************************************************************

  4. ;* Copyright (C) 2004-2005 Michael Niedermayer, Loren Merritt

  5. ;* Copyright (C) 2003-2008 x264 project

  6. ;*

  7. ;* Authors: Laurent Aimar <fenrir@via.ecp.fr>

  8. ;*          Loren Merritt <lorenm@u.washington.edu>

  9. ;*          Holger Lubitz <hal@duncan.ol.sub.de>

  10. ;*          Min Chen <chenm001.163.com>

  11. ;*

  12. ;* This file is part of Libav.

  13. ;*

  14. ;* Libav is free software; you can redistribute it and/or

  15. ;* modify it under the terms of the GNU Lesser General Public

  16. ;* License as published by the Free Software Foundation; either

  17. ;* version 2.1 of the License, or (at your option) any later version.

  18. ;*

  19. ;* Libav is distributed in the hope that it will be useful,

  20. ;* but WITHOUT ANY WARRANTY; without even the implied warranty of

  21. ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  22. ;* Lesser General Public License for more details.

  23. ;*

  24. ;* You should have received a copy of the GNU Lesser General Public

  25. ;* License along with Libav; if not, write to the Free Software

  26. ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  27. ;*****************************************************************************

  28. 

  29. %include "x86inc.asm"

  30. %include "x86util.asm"

  31. 

  32. SECTION_RODATA

  33. 

  34. ; FIXME this table is a duplicate from h264data.h, and will be removed once the tables from, h264 have been split

  35. scan8_mem: db  4+ 1*8, 5+ 1*8, 4+ 2*8, 5+ 2*8

  36.            db  6+ 1*8, 7+ 1*8, 6+ 2*8, 7+ 2*8

  37.            db  4+ 3*8, 5+ 3*8, 4+ 4*8, 5+ 4*8

  38.            db  6+ 3*8, 7+ 3*8, 6+ 4*8, 7+ 4*8

  39.            db  4+ 6*8, 5+ 6*8, 4+ 7*8, 5+ 7*8

  40.            db  6+ 6*8, 7+ 6*8, 6+ 7*8, 7+ 7*8

  41.            db  4+ 8*8, 5+ 8*8, 4+ 9*8, 5+ 9*8

  42.            db  6+ 8*8, 7+ 8*8, 6+ 9*8, 7+ 9*8

  43.            db  4+11*8, 5+11*8, 4+12*8, 5+12*8

  44.            db  6+11*8, 7+11*8, 6+12*8, 7+12*8

  45.            db  4+13*8, 5+13*8, 4+14*8, 5+14*8

  46.            db  6+13*8, 7+13*8, 6+14*8, 7+14*8

  47. %ifdef PIC

  48. %define npicregs 1

  49. %define scan8 picregq

  50. %else

  51. %define npicregs 0

  52. %define scan8 scan8_mem

  53. %endif

  54. 

  55. cextern pw_32

  56. cextern pw_1

  57. 

  58. SECTION .text

  59. 

  60. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  61. %macro IDCT4_ADD 3

  62.     ; Load dct coeffs

  63.     movq         m0, [%2]

  64.     movq         m1, [%2+8]

  65.     movq         m2, [%2+16]

  66.     movq         m3, [%2+24]

  67. 

  68.     IDCT4_1D      w, 0, 1, 2, 3, 4, 5

  69.     mova         m6, [pw_32]

  70.     TRANSPOSE4x4W 0, 1, 2, 3, 4

  71.     paddw        m0, m6

  72.     IDCT4_1D      w, 0, 1, 2, 3, 4, 5

  73.     pxor         m7, m7

  74. 

  75.     STORE_DIFFx2 m0, m1, m4, m5, m7, 6, %1, %3

  76.     lea          %1, [%1+%3*2]

  77.     STORE_DIFFx2 m2, m3, m4, m5, m7, 6, %1, %3

  78. %endmacro

  79. 

  80. INIT_MMX

  81. ; ff_h264_idct_add_mmx(uint8_t *dst, int16_t *block, int stride)

  82. cglobal h264_idct_add_8_mmx, 3, 3, 0

  83.     IDCT4_ADD    r0, r1, r2

  84.     RET

  85. 

  86. %macro IDCT8_1D 2

  87.     mova         m0, m1

  88.     psraw        m1, 1

  89.     mova         m4, m5

  90.     psraw        m4, 1

  91.     paddw        m4, m5

  92.     paddw        m1, m0

  93.     paddw        m4, m7

  94.     paddw        m1, m5

  95.     psubw        m4, m0

  96.     paddw        m1, m3

  97. 

  98.     psubw        m0, m3

  99.     psubw        m5, m3

  100.     psraw        m3, 1

  101.     paddw        m0, m7

  102.     psubw        m5, m7

  103.     psraw        m7, 1

  104.     psubw        m0, m3

  105.     psubw        m5, m7

  106. 

  107.     mova         m7, m1

  108.     psraw        m1, 2

  109.     mova         m3, m4

  110.     psraw        m3, 2

  111.     paddw        m3, m0

  112.     psraw        m0, 2

  113.     paddw        m1, m5

  114.     psraw        m5, 2

  115.     psubw        m0, m4

  116.     psubw        m7, m5

  117. 

  118.     mova         m5, m6

  119.     psraw        m6, 1

  120.     mova         m4, m2

  121.     psraw        m4, 1

  122.     paddw        m6, m2

  123.     psubw        m4, m5

  124. 

  125.     mova         m2, %1

  126.     mova         m5, %2

  127.     SUMSUB_BA    w, 5, 2

  128.     SUMSUB_BA    w, 6, 5

  129.     SUMSUB_BA    w, 4, 2

  130.     SUMSUB_BA    w, 7, 6

  131.     SUMSUB_BA    w, 0, 4

  132.     SUMSUB_BA    w, 3, 2

  133.     SUMSUB_BA    w, 1, 5

  134.     SWAP         7, 6, 4, 5, 2, 3, 1, 0 ; 70315246 -> 01234567

  135. %endmacro

  136. 

  137. %macro IDCT8_1D_FULL 1

  138.     mova         m7, [%1+112]

  139.     mova         m6, [%1+ 96]

  140.     mova         m5, [%1+ 80]

  141.     mova         m3, [%1+ 48]

  142.     mova         m2, [%1+ 32]

  143.     mova         m1, [%1+ 16]

  144.     IDCT8_1D   [%1], [%1+ 64]

  145. %endmacro

  146. 

  147. ; %1=int16_t *block, %2=int16_t *dstblock

  148. %macro IDCT8_ADD_MMX_START 2

  149.     IDCT8_1D_FULL %1

  150.     mova       [%1], m7

  151.     TRANSPOSE4x4W 0, 1, 2, 3, 7

  152.     mova         m7, [%1]

  153.     mova    [%2   ], m0

  154.     mova    [%2+16], m1

  155.     mova    [%2+32], m2

  156.     mova    [%2+48], m3

  157.     TRANSPOSE4x4W 4, 5, 6, 7, 3

  158.     mova    [%2+ 8], m4

  159.     mova    [%2+24], m5

  160.     mova    [%2+40], m6

  161.     mova    [%2+56], m7

  162. %endmacro

  163. 

  164. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  165. %macro IDCT8_ADD_MMX_END 3

  166.     IDCT8_1D_FULL %2

  167.     mova    [%2   ], m5

  168.     mova    [%2+16], m6

  169.     mova    [%2+32], m7

  170. 

  171.     pxor         m7, m7

  172.     STORE_DIFFx2 m0, m1, m5, m6, m7, 6, %1, %3

  173.     lea          %1, [%1+%3*2]

  174.     STORE_DIFFx2 m2, m3, m5, m6, m7, 6, %1, %3

  175.     mova         m0, [%2   ]

  176.     mova         m1, [%2+16]

  177.     mova         m2, [%2+32]

  178.     lea          %1, [%1+%3*2]

  179.     STORE_DIFFx2 m4, m0, m5, m6, m7, 6, %1, %3

  180.     lea          %1, [%1+%3*2]

  181.     STORE_DIFFx2 m1, m2, m5, m6, m7, 6, %1, %3

  182. %endmacro

  183. 

  184. INIT_MMX

  185. ; ff_h264_idct8_add_mmx(uint8_t *dst, int16_t *block, int stride)

  186. cglobal h264_idct8_add_8_mmx, 3, 4, 0

  187.     %assign pad 128+4-(stack_offset&7)

  188.     SUB         rsp, pad

  189. 

  190.     add   word [r1], 32

  191.     IDCT8_ADD_MMX_START r1  , rsp

  192.     IDCT8_ADD_MMX_START r1+8, rsp+64

  193.     lea          r3, [r0+4]

  194.     IDCT8_ADD_MMX_END   r0  , rsp,   r2

  195.     IDCT8_ADD_MMX_END   r3  , rsp+8, r2

  196. 

  197.     ADD         rsp, pad

  198.     RET

  199. 

  200. ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride

  201. %macro IDCT8_ADD_SSE 4

  202.     IDCT8_1D_FULL %2

  203. %if ARCH_X86_64

  204.     TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, 8

  205. %else

  206.     TRANSPOSE8x8W 0, 1, 2, 3, 4, 5, 6, 7, [%2], [%2+16]

  207. %endif

  208.     paddw        m0, [pw_32]

  209. 

  210. %if ARCH_X86_64 == 0

  211.     mova    [%2   ], m0

  212.     mova    [%2+16], m4

  213.     IDCT8_1D   [%2], [%2+ 16]

  214.     mova    [%2   ], m6

  215.     mova    [%2+16], m7

  216. %else

  217.     SWAP          0, 8

  218.     SWAP          4, 9

  219.     IDCT8_1D     m8, m9

  220.     SWAP          6, 8

  221.     SWAP          7, 9

  222. %endif

  223. 

  224.     pxor         m7, m7

  225.     lea          %4, [%3*3]

  226.     STORE_DIFF   m0, m6, m7, [%1     ]

  227.     STORE_DIFF   m1, m6, m7, [%1+%3  ]

  228.     STORE_DIFF   m2, m6, m7, [%1+%3*2]

  229.     STORE_DIFF   m3, m6, m7, [%1+%4  ]

  230. %if ARCH_X86_64 == 0

  231.     mova         m0, [%2   ]

  232.     mova         m1, [%2+16]

  233. %else

  234.     SWAP          0, 8

  235.     SWAP          1, 9

  236. %endif

  237.     lea          %1, [%1+%3*4]

  238.     STORE_DIFF   m4, m6, m7, [%1     ]

  239.     STORE_DIFF   m5, m6, m7, [%1+%3  ]

  240.     STORE_DIFF   m0, m6, m7, [%1+%3*2]

  241.     STORE_DIFF   m1, m6, m7, [%1+%4  ]

  242. %endmacro

  243. 

  244. INIT_XMM

  245. ; ff_h264_idct8_add_sse2(uint8_t *dst, int16_t *block, int stride)

  246. cglobal h264_idct8_add_8_sse2, 3, 4, 10

  247.     IDCT8_ADD_SSE r0, r1, r2, r3

  248.     RET

  249. 

  250. %macro DC_ADD_MMX2_INIT 2-3

  251. %if %0 == 2

  252.     movsx        %1, word [%1]

  253.     add          %1, 32

  254.     sar          %1, 6

  255.     movd         m0, %1d

  256.     lea          %1, [%2*3]

  257. %else

  258.     add          %3, 32

  259.     sar          %3, 6

  260.     movd         m0, %3d

  261.     lea          %3, [%2*3]

  262. %endif

  263.     pshufw       m0, m0, 0

  264.     pxor         m1, m1

  265.     psubw        m1, m0

  266.     packuswb     m0, m0

  267.     packuswb     m1, m1

  268. %endmacro

  269. 

  270. %macro DC_ADD_MMX2_OP 4

  271.     %1           m2, [%2     ]

  272.     %1           m3, [%2+%3  ]

  273.     %1           m4, [%2+%3*2]

  274.     %1           m5, [%2+%4  ]

  275.     paddusb      m2, m0

  276.     paddusb      m3, m0

  277.     paddusb      m4, m0

  278.     paddusb      m5, m0

  279.     psubusb      m2, m1

  280.     psubusb      m3, m1

  281.     psubusb      m4, m1

  282.     psubusb      m5, m1

  283.     %1    [%2     ], m2

  284.     %1    [%2+%3  ], m3

  285.     %1    [%2+%3*2], m4

  286.     %1    [%2+%4  ], m5

  287. %endmacro

  288. 

  289. INIT_MMX

  290. ; ff_h264_idct_dc_add_mmx2(uint8_t *dst, int16_t *block, int stride)

  291. cglobal h264_idct_dc_add_8_mmx2, 3, 3, 0

  292.     DC_ADD_MMX2_INIT r1, r2

  293.     DC_ADD_MMX2_OP movh, r0, r2, r1

  294.     RET

  295. 

  296. ; ff_h264_idct8_dc_add_mmx2(uint8_t *dst, int16_t *block, int stride)

  297. cglobal h264_idct8_dc_add_8_mmx2, 3, 3, 0

  298.     DC_ADD_MMX2_INIT r1, r2

  299.     DC_ADD_MMX2_OP mova, r0, r2, r1

  300.     lea          r0, [r0+r2*4]

  301.     DC_ADD_MMX2_OP mova, r0, r2, r1

  302.     RET

  303. 

  304. ; ff_h264_idct_add16_mmx(uint8_t *dst, const int *block_offset,

  305. ;             DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  306. cglobal h264_idct_add16_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  307.     xor          r5, r5

  308. %ifdef PIC

  309.     lea     picregq, [scan8_mem]

  310. %endif

  311. .nextblock

  312.     movzx        r6, byte [scan8+r5]

  313.     movzx        r6, byte [r4+r6]

  314.     test         r6, r6

  315.     jz .skipblock

  316.     mov         r6d, dword [r1+r5*4]

  317.     lea          r6, [r0+r6]

  318.     IDCT4_ADD    r6, r2, r3

  319. .skipblock

  320.     inc          r5

  321.     add          r2, 32

  322.     cmp          r5, 16

  323.     jl .nextblock

  324.     REP_RET

  325. 

  326. ; ff_h264_idct8_add4_mmx(uint8_t *dst, const int *block_offset,

  327. ;                        DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  328. cglobal h264_idct8_add4_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  329.     %assign pad 128+4-(stack_offset&7)

  330.     SUB         rsp, pad

  331. 

  332.     xor          r5, r5

  333. %ifdef PIC

  334.     lea     picregq, [scan8_mem]

  335. %endif

  336. .nextblock

  337.     movzx        r6, byte [scan8+r5]

  338.     movzx        r6, byte [r4+r6]

  339.     test         r6, r6

  340.     jz .skipblock

  341.     mov         r6d, dword [r1+r5*4]

  342.     add          r6, r0

  343.     add   word [r2], 32

  344.     IDCT8_ADD_MMX_START r2  , rsp

  345.     IDCT8_ADD_MMX_START r2+8, rsp+64

  346.     IDCT8_ADD_MMX_END   r6  , rsp,   r3

  347.     mov         r6d, dword [r1+r5*4]

  348.     lea          r6, [r0+r6+4]

  349.     IDCT8_ADD_MMX_END   r6  , rsp+8, r3

  350. .skipblock

  351.     add          r5, 4

  352.     add          r2, 128

  353.     cmp          r5, 16

  354.     jl .nextblock

  355.     ADD         rsp, pad

  356.     RET

  357. 

  358. ; ff_h264_idct_add16_mmx2(uint8_t *dst, const int *block_offset,

  359. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  360. cglobal h264_idct_add16_8_mmx2, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  361.     xor          r5, r5

  362. %ifdef PIC

  363.     lea     picregq, [scan8_mem]

  364. %endif

  365. .nextblock

  366.     movzx        r6, byte [scan8+r5]

  367.     movzx        r6, byte [r4+r6]

  368.     test         r6, r6

  369.     jz .skipblock

  370.     cmp          r6, 1

  371.     jnz .no_dc

  372.     movsx        r6, word [r2]

  373.     test         r6, r6

  374.     jz .no_dc

  375.     DC_ADD_MMX2_INIT r2, r3, r6

  376. %if ARCH_X86_64 == 0

  377. %define dst2q r1

  378. %define dst2d r1d

  379. %endif

  380.     mov       dst2d, dword [r1+r5*4]

  381.     lea       dst2q, [r0+dst2q]

  382.     DC_ADD_MMX2_OP movh, dst2q, r3, r6

  383. %if ARCH_X86_64 == 0

  384.     mov          r1, r1m

  385. %endif

  386.     inc          r5

  387.     add          r2, 32

  388.     cmp          r5, 16

  389.     jl .nextblock

  390.     REP_RET

  391. .no_dc

  392.     mov         r6d, dword [r1+r5*4]

  393.     add          r6, r0

  394.     IDCT4_ADD    r6, r2, r3

  395. .skipblock

  396.     inc          r5

  397.     add          r2, 32

  398.     cmp          r5, 16

  399.     jl .nextblock

  400.     REP_RET

  401. 

  402. ; ff_h264_idct_add16intra_mmx(uint8_t *dst, const int *block_offset,

  403. ;                             DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  404. cglobal h264_idct_add16intra_8_mmx, 5, 7 + npicregs, 0, dst, block_offset, block, stride, nnzc, cntr, coeff, picreg

  405.     xor          r5, r5

  406. %ifdef PIC

  407.     lea     picregq, [scan8_mem]

  408. %endif

  409. .nextblock

  410.     movzx        r6, byte [scan8+r5]

  411.     movzx        r6, byte [r4+r6]

  412.     or          r6w, word [r2]

  413.     test         r6, r6

  414.     jz .skipblock

  415.     mov         r6d, dword [r1+r5*4]

  416.     add          r6, r0

  417.     IDCT4_ADD    r6, r2, r3

  418. .skipblock

  419.     inc          r5

  420.     add          r2, 32

  421.     cmp          r5, 16

  422.     jl .nextblock

  423.     REP_RET

  424. 

  425. ; ff_h264_idct_add16intra_mmx2(uint8_t *dst, const int *block_offset,

  426. ;                              DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  427. cglobal h264_idct_add16intra_8_mmx2, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  428.     xor          r5, r5

  429. %ifdef PIC

  430.     lea     picregq, [scan8_mem]

  431. %endif

  432. .nextblock

  433.     movzx        r6, byte [scan8+r5]

  434.     movzx        r6, byte [r4+r6]

  435.     test         r6, r6

  436.     jz .try_dc

  437.     mov         r6d, dword [r1+r5*4]

  438.     lea          r6, [r0+r6]

  439.     IDCT4_ADD    r6, r2, r3

  440.     inc          r5

  441.     add          r2, 32

  442.     cmp          r5, 16

  443.     jl .nextblock

  444.     REP_RET

  445. .try_dc

  446.     movsx        r6, word [r2]

  447.     test         r6, r6

  448.     jz .skipblock

  449.     DC_ADD_MMX2_INIT r2, r3, r6

  450. %if ARCH_X86_64 == 0

  451. %define dst2q r1

  452. %define dst2d r1d

  453. %endif

  454.     mov       dst2d, dword [r1+r5*4]

  455.     add       dst2q, r0

  456.     DC_ADD_MMX2_OP movh, dst2q, r3, r6

  457. %if ARCH_X86_64 == 0

  458.     mov          r1, r1m

  459. %endif

  460. .skipblock

  461.     inc          r5

  462.     add          r2, 32

  463.     cmp          r5, 16

  464.     jl .nextblock

  465.     REP_RET

  466. 

  467. ; ff_h264_idct8_add4_mmx2(uint8_t *dst, const int *block_offset,

  468. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  469. cglobal h264_idct8_add4_8_mmx2, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  470.     %assign pad 128+4-(stack_offset&7)

  471.     SUB         rsp, pad

  472. 

  473.     xor          r5, r5

  474. %ifdef PIC

  475.     lea     picregq, [scan8_mem]

  476. %endif

  477. .nextblock

  478.     movzx        r6, byte [scan8+r5]

  479.     movzx        r6, byte [r4+r6]

  480.     test         r6, r6

  481.     jz .skipblock

  482.     cmp          r6, 1

  483.     jnz .no_dc

  484.     movsx        r6, word [r2]

  485.     test         r6, r6

  486.     jz .no_dc

  487.     DC_ADD_MMX2_INIT r2, r3, r6

  488. %if ARCH_X86_64 == 0

  489. %define dst2q r1

  490. %define dst2d r1d

  491. %endif

  492.     mov       dst2d, dword [r1+r5*4]

  493.     lea       dst2q, [r0+dst2q]

  494.     DC_ADD_MMX2_OP mova, dst2q, r3, r6

  495.     lea       dst2q, [dst2q+r3*4]

  496.     DC_ADD_MMX2_OP mova, dst2q, r3, r6

  497. %if ARCH_X86_64 == 0

  498.     mov          r1, r1m

  499. %endif

  500.     add          r5, 4

  501.     add          r2, 128

  502.     cmp          r5, 16

  503.     jl .nextblock

  504. 

  505.     ADD         rsp, pad

  506.     RET

  507. .no_dc

  508.     mov         r6d, dword [r1+r5*4]

  509.     add          r6, r0

  510.     add   word [r2], 32

  511.     IDCT8_ADD_MMX_START r2  , rsp

  512.     IDCT8_ADD_MMX_START r2+8, rsp+64

  513.     IDCT8_ADD_MMX_END   r6  , rsp,   r3

  514.     mov         r6d, dword [r1+r5*4]

  515.     lea          r6, [r0+r6+4]

  516.     IDCT8_ADD_MMX_END   r6  , rsp+8, r3

  517. .skipblock

  518.     add          r5, 4

  519.     add          r2, 128

  520.     cmp          r5, 16

  521.     jl .nextblock

  522. 

  523.     ADD         rsp, pad

  524.     RET

  525. 

  526. INIT_XMM

  527. ; ff_h264_idct8_add4_sse2(uint8_t *dst, const int *block_offset,

  528. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  529. cglobal h264_idct8_add4_8_sse2, 5, 8 + npicregs, 10, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  530.     xor          r5, r5

  531. %ifdef PIC

  532.     lea     picregq, [scan8_mem]

  533. %endif

  534. .nextblock

  535.     movzx        r6, byte [scan8+r5]

  536.     movzx        r6, byte [r4+r6]

  537.     test         r6, r6

  538.     jz .skipblock

  539.     cmp          r6, 1

  540.     jnz .no_dc

  541.     movsx        r6, word [r2]

  542.     test         r6, r6

  543.     jz .no_dc

  544. INIT_MMX

  545.     DC_ADD_MMX2_INIT r2, r3, r6

  546. %if ARCH_X86_64 == 0

  547. %define dst2q r1

  548. %define dst2d r1d

  549. %endif

  550.     mov       dst2d, dword [r1+r5*4]

  551.     add       dst2q, r0

  552.     DC_ADD_MMX2_OP mova, dst2q, r3, r6

  553.     lea       dst2q, [dst2q+r3*4]

  554.     DC_ADD_MMX2_OP mova, dst2q, r3, r6

  555. %if ARCH_X86_64 == 0

  556.     mov          r1, r1m

  557. %endif

  558.     add          r5, 4

  559.     add          r2, 128

  560.     cmp          r5, 16

  561.     jl .nextblock

  562.     REP_RET

  563. .no_dc

  564. INIT_XMM

  565.     mov       dst2d, dword [r1+r5*4]

  566.     add       dst2q, r0

  567.     IDCT8_ADD_SSE dst2q, r2, r3, r6

  568. %if ARCH_X86_64 == 0

  569.     mov          r1, r1m

  570. %endif

  571. .skipblock

  572.     add          r5, 4

  573.     add          r2, 128

  574.     cmp          r5, 16

  575.     jl .nextblock

  576.     REP_RET

  577. 

  578. INIT_MMX

  579. h264_idct_add8_mmx_plane:

  580. .nextblock

  581.     movzx        r6, byte [scan8+r5]

  582.     movzx        r6, byte [r4+r6]

  583.     or          r6w, word [r2]

  584.     test         r6, r6

  585.     jz .skipblock

  586. %if ARCH_X86_64

  587.     mov         r0d, dword [r1+r5*4]

  588.     add          r0, [dst2q]

  589. %else

  590.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  591.     mov          r0, [r0]

  592.     add          r0, dword [r1+r5*4]

  593. %endif

  594.     IDCT4_ADD    r0, r2, r3

  595. .skipblock

  596.     inc          r5

  597.     add          r2, 32

  598.     test         r5, 3

  599.     jnz .nextblock

  600.     rep ret

  601. 

  602. ; ff_h264_idct_add8_mmx(uint8_t **dest, const int *block_offset,

  603. ;                       DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  604. cglobal h264_idct_add8_8_mmx, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  605.     mov          r5, 16

  606.     add          r2, 512

  607. %ifdef PIC

  608.     lea     picregq, [scan8_mem]

  609. %endif

  610. %if ARCH_X86_64

  611.     mov       dst2q, r0

  612. %endif

  613.     call         h264_idct_add8_mmx_plane

  614.     mov          r5, 32

  615.     add          r2, 384

  616. %if ARCH_X86_64

  617.     add       dst2q, gprsize

  618. %else

  619.     add        r0mp, gprsize

  620. %endif

  621.     call         h264_idct_add8_mmx_plane

  622.     RET

  623. 

  624. h264_idct_add8_mmx2_plane

  625. .nextblock

  626.     movzx        r6, byte [scan8+r5]

  627.     movzx        r6, byte [r4+r6]

  628.     test         r6, r6

  629.     jz .try_dc

  630. %if ARCH_X86_64

  631.     mov         r0d, dword [r1+r5*4]

  632.     add          r0, [dst2q]

  633. %else

  634.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  635.     mov          r0, [r0]

  636.     add          r0, dword [r1+r5*4]

  637. %endif

  638.     IDCT4_ADD    r0, r2, r3

  639.     inc          r5

  640.     add          r2, 32

  641.     test         r5, 3

  642.     jnz .nextblock

  643.     rep ret

  644. .try_dc

  645.     movsx        r6, word [r2]

  646.     test         r6, r6

  647.     jz .skipblock

  648.     DC_ADD_MMX2_INIT r2, r3, r6

  649. %if ARCH_X86_64

  650.     mov         r0d, dword [r1+r5*4]

  651.     add          r0, [dst2q]

  652. %else

  653.     mov          r0, r1m ; XXX r1m here is actually r0m of the calling func

  654.     mov          r0, [r0]

  655.     add          r0, dword [r1+r5*4]

  656. %endif

  657.     DC_ADD_MMX2_OP movh, r0, r3, r6

  658. .skipblock

  659.     inc          r5

  660.     add          r2, 32

  661.     test         r5, 3

  662.     jnz .nextblock

  663.     rep ret

  664. 

  665. ; ff_h264_idct_add8_mmx2(uint8_t **dest, const int *block_offset,

  666. ;                        DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  667. cglobal h264_idct_add8_8_mmx2, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg

  668.     mov          r5, 16

  669.     add          r2, 512

  670. %if ARCH_X86_64

  671.     mov       dst2q, r0

  672. %endif

  673. %ifdef PIC

  674.     lea     picregq, [scan8_mem]

  675. %endif

  676.     call h264_idct_add8_mmx2_plane

  677.     mov          r5, 32

  678.     add          r2, 384

  679. %if ARCH_X86_64

  680.     add       dst2q, gprsize

  681. %else

  682.     add        r0mp, gprsize

  683. %endif

  684.     call h264_idct_add8_mmx2_plane

  685.     RET

  686. 

  687. INIT_MMX

  688. ; r0 = uint8_t *dst, r2 = int16_t *block, r3 = int stride, r6=clobbered

  689. h264_idct_dc_add8_mmx2:

  690.     movd         m0, [r2   ]          ;  0 0 X D

  691.     punpcklwd    m0, [r2+32]          ;  x X d D

  692.     paddsw       m0, [pw_32]

  693.     psraw        m0, 6

  694.     punpcklwd    m0, m0               ;  d d D D

  695.     pxor         m1, m1               ;  0 0 0 0

  696.     psubw        m1, m0               ; -d-d-D-D

  697.     packuswb     m0, m1               ; -d-d-D-D d d D D

  698.     pshufw       m1, m0, 0xFA         ; -d-d-d-d-D-D-D-D

  699.     punpcklwd    m0, m0               ;  d d d d D D D D

  700.     lea          r6, [r3*3]

  701.     DC_ADD_MMX2_OP movq, r0, r3, r6

  702.     ret

  703. 

  704. ALIGN 16

  705. INIT_XMM

  706. ; r0 = uint8_t *dst (clobbered), r2 = int16_t *block, r3 = int stride

  707. x264_add8x4_idct_sse2:

  708.     movq   m0, [r2+ 0]

  709.     movq   m1, [r2+ 8]

  710.     movq   m2, [r2+16]

  711.     movq   m3, [r2+24]

  712.     movhps m0, [r2+32]

  713.     movhps m1, [r2+40]

  714.     movhps m2, [r2+48]

  715.     movhps m3, [r2+56]

  716.     IDCT4_1D w,0,1,2,3,4,5

  717.     TRANSPOSE2x4x4W 0,1,2,3,4

  718.     paddw m0, [pw_32]

  719.     IDCT4_1D w,0,1,2,3,4,5

  720.     pxor  m7, m7

  721.     STORE_DIFFx2 m0, m1, m4, m5, m7, 6, r0, r3

  722.     lea   r0, [r0+r3*2]

  723.     STORE_DIFFx2 m2, m3, m4, m5, m7, 6, r0, r3

  724.     ret

  725. 

  726. %macro add16_sse2_cycle 2

  727.     movzx       r0, word [r4+%2]

  728.     test        r0, r0

  729.     jz .cycle%1end

  730.     mov        r0d, dword [r1+%1*8]

  731. %if ARCH_X86_64

  732.     add         r0, r5

  733. %else

  734.     add         r0, r0m

  735. %endif

  736.     call        x264_add8x4_idct_sse2

  737. .cycle%1end

  738. %if %1 < 7

  739.     add         r2, 64

  740. %endif

  741. %endmacro

  742. 

  743. ; ff_h264_idct_add16_sse2(uint8_t *dst, const int *block_offset,

  744. ;                         DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  745. cglobal h264_idct_add16_8_sse2, 5, 5 + ARCH_X86_64, 8

  746. %if ARCH_X86_64

  747.     mov         r5, r0

  748. %endif

  749.     ; unrolling of the loop leads to an average performance gain of

  750.     ; 20-25%

  751.     add16_sse2_cycle 0, 0xc

  752.     add16_sse2_cycle 1, 0x14

  753.     add16_sse2_cycle 2, 0xe

  754.     add16_sse2_cycle 3, 0x16

  755.     add16_sse2_cycle 4, 0x1c

  756.     add16_sse2_cycle 5, 0x24

  757.     add16_sse2_cycle 6, 0x1e

  758.     add16_sse2_cycle 7, 0x26

  759.     RET

  760. 

  761. %macro add16intra_sse2_cycle 2

  762.     movzx       r0, word [r4+%2]

  763.     test        r0, r0

  764.     jz .try%1dc

  765.     mov        r0d, dword [r1+%1*8]

  766. %if ARCH_X86_64

  767.     add         r0, r7

  768. %else

  769.     add         r0, r0m

  770. %endif

  771.     call        x264_add8x4_idct_sse2

  772.     jmp .cycle%1end

  773. .try%1dc

  774.     movsx       r0, word [r2   ]

  775.     or         r0w, word [r2+32]

  776.     jz .cycle%1end

  777.     mov        r0d, dword [r1+%1*8]

  778. %if ARCH_X86_64

  779.     add         r0, r7

  780. %else

  781.     add         r0, r0m

  782. %endif

  783.     call        h264_idct_dc_add8_mmx2

  784. .cycle%1end

  785. %if %1 < 7

  786.     add         r2, 64

  787. %endif

  788. %endmacro

  789. 

  790. ; ff_h264_idct_add16intra_sse2(uint8_t *dst, const int *block_offset,

  791. ;                              DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  792. cglobal h264_idct_add16intra_8_sse2, 5, 7 + ARCH_X86_64, 8

  793. %if ARCH_X86_64

  794.     mov         r7, r0

  795. %endif

  796.     add16intra_sse2_cycle 0, 0xc

  797.     add16intra_sse2_cycle 1, 0x14

  798.     add16intra_sse2_cycle 2, 0xe

  799.     add16intra_sse2_cycle 3, 0x16

  800.     add16intra_sse2_cycle 4, 0x1c

  801.     add16intra_sse2_cycle 5, 0x24

  802.     add16intra_sse2_cycle 6, 0x1e

  803.     add16intra_sse2_cycle 7, 0x26

  804.     RET

  805. 

  806. %macro add8_sse2_cycle 2

  807.     movzx       r0, word [r4+%2]

  808.     test        r0, r0

  809.     jz .try%1dc

  810. %if ARCH_X86_64

  811.     mov        r0d, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  812.     add         r0, [r7]

  813. %else

  814.     mov         r0, r0m

  815.     mov         r0, [r0]

  816.     add         r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  817. %endif

  818.     call        x264_add8x4_idct_sse2

  819.     jmp .cycle%1end

  820. .try%1dc

  821.     movsx       r0, word [r2   ]

  822.     or         r0w, word [r2+32]

  823.     jz .cycle%1end

  824. %if ARCH_X86_64

  825.     mov        r0d, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  826.     add         r0, [r7]

  827. %else

  828.     mov         r0, r0m

  829.     mov         r0, [r0]

  830.     add         r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]

  831. %endif

  832.     call        h264_idct_dc_add8_mmx2

  833. .cycle%1end

  834. %if %1 == 1

  835.     add         r2, 384+64

  836. %elif %1 < 3

  837.     add         r2, 64

  838. %endif

  839. %endmacro

  840. 

  841. ; ff_h264_idct_add8_sse2(uint8_t **dest, const int *block_offset,

  842. ;                        DCTELEM *block, int stride, const uint8_t nnzc[6*8])

  843. cglobal h264_idct_add8_8_sse2, 5, 7 + ARCH_X86_64, 8

  844.     add          r2, 512

  845. %if ARCH_X86_64

  846.     mov          r7, r0

  847. %endif

  848.     add8_sse2_cycle 0, 0x34

  849.     add8_sse2_cycle 1, 0x3c

  850. %if ARCH_X86_64

  851.     add          r7, gprsize

  852. %else

  853.     add        r0mp, gprsize

  854. %endif

  855.     add8_sse2_cycle 2, 0x5c

  856.     add8_sse2_cycle 3, 0x64

  857.     RET

  858. 

  859. ;void ff_h264_luma_dc_dequant_idct_mmx(DCTELEM *output, DCTELEM *input, int qmul)

  860. 

  861. %macro WALSH4_1D 5

  862.     SUMSUB_BADC w, %4, %3, %2, %1, %5

  863.     SUMSUB_BADC w, %4, %2, %3, %1, %5

  864.     SWAP %1, %4, %3

  865. %endmacro

  866. 

  867. %macro DEQUANT_MMX 3

  868.     mova        m7, [pw_1]

  869.     mova        m4, %1

  870.     punpcklwd   %1, m7

  871.     punpckhwd   m4, m7

  872.     mova        m5, %2

  873.     punpcklwd   %2, m7

  874.     punpckhwd   m5, m7

  875.     movd        m7, t3d

  876.     punpckldq   m7, m7

  877.     pmaddwd     %1, m7

  878.     pmaddwd     %2, m7

  879.     pmaddwd     m4, m7

  880.     pmaddwd     m5, m7

  881.     psrad       %1, %3

  882.     psrad       %2, %3

  883.     psrad       m4, %3

  884.     psrad       m5, %3

  885.     packssdw    %1, m4

  886.     packssdw    %2, m5

  887. %endmacro

  888. 

  889. %macro STORE_WORDS_MMX 5

  890.     movd  t0d, %1

  891.     psrlq  %1, 32

  892.     movd  t1d, %1

  893.     mov [t2+%2*32], t0w

  894.     mov [t2+%4*32], t1w

  895.     shr   t0d, 16

  896.     shr   t1d, 16

  897.     mov [t2+%3*32], t0w

  898.     mov [t2+%5*32], t1w

  899. %endmacro

  900. 

  901. %macro DEQUANT_STORE_MMX 1

  902.     DEQUANT_MMX m0, m1, %1

  903.     STORE_WORDS_MMX m0,  0,  1,  4,  5

  904.     STORE_WORDS_MMX m1,  2,  3,  6,  7

  905. 

  906.     DEQUANT_MMX m2, m3, %1

  907.     STORE_WORDS_MMX m2,  8,  9, 12, 13

  908.     STORE_WORDS_MMX m3, 10, 11, 14, 15

  909. %endmacro

  910. 

  911. %macro STORE_WORDS_SSE 9

  912.     movd  t0d, %1

  913.     psrldq  %1, 4

  914.     movd  t1d, %1

  915.     psrldq  %1, 4

  916.     mov [t2+%2*32], t0w

  917.     mov [t2+%4*32], t1w

  918.     shr   t0d, 16

  919.     shr   t1d, 16

  920.     mov [t2+%3*32], t0w

  921.     mov [t2+%5*32], t1w

  922.     movd  t0d, %1

  923.     psrldq  %1, 4

  924.     movd  t1d, %1

  925.     mov [t2+%6*32], t0w

  926.     mov [t2+%8*32], t1w

  927.     shr   t0d, 16

  928.     shr   t1d, 16

  929.     mov [t2+%7*32], t0w

  930.     mov [t2+%9*32], t1w

  931. %endmacro

  932. 

  933. %macro DEQUANT_STORE_SSE2 1

  934.     movd      xmm4, t3d

  935.     movq      xmm5, [pw_1]

  936.     pshufd    xmm4, xmm4, 0

  937.     movq2dq   xmm0, m0

  938.     movq2dq   xmm1, m1

  939.     movq2dq   xmm2, m2

  940.     movq2dq   xmm3, m3

  941.     punpcklwd xmm0, xmm5

  942.     punpcklwd xmm1, xmm5

  943.     punpcklwd xmm2, xmm5

  944.     punpcklwd xmm3, xmm5

  945.     pmaddwd   xmm0, xmm4

  946.     pmaddwd   xmm1, xmm4

  947.     pmaddwd   xmm2, xmm4

  948.     pmaddwd   xmm3, xmm4

  949.     psrad     xmm0, %1

  950.     psrad     xmm1, %1

  951.     psrad     xmm2, %1

  952.     psrad     xmm3, %1

  953.     packssdw  xmm0, xmm1

  954.     packssdw  xmm2, xmm3

  955.     STORE_WORDS_SSE xmm0,  0,  1,  4,  5,  2,  3,  6,  7

  956.     STORE_WORDS_SSE xmm2,  8,  9, 12, 13, 10, 11, 14, 15

  957. %endmacro

  958. 

  959. %macro IDCT_DC_DEQUANT 2

  960. cglobal h264_luma_dc_dequant_idct_%1, 3,4,%2

  961.     ; manually spill XMM registers for Win64 because

  962.     ; the code here is initialized with INIT_MMX

  963.     WIN64_SPILL_XMM %2

  964.     movq        m3, [r1+24]

  965.     movq        m2, [r1+16]

  966.     movq        m1, [r1+ 8]

  967.     movq        m0, [r1+ 0]

  968.     WALSH4_1D    0,1,2,3,4

  969.     TRANSPOSE4x4W 0,1,2,3,4

  970.     WALSH4_1D    0,1,2,3,4

  971. 

  972. ; shift, tmp, output, qmul

  973. %if WIN64

  974.     DECLARE_REG_TMP 0,3,1,2

  975.     ; we can't avoid this, because r0 is the shift register (ecx) on win64

  976.     xchg        r0, t2

  977. %elif ARCH_X86_64

  978.     DECLARE_REG_TMP 3,1,0,2

  979. %else

  980.     DECLARE_REG_TMP 1,3,0,2

  981. %endif

  982. 

  983.     cmp        t3d, 32767

  984.     jg .big_qmul

  985.     add        t3d, 128 << 16

  986. %ifidn %1,mmx

  987.     DEQUANT_STORE_MMX 8

  988. %else

  989.     DEQUANT_STORE_SSE2 8

  990. %endif

  991.     RET

  992. .big_qmul:

  993.     bsr        t0d, t3d

  994.     add        t3d, 128 << 16

  995.     mov        t1d, 7

  996.     cmp        t0d, t1d

  997.     cmovg      t0d, t1d

  998.     inc        t1d

  999.     shr        t3d, t0b

  1000.     sub        t1d, t0d

  1001. %ifidn %1,mmx

  1002.     movd        m6, t1d

  1003.     DEQUANT_STORE_MMX m6

  1004. %else

  1005.     movd      xmm6, t1d

  1006.     DEQUANT_STORE_SSE2 xmm6

  1007. %endif

  1008.     RET

  1009. %endmacro

  1010. 

  1011. INIT_MMX

  1012. IDCT_DC_DEQUANT mmx, 0

  1013. IDCT_DC_DEQUANT sse2, 7