falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
63427 Commits
32 Branches
411MB
Tree: ad733089b0
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'ad733089b0'
${ noResults }
FFmpeg/libavcodec/x86/dsputilenc_mmx.c

866 lines
26KB
Raw Blame History 

  1. /*

  2.  * MMX optimized DSP utils

  3.  * Copyright (c) 2000, 2001 Fabrice Bellard

  4.  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>

  5.  *

  6.  * MMX optimization by Nick Kurshev <nickols_k@mail.ru>

  7.  *

  8.  * This file is part of FFmpeg.

  9.  *

  10.  * FFmpeg is free software; you can redistribute it and/or

  11.  * modify it under the terms of the GNU Lesser General Public

  12.  * License as published by the Free Software Foundation; either

  13.  * version 2.1 of the License, or (at your option) any later version.

  14.  *

  15.  * FFmpeg is distributed in the hope that it will be useful,

  16.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  17.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

  18.  * Lesser General Public License for more details.

  19.  *

  20.  * You should have received a copy of the GNU Lesser General Public

  21.  * License along with FFmpeg; if not, write to the Free Software

  22.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

  23.  */

  24. 

  25. #include "libavutil/attributes.h"

  26. #include "libavutil/cpu.h"

  27. #include "libavutil/x86/asm.h"

  28. #include "libavutil/x86/cpu.h"

  29. #include "libavcodec/dct.h"

  30. #include "libavcodec/dsputil.h"

  31. #include "libavcodec/mpegvideo.h"

  32. #include "libavcodec/mathops.h"

  33. #include "dsputil_x86.h"

  34. 

  35. void ff_get_pixels_mmx(int16_t *block, const uint8_t *pixels, int line_size);

  36. void ff_get_pixels_sse2(int16_t *block, const uint8_t *pixels, int line_size);

  37. void ff_diff_pixels_mmx(int16_t *block, const uint8_t *s1, const uint8_t *s2,

  38.                         int stride);

  39. int ff_pix_sum16_mmx(uint8_t *pix, int line_size);

  40. int ff_pix_norm1_mmx(uint8_t *pix, int line_size);

  41. int ff_sum_abs_dctelem_mmx(int16_t *block);

  42. int ff_sum_abs_dctelem_mmxext(int16_t *block);

  43. int ff_sum_abs_dctelem_sse2(int16_t *block);

  44. int ff_sum_abs_dctelem_ssse3(int16_t *block);

  45. int ff_sse8_mmx(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,

  46.                 int line_size, int h);

  47. int ff_sse16_mmx(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,

  48.                  int line_size, int h);

  49. 

  50. #if HAVE_INLINE_ASM

  51. 

  52. static int hf_noise8_mmx(uint8_t *pix1, int line_size, int h)

  53. {

  54.     int tmp;

  55. 

  56.     __asm__ volatile (

  57.         "movl %3, %%ecx\n"

  58.         "pxor %%mm7, %%mm7\n"

  59.         "pxor %%mm6, %%mm6\n"

  60. 

  61.         "movq (%0), %%mm0\n"

  62.         "movq %%mm0, %%mm1\n"

  63.         "psllq $8, %%mm0\n"

  64.         "psrlq $8, %%mm1\n"

  65.         "psrlq $8, %%mm0\n"

  66.         "movq %%mm0, %%mm2\n"

  67.         "movq %%mm1, %%mm3\n"

  68.         "punpcklbw %%mm7, %%mm0\n"

  69.         "punpcklbw %%mm7, %%mm1\n"

  70.         "punpckhbw %%mm7, %%mm2\n"

  71.         "punpckhbw %%mm7, %%mm3\n"

  72.         "psubw %%mm1, %%mm0\n"

  73.         "psubw %%mm3, %%mm2\n"

  74. 

  75.         "add %2, %0\n"

  76. 

  77.         "movq (%0), %%mm4\n"

  78.         "movq %%mm4, %%mm1\n"

  79.         "psllq $8, %%mm4\n"

  80.         "psrlq $8, %%mm1\n"

  81.         "psrlq $8, %%mm4\n"

  82.         "movq %%mm4, %%mm5\n"

  83.         "movq %%mm1, %%mm3\n"

  84.         "punpcklbw %%mm7, %%mm4\n"

  85.         "punpcklbw %%mm7, %%mm1\n"

  86.         "punpckhbw %%mm7, %%mm5\n"

  87.         "punpckhbw %%mm7, %%mm3\n"

  88.         "psubw %%mm1, %%mm4\n"

  89.         "psubw %%mm3, %%mm5\n"

  90.         "psubw %%mm4, %%mm0\n"

  91.         "psubw %%mm5, %%mm2\n"

  92.         "pxor %%mm3, %%mm3\n"

  93.         "pxor %%mm1, %%mm1\n"

  94.         "pcmpgtw %%mm0, %%mm3\n\t"

  95.         "pcmpgtw %%mm2, %%mm1\n\t"

  96.         "pxor %%mm3, %%mm0\n"

  97.         "pxor %%mm1, %%mm2\n"

  98.         "psubw %%mm3, %%mm0\n"

  99.         "psubw %%mm1, %%mm2\n"

  100.         "paddw %%mm0, %%mm2\n"

  101.         "paddw %%mm2, %%mm6\n"

  102. 

  103.         "add %2, %0\n"

  104.         "1:\n"

  105. 

  106.         "movq (%0), %%mm0\n"

  107.         "movq %%mm0, %%mm1\n"

  108.         "psllq $8, %%mm0\n"

  109.         "psrlq $8, %%mm1\n"

  110.         "psrlq $8, %%mm0\n"

  111.         "movq %%mm0, %%mm2\n"

  112.         "movq %%mm1, %%mm3\n"

  113.         "punpcklbw %%mm7, %%mm0\n"

  114.         "punpcklbw %%mm7, %%mm1\n"

  115.         "punpckhbw %%mm7, %%mm2\n"

  116.         "punpckhbw %%mm7, %%mm3\n"

  117.         "psubw %%mm1, %%mm0\n"

  118.         "psubw %%mm3, %%mm2\n"

  119.         "psubw %%mm0, %%mm4\n"

  120.         "psubw %%mm2, %%mm5\n"

  121.         "pxor  %%mm3, %%mm3\n"

  122.         "pxor  %%mm1, %%mm1\n"

  123.         "pcmpgtw %%mm4, %%mm3\n\t"

  124.         "pcmpgtw %%mm5, %%mm1\n\t"

  125.         "pxor  %%mm3, %%mm4\n"

  126.         "pxor  %%mm1, %%mm5\n"

  127.         "psubw %%mm3, %%mm4\n"

  128.         "psubw %%mm1, %%mm5\n"

  129.         "paddw %%mm4, %%mm5\n"

  130.         "paddw %%mm5, %%mm6\n"

  131. 

  132.         "add %2, %0\n"

  133. 

  134.         "movq (%0), %%mm4\n"

  135.         "movq      %%mm4, %%mm1\n"

  136.         "psllq $8, %%mm4\n"

  137.         "psrlq $8, %%mm1\n"

  138.         "psrlq $8, %%mm4\n"

  139.         "movq      %%mm4, %%mm5\n"

  140.         "movq      %%mm1, %%mm3\n"

  141.         "punpcklbw %%mm7, %%mm4\n"

  142.         "punpcklbw %%mm7, %%mm1\n"

  143.         "punpckhbw %%mm7, %%mm5\n"

  144.         "punpckhbw %%mm7, %%mm3\n"

  145.         "psubw     %%mm1, %%mm4\n"

  146.         "psubw     %%mm3, %%mm5\n"

  147.         "psubw     %%mm4, %%mm0\n"

  148.         "psubw     %%mm5, %%mm2\n"

  149.         "pxor      %%mm3, %%mm3\n"

  150.         "pxor      %%mm1, %%mm1\n"

  151.         "pcmpgtw   %%mm0, %%mm3\n\t"

  152.         "pcmpgtw   %%mm2, %%mm1\n\t"

  153.         "pxor      %%mm3, %%mm0\n"

  154.         "pxor      %%mm1, %%mm2\n"

  155.         "psubw     %%mm3, %%mm0\n"

  156.         "psubw     %%mm1, %%mm2\n"

  157.         "paddw     %%mm0, %%mm2\n"

  158.         "paddw     %%mm2, %%mm6\n"

  159. 

  160.         "add  %2, %0\n"

  161.         "subl $2, %%ecx\n"

  162.         " jnz 1b\n"

  163. 

  164.         "movq      %%mm6, %%mm0\n"

  165.         "punpcklwd %%mm7, %%mm0\n"

  166.         "punpckhwd %%mm7, %%mm6\n"

  167.         "paddd     %%mm0, %%mm6\n"

  168. 

  169.         "movq  %%mm6, %%mm0\n"

  170.         "psrlq $32,   %%mm6\n"

  171.         "paddd %%mm6, %%mm0\n"

  172.         "movd  %%mm0, %1\n"

  173.         : "+r" (pix1), "=r" (tmp)

  174.         : "r" ((x86_reg) line_size), "g" (h - 2)

  175.         : "%ecx");

  176. 

  177.     return tmp;

  178. }

  179. 

  180. static int hf_noise16_mmx(uint8_t *pix1, int line_size, int h)

  181. {

  182.     int tmp;

  183.     uint8_t *pix = pix1;

  184. 

  185.     __asm__ volatile (

  186.         "movl %3, %%ecx\n"

  187.         "pxor %%mm7, %%mm7\n"

  188.         "pxor %%mm6, %%mm6\n"

  189. 

  190.         "movq (%0), %%mm0\n"

  191.         "movq 1(%0), %%mm1\n"

  192.         "movq %%mm0, %%mm2\n"

  193.         "movq %%mm1, %%mm3\n"

  194.         "punpcklbw %%mm7, %%mm0\n"

  195.         "punpcklbw %%mm7, %%mm1\n"

  196.         "punpckhbw %%mm7, %%mm2\n"

  197.         "punpckhbw %%mm7, %%mm3\n"

  198.         "psubw %%mm1, %%mm0\n"

  199.         "psubw %%mm3, %%mm2\n"

  200. 

  201.         "add %2, %0\n"

  202. 

  203.         "movq (%0), %%mm4\n"

  204.         "movq 1(%0), %%mm1\n"

  205.         "movq %%mm4, %%mm5\n"

  206.         "movq %%mm1, %%mm3\n"

  207.         "punpcklbw %%mm7, %%mm4\n"

  208.         "punpcklbw %%mm7, %%mm1\n"

  209.         "punpckhbw %%mm7, %%mm5\n"

  210.         "punpckhbw %%mm7, %%mm3\n"

  211.         "psubw %%mm1, %%mm4\n"

  212.         "psubw %%mm3, %%mm5\n"

  213.         "psubw %%mm4, %%mm0\n"

  214.         "psubw %%mm5, %%mm2\n"

  215.         "pxor %%mm3, %%mm3\n"

  216.         "pxor %%mm1, %%mm1\n"

  217.         "pcmpgtw %%mm0, %%mm3\n\t"

  218.         "pcmpgtw %%mm2, %%mm1\n\t"

  219.         "pxor %%mm3, %%mm0\n"

  220.         "pxor %%mm1, %%mm2\n"

  221.         "psubw %%mm3, %%mm0\n"

  222.         "psubw %%mm1, %%mm2\n"

  223.         "paddw %%mm0, %%mm2\n"

  224.         "paddw %%mm2, %%mm6\n"

  225. 

  226.         "add %2, %0\n"

  227.         "1:\n"

  228. 

  229.         "movq (%0), %%mm0\n"

  230.         "movq 1(%0), %%mm1\n"

  231.         "movq %%mm0, %%mm2\n"

  232.         "movq %%mm1, %%mm3\n"

  233.         "punpcklbw %%mm7, %%mm0\n"

  234.         "punpcklbw %%mm7, %%mm1\n"

  235.         "punpckhbw %%mm7, %%mm2\n"

  236.         "punpckhbw %%mm7, %%mm3\n"

  237.         "psubw %%mm1, %%mm0\n"

  238.         "psubw %%mm3, %%mm2\n"

  239.         "psubw %%mm0, %%mm4\n"

  240.         "psubw %%mm2, %%mm5\n"

  241.         "pxor %%mm3, %%mm3\n"

  242.         "pxor %%mm1, %%mm1\n"

  243.         "pcmpgtw %%mm4, %%mm3\n\t"

  244.         "pcmpgtw %%mm5, %%mm1\n\t"

  245.         "pxor %%mm3, %%mm4\n"

  246.         "pxor %%mm1, %%mm5\n"

  247.         "psubw %%mm3, %%mm4\n"

  248.         "psubw %%mm1, %%mm5\n"

  249.         "paddw %%mm4, %%mm5\n"

  250.         "paddw %%mm5, %%mm6\n"

  251. 

  252.         "add %2, %0\n"

  253. 

  254.         "movq (%0), %%mm4\n"

  255.         "movq 1(%0), %%mm1\n"

  256.         "movq %%mm4, %%mm5\n"

  257.         "movq %%mm1, %%mm3\n"

  258.         "punpcklbw %%mm7, %%mm4\n"

  259.         "punpcklbw %%mm7, %%mm1\n"

  260.         "punpckhbw %%mm7, %%mm5\n"

  261.         "punpckhbw %%mm7, %%mm3\n"

  262.         "psubw %%mm1, %%mm4\n"

  263.         "psubw %%mm3, %%mm5\n"

  264.         "psubw %%mm4, %%mm0\n"

  265.         "psubw %%mm5, %%mm2\n"

  266.         "pxor %%mm3, %%mm3\n"

  267.         "pxor %%mm1, %%mm1\n"

  268.         "pcmpgtw %%mm0, %%mm3\n\t"

  269.         "pcmpgtw %%mm2, %%mm1\n\t"

  270.         "pxor %%mm3, %%mm0\n"

  271.         "pxor %%mm1, %%mm2\n"

  272.         "psubw %%mm3, %%mm0\n"

  273.         "psubw %%mm1, %%mm2\n"

  274.         "paddw %%mm0, %%mm2\n"

  275.         "paddw %%mm2, %%mm6\n"

  276. 

  277.         "add %2, %0\n"

  278.         "subl $2, %%ecx\n"

  279.         " jnz 1b\n"

  280. 

  281.         "movq %%mm6, %%mm0\n"

  282.         "punpcklwd %%mm7, %%mm0\n"

  283.         "punpckhwd %%mm7, %%mm6\n"

  284.         "paddd %%mm0, %%mm6\n"

  285. 

  286.         "movq %%mm6, %%mm0\n"

  287.         "psrlq $32, %%mm6\n"

  288.         "paddd %%mm6, %%mm0\n"

  289.         "movd %%mm0, %1\n"

  290.         : "+r" (pix1), "=r" (tmp)

  291.         : "r" ((x86_reg) line_size), "g" (h - 2)

  292.         : "%ecx");

  293. 

  294.     return tmp + hf_noise8_mmx(pix + 8, line_size, h);

  295. }

  296. 

  297. static int nsse16_mmx(MpegEncContext *c, uint8_t *pix1, uint8_t *pix2,

  298.                       int line_size, int h)

  299. {

  300.     int score1, score2;

  301. 

  302.     if (c)

  303.         score1 = c->dsp.sse[0](c, pix1, pix2, line_size, h);

  304.     else

  305.         score1 = ff_sse16_mmx(c, pix1, pix2, line_size, h);

  306.     score2 = hf_noise16_mmx(pix1, line_size, h) -

  307.              hf_noise16_mmx(pix2, line_size, h);

  308. 

  309.     if (c)

  310.         return score1 + FFABS(score2) * c->avctx->nsse_weight;

  311.     else

  312.         return score1 + FFABS(score2) * 8;

  313. }

  314. 

  315. static int nsse8_mmx(MpegEncContext *c, uint8_t *pix1, uint8_t *pix2,

  316.                      int line_size, int h)

  317. {

  318.     int score1 = ff_sse8_mmx(c, pix1, pix2, line_size, h);

  319.     int score2 = hf_noise8_mmx(pix1, line_size, h) -

  320.                  hf_noise8_mmx(pix2, line_size, h);

  321. 

  322.     if (c)

  323.         return score1 + FFABS(score2) * c->avctx->nsse_weight;

  324.     else

  325.         return score1 + FFABS(score2) * 8;

  326. }

  327. 

  328. static int vsad_intra16_mmx(MpegEncContext *v, uint8_t *pix, uint8_t *dummy,

  329.                             int line_size, int h)

  330. {

  331.     int tmp;

  332. 

  333.     av_assert2((((int) pix) & 7) == 0);

  334.     av_assert2((line_size & 7) == 0);

  335. 

  336. #define SUM(in0, in1, out0, out1)               \

  337.     "movq (%0), %%mm2\n"                        \

  338.     "movq 8(%0), %%mm3\n"                       \

  339.     "add %2,%0\n"                               \

  340.     "movq %%mm2, " #out0 "\n"                   \

  341.     "movq %%mm3, " #out1 "\n"                   \

  342.     "psubusb " #in0 ", %%mm2\n"                 \

  343.     "psubusb " #in1 ", %%mm3\n"                 \

  344.     "psubusb " #out0 ", " #in0 "\n"             \

  345.     "psubusb " #out1 ", " #in1 "\n"             \

  346.     "por %%mm2, " #in0 "\n"                     \

  347.     "por %%mm3, " #in1 "\n"                     \

  348.     "movq " #in0 ", %%mm2\n"                    \

  349.     "movq " #in1 ", %%mm3\n"                    \

  350.     "punpcklbw %%mm7, " #in0 "\n"               \

  351.     "punpcklbw %%mm7, " #in1 "\n"               \

  352.     "punpckhbw %%mm7, %%mm2\n"                  \

  353.     "punpckhbw %%mm7, %%mm3\n"                  \

  354.     "paddw " #in1 ", " #in0 "\n"                \

  355.     "paddw %%mm3, %%mm2\n"                      \

  356.     "paddw %%mm2, " #in0 "\n"                   \

  357.     "paddw " #in0 ", %%mm6\n"

  358. 

  359. 

  360.     __asm__ volatile (

  361.         "movl    %3, %%ecx\n"

  362.         "pxor %%mm6, %%mm6\n"

  363.         "pxor %%mm7, %%mm7\n"

  364.         "movq  (%0), %%mm0\n"

  365.         "movq 8(%0), %%mm1\n"

  366.         "add %2, %0\n"

  367.         "jmp 2f\n"

  368.         "1:\n"

  369. 

  370.         SUM(%%mm4, %%mm5, %%mm0, %%mm1)

  371.         "2:\n"

  372.         SUM(%%mm0, %%mm1, %%mm4, %%mm5)

  373. 

  374.         "subl $2, %%ecx\n"

  375.         "jnz 1b\n"

  376. 

  377.         "movq  %%mm6, %%mm0\n"

  378.         "psrlq $32,   %%mm6\n"

  379.         "paddw %%mm6, %%mm0\n"

  380.         "movq  %%mm0, %%mm6\n"

  381.         "psrlq $16,   %%mm0\n"

  382.         "paddw %%mm6, %%mm0\n"

  383.         "movd  %%mm0, %1\n"

  384.         : "+r" (pix), "=r" (tmp)

  385.         : "r" ((x86_reg) line_size), "m" (h)

  386.         : "%ecx");

  387. 

  388.     return tmp & 0xFFFF;

  389. }

  390. #undef SUM

  391. 

  392. static int vsad_intra16_mmxext(MpegEncContext *v, uint8_t *pix, uint8_t *dummy,

  393.                                int line_size, int h)

  394. {

  395.     int tmp;

  396. 

  397.     av_assert2((((int) pix) & 7) == 0);

  398.     av_assert2((line_size & 7) == 0);

  399. 

  400. #define SUM(in0, in1, out0, out1)               \

  401.     "movq (%0), " #out0 "\n"                    \

  402.     "movq 8(%0), " #out1 "\n"                   \

  403.     "add %2, %0\n"                              \

  404.     "psadbw " #out0 ", " #in0 "\n"              \

  405.     "psadbw " #out1 ", " #in1 "\n"              \

  406.     "paddw " #in1 ", " #in0 "\n"                \

  407.     "paddw " #in0 ", %%mm6\n"

  408. 

  409.     __asm__ volatile (

  410.         "movl %3, %%ecx\n"

  411.         "pxor %%mm6, %%mm6\n"

  412.         "pxor %%mm7, %%mm7\n"

  413.         "movq (%0), %%mm0\n"

  414.         "movq 8(%0), %%mm1\n"

  415.         "add %2, %0\n"

  416.         "jmp 2f\n"

  417.         "1:\n"

  418. 

  419.         SUM(%%mm4, %%mm5, %%mm0, %%mm1)

  420.         "2:\n"

  421.         SUM(%%mm0, %%mm1, %%mm4, %%mm5)

  422. 

  423.         "subl $2, %%ecx\n"

  424.         "jnz 1b\n"

  425. 

  426.         "movd %%mm6, %1\n"

  427.         : "+r" (pix), "=r" (tmp)

  428.         : "r" ((x86_reg) line_size), "m" (h)

  429.         : "%ecx");

  430. 

  431.     return tmp;

  432. }

  433. #undef SUM

  434. 

  435. static int vsad16_mmx(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,

  436.                       int line_size, int h)

  437. {

  438.     int tmp;

  439. 

  440.     av_assert2((((int) pix1) & 7) == 0);

  441.     av_assert2((((int) pix2) & 7) == 0);

  442.     av_assert2((line_size & 7) == 0);

  443. 

  444. #define SUM(in0, in1, out0, out1)       \

  445.     "movq (%0), %%mm2\n"                \

  446.     "movq (%1), " #out0 "\n"            \

  447.     "movq 8(%0), %%mm3\n"               \

  448.     "movq 8(%1), " #out1 "\n"           \

  449.     "add %3, %0\n"                      \

  450.     "add %3, %1\n"                      \

  451.     "psubb " #out0 ", %%mm2\n"          \

  452.     "psubb " #out1 ", %%mm3\n"          \

  453.     "pxor %%mm7, %%mm2\n"               \

  454.     "pxor %%mm7, %%mm3\n"               \

  455.     "movq %%mm2, " #out0 "\n"           \

  456.     "movq %%mm3, " #out1 "\n"           \

  457.     "psubusb " #in0 ", %%mm2\n"         \

  458.     "psubusb " #in1 ", %%mm3\n"         \

  459.     "psubusb " #out0 ", " #in0 "\n"     \

  460.     "psubusb " #out1 ", " #in1 "\n"     \

  461.     "por %%mm2, " #in0 "\n"             \

  462.     "por %%mm3, " #in1 "\n"             \

  463.     "movq " #in0 ", %%mm2\n"            \

  464.     "movq " #in1 ", %%mm3\n"            \

  465.     "punpcklbw %%mm7, " #in0 "\n"       \

  466.     "punpcklbw %%mm7, " #in1 "\n"       \

  467.     "punpckhbw %%mm7, %%mm2\n"          \

  468.     "punpckhbw %%mm7, %%mm3\n"          \

  469.     "paddw " #in1 ", " #in0 "\n"        \

  470.     "paddw %%mm3, %%mm2\n"              \

  471.     "paddw %%mm2, " #in0 "\n"           \

  472.     "paddw " #in0 ", %%mm6\n"

  473. 

  474. 

  475.     __asm__ volatile (

  476.         "movl %4, %%ecx\n"

  477.         "pxor %%mm6, %%mm6\n"

  478.         "pcmpeqw %%mm7, %%mm7\n"

  479.         "psllw $15, %%mm7\n"

  480.         "packsswb %%mm7, %%mm7\n"

  481.         "movq (%0), %%mm0\n"

  482.         "movq (%1), %%mm2\n"

  483.         "movq 8(%0), %%mm1\n"

  484.         "movq 8(%1), %%mm3\n"

  485.         "add %3, %0\n"

  486.         "add %3, %1\n"

  487.         "psubb %%mm2, %%mm0\n"

  488.         "psubb %%mm3, %%mm1\n"

  489.         "pxor %%mm7, %%mm0\n"

  490.         "pxor %%mm7, %%mm1\n"

  491.         "jmp 2f\n"

  492.         "1:\n"

  493. 

  494.         SUM(%%mm4, %%mm5, %%mm0, %%mm1)

  495.         "2:\n"

  496.         SUM(%%mm0, %%mm1, %%mm4, %%mm5)

  497. 

  498.         "subl $2, %%ecx\n"

  499.         "jnz 1b\n"

  500. 

  501.         "movq %%mm6, %%mm0\n"

  502.         "psrlq $32, %%mm6\n"

  503.         "paddw %%mm6, %%mm0\n"

  504.         "movq %%mm0, %%mm6\n"

  505.         "psrlq $16, %%mm0\n"

  506.         "paddw %%mm6, %%mm0\n"

  507.         "movd %%mm0, %2\n"

  508.         : "+r" (pix1), "+r" (pix2), "=r" (tmp)

  509.         : "r" ((x86_reg) line_size), "m" (h)

  510.         : "%ecx");

  511. 

  512.     return tmp & 0x7FFF;

  513. }

  514. #undef SUM

  515. 

  516. static int vsad16_mmxext(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,

  517.                          int line_size, int h)

  518. {

  519.     int tmp;

  520. 

  521.     av_assert2((((int) pix1) & 7) == 0);

  522.     av_assert2((((int) pix2) & 7) == 0);

  523.     av_assert2((line_size & 7) == 0);

  524. 

  525. #define SUM(in0, in1, out0, out1)               \

  526.     "movq (%0), " #out0 "\n"                    \

  527.     "movq (%1), %%mm2\n"                        \

  528.     "movq 8(%0), " #out1 "\n"                   \

  529.     "movq 8(%1), %%mm3\n"                       \

  530.     "add %3, %0\n"                              \

  531.     "add %3, %1\n"                              \

  532.     "psubb %%mm2, " #out0 "\n"                  \

  533.     "psubb %%mm3, " #out1 "\n"                  \

  534.     "pxor %%mm7, " #out0 "\n"                   \

  535.     "pxor %%mm7, " #out1 "\n"                   \

  536.     "psadbw " #out0 ", " #in0 "\n"              \

  537.     "psadbw " #out1 ", " #in1 "\n"              \

  538.     "paddw " #in1 ", " #in0 "\n"                \

  539.     "paddw " #in0 ", %%mm6\n    "

  540. 

  541.     __asm__ volatile (

  542.         "movl %4, %%ecx\n"

  543.         "pxor %%mm6, %%mm6\n"

  544.         "pcmpeqw %%mm7, %%mm7\n"

  545.         "psllw $15, %%mm7\n"

  546.         "packsswb %%mm7, %%mm7\n"

  547.         "movq (%0), %%mm0\n"

  548.         "movq (%1), %%mm2\n"

  549.         "movq 8(%0), %%mm1\n"

  550.         "movq 8(%1), %%mm3\n"

  551.         "add %3, %0\n"

  552.         "add %3, %1\n"

  553.         "psubb %%mm2, %%mm0\n"

  554.         "psubb %%mm3, %%mm1\n"

  555.         "pxor %%mm7, %%mm0\n"

  556.         "pxor %%mm7, %%mm1\n"

  557.         "jmp 2f\n"

  558.         "1:\n"

  559. 

  560.         SUM(%%mm4, %%mm5, %%mm0, %%mm1)

  561.         "2:\n"

  562.         SUM(%%mm0, %%mm1, %%mm4, %%mm5)

  563. 

  564.         "subl $2, %%ecx\n"

  565.         "jnz 1b\n"

  566. 

  567.         "movd %%mm6, %2\n"

  568.         : "+r" (pix1), "+r" (pix2), "=r" (tmp)

  569.         : "r" ((x86_reg) line_size), "m" (h)

  570.         : "%ecx");

  571. 

  572.     return tmp;

  573. }

  574. #undef SUM

  575. 

  576. static void diff_bytes_mmx(uint8_t *dst, const uint8_t *src1, const uint8_t *src2, int w)

  577. {

  578.     x86_reg i = 0;

  579. 

  580.     if (w >= 16)

  581.     __asm__ volatile (

  582.         "1:                             \n\t"

  583.         "movq  (%2, %0), %%mm0          \n\t"

  584.         "movq  (%1, %0), %%mm1          \n\t"

  585.         "psubb %%mm0, %%mm1             \n\t"

  586.         "movq %%mm1, (%3, %0)           \n\t"

  587.         "movq 8(%2, %0), %%mm0          \n\t"

  588.         "movq 8(%1, %0), %%mm1          \n\t"

  589.         "psubb %%mm0, %%mm1             \n\t"

  590.         "movq %%mm1, 8(%3, %0)          \n\t"

  591.         "add $16, %0                    \n\t"

  592.         "cmp %4, %0                     \n\t"

  593.         " jb 1b                         \n\t"

  594.         : "+r" (i)

  595.         : "r" (src1), "r" (src2), "r" (dst), "r" ((x86_reg) w - 15));

  596. 

  597.     for (; i < w; i++)

  598.         dst[i + 0] = src1[i + 0] - src2[i + 0];

  599. }

  600. 

  601. static void sub_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *src1,

  602.                                               const uint8_t *src2, int w,

  603.                                               int *left, int *left_top)

  604. {

  605.     x86_reg i = 0;

  606.     uint8_t l, lt;

  607. 

  608.     __asm__ volatile (

  609.         "movq  (%1, %0), %%mm0          \n\t" // LT

  610.         "psllq $8, %%mm0                \n\t"

  611.         "1:                             \n\t"

  612.         "movq  (%1, %0), %%mm1          \n\t" // T

  613.         "movq  -1(%2, %0), %%mm2        \n\t" // L

  614.         "movq  (%2, %0), %%mm3          \n\t" // X

  615.         "movq %%mm2, %%mm4              \n\t" // L

  616.         "psubb %%mm0, %%mm2             \n\t"

  617.         "paddb %%mm1, %%mm2             \n\t" // L + T - LT

  618.         "movq %%mm4, %%mm5              \n\t" // L

  619.         "pmaxub %%mm1, %%mm4            \n\t" // max(T, L)

  620.         "pminub %%mm5, %%mm1            \n\t" // min(T, L)

  621.         "pminub %%mm2, %%mm4            \n\t"

  622.         "pmaxub %%mm1, %%mm4            \n\t"

  623.         "psubb %%mm4, %%mm3             \n\t" // dst - pred

  624.         "movq %%mm3, (%3, %0)           \n\t"

  625.         "add $8, %0                     \n\t"

  626.         "movq -1(%1, %0), %%mm0         \n\t" // LT

  627.         "cmp %4, %0                     \n\t"

  628.         " jb 1b                         \n\t"

  629.         : "+r" (i)

  630.         : "r" (src1), "r" (src2), "r" (dst), "r" ((x86_reg) w));

  631. 

  632.     l  = *left;

  633.     lt = *left_top;

  634. 

  635.     dst[0] = src2[0] - mid_pred(l, src1[0], (l + src1[0] - lt) & 0xFF);

  636. 

  637.     *left_top = src1[w - 1];

  638.     *left     = src2[w - 1];

  639. }

  640. 

  641. static int ssd_int8_vs_int16_mmx(const int8_t *pix1, const int16_t *pix2,

  642.                                  int size)

  643. {

  644.     int sum;

  645.     x86_reg i = size;

  646. 

  647.     __asm__ volatile (

  648.         "pxor %%mm4, %%mm4 \n"

  649.         "1: \n"

  650.         "sub $8, %0 \n"

  651.         "movq (%2, %0), %%mm2 \n"

  652.         "movq (%3, %0, 2), %%mm0 \n"

  653.         "movq 8(%3, %0, 2), %%mm1 \n"

  654.         "punpckhbw %%mm2, %%mm3 \n"

  655.         "punpcklbw %%mm2, %%mm2 \n"

  656.         "psraw $8, %%mm3 \n"

  657.         "psraw $8, %%mm2 \n"

  658.         "psubw %%mm3, %%mm1 \n"

  659.         "psubw %%mm2, %%mm0 \n"

  660.         "pmaddwd %%mm1, %%mm1 \n"

  661.         "pmaddwd %%mm0, %%mm0 \n"

  662.         "paddd %%mm1, %%mm4 \n"

  663.         "paddd %%mm0, %%mm4 \n"

  664.         "jg 1b \n"

  665.         "movq %%mm4, %%mm3 \n"

  666.         "psrlq $32, %%mm3 \n"

  667.         "paddd %%mm3, %%mm4 \n"

  668.         "movd %%mm4, %1 \n"

  669.         : "+r" (i), "=r" (sum)

  670.         : "r" (pix1), "r" (pix2));

  671. 

  672.     return sum;

  673. }

  674. 

  675. #define PHADDD(a, t)                            \

  676.     "movq  " #a ", " #t "               \n\t"   \

  677.     "psrlq    $32, " #a "               \n\t"   \

  678.     "paddd " #t ", " #a "               \n\t"

  679. 

  680. /*

  681.  * pmulhw:   dst[0 - 15] = (src[0 - 15] * dst[0 - 15])[16 - 31]

  682.  * pmulhrw:  dst[0 - 15] = (src[0 - 15] * dst[0 - 15] + 0x8000)[16 - 31]

  683.  * pmulhrsw: dst[0 - 15] = (src[0 - 15] * dst[0 - 15] + 0x4000)[15 - 30]

  684.  */

  685. #define PMULHRW(x, y, s, o)                     \

  686.     "pmulhw " #s ", " #x "              \n\t"   \

  687.     "pmulhw " #s ", " #y "              \n\t"   \

  688.     "paddw  " #o ", " #x "              \n\t"   \

  689.     "paddw  " #o ", " #y "              \n\t"   \

  690.     "psraw      $1, " #x "              \n\t"   \

  691.     "psraw      $1, " #y "              \n\t"

  692. #define DEF(x) x ## _mmx

  693. #define SET_RND MOVQ_WONE

  694. #define SCALE_OFFSET 1

  695. 

  696. #include "dsputil_qns_template.c"

  697. 

  698. #undef DEF

  699. #undef SET_RND

  700. #undef SCALE_OFFSET

  701. #undef PMULHRW

  702. 

  703. #define DEF(x) x ## _3dnow

  704. #define SET_RND(x)

  705. #define SCALE_OFFSET 0

  706. #define PMULHRW(x, y, s, o)                     \

  707.     "pmulhrw " #s ", " #x "             \n\t"   \

  708.     "pmulhrw " #s ", " #y "             \n\t"

  709. 

  710. #include "dsputil_qns_template.c"

  711. 

  712. #undef DEF

  713. #undef SET_RND

  714. #undef SCALE_OFFSET

  715. #undef PMULHRW

  716. 

  717. #if HAVE_SSSE3_INLINE

  718. #undef PHADDD

  719. #define DEF(x) x ## _ssse3

  720. #define SET_RND(x)

  721. #define SCALE_OFFSET -1

  722. 

  723. #define PHADDD(a, t)                            \

  724.     "pshufw $0x0E, " #a ", " #t "       \n\t"   \

  725.     /* faster than phaddd on core2 */           \

  726.     "paddd " #t ", " #a "               \n\t"

  727. 

  728. #define PMULHRW(x, y, s, o)                     \

  729.     "pmulhrsw " #s ", " #x "            \n\t"   \

  730.     "pmulhrsw " #s ", " #y "            \n\t"

  731. 

  732. #include "dsputil_qns_template.c"

  733. 

  734. #undef DEF

  735. #undef SET_RND

  736. #undef SCALE_OFFSET

  737. #undef PMULHRW

  738. #undef PHADDD

  739. #endif /* HAVE_SSSE3_INLINE */

  740. 

  741. #endif /* HAVE_INLINE_ASM */

  742. 

  743. int ff_sse16_sse2(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,

  744.                   int line_size, int h);

  745. 

  746. #define hadamard_func(cpu)                                              \

  747.     int ff_hadamard8_diff_ ## cpu(MpegEncContext *s, uint8_t *src1,     \

  748.                                   uint8_t *src2, int stride, int h);    \

  749.     int ff_hadamard8_diff16_ ## cpu(MpegEncContext *s, uint8_t *src1,   \

  750.                                     uint8_t *src2, int stride, int h);

  751. 

  752. hadamard_func(mmx)

  753. hadamard_func(mmxext)

  754. hadamard_func(sse2)

  755. hadamard_func(ssse3)

  756. 

  757. av_cold void ff_dsputilenc_init_mmx(DSPContext *c, AVCodecContext *avctx,

  758.                                     unsigned high_bit_depth)

  759. {

  760.     int cpu_flags = av_get_cpu_flags();

  761.     const int dct_algo = avctx->dct_algo;

  762. 

  763.     if (EXTERNAL_MMX(cpu_flags)) {

  764.         if (!high_bit_depth)

  765.             c->get_pixels = ff_get_pixels_mmx;

  766.         c->diff_pixels = ff_diff_pixels_mmx;

  767.         c->pix_sum     = ff_pix_sum16_mmx;

  768.         c->pix_norm1   = ff_pix_norm1_mmx;

  769.         c->sse[0]      = ff_sse16_mmx;

  770.         c->sse[1]      = ff_sse8_mmx;

  771.     }

  772. 

  773.     if (EXTERNAL_SSE2(cpu_flags))

  774.         if (!high_bit_depth)

  775.             c->get_pixels = ff_get_pixels_sse2;

  776. 

  777. #if HAVE_INLINE_ASM

  778.     if (INLINE_MMX(cpu_flags)) {

  779.         if (!high_bit_depth &&

  780.             (dct_algo == FF_DCT_AUTO || dct_algo == FF_DCT_MMX))

  781.             c->fdct = ff_fdct_mmx;

  782. 

  783.         c->diff_bytes      = diff_bytes_mmx;

  784.         c->vsad[4] = vsad_intra16_mmx;

  785. 

  786.         c->nsse[0] = nsse16_mmx;

  787.         c->nsse[1] = nsse8_mmx;

  788.         if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {

  789.             c->vsad[0]      = vsad16_mmx;

  790.             c->try_8x8basis = try_8x8basis_mmx;

  791.         }

  792.         c->add_8x8basis = add_8x8basis_mmx;

  793. 

  794.         c->ssd_int8_vs_int16 = ssd_int8_vs_int16_mmx;

  795.     }

  796. 

  797.     if (INLINE_AMD3DNOW(cpu_flags)) {

  798.         if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {

  799.             c->try_8x8basis = try_8x8basis_3dnow;

  800.         }

  801.         c->add_8x8basis = add_8x8basis_3dnow;

  802.     }

  803. 

  804.     if (INLINE_MMXEXT(cpu_flags)) {

  805.         if (!high_bit_depth &&

  806.             (dct_algo == FF_DCT_AUTO || dct_algo == FF_DCT_MMX))

  807.             c->fdct = ff_fdct_mmxext;

  808. 

  809.         c->vsad[4]         = vsad_intra16_mmxext;

  810. 

  811.         if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {

  812.             c->vsad[0] = vsad16_mmxext;

  813.         }

  814. 

  815.         c->sub_hfyu_median_prediction = sub_hfyu_median_prediction_mmxext;

  816.     }

  817. 

  818.     if (INLINE_SSE2(cpu_flags)) {

  819.         if (!high_bit_depth &&

  820.             (dct_algo == FF_DCT_AUTO || dct_algo == FF_DCT_MMX))

  821.             c->fdct = ff_fdct_sse2;

  822.     }

  823. 

  824. #if HAVE_SSSE3_INLINE

  825.     if (INLINE_SSSE3(cpu_flags)) {

  826.         if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {

  827.             c->try_8x8basis = try_8x8basis_ssse3;

  828.         }

  829.         c->add_8x8basis    = add_8x8basis_ssse3;

  830.     }

  831. #endif

  832. #endif /* HAVE_INLINE_ASM */

  833. 

  834.     if (EXTERNAL_MMX(cpu_flags)) {

  835.         c->hadamard8_diff[0] = ff_hadamard8_diff16_mmx;

  836.         c->hadamard8_diff[1] = ff_hadamard8_diff_mmx;

  837.         c->sum_abs_dctelem   = ff_sum_abs_dctelem_mmx;

  838.     }

  839. 

  840.     if (EXTERNAL_MMXEXT(cpu_flags)) {

  841.         c->hadamard8_diff[0] = ff_hadamard8_diff16_mmxext;

  842.         c->hadamard8_diff[1] = ff_hadamard8_diff_mmxext;

  843.         c->sum_abs_dctelem   = ff_sum_abs_dctelem_mmxext;

  844.     }

  845. 

  846.     if (EXTERNAL_SSE2(cpu_flags)) {

  847.         c->sse[0] = ff_sse16_sse2;

  848.         c->sum_abs_dctelem   = ff_sum_abs_dctelem_sse2;

  849. 

  850. #if HAVE_ALIGNED_STACK

  851.         c->hadamard8_diff[0] = ff_hadamard8_diff16_sse2;

  852.         c->hadamard8_diff[1] = ff_hadamard8_diff_sse2;

  853. #endif

  854.     }

  855. 

  856.     if (EXTERNAL_SSSE3(cpu_flags)) {

  857.         c->sum_abs_dctelem   = ff_sum_abs_dctelem_ssse3;

  858. #if HAVE_ALIGNED_STACK

  859.         c->hadamard8_diff[0] = ff_hadamard8_diff16_ssse3;

  860.         c->hadamard8_diff[1] = ff_hadamard8_diff_ssse3;

  861. #endif

  862.     }

  863. 

  864.     ff_dsputil_init_pix_mmx(c, avctx);

  865. }