falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
Browse Source

h264: do not use 422 functions for monochrome 

Fixes invalid memory access.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC:libav-stable@libav.org
tags/n2.2-rc1
Anton Khirnov 11 years ago
parent
9eef9eb301
commit
a03a642d5c
6 changed files with 30 additions and 30 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -1
      libavcodec/arm/h264dsp_init_arm.c
  2. +6
    -6
      libavcodec/h264dsp.c
  3. +5
    -5
      libavcodec/h264pred.c
  4. +1
    -1
      libavcodec/ppc/h264dsp.c
  5. +10
    -10
      libavcodec/x86/h264_intrapred_init.c
  6. +7
    -7
      libavcodec/x86/h264dsp_init.c

+ 1
- 1
libavcodec/arm/h264dsp_init_arm.c View File

@@ -91,7 +91,7 @@ static av_cold void h264dsp_init_neon(H264DSPContext *c, const int bit_depth,
c->h264_idct_dc_add = ff_h264_idct_dc_add_neon;
c->h264_idct_add16 = ff_h264_idct_add16_neon;
c->h264_idct_add16intra = ff_h264_idct_add16intra_neon;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_neon;
c->h264_idct8_add = ff_h264_idct8_add_neon;
c->h264_idct8_dc_add = ff_h264_idct8_dc_add_neon;


+ 6
- 6
libavcodec/h264dsp.c View File

@@ -104,13 +104,13 @@ av_cold void ff_h264dsp_init(H264DSPContext *c, const int bit_depth,
c->h264_idct8_dc_add= FUNC(ff_h264_idct8_dc_add, depth);\
c->h264_idct_add16 = FUNC(ff_h264_idct_add16, depth);\
c->h264_idct8_add4 = FUNC(ff_h264_idct8_add4, depth);\
if (chroma_format_idc == 1)\
if (chroma_format_idc <= 1)\
c->h264_idct_add8 = FUNC(ff_h264_idct_add8, depth);\
else\
c->h264_idct_add8 = FUNC(ff_h264_idct_add8_422, depth);\
c->h264_idct_add16intra= FUNC(ff_h264_idct_add16intra, depth);\
c->h264_luma_dc_dequant_idct= FUNC(ff_h264_luma_dc_dequant_idct, depth);\
if (chroma_format_idc == 1)\
if (chroma_format_idc <= 1)\
c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma_dc_dequant_idct, depth);\
else\
c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma422_dc_dequant_idct, depth);\
@@ -131,20 +131,20 @@ av_cold void ff_h264dsp_init(H264DSPContext *c, const int bit_depth,
c->h264_h_loop_filter_luma_intra= FUNC(h264_h_loop_filter_luma_intra, depth);\
c->h264_h_loop_filter_luma_mbaff_intra= FUNC(h264_h_loop_filter_luma_mbaff_intra, depth);\
c->h264_v_loop_filter_chroma= FUNC(h264_v_loop_filter_chroma, depth);\
if (chroma_format_idc == 1)\
if (chroma_format_idc <= 1)\
c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma, depth);\
else\
c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma422, depth);\
if (chroma_format_idc == 1)\
if (chroma_format_idc <= 1)\
c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma_mbaff, depth);\
else\
c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma422_mbaff, depth);\
c->h264_v_loop_filter_chroma_intra= FUNC(h264_v_loop_filter_chroma_intra, depth);\
if (chroma_format_idc == 1)\
if (chroma_format_idc <= 1)\
c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma_intra, depth);\
else\
c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma422_intra, depth);\
if (chroma_format_idc == 1)\
if (chroma_format_idc <= 1)\
c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma_mbaff_intra, depth);\
else\
c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma422_mbaff_intra, depth);\


+ 5
- 5
libavcodec/h264pred.c View File

@@ -472,7 +472,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
h->pred8x8l[TOP_DC_PRED ]= FUNCC(pred8x8l_top_dc , depth);\
h->pred8x8l[DC_128_PRED ]= FUNCC(pred8x8l_128_dc , depth);\
\
if (chroma_format_idc == 1) {\
if (chroma_format_idc <= 1) {\
h->pred8x8[VERT_PRED8x8 ]= FUNCC(pred8x8_vertical , depth);\
h->pred8x8[HOR_PRED8x8 ]= FUNCC(pred8x8_horizontal , depth);\
} else {\
@@ -480,7 +480,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
h->pred8x8[HOR_PRED8x8 ]= FUNCC(pred8x16_horizontal , depth);\
}\
if (codec_id != AV_CODEC_ID_VP8) {\
if (chroma_format_idc == 1) {\
if (chroma_format_idc <= 1) {\
h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x8_plane , depth);\
} else {\
h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x16_plane , depth);\
@@ -488,7 +488,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
} else\
h->pred8x8[PLANE_PRED8x8]= FUNCD(pred8x8_tm_vp8);\
if(codec_id != AV_CODEC_ID_RV40 && codec_id != AV_CODEC_ID_VP8){\
if (chroma_format_idc == 1) {\
if (chroma_format_idc <= 1) {\
h->pred8x8[DC_PRED8x8 ]= FUNCC(pred8x8_dc , depth);\
h->pred8x8[LEFT_DC_PRED8x8]= FUNCC(pred8x8_left_dc , depth);\
h->pred8x8[TOP_DC_PRED8x8 ]= FUNCC(pred8x8_top_dc , depth);\
@@ -514,7 +514,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
h->pred8x8[DC_129_PRED8x8]= FUNCC(pred8x8_129_dc , depth);\
}\
}\
if (chroma_format_idc == 1) {\
if (chroma_format_idc <= 1) {\
h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x8_128_dc , depth);\
} else {\
h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x16_128_dc , depth);\
@@ -548,7 +548,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
h->pred4x4_add [ HOR_PRED ]= FUNCC(pred4x4_horizontal_add , depth);\
h->pred8x8l_add [VERT_PRED ]= FUNCC(pred8x8l_vertical_add , depth);\
h->pred8x8l_add [ HOR_PRED ]= FUNCC(pred8x8l_horizontal_add , depth);\
if (chroma_format_idc == 1) {\
if (chroma_format_idc <= 1) {\
h->pred8x8_add [VERT_PRED8x8]= FUNCC(pred8x8_vertical_add , depth);\
h->pred8x8_add [ HOR_PRED8x8]= FUNCC(pred8x8_horizontal_add , depth);\
} else {\


+ 1
- 1
libavcodec/ppc/h264dsp.c View File

@@ -750,7 +750,7 @@ av_cold void ff_h264dsp_init_ppc(H264DSPContext *c, const int bit_depth,

if (bit_depth == 8) {
c->h264_idct_add = h264_idct_add_altivec;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
c->h264_idct_add8 = h264_idct_add8_altivec;
c->h264_idct_add16 = h264_idct_add16_altivec;
c->h264_idct_add16intra = h264_idct_add16intra_altivec;


+ 10
- 10
libavcodec/x86/h264_intrapred_init.c View File

@@ -191,7 +191,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
if (EXTERNAL_MMX(cpu_flags)) {
h->pred16x16[VERT_PRED8x8 ] = ff_pred16x16_vertical_8_mmx;
h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_mmx;
if (chroma_format_idc == 1) {
if (chroma_format_idc <= 1) {
h->pred8x8 [VERT_PRED8x8 ] = ff_pred8x8_vertical_8_mmx;
h->pred8x8 [HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_mmx;
}
@@ -200,7 +200,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_mmx;
h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_mmx;
} else {
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmx;
if (codec_id == AV_CODEC_ID_SVQ3) {
if (cpu_flags & AV_CPU_FLAG_CMOV)
@@ -216,7 +216,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
if (EXTERNAL_MMXEXT(cpu_flags)) {
h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_mmxext;
h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_8_mmxext;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
h->pred8x8[HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_mmxext;
h->pred8x8l [TOP_DC_PRED ] = ff_pred8x8l_top_dc_8_mmxext;
h->pred8x8l [DC_PRED ] = ff_pred8x8l_dc_8_mmxext;
@@ -241,7 +241,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
h->pred4x4 [HOR_UP_PRED ] = ff_pred4x4_horizontal_up_8_mmxext;
}
if (codec_id == AV_CODEC_ID_SVQ3 || codec_id == AV_CODEC_ID_H264) {
if (chroma_format_idc == 1) {
if (chroma_format_idc <= 1) {
h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_8_mmxext;
h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_8_mmxext;
}
@@ -253,7 +253,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_mmxext;
h->pred4x4 [VERT_PRED ] = ff_pred4x4_vertical_vp8_8_mmxext;
} else {
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmxext;
if (codec_id == AV_CODEC_ID_SVQ3) {
h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_plane_svq3_8_mmxext;
@@ -280,7 +280,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_tm_vp8_8_sse2;
h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_sse2;
} else {
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_sse2;
if (codec_id == AV_CODEC_ID_SVQ3) {
h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_sse2;
@@ -295,7 +295,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
if (EXTERNAL_SSSE3(cpu_flags)) {
h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_ssse3;
h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_8_ssse3;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
h->pred8x8 [HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_ssse3;
h->pred8x8l [TOP_DC_PRED ] = ff_pred8x8l_top_dc_8_ssse3;
h->pred8x8l [DC_PRED ] = ff_pred8x8l_dc_8_ssse3;
@@ -311,7 +311,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_ssse3;
h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_ssse3;
} else {
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_ssse3;
if (codec_id == AV_CODEC_ID_SVQ3) {
h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_ssse3;
@@ -327,7 +327,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
h->pred4x4[DC_PRED ] = ff_pred4x4_dc_10_mmxext;
h->pred4x4[HOR_UP_PRED ] = ff_pred4x4_horizontal_up_10_mmxext;

if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_10_mmxext;

h->pred8x8l[DC_128_PRED ] = ff_pred8x8l_128_dc_10_mmxext;
@@ -346,7 +346,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
h->pred4x4[VERT_RIGHT_PRED ] = ff_pred4x4_vertical_right_10_sse2;
h->pred4x4[HOR_DOWN_PRED ] = ff_pred4x4_horizontal_down_10_sse2;

if (chroma_format_idc == 1) {
if (chroma_format_idc <= 1) {
h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_10_sse2;
h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_10_sse2;
h->pred8x8[PLANE_PRED8x8 ] = ff_pred8x8_plane_10_sse2;


+ 7
- 7
libavcodec/x86/h264dsp_init.c View File

@@ -213,7 +213,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
{
int cpu_flags = av_get_cpu_flags();

if (chroma_format_idc == 1 && EXTERNAL_MMXEXT(cpu_flags))
if (chroma_format_idc <= 1 && EXTERNAL_MMXEXT(cpu_flags))
c->h264_loop_filter_strength = ff_h264_loop_filter_strength_mmxext;

if (bit_depth == 8) {
@@ -225,7 +225,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,

c->h264_idct_add16 = ff_h264_idct_add16_8_mmx;
c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmx;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_8_mmx;
c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmx;
if (cpu_flags & AV_CPU_FLAG_CMOV)
@@ -236,13 +236,13 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
c->h264_idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext;
c->h264_idct_add16 = ff_h264_idct_add16_8_mmxext;
c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmxext;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_8_mmxext;
c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmxext;

c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_8_mmxext;
c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_mmxext;
if (chroma_format_idc == 1) {
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_8_mmxext;
c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_mmxext;
}
@@ -265,7 +265,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,

c->h264_idct_add16 = ff_h264_idct_add16_8_sse2;
c->h264_idct8_add4 = ff_h264_idct8_add4_8_sse2;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_8_sse2;
c->h264_idct_add16intra = ff_h264_idct_add16intra_8_sse2;
c->h264_luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2;
@@ -308,7 +308,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_sse2;

c->h264_idct_add16 = ff_h264_idct_add16_10_sse2;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_10_sse2;
c->h264_idct_add16intra = ff_h264_idct_add16intra_10_sse2;
#if HAVE_ALIGNED_STACK
@@ -348,7 +348,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_avx;

c->h264_idct_add16 = ff_h264_idct_add16_10_avx;
if (chroma_format_idc == 1)
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_10_avx;
c->h264_idct_add16intra = ff_h264_idct_add16intra_10_avx;
#if HAVE_ALIGNED_STACK


Write Preview
Loading…
Cancel
Save