vf_histogram: avoid floats, unbreak fate

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavfilter/vf_histogram.c

@@ -197,7 +197,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
                 h->max_hval = FFMAX(h->max_hval, h->histogram[i]);
 
             for (i = 0; i < outlink->w; i++) {
-                int col_height = h->level_height - (float)h->histogram[i] / h->max_hval * h->level_height;
+                int col_height = h->level_height - (h->histogram[i] * (int64_t)h->level_height + h->max_hval - 1) / h->max_hval;
 
                 for (j = h->level_height - 1; j >= col_height; j--) {
                     if (h->display_mode) {

tests/ref/fate/filter-histogram-levels

@@ -10,7 +10,7 @@
 0,          8,          8,        1,   488448, 0xbbe0ba18
 0,          9,          9,        1,   488448, 0x29d7e5a6
 0,         10,         10,        1,   488448, 0x9428b9ee
-0,         11,         11,        1,   488448, 0x83b3cc4f
+0,         11,         11,        1,   488448, 0xe0dacb50
 0,         12,         12,        1,   488448, 0x6fbe351f
 0,         13,         13,        1,   488448, 0x27b165e0
 0,         14,         14,        1,   488448, 0xc1301e0c
@@ -26,7 +26,7 @@
 0,         24,         24,        1,   488448, 0x8fdf8efd
 0,         25,         25,        1,   488448, 0x11ad9232
 0,         26,         26,        1,   488448, 0x26bd6353
-0,         27,         27,        1,   488448, 0xfcb3873d
+0,         27,         27,        1,   488448, 0xa6ab863e
 0,         28,         28,        1,   488448, 0x4c14852e
 0,         29,         29,        1,   488448, 0x738ce85a
 0,         30,         30,        1,   488448, 0xdb84785a