falkTX
/
FFmpeg
mirror of https://github.com/falkTX/FFmpeg.git
1
0
Fork 0
Code Issues 0 Releases 338 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
100686 Commits
32 Branches
824MB
Tree: b51eae1abe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'b51eae1abe'
${ noResults }
FFmpeg/libavfilter/vf_signature.c

773 lines
28KB
Raw Blame History 

  1. /*

  2.  * Copyright (c) 2017 Gerion Entrup

  3.  *

  4.  * This file is part of FFmpeg.

  5.  *

  6.  * FFmpeg is free software; you can redistribute it and/or modify

  7.  * it under the terms of the GNU General Public License as published by

  8.  * the Free Software Foundation; either version 2 of the License, or

  9.  * (at your option) any later version.

  10.  *

  11.  * FFmpeg is distributed in the hope that it will be useful,

  12.  * but WITHOUT ANY WARRANTY; without even the implied warranty of

  13.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

  14.  * GNU General Public License for more details.

  15.  *

  16.  * You should have received a copy of the GNU General Public License along

  17.  * with FFmpeg; if not, write to the Free Software Foundation, Inc.,

  18.  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

  19.  */

  20. 

  21. /**

  22.  * @file

  23.  * MPEG-7 video signature calculation and lookup filter

  24.  * @see http://epubs.surrey.ac.uk/531590/1/MPEG-7%20Video%20Signature%20Author%27s%20Copy.pdf

  25.  */

  26. 

  27. #include <float.h>

  28. #include "libavcodec/put_bits.h"

  29. #include "libavformat/avformat.h"

  30. #include "libavutil/opt.h"

  31. #include "libavutil/avstring.h"

  32. #include "libavutil/intreadwrite.h"

  33. #include "libavutil/timestamp.h"

  34. #include "avfilter.h"

  35. #include "internal.h"

  36. #include "signature.h"

  37. #include "signature_lookup.c"

  38. 

  39. #define OFFSET(x) offsetof(SignatureContext, x)

  40. #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM

  41. #define BLOCK_LCM (int64_t) 476985600

  42. 

  43. static const AVOption signature_options[] = {

  44.     { "detectmode", "set the detectmode",

  45.         OFFSET(mode),         AV_OPT_TYPE_INT,    {.i64 = MODE_OFF}, 0, NB_LOOKUP_MODE-1, FLAGS, "mode" },

  46.         { "off",  NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_OFF},  0, 0, .flags = FLAGS, "mode" },

  47.         { "full", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_FULL}, 0, 0, .flags = FLAGS, "mode" },

  48.         { "fast", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_FAST}, 0, 0, .flags = FLAGS, "mode" },

  49.     { "nb_inputs",  "number of inputs",

  50.         OFFSET(nb_inputs),    AV_OPT_TYPE_INT,    {.i64 = 1},        1, INT_MAX,          FLAGS },

  51.     { "filename",   "filename for output files",

  52.         OFFSET(filename),     AV_OPT_TYPE_STRING, {.str = ""},       0, NB_FORMATS-1,     FLAGS },

  53.     { "format",     "set output format",

  54.         OFFSET(format),       AV_OPT_TYPE_INT,    {.i64 = FORMAT_BINARY}, 0, 1,           FLAGS , "format" },

  55.         { "binary", 0, 0, AV_OPT_TYPE_CONST, {.i64=FORMAT_BINARY}, 0, 0, FLAGS, "format" },

  56.         { "xml",    0, 0, AV_OPT_TYPE_CONST, {.i64=FORMAT_XML},    0, 0, FLAGS, "format" },

  57.     { "th_d",       "threshold to detect one word as similar",

  58.         OFFSET(thworddist),   AV_OPT_TYPE_INT,    {.i64 = 9000},     1, INT_MAX,          FLAGS },

  59.     { "th_dc",      "threshold to detect all words as similar",

  60.         OFFSET(thcomposdist), AV_OPT_TYPE_INT,    {.i64 = 60000},    1, INT_MAX,          FLAGS },

  61.     { "th_xh",      "threshold to detect frames as similar",

  62.         OFFSET(thl1),         AV_OPT_TYPE_INT,    {.i64 = 116},      1, INT_MAX,          FLAGS },

  63.     { "th_di",      "minimum length of matching sequence in frames",

  64.         OFFSET(thdi),         AV_OPT_TYPE_INT,    {.i64 = 0},        0, INT_MAX,          FLAGS },

  65.     { "th_it",      "threshold for relation of good to all frames",

  66.         OFFSET(thit),         AV_OPT_TYPE_DOUBLE, {.dbl = 0.5},    0.0, 1.0,              FLAGS },

  67.     { NULL }

  68. };

  69. 

  70. AVFILTER_DEFINE_CLASS(signature);

  71. 

  72. static int query_formats(AVFilterContext *ctx)

  73. {

  74.     /* all formats with a separate gray value */

  75.     static const enum AVPixelFormat pix_fmts[] = {

  76.         AV_PIX_FMT_GRAY8,

  77.         AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,

  78.         AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,

  79.         AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,

  80.         AV_PIX_FMT_YUVJ411P, AV_PIX_FMT_YUVJ420P,

  81.         AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P,

  82.         AV_PIX_FMT_YUVJ440P,

  83.         AV_PIX_FMT_NV12, AV_PIX_FMT_NV21,

  84.         AV_PIX_FMT_NONE

  85.     };

  86. 

  87.     return ff_set_common_formats(ctx, ff_make_format_list(pix_fmts));

  88. }

  89. 

  90. static int config_input(AVFilterLink *inlink)

  91. {

  92.     AVFilterContext *ctx = inlink->dst;

  93.     SignatureContext *sic = ctx->priv;

  94.     StreamContext *sc = &(sic->streamcontexts[FF_INLINK_IDX(inlink)]);

  95. 

  96.     sc->time_base = inlink->time_base;

  97.     /* test for overflow */

  98.     sc->divide = (((uint64_t) inlink->w/32) * (inlink->w/32 + 1) * (inlink->h/32 * inlink->h/32 + 1) > INT64_MAX / (BLOCK_LCM * 255));

  99.     if (sc->divide) {

  100.         av_log(ctx, AV_LOG_WARNING, "Input dimension too high for precise calculation, numbers will be rounded.\n");

  101.     }

  102.     sc->w = inlink->w;

  103.     sc->h = inlink->h;

  104.     return 0;

  105. }

  106. 

  107. static int get_block_size(const Block *b)

  108. {

  109.     return (b->to.y - b->up.y + 1) * (b->to.x - b->up.x + 1);

  110. }

  111. 

  112. static uint64_t get_block_sum(StreamContext *sc, uint64_t intpic[32][32], const Block *b)

  113. {

  114.     uint64_t sum = 0;

  115. 

  116.     int x0, y0, x1, y1;

  117. 

  118.     x0 = b->up.x;

  119.     y0 = b->up.y;

  120.     x1 = b->to.x;

  121.     y1 = b->to.y;

  122. 

  123.     if (x0-1 >= 0 && y0-1 >= 0) {

  124.         sum = intpic[y1][x1] + intpic[y0-1][x0-1] - intpic[y1][x0-1] - intpic[y0-1][x1];

  125.     } else if (x0-1 >= 0) {

  126.         sum = intpic[y1][x1] - intpic[y1][x0-1];

  127.     } else if (y0-1 >= 0) {

  128.         sum = intpic[y1][x1] - intpic[y0-1][x1];

  129.     } else {

  130.         sum = intpic[y1][x1];

  131.     }

  132.     return sum;

  133. }

  134. 

  135. static int cmp(const void *x, const void *y)

  136. {

  137.     const uint64_t *a = x, *b = y;

  138.     return *a < *b ? -1 : ( *a > *b ? 1 : 0 );

  139. }

  140. 

  141. /**

  142.  * sets the bit at position pos to 1 in data

  143.  */

  144. static void set_bit(uint8_t* data, size_t pos)

  145. {

  146.     uint8_t mask = 1 << 7-(pos%8);

  147.     data[pos/8] |= mask;

  148. }

  149. 

  150. static int filter_frame(AVFilterLink *inlink, AVFrame *picref)

  151. {

  152.     AVFilterContext *ctx = inlink->dst;

  153.     SignatureContext *sic = ctx->priv;

  154.     StreamContext *sc = &(sic->streamcontexts[FF_INLINK_IDX(inlink)]);

  155.     FineSignature* fs;

  156. 

  157.     static const uint8_t pot3[5] = { 3*3*3*3, 3*3*3, 3*3, 3, 1 };

  158.     /* indexes of words : 210,217,219,274,334  44,175,233,270,273  57,70,103,237,269  100,285,295,337,354  101,102,111,275,296

  159.     s2usw = sorted to unsorted wordvec: 44 is at index 5, 57 at index 10...

  160.     */

  161.     static const unsigned int wordvec[25] = {44,57,70,100,101,102,103,111,175,210,217,219,233,237,269,270,273,274,275,285,295,296,334,337,354};

  162.     static const uint8_t      s2usw[25]   = { 5,10,11, 15, 20, 21, 12, 22,  6,  0,  1,  2,  7, 13, 14,  8,  9,  3, 23, 16, 17, 24,  4, 18, 19};

  163. 

  164.     uint8_t wordt2b[5] = { 0, 0, 0, 0, 0 }; /* word ternary to binary */

  165.     uint64_t intpic[32][32];

  166.     uint64_t rowcount;

  167.     uint8_t *p = picref->data[0];

  168.     int inti, intj;

  169.     int *intjlut;

  170. 

  171.     uint64_t conflist[DIFFELEM_SIZE];

  172.     int f = 0, g = 0, w = 0;

  173.     int32_t dh1 = 1, dh2 = 1, dw1 = 1, dw2 = 1, a, b;

  174.     int64_t denom;

  175.     int i, j, k, ternary;

  176.     uint64_t blocksum;

  177.     int blocksize;

  178.     int64_t th; /* threshold */

  179.     int64_t sum;

  180. 

  181.     int64_t precfactor = (sc->divide) ? 65536 : BLOCK_LCM;

  182. 

  183.     /* initialize fs */

  184.     if (sc->curfinesig) {

  185.         fs = av_mallocz(sizeof(FineSignature));

  186.         if (!fs)

  187.             return AVERROR(ENOMEM);

  188.         sc->curfinesig->next = fs;

  189.         fs->prev = sc->curfinesig;

  190.         sc->curfinesig = fs;

  191.     } else {

  192.         fs = sc->curfinesig = sc->finesiglist;

  193.         sc->curcoarsesig1->first = fs;

  194.     }

  195. 

  196.     fs->pts = picref->pts;

  197.     fs->index = sc->lastindex++;

  198. 

  199.     memset(intpic, 0, sizeof(uint64_t)*32*32);

  200.     intjlut = av_malloc_array(inlink->w, sizeof(int));

  201.     if (!intjlut)

  202.         return AVERROR(ENOMEM);

  203.     for (i = 0; i < inlink->w; i++) {

  204.         intjlut[i] = (i*32)/inlink->w;

  205.     }

  206. 

  207.     for (i = 0; i < inlink->h; i++) {

  208.         inti = (i*32)/inlink->h;

  209.         for (j = 0; j < inlink->w; j++) {

  210.             intj = intjlut[j];

  211.             intpic[inti][intj] += p[j];

  212.         }

  213.         p += picref->linesize[0];

  214.     }

  215.     av_freep(&intjlut);

  216. 

  217.     /* The following calculates a summed area table (intpic) and brings the numbers

  218.      * in intpic to the same denominator.

  219.      * So you only have to handle the numinator in the following sections.

  220.      */

  221.     dh1 = inlink->h / 32;

  222.     if (inlink->h % 32)

  223.         dh2 = dh1 + 1;

  224.     dw1 = inlink->w / 32;

  225.     if (inlink->w % 32)

  226.         dw2 = dw1 + 1;

  227.     denom = (sc->divide) ? dh1 * dh2 * dw1 * dw2 : 1;

  228. 

  229.     for (i = 0; i < 32; i++) {

  230.         rowcount = 0;

  231.         a = 1;

  232.         if (dh2 > 1) {

  233.             a = ((inlink->h*(i+1))%32 == 0) ? (inlink->h*(i+1))/32 - 1 : (inlink->h*(i+1))/32;

  234.             a -= ((inlink->h*i)%32 == 0) ? (inlink->h*i)/32 - 1 : (inlink->h*i)/32;

  235.             a = (a == dh1)? dh2 : dh1;

  236.         }

  237.         for (j = 0; j < 32; j++) {

  238.             b = 1;

  239.             if (dw2 > 1) {

  240.                 b = ((inlink->w*(j+1))%32 == 0) ? (inlink->w*(j+1))/32 - 1 : (inlink->w*(j+1))/32;

  241.                 b -= ((inlink->w*j)%32 == 0) ? (inlink->w*j)/32 - 1 : (inlink->w*j)/32;

  242.                 b = (b == dw1)? dw2 : dw1;

  243.             }

  244.             rowcount += intpic[i][j] * a * b * precfactor / denom;

  245.             if (i > 0) {

  246.                 intpic[i][j] = intpic[i-1][j] + rowcount;

  247.             } else {

  248.                 intpic[i][j] = rowcount;

  249.             }

  250.         }

  251.     }

  252. 

  253.     denom = (sc->divide) ? 1 : dh1 * dh2 * dw1 * dw2;

  254. 

  255.     for (i = 0; i < ELEMENT_COUNT; i++) {

  256.         const ElemCat* elemcat = elements[i];

  257.         int64_t* elemsignature;

  258.         uint64_t* sortsignature;

  259. 

  260.         elemsignature = av_malloc_array(elemcat->elem_count, sizeof(int64_t));

  261.         if (!elemsignature)

  262.             return AVERROR(ENOMEM);

  263.         sortsignature = av_malloc_array(elemcat->elem_count, sizeof(int64_t));

  264.         if (!sortsignature) {

  265.             av_freep(&elemsignature);

  266.             return AVERROR(ENOMEM);

  267.         }

  268. 

  269.         for (j = 0; j < elemcat->elem_count; j++) {

  270.             blocksum = 0;

  271.             blocksize = 0;

  272.             for (k = 0; k < elemcat->left_count; k++) {

  273.                 blocksum += get_block_sum(sc, intpic, &elemcat->blocks[j*elemcat->block_count+k]);

  274.                 blocksize += get_block_size(&elemcat->blocks[j*elemcat->block_count+k]);

  275.             }

  276.             sum = blocksum / blocksize;

  277.             if (elemcat->av_elem) {

  278.                 sum -= 128 * precfactor * denom;

  279.             } else {

  280.                 blocksum = 0;

  281.                 blocksize = 0;

  282.                 for (; k < elemcat->block_count; k++) {

  283.                     blocksum += get_block_sum(sc, intpic, &elemcat->blocks[j*elemcat->block_count+k]);

  284.                     blocksize += get_block_size(&elemcat->blocks[j*elemcat->block_count+k]);

  285.                 }

  286.                 sum -= blocksum / blocksize;

  287.                 conflist[g++] = FFABS(sum * 8 / (precfactor * denom));

  288.             }

  289. 

  290.             elemsignature[j] = sum;

  291.             sortsignature[j] = FFABS(sum);

  292.         }

  293. 

  294.         /* get threshold */

  295.         qsort(sortsignature, elemcat->elem_count, sizeof(uint64_t), cmp);

  296.         th = sortsignature[(int) (elemcat->elem_count*0.333)];

  297. 

  298.         /* ternarize */

  299.         for (j = 0; j < elemcat->elem_count; j++) {

  300.             if (elemsignature[j] < -th) {

  301.                 ternary = 0;

  302.             } else if (elemsignature[j] <= th) {

  303.                 ternary = 1;

  304.             } else {

  305.                 ternary = 2;

  306.             }

  307.             fs->framesig[f/5] += ternary * pot3[f%5];

  308. 

  309.             if (f == wordvec[w]) {

  310.                 fs->words[s2usw[w]/5] += ternary * pot3[wordt2b[s2usw[w]/5]++];

  311.                 if (w < 24)

  312.                     w++;

  313.             }

  314.             f++;

  315.         }

  316.         av_freep(&elemsignature);

  317.         av_freep(&sortsignature);

  318.     }

  319. 

  320.     /* confidence */

  321.     qsort(conflist, DIFFELEM_SIZE, sizeof(uint64_t), cmp);

  322.     fs->confidence = FFMIN(conflist[DIFFELEM_SIZE/2], 255);

  323. 

  324.     /* coarsesignature */

  325.     if (sc->coarsecount == 0) {

  326.         if (sc->curcoarsesig2) {

  327.             sc->curcoarsesig1 = av_mallocz(sizeof(CoarseSignature));

  328.             if (!sc->curcoarsesig1)

  329.                 return AVERROR(ENOMEM);

  330.             sc->curcoarsesig1->first = fs;

  331.             sc->curcoarsesig2->next = sc->curcoarsesig1;

  332.             sc->coarseend = sc->curcoarsesig1;

  333.         }

  334.     }

  335.     if (sc->coarsecount == 45) {

  336.         sc->midcoarse = 1;

  337.         sc->curcoarsesig2 = av_mallocz(sizeof(CoarseSignature));

  338.         if (!sc->curcoarsesig2)

  339.             return AVERROR(ENOMEM);

  340.         sc->curcoarsesig2->first = fs;

  341.         sc->curcoarsesig1->next = sc->curcoarsesig2;

  342.         sc->coarseend = sc->curcoarsesig2;

  343.     }

  344.     for (i = 0; i < 5; i++) {

  345.         set_bit(sc->curcoarsesig1->data[i], fs->words[i]);

  346.     }

  347.     /* assuming the actual frame is the last */

  348.     sc->curcoarsesig1->last = fs;

  349.     if (sc->midcoarse) {

  350.         for (i = 0; i < 5; i++) {

  351.             set_bit(sc->curcoarsesig2->data[i], fs->words[i]);

  352.         }

  353.         sc->curcoarsesig2->last = fs;

  354.     }

  355. 

  356.     sc->coarsecount = (sc->coarsecount+1)%90;

  357. 

  358.     /* debug printing finesignature */

  359.     if (av_log_get_level() == AV_LOG_DEBUG) {

  360.         av_log(ctx, AV_LOG_DEBUG, "input %d, confidence: %d\n", FF_INLINK_IDX(inlink), fs->confidence);

  361. 

  362.         av_log(ctx, AV_LOG_DEBUG, "words:");

  363.         for (i = 0; i < 5; i++) {

  364.             av_log(ctx, AV_LOG_DEBUG, " %d:", fs->words[i] );

  365.             av_log(ctx, AV_LOG_DEBUG, " %d", fs->words[i] / pot3[0] );

  366.             for (j = 1; j < 5; j++)

  367.                 av_log(ctx, AV_LOG_DEBUG, ",%d", fs->words[i] % pot3[j-1] / pot3[j] );

  368.             av_log(ctx, AV_LOG_DEBUG, ";");

  369.         }

  370.         av_log(ctx, AV_LOG_DEBUG, "\n");

  371. 

  372.         av_log(ctx, AV_LOG_DEBUG, "framesignature:");

  373.         for (i = 0; i < SIGELEM_SIZE/5; i++) {

  374.             av_log(ctx, AV_LOG_DEBUG, " %d", fs->framesig[i] / pot3[0] );

  375.             for (j = 1; j < 5; j++)

  376.                 av_log(ctx, AV_LOG_DEBUG, ",%d", fs->framesig[i] % pot3[j-1] / pot3[j] );

  377.         }

  378.         av_log(ctx, AV_LOG_DEBUG, "\n");

  379.     }

  380. 

  381.     if (FF_INLINK_IDX(inlink) == 0)

  382.         return ff_filter_frame(inlink->dst->outputs[0], picref);

  383.     return 1;

  384. }

  385. 

  386. static int xml_export(AVFilterContext *ctx, StreamContext *sc, const char* filename)

  387. {

  388.     FineSignature* fs;

  389.     CoarseSignature* cs;

  390.     int i, j;

  391.     FILE* f;

  392.     unsigned int pot3[5] = { 3*3*3*3, 3*3*3, 3*3, 3, 1 };

  393. 

  394.     f = fopen(filename, "w");

  395.     if (!f) {

  396.         int err = AVERROR(EINVAL);

  397.         char buf[128];

  398.         av_strerror(err, buf, sizeof(buf));

  399.         av_log(ctx, AV_LOG_ERROR, "cannot open xml file %s: %s\n", filename, buf);

  400.         return err;

  401.     }

  402. 

  403.     /* header */

  404.     fprintf(f, "<?xml version='1.0' encoding='ASCII' ?>\n");

  405.     fprintf(f, "<Mpeg7 xmlns=\"urn:mpeg:mpeg7:schema:2001\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"urn:mpeg:mpeg7:schema:2001 schema/Mpeg7-2001.xsd\">\n");

  406.     fprintf(f, "  <DescriptionUnit xsi:type=\"DescriptorCollectionType\">\n");

  407.     fprintf(f, "    <Descriptor xsi:type=\"VideoSignatureType\">\n");

  408.     fprintf(f, "      <VideoSignatureRegion>\n");

  409.     fprintf(f, "        <VideoSignatureSpatialRegion>\n");

  410.     fprintf(f, "          <Pixel>0 0 </Pixel>\n");

  411.     fprintf(f, "          <Pixel>%d %d </Pixel>\n", sc->w - 1, sc->h - 1);

  412.     fprintf(f, "        </VideoSignatureSpatialRegion>\n");

  413.     fprintf(f, "        <StartFrameOfSpatialRegion>0</StartFrameOfSpatialRegion>\n");

  414.     /* hoping num is 1, other values are vague */

  415.     fprintf(f, "        <MediaTimeUnit>%d</MediaTimeUnit>\n", sc->time_base.den / sc->time_base.num);

  416.     fprintf(f, "        <MediaTimeOfSpatialRegion>\n");

  417.     fprintf(f, "          <StartMediaTimeOfSpatialRegion>0</StartMediaTimeOfSpatialRegion>\n");

  418.     fprintf(f, "          <EndMediaTimeOfSpatialRegion>%" PRIu64 "</EndMediaTimeOfSpatialRegion>\n", sc->coarseend->last->pts);

  419.     fprintf(f, "        </MediaTimeOfSpatialRegion>\n");

  420. 

  421.     /* coarsesignatures */

  422.     for (cs = sc->coarsesiglist; cs; cs = cs->next) {

  423.         fprintf(f, "        <VSVideoSegment>\n");

  424.         fprintf(f, "          <StartFrameOfSegment>%" PRIu32 "</StartFrameOfSegment>\n", cs->first->index);

  425.         fprintf(f, "          <EndFrameOfSegment>%" PRIu32 "</EndFrameOfSegment>\n", cs->last->index);

  426.         fprintf(f, "          <MediaTimeOfSegment>\n");

  427.         fprintf(f, "            <StartMediaTimeOfSegment>%" PRIu64 "</StartMediaTimeOfSegment>\n", cs->first->pts);

  428.         fprintf(f, "            <EndMediaTimeOfSegment>%" PRIu64 "</EndMediaTimeOfSegment>\n", cs->last->pts);

  429.         fprintf(f, "          </MediaTimeOfSegment>\n");

  430.         for (i = 0; i < 5; i++) {

  431.             fprintf(f, "          <BagOfWords>");

  432.             for (j = 0; j < 31; j++) {

  433.                 uint8_t n = cs->data[i][j];

  434.                 if (j < 30) {

  435.                     fprintf(f, "%d  %d  %d  %d  %d  %d  %d  %d  ", (n & 0x80) >> 7,

  436.                                                                    (n & 0x40) >> 6,

  437.                                                                    (n & 0x20) >> 5,

  438.                                                                    (n & 0x10) >> 4,

  439.                                                                    (n & 0x08) >> 3,

  440.                                                                    (n & 0x04) >> 2,

  441.                                                                    (n & 0x02) >> 1,

  442.                                                                    (n & 0x01));

  443.                 } else {

  444.                     /* print only 3 bit in last byte */

  445.                     fprintf(f, "%d  %d  %d ", (n & 0x80) >> 7,

  446.                                               (n & 0x40) >> 6,

  447.                                               (n & 0x20) >> 5);

  448.                 }

  449.             }

  450.             fprintf(f, "</BagOfWords>\n");

  451.         }

  452.         fprintf(f, "        </VSVideoSegment>\n");

  453.     }

  454. 

  455.     /* finesignatures */

  456.     for (fs = sc->finesiglist; fs; fs = fs->next) {

  457.         fprintf(f, "        <VideoFrame>\n");

  458.         fprintf(f, "          <MediaTimeOfFrame>%" PRIu64 "</MediaTimeOfFrame>\n", fs->pts);

  459.         /* confidence */

  460.         fprintf(f, "          <FrameConfidence>%d</FrameConfidence>\n", fs->confidence);

  461.         /* words */

  462.         fprintf(f, "          <Word>");

  463.         for (i = 0; i < 5; i++) {

  464.             fprintf(f, "%d ", fs->words[i]);

  465.             if (i < 4) {

  466.                 fprintf(f, " ");

  467.             }

  468.         }

  469.         fprintf(f, "</Word>\n");

  470.         /* framesignature */

  471.         fprintf(f, "          <FrameSignature>");

  472.         for (i = 0; i< SIGELEM_SIZE/5; i++) {

  473.             if (i > 0) {

  474.                 fprintf(f, " ");

  475.             }

  476.             fprintf(f, "%d ", fs->framesig[i] / pot3[0]);

  477.             for (j = 1; j < 5; j++)

  478.                 fprintf(f, " %d ", fs->framesig[i] % pot3[j-1] / pot3[j] );

  479.         }

  480.         fprintf(f, "</FrameSignature>\n");

  481.         fprintf(f, "        </VideoFrame>\n");

  482.     }

  483.     fprintf(f, "      </VideoSignatureRegion>\n");

  484.     fprintf(f, "    </Descriptor>\n");

  485.     fprintf(f, "  </DescriptionUnit>\n");

  486.     fprintf(f, "</Mpeg7>\n");

  487. 

  488.     fclose(f);

  489.     return 0;

  490. }

  491. 

  492. static int binary_export(AVFilterContext *ctx, StreamContext *sc, const char* filename)

  493. {

  494.     FILE* f;

  495.     FineSignature* fs;

  496.     CoarseSignature* cs;

  497.     uint32_t numofsegments = (sc->lastindex + 44)/45;

  498.     int i, j;

  499.     PutBitContext buf;

  500.     /* buffer + header + coarsesignatures + finesignature */

  501.     int len = (512 + 6 * 32 + 3*16 + 2 +

  502.         numofsegments * (4*32 + 1 + 5*243) +

  503.         sc->lastindex * (2 + 32 + 6*8 + 608)) / 8;

  504.     uint8_t* buffer = av_malloc_array(len, sizeof(uint8_t));

  505.     if (!buffer)

  506.         return AVERROR(ENOMEM);

  507. 

  508.     f = fopen(filename, "wb");

  509.     if (!f) {

  510.         int err = AVERROR(EINVAL);

  511.         char buf[128];

  512.         av_strerror(err, buf, sizeof(buf));

  513.         av_log(ctx, AV_LOG_ERROR, "cannot open file %s: %s\n", filename, buf);

  514.         av_freep(&buffer);

  515.         return err;

  516.     }

  517.     init_put_bits(&buf, buffer, len);

  518. 

  519.     put_bits32(&buf, 1); /* NumOfSpatial Regions, only 1 supported */

  520.     put_bits(&buf, 1, 1); /* SpatialLocationFlag, always the whole image */

  521.     put_bits32(&buf, 0); /* PixelX,1 PixelY,1, 0,0 */

  522.     put_bits(&buf, 16, sc->w-1 & 0xFFFF); /* PixelX,2 */

  523.     put_bits(&buf, 16, sc->h-1 & 0xFFFF); /* PixelY,2 */

  524.     put_bits32(&buf, 0); /* StartFrameOfSpatialRegion */

  525.     put_bits32(&buf, sc->lastindex); /* NumOfFrames */

  526.     /* hoping num is 1, other values are vague */

  527.     /* den/num might be greater than 16 bit, so cutting it */

  528.     put_bits(&buf, 16, 0xFFFF & (sc->time_base.den / sc->time_base.num)); /* MediaTimeUnit */

  529.     put_bits(&buf, 1, 1); /* MediaTimeFlagOfSpatialRegion */

  530.     put_bits32(&buf, 0); /* StartMediaTimeOfSpatialRegion */

  531.     put_bits32(&buf, 0xFFFFFFFF & sc->coarseend->last->pts); /* EndMediaTimeOfSpatialRegion */

  532.     put_bits32(&buf, numofsegments); /* NumOfSegments */

  533.     /* coarsesignatures */

  534.     for (cs = sc->coarsesiglist; cs; cs = cs->next) {

  535.         put_bits32(&buf, cs->first->index); /* StartFrameOfSegment */

  536.         put_bits32(&buf, cs->last->index); /* EndFrameOfSegment */

  537.         put_bits(&buf, 1, 1); /* MediaTimeFlagOfSegment */

  538.         put_bits32(&buf, 0xFFFFFFFF & cs->first->pts); /* StartMediaTimeOfSegment */

  539.         put_bits32(&buf, 0xFFFFFFFF & cs->last->pts); /* EndMediaTimeOfSegment */

  540.         for (i = 0; i < 5; i++) {

  541.             /* put 243 bits ( = 7 * 32 + 19 = 8 * 28 + 19) into buffer */

  542.             for (j = 0; j < 30; j++) {

  543.                 put_bits(&buf, 8, cs->data[i][j]);

  544.             }

  545.             put_bits(&buf, 3, cs->data[i][30] >> 5);

  546.         }

  547.     }

  548.     /* finesignatures */

  549.     put_bits(&buf, 1, 0); /* CompressionFlag, only 0 supported */

  550.     for (fs = sc->finesiglist; fs; fs = fs->next) {

  551.         put_bits(&buf, 1, 1); /* MediaTimeFlagOfFrame */

  552.         put_bits32(&buf, 0xFFFFFFFF & fs->pts); /* MediaTimeOfFrame */

  553.         put_bits(&buf, 8, fs->confidence); /* FrameConfidence */

  554.         for (i = 0; i < 5; i++) {

  555.             put_bits(&buf, 8, fs->words[i]); /* Words */

  556.         }

  557.         /* framesignature */

  558.         for (i = 0; i < SIGELEM_SIZE/5; i++) {

  559.             put_bits(&buf, 8, fs->framesig[i]);

  560.         }

  561.     }

  562. 

  563.     flush_put_bits(&buf);

  564.     fwrite(buffer, 1, put_bits_count(&buf)/8, f);

  565.     fclose(f);

  566.     av_freep(&buffer);

  567.     return 0;

  568. }

  569. 

  570. static int export(AVFilterContext *ctx, StreamContext *sc, int input)

  571. {

  572.     SignatureContext* sic = ctx->priv;

  573.     char filename[1024];

  574. 

  575.     if (sic->nb_inputs > 1) {

  576.         /* error already handled */

  577.         av_assert0(av_get_frame_filename(filename, sizeof(filename), sic->filename, input) == 0);

  578.     } else {

  579.         if (av_strlcpy(filename, sic->filename, sizeof(filename)) >= sizeof(filename))

  580.             return AVERROR(EINVAL);

  581.     }

  582.     if (sic->format == FORMAT_XML) {

  583.         return xml_export(ctx, sc, filename);

  584.     } else {

  585.         return binary_export(ctx, sc, filename);

  586.     }

  587. }

  588. 

  589. static int request_frame(AVFilterLink *outlink)

  590. {

  591.     AVFilterContext *ctx = outlink->src;

  592.     SignatureContext *sic = ctx->priv;

  593.     StreamContext *sc, *sc2;

  594.     MatchingInfo match;

  595.     int i, j, ret;

  596.     int lookup = 1; /* indicates wheather EOF of all files is reached */

  597. 

  598.     /* process all inputs */

  599.     for (i = 0; i < sic->nb_inputs; i++){

  600.         sc = &(sic->streamcontexts[i]);

  601. 

  602.         ret = ff_request_frame(ctx->inputs[i]);

  603. 

  604.         /* return if unexpected error occurs in input stream */

  605.         if (ret < 0 && ret != AVERROR_EOF)

  606.             return ret;

  607. 

  608.         /* export signature at EOF */

  609.         if (ret == AVERROR_EOF && !sc->exported) {

  610.             /* export if wanted */

  611.             if (strlen(sic->filename) > 0) {

  612.                 if (export(ctx, sc, i) < 0)

  613.                     return ret;

  614.             }

  615.             sc->exported = 1;

  616.         }

  617.         lookup &= sc->exported;

  618.     }

  619. 

  620.     /* signature lookup */

  621.     if (lookup && sic->mode != MODE_OFF) {

  622.         /* iterate over every pair */

  623.         for (i = 0; i < sic->nb_inputs; i++) {

  624.             sc = &(sic->streamcontexts[i]);

  625.             for (j = i+1; j < sic->nb_inputs; j++) {

  626.                 sc2 = &(sic->streamcontexts[j]);

  627.                 match = lookup_signatures(ctx, sic, sc, sc2, sic->mode);

  628.                 if (match.score != 0) {

  629.                     av_log(ctx, AV_LOG_INFO, "matching of video %d at %f and %d at %f, %d frames matching\n",

  630.                             i, ((double) match.first->pts * sc->time_base.num) / sc->time_base.den,

  631.                             j, ((double) match.second->pts * sc2->time_base.num) / sc2->time_base.den,

  632.                             match.matchframes);

  633.                     if (match.whole)

  634.                         av_log(ctx, AV_LOG_INFO, "whole video matching\n");

  635.                 } else {

  636.                     av_log(ctx, AV_LOG_INFO, "no matching of video %d and %d\n", i, j);

  637.                 }

  638.             }

  639.         }

  640.     }

  641. 

  642.     return ret;

  643. }

  644. 

  645. static av_cold int init(AVFilterContext *ctx)

  646. {

  647. 

  648.     SignatureContext *sic = ctx->priv;

  649.     StreamContext *sc;

  650.     int i, ret;

  651.     char tmp[1024];

  652. 

  653.     sic->streamcontexts = av_mallocz(sic->nb_inputs * sizeof(StreamContext));

  654.     if (!sic->streamcontexts)

  655.         return AVERROR(ENOMEM);

  656. 

  657.     for (i = 0; i < sic->nb_inputs; i++) {

  658.         AVFilterPad pad = {

  659.             .type = AVMEDIA_TYPE_VIDEO,

  660.             .name = av_asprintf("in%d", i),

  661.             .config_props = config_input,

  662.             .filter_frame = filter_frame,

  663.         };

  664. 

  665.         if (!pad.name)

  666.             return AVERROR(ENOMEM);

  667.         if ((ret = ff_insert_inpad(ctx, i, &pad)) < 0) {

  668.             av_freep(&pad.name);

  669.             return ret;

  670.         }

  671. 

  672.         sc = &(sic->streamcontexts[i]);

  673. 

  674.         sc->lastindex = 0;

  675.         sc->finesiglist = av_mallocz(sizeof(FineSignature));

  676.         if (!sc->finesiglist)

  677.             return AVERROR(ENOMEM);

  678.         sc->curfinesig = NULL;

  679. 

  680.         sc->coarsesiglist = av_mallocz(sizeof(CoarseSignature));

  681.         if (!sc->coarsesiglist)

  682.             return AVERROR(ENOMEM);

  683.         sc->curcoarsesig1 = sc->coarsesiglist;

  684.         sc->coarseend = sc->coarsesiglist;

  685.         sc->coarsecount = 0;

  686.         sc->midcoarse = 0;

  687.     }

  688. 

  689.     /* check filename */

  690.     if (sic->nb_inputs > 1 && strlen(sic->filename) > 0 && av_get_frame_filename(tmp, sizeof(tmp), sic->filename, 0) == -1) {

  691.         av_log(ctx, AV_LOG_ERROR, "The filename must contain %%d or %%0nd, if you have more than one input.\n");

  692.         return AVERROR(EINVAL);

  693.     }

  694. 

  695.     return 0;

  696. }

  697. 

  698. 

  699. 

  700. static av_cold void uninit(AVFilterContext *ctx)

  701. {

  702.     SignatureContext *sic = ctx->priv;

  703.     StreamContext *sc;

  704.     void* tmp;

  705.     FineSignature* finsig;

  706.     CoarseSignature* cousig;

  707.     int i;

  708. 

  709. 

  710.     /* free the lists */

  711.     if (sic->streamcontexts != NULL) {

  712.         for (i = 0; i < sic->nb_inputs; i++) {

  713.             sc = &(sic->streamcontexts[i]);

  714.             finsig = sc->finesiglist;

  715.             cousig = sc->coarsesiglist;

  716. 

  717.             while (finsig) {

  718.                 tmp = finsig;

  719.                 finsig = finsig->next;

  720.                 av_freep(&tmp);

  721.             }

  722.             sc->finesiglist = NULL;

  723. 

  724.             while (cousig) {

  725.                 tmp = cousig;

  726.                 cousig = cousig->next;

  727.                 av_freep(&tmp);

  728.             }

  729.             sc->coarsesiglist = NULL;

  730.         }

  731.         av_freep(&sic->streamcontexts);

  732.     }

  733.     for (unsigned i = 0; i < ctx->nb_inputs; i++)

  734.         av_freep(&ctx->input_pads[i].name);

  735. }

  736. 

  737. static int config_output(AVFilterLink *outlink)

  738. {

  739.     AVFilterContext *ctx = outlink->src;

  740.     AVFilterLink *inlink = ctx->inputs[0];

  741. 

  742.     outlink->time_base = inlink->time_base;

  743.     outlink->frame_rate = inlink->frame_rate;

  744.     outlink->sample_aspect_ratio = inlink->sample_aspect_ratio;

  745.     outlink->w = inlink->w;

  746.     outlink->h = inlink->h;

  747. 

  748.     return 0;

  749. }

  750. 

  751. static const AVFilterPad signature_outputs[] = {

  752.     {

  753.         .name          = "default",

  754.         .type          = AVMEDIA_TYPE_VIDEO,

  755.         .request_frame = request_frame,

  756.         .config_props  = config_output,

  757.     },

  758.     { NULL }

  759. };

  760. 

  761. AVFilter ff_vf_signature = {

  762.     .name          = "signature",

  763.     .description   = NULL_IF_CONFIG_SMALL("Calculate the MPEG-7 video signature"),

  764.     .priv_size     = sizeof(SignatureContext),

  765.     .priv_class    = &signature_class,

  766.     .init          = init,

  767.     .uninit        = uninit,

  768.     .query_formats = query_formats,

  769.     .outputs       = signature_outputs,

  770.     .inputs        = NULL,

  771.     .flags         = AVFILTER_FLAG_DYNAMIC_INPUTS,

  772. };