You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

497 lines
20KB

  1. # Copyright (c) 2019 Guo Yejun
  2. #
  3. # This file is part of FFmpeg.
  4. #
  5. # FFmpeg is free software; you can redistribute it and/or
  6. # modify it under the terms of the GNU Lesser General Public
  7. # License as published by the Free Software Foundation; either
  8. # version 2.1 of the License, or (at your option) any later version.
  9. #
  10. # FFmpeg is distributed in the hope that it will be useful,
  11. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13. # Lesser General Public License for more details.
  14. #
  15. # You should have received a copy of the GNU Lesser General Public
  16. # License along with FFmpeg; if not, write to the Free Software
  17. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  18. # ==============================================================================
  19. import tensorflow as tf
  20. import numpy as np
  21. import sys, struct
  22. import convert_header as header
  23. __all__ = ['convert_from_tensorflow']
  24. class Operand(object):
  25. IOTYPE_INPUT = 1
  26. IOTYPE_OUTPUT = 2
  27. IOTYPE_INTERMEDIATE = IOTYPE_INPUT | IOTYPE_OUTPUT
  28. DTYPE_FLOAT = 1
  29. DTYPE_UINT8 = 4
  30. index = 0
  31. def __init__(self, name, dtype, dims):
  32. self.name = name
  33. self.dtype = dtype
  34. self.dims = dims
  35. self.iotype = 0
  36. self.used_count = 0
  37. self.index = Operand.index
  38. Operand.index = Operand.index + 1
  39. self.iotype2str = {Operand.IOTYPE_INPUT: 'in', Operand.IOTYPE_OUTPUT: 'out', Operand.IOTYPE_INTERMEDIATE: 'inout'}
  40. self.dtype2str = {Operand.DTYPE_FLOAT: 'DT_FLOAT', Operand.DTYPE_UINT8: 'DT_UINT8'}
  41. def add_iotype(self, iotype):
  42. self.iotype = self.iotype | iotype
  43. if iotype == Operand.IOTYPE_INPUT:
  44. self.used_count = self.used_count + 1
  45. def __str__(self):
  46. return "{}: (name: {}, iotype: {}, dtype: {}, dims: ({},{},{},{}) used_count: {})".format(self.index,
  47. self.name, self.iotype2str[self.iotype], self.dtype2str[self.dtype],
  48. self.dims[0], self.dims[1], self.dims[2], self.dims[3], self.used_count)
  49. def __lt__(self, other):
  50. return self.index < other.index
  51. class TFConverter:
  52. def __init__(self, graph_def, nodes, outfile, dump4tb):
  53. self.graph_def = graph_def
  54. self.nodes = nodes
  55. self.outfile = outfile
  56. self.dump4tb = dump4tb
  57. self.layer_number = 0
  58. self.output_names = []
  59. self.name_node_dict = {}
  60. self.edges = {}
  61. self.conv_activations = {'Relu':0, 'Tanh':1, 'Sigmoid':2, 'None':3, 'LeakyRelu':4}
  62. self.conv_paddings = {'VALID':0, 'SAME':1}
  63. self.pool_paddings = {'VALID':0, 'SAME':1}
  64. self.converted_nodes = set()
  65. self.conv2d_scope_names = set()
  66. self.conv2d_scopename_inputname_dict = {}
  67. self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3, 'Maximum':4,
  68. 'MathBinary':5, 'MathUnary':6, 'AvgPool':7}
  69. self.mathbin2code = {'Sub':0, 'Add':1, 'Mul':2, 'RealDiv':3, 'Minimum':4}
  70. self.mathun2code = {'Abs':0, 'Sin':1, 'Cos':2, 'Tan':3, 'Asin':4,
  71. 'Acos':5, 'Atan':6, 'Sinh':7, 'Cosh':8, 'Tanh':9, 'Asinh':10,
  72. 'Acosh':11, 'Atanh':12, 'Ceil':13, 'Floor':14, 'Round':15}
  73. self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
  74. self.name_operand_dict = {}
  75. def add_operand(self, name, type):
  76. node = self.name_node_dict[name]
  77. if name not in self.name_operand_dict:
  78. dtype = node.attr['dtype'].type
  79. if dtype == 0:
  80. dtype = node.attr['T'].type
  81. dims = [-1,-1,-1,-1]
  82. if 'shape' in node.attr:
  83. dims[0] = node.attr['shape'].shape.dim[0].size
  84. dims[1] = node.attr['shape'].shape.dim[1].size
  85. dims[2] = node.attr['shape'].shape.dim[2].size
  86. dims[3] = node.attr['shape'].shape.dim[3].size
  87. operand = Operand(name, dtype, dims)
  88. self.name_operand_dict[name] = operand;
  89. self.name_operand_dict[name].add_iotype(type)
  90. return self.name_operand_dict[name].index
  91. def dump_for_tensorboard(self):
  92. graph = tf.get_default_graph()
  93. tf.import_graph_def(self.graph_def, name="")
  94. tf.summary.FileWriter('/tmp/graph', graph)
  95. print('graph saved, run "tensorboard --logdir=/tmp/graph" to see it')
  96. def get_conv2d_params(self, conv2d_scope_name):
  97. knode = self.name_node_dict[conv2d_scope_name + '/kernel']
  98. bnode = self.name_node_dict[conv2d_scope_name + '/bias']
  99. if conv2d_scope_name + '/dilation_rate' in self.name_node_dict:
  100. dnode = self.name_node_dict[conv2d_scope_name + '/dilation_rate']
  101. else:
  102. dnode = None
  103. # the BiasAdd name is possible be changed into the output name,
  104. # if activation is None, and BiasAdd.next is the last op which is Identity
  105. if conv2d_scope_name + '/BiasAdd' in self.edges:
  106. anode = self.edges[conv2d_scope_name + '/BiasAdd'][0]
  107. if anode.op not in self.conv_activations:
  108. anode = None
  109. else:
  110. anode = None
  111. return knode, bnode, dnode, anode
  112. def dump_complex_conv2d_to_file(self, node, f):
  113. assert(node.op == 'Conv2D')
  114. self.layer_number = self.layer_number + 1
  115. self.converted_nodes.add(node.name)
  116. scope_name = TFConverter.get_scope_name(node.name)
  117. #knode for kernel, bnode for bias, dnode for dilation, anode for activation
  118. knode, bnode, dnode, anode = self.get_conv2d_params(scope_name)
  119. if dnode is not None:
  120. dilation = struct.unpack('i', dnode.attr['value'].tensor.tensor_content[0:4])[0]
  121. else:
  122. dilation = 1
  123. if anode is not None:
  124. activation = anode.op
  125. else:
  126. activation = 'None'
  127. padding = node.attr['padding'].s.decode("utf-8")
  128. # conv2d with dilation > 1 generates tens of nodes, not easy to parse them, so use this tricky method.
  129. if dilation > 1 and scope_name + '/stack' in self.name_node_dict:
  130. if self.name_node_dict[scope_name + '/stack'].op == "Const":
  131. padding = 'SAME'
  132. padding = self.conv_paddings[padding]
  133. ktensor = knode.attr['value'].tensor
  134. filter_height = ktensor.tensor_shape.dim[0].size
  135. filter_width = ktensor.tensor_shape.dim[1].size
  136. in_channels = ktensor.tensor_shape.dim[2].size
  137. out_channels = ktensor.tensor_shape.dim[3].size
  138. kernel = np.frombuffer(ktensor.tensor_content, dtype=np.float32)
  139. kernel = kernel.reshape(filter_height, filter_width, in_channels, out_channels)
  140. kernel = np.transpose(kernel, [3, 0, 1, 2])
  141. has_bias = 1
  142. np.array([self.op2code[node.op], dilation, padding, self.conv_activations[activation], in_channels, out_channels, filter_height, has_bias], dtype=np.uint32).tofile(f)
  143. kernel.tofile(f)
  144. btensor = bnode.attr['value'].tensor
  145. if btensor.tensor_shape.dim[0].size == 1:
  146. bias = struct.pack("f", btensor.float_val[0])
  147. else:
  148. bias = btensor.tensor_content
  149. f.write(bias)
  150. input_name = self.conv2d_scopename_inputname_dict[scope_name]
  151. input_operand_index = self.add_operand(input_name, Operand.IOTYPE_INPUT)
  152. if anode is not None:
  153. output_operand_index = self.add_operand(anode.name, Operand.IOTYPE_OUTPUT)
  154. else:
  155. output_operand_index = self.add_operand(self.edges[bnode.name][0].name, Operand.IOTYPE_OUTPUT)
  156. np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
  157. def dump_simple_conv2d_to_file(self, node, f):
  158. assert(node.op == 'Conv2D')
  159. self.layer_number = self.layer_number + 1
  160. self.converted_nodes.add(node.name)
  161. node0 = self.name_node_dict[node.input[0]]
  162. node1 = self.name_node_dict[node.input[1]]
  163. if node0.op == 'Const':
  164. knode = node0
  165. input_name = node.input[1]
  166. else:
  167. knode = node1
  168. input_name = node.input[0]
  169. ktensor = knode.attr['value'].tensor
  170. filter_height = ktensor.tensor_shape.dim[0].size
  171. filter_width = ktensor.tensor_shape.dim[1].size
  172. in_channels = ktensor.tensor_shape.dim[2].size
  173. out_channels = ktensor.tensor_shape.dim[3].size
  174. if filter_height * filter_width * in_channels * out_channels == 1:
  175. kernel = np.float32(ktensor.float_val[0])
  176. else:
  177. kernel = np.frombuffer(ktensor.tensor_content, dtype=np.float32)
  178. kernel = kernel.reshape(filter_height, filter_width, in_channels, out_channels)
  179. kernel = np.transpose(kernel, [3, 0, 1, 2])
  180. has_bias = 0
  181. dilation = 1
  182. padding = node.attr['padding'].s.decode("utf-8")
  183. np.array([self.op2code[node.op], dilation, self.conv_paddings[padding], self.conv_activations['None'],
  184. in_channels, out_channels, filter_height, has_bias], dtype=np.uint32).tofile(f)
  185. kernel.tofile(f)
  186. input_operand_index = self.add_operand(input_name, Operand.IOTYPE_INPUT)
  187. output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
  188. np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
  189. def dump_depth2space_to_file(self, node, f):
  190. assert(node.op == 'DepthToSpace')
  191. self.layer_number = self.layer_number + 1
  192. block_size = node.attr['block_size'].i
  193. np.array([self.op2code[node.op], block_size], dtype=np.uint32).tofile(f)
  194. self.converted_nodes.add(node.name)
  195. input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
  196. output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
  197. np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
  198. def dump_mirrorpad_to_file(self, node, f):
  199. assert(node.op == 'MirrorPad')
  200. self.layer_number = self.layer_number + 1
  201. mode = node.attr['mode'].s
  202. mode = self.mirrorpad_mode[mode.decode("utf-8")]
  203. np.array([self.op2code[node.op], mode], dtype=np.uint32).tofile(f)
  204. pnode = self.name_node_dict[node.input[1]]
  205. self.converted_nodes.add(pnode.name)
  206. paddings = pnode.attr['value'].tensor.tensor_content
  207. f.write(paddings)
  208. self.converted_nodes.add(node.name)
  209. input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
  210. output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
  211. np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
  212. def dump_maximum_to_file(self, node, f):
  213. assert(node.op == 'Maximum')
  214. self.layer_number = self.layer_number + 1
  215. ynode = self.name_node_dict[node.input[1]]
  216. y = ynode.attr['value'].tensor.float_val[0]
  217. np.array([self.op2code[node.op]], dtype=np.uint32).tofile(f)
  218. np.array([y], dtype=np.float32).tofile(f)
  219. self.converted_nodes.add(node.name)
  220. input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
  221. output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
  222. np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
  223. def dump_mathbinary_to_file(self, node, f):
  224. self.layer_number = self.layer_number + 1
  225. self.converted_nodes.add(node.name)
  226. i0_node = self.name_node_dict[node.input[0]]
  227. i1_node = self.name_node_dict[node.input[1]]
  228. np.array([self.op2code['MathBinary'], self.mathbin2code[node.op]], dtype=np.uint32).tofile(f)
  229. if i0_node.op == 'Const':
  230. scalar = i0_node.attr['value'].tensor.float_val[0]
  231. np.array([1], dtype=np.uint32).tofile(f) # broadcast: 1
  232. np.array([scalar], dtype=np.float32).tofile(f)
  233. np.array([0], dtype=np.uint32).tofile(f) # broadcast: 0
  234. input_operand_index = self.add_operand(i1_node.name, Operand.IOTYPE_INPUT)
  235. np.array([input_operand_index], dtype=np.uint32).tofile(f)
  236. elif i1_node.op == 'Const':
  237. scalar = i1_node.attr['value'].tensor.float_val[0]
  238. np.array([0], dtype=np.uint32).tofile(f)
  239. input_operand_index = self.add_operand(i0_node.name, Operand.IOTYPE_INPUT)
  240. np.array([input_operand_index], dtype=np.uint32).tofile(f)
  241. np.array([1], dtype=np.uint32).tofile(f)
  242. np.array([scalar], dtype=np.float32).tofile(f)
  243. else:
  244. np.array([0], dtype=np.uint32).tofile(f)
  245. input_operand_index = self.add_operand(i0_node.name, Operand.IOTYPE_INPUT)
  246. np.array([input_operand_index], dtype=np.uint32).tofile(f)
  247. np.array([0], dtype=np.uint32).tofile(f)
  248. input_operand_index = self.add_operand(i1_node.name, Operand.IOTYPE_INPUT)
  249. np.array([input_operand_index], dtype=np.uint32).tofile(f)
  250. output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
  251. np.array([output_operand_index], dtype=np.uint32).tofile(f)
  252. def dump_mathunary_to_file(self, node, f):
  253. self.layer_number = self.layer_number + 1
  254. self.converted_nodes.add(node.name)
  255. i0_node = self.name_node_dict[node.input[0]]
  256. np.array([self.op2code['MathUnary'], self.mathun2code[node.op]], dtype=np.uint32).tofile(f)
  257. input_operand_index = self.add_operand(i0_node.name, Operand.IOTYPE_INPUT)
  258. np.array([input_operand_index], dtype=np.uint32).tofile(f)
  259. output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
  260. np.array([output_operand_index],dtype=np.uint32).tofile(f)
  261. def dump_avg_pool_to_file(self, node, f):
  262. assert(node.op == 'AvgPool')
  263. self.layer_number = self.layer_number + 1
  264. self.converted_nodes.add(node.name)
  265. node0 = self.name_node_dict[node.input[0]]
  266. strides = node.attr['strides']
  267. # Tensorflow do not support pooling strides in batch dimension and
  268. # current native NN do not support pooling strides in channel dimension, added assert() here.
  269. assert(strides.list.i[1]==strides.list.i[2])
  270. assert(strides.list.i[0]==1)
  271. assert(strides.list.i[3]==1)
  272. strides = strides.list.i[1]
  273. filter_node = node.attr['ksize']
  274. input_name = node.input[0]
  275. # Tensorflow do not support pooling ksize in batch dimension and channel dimension.
  276. assert(filter_node.list.i[0]==1)
  277. assert(filter_node.list.i[3]==1)
  278. filter_height = filter_node.list.i[1]
  279. filter_width = filter_node.list.i[2]
  280. padding = node.attr['padding'].s.decode("utf-8")
  281. np.array([self.op2code[node.op], strides, self.pool_paddings[padding], filter_height],
  282. dtype=np.uint32).tofile(f)
  283. input_operand_index = self.add_operand(input_name, Operand.IOTYPE_INPUT)
  284. output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
  285. np.array([input_operand_index, output_operand_index],dtype=np.uint32).tofile(f)
  286. def dump_layers_to_file(self, f):
  287. for node in self.nodes:
  288. if node.name in self.converted_nodes:
  289. continue
  290. # conv2d with dilation generates very complex nodes, so handle it in special
  291. if self.in_conv2d_scope(node.name):
  292. if node.op == 'Conv2D':
  293. self.dump_complex_conv2d_to_file(node, f)
  294. continue
  295. if node.op == 'Conv2D':
  296. self.dump_simple_conv2d_to_file(node, f)
  297. if node.op == 'AvgPool':
  298. self.dump_avg_pool_to_file(node, f)
  299. elif node.op == 'DepthToSpace':
  300. self.dump_depth2space_to_file(node, f)
  301. elif node.op == 'MirrorPad':
  302. self.dump_mirrorpad_to_file(node, f)
  303. elif node.op == 'Maximum':
  304. self.dump_maximum_to_file(node, f)
  305. elif node.op in self.mathbin2code:
  306. self.dump_mathbinary_to_file(node, f)
  307. elif node.op in self.mathun2code:
  308. self.dump_mathunary_to_file(node, f)
  309. def dump_operands_to_file(self, f):
  310. operands = sorted(self.name_operand_dict.values())
  311. for operand in operands:
  312. #print('{}'.format(operand))
  313. np.array([operand.index, len(operand.name)], dtype=np.uint32).tofile(f)
  314. f.write(operand.name.encode('utf-8'))
  315. np.array([operand.iotype, operand.dtype], dtype=np.uint32).tofile(f)
  316. np.array([operand.dims[0], operand.dims[1], operand.dims[2], operand.dims[3]], dtype=np.uint32).tofile(f)
  317. def dump_to_file(self):
  318. with open(self.outfile, 'wb') as f:
  319. f.write(header.str.encode('utf-8'))
  320. np.array([header.major, header.minor], dtype=np.uint32).tofile(f)
  321. self.dump_layers_to_file(f)
  322. self.dump_operands_to_file(f)
  323. np.array([self.layer_number, len(self.name_operand_dict)], dtype=np.uint32).tofile(f)
  324. def generate_name_node_dict(self):
  325. for node in self.nodes:
  326. self.name_node_dict[node.name] = node
  327. def generate_output_names(self):
  328. used_names = []
  329. for node in self.nodes:
  330. for input in node.input:
  331. used_names.append(input)
  332. for node in self.nodes:
  333. if node.name not in used_names:
  334. self.output_names.append(node.name)
  335. def remove_identity(self):
  336. id_nodes = []
  337. id_dict = {}
  338. for node in self.nodes:
  339. if node.op == 'Identity':
  340. name = node.name
  341. input = node.input[0]
  342. id_nodes.append(node)
  343. # do not change the output name
  344. if name in self.output_names:
  345. self.name_node_dict[input].name = name
  346. self.name_node_dict[name] = self.name_node_dict[input]
  347. del self.name_node_dict[input]
  348. else:
  349. id_dict[name] = input
  350. for idnode in id_nodes:
  351. self.nodes.remove(idnode)
  352. for node in self.nodes:
  353. for i in range(len(node.input)):
  354. input = node.input[i]
  355. if input in id_dict:
  356. node.input[i] = id_dict[input]
  357. def generate_edges(self):
  358. for node in self.nodes:
  359. for input in node.input:
  360. if input in self.edges:
  361. self.edges[input].append(node)
  362. else:
  363. self.edges[input] = [node]
  364. @staticmethod
  365. def get_scope_name(name):
  366. index = name.rfind('/')
  367. if index == -1:
  368. return ""
  369. return name[0:index]
  370. def in_conv2d_scope(self, name):
  371. inner_scope = TFConverter.get_scope_name(name)
  372. if inner_scope == "":
  373. return False;
  374. for scope in self.conv2d_scope_names:
  375. index = inner_scope.find(scope)
  376. if index == 0:
  377. return True
  378. return False
  379. def generate_conv2d_scope_info(self):
  380. # mostly, conv2d is a sub block in graph, get the scope name
  381. for node in self.nodes:
  382. if node.op == 'Conv2D':
  383. scope = TFConverter.get_scope_name(node.name)
  384. # for the case tf.nn.conv2d is called directly
  385. if scope == '':
  386. continue
  387. # for the case tf.nn.conv2d is called within a scope
  388. if scope + '/kernel' not in self.name_node_dict:
  389. continue
  390. self.conv2d_scope_names.add(scope)
  391. # get the input name to the conv2d sub block
  392. for node in self.nodes:
  393. scope = TFConverter.get_scope_name(node.name)
  394. if scope in self.conv2d_scope_names:
  395. if node.op == 'Conv2D' or node.op == 'Shape':
  396. for inp in node.input:
  397. if TFConverter.get_scope_name(inp) != scope:
  398. self.conv2d_scopename_inputname_dict[scope] = inp
  399. def run(self):
  400. self.generate_name_node_dict()
  401. self.generate_output_names()
  402. self.remove_identity()
  403. self.generate_edges()
  404. self.generate_conv2d_scope_info()
  405. if self.dump4tb:
  406. self.dump_for_tensorboard()
  407. self.dump_to_file()
  408. def convert_from_tensorflow(infile, outfile, dump4tb):
  409. with open(infile, 'rb') as f:
  410. # read the file in .proto format
  411. graph_def = tf.GraphDef()
  412. graph_def.ParseFromString(f.read())
  413. nodes = graph_def.node
  414. converter = TFConverter(graph_def, nodes, outfile, dump4tb)
  415. converter.run()