tiny_dnn 1.0.0
A header only, dependency-free deep learning framework in C++11
Loading...
Searching...
No Matches
quantized_convolutional_layer.h
1/*
2 Copyright (c) 2013, Taiga Nomi
3 All rights reserved.
4
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in the
11 documentation and/or other materials provided with the distribution.
12 * Neither the name of the <organization> nor the
13 names of its contributors may be used to endorse or promote products
14 derived from this software without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
17 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26*/
27#pragma once
28
29#include <vector>
30#include <string>
31#include <algorithm>
32
33#include "tiny_dnn/core/backend_tiny.h"
34#include "tiny_dnn/core/backend_nnp.h"
35#include "tiny_dnn/core/backend_dnn.h"
36#ifdef CNN_USE_AVX
37#include "tiny_dnn/core/backend_avx.h"
38#endif
39
40#include "tiny_dnn/util/util.h"
41#include "tiny_dnn/util/image.h"
42#include "tiny_dnn/activations/activation_function.h"
43
44using namespace tiny_dnn::core;
45
46namespace tiny_dnn {
47
53template<typename Activation = activation::identity>
55 public:
57 CNN_USE_LAYER_MEMBERS;
58
75 serial_size_t in_height,
76 serial_size_t window_size,
77 serial_size_t in_channels,
78 serial_size_t out_channels,
79 padding pad_type = padding::valid,
80 bool has_bias = true,
81 serial_size_t w_stride = 1,
82 serial_size_t h_stride = 1,
83 backend_t backend_type = core::backend_t::internal)
84 : Base(std_input_order(has_bias)) {
85 conv_set_params(shape3d(in_width, in_height, in_channels),
87 out_channels, pad_type, has_bias,
88 w_stride, h_stride);
89 init_backend(backend_type);
90 }
91
109 serial_size_t in_height,
110 serial_size_t window_width,
111 serial_size_t window_height,
112 serial_size_t in_channels,
113 serial_size_t out_channels,
114 padding pad_type = padding::valid,
115 bool has_bias = true,
116 serial_size_t w_stride = 1,
117 serial_size_t h_stride = 1,
118 backend_t backend_type = core::backend_t::internal)
119 : Base(std_input_order(has_bias)) {
120 conv_set_params(shape3d(in_width, in_height, in_channels),
122 out_channels, pad_type, has_bias,
123 w_stride, h_stride);
124 init_backend(backend_type);
125 }
126
144 serial_size_t in_height,
145 serial_size_t window_size,
146 serial_size_t in_channels,
147 serial_size_t out_channels,
149 padding pad_type = padding::valid,
150 bool has_bias = true,
151 serial_size_t w_stride = 1,
152 serial_size_t h_stride = 1,
153 backend_t backend_type = core::backend_t::internal)
154 : Base(std_input_order(has_bias)) {
155 conv_set_params(shape3d(in_width, in_height, in_channels),
157 out_channels, pad_type, has_bias,
158 w_stride, h_stride,
160 init_backend(backend_type);
161 }
162
181 serial_size_t in_height,
182 serial_size_t window_width,
183 serial_size_t window_height,
184 serial_size_t in_channels,
185 serial_size_t out_channels,
187 padding pad_type = padding::valid,
188 bool has_bias = true,
189 serial_size_t w_stride = 1,
190 serial_size_t h_stride = 1,
191 backend_t backend_type = core::backend_t::internal)
192 : Base(has_bias ? 3 : 2, 1, std_input_order(has_bias)) {
193 conv_set_params(shape3d(in_width, in_height, in_channels),
195 out_channels, pad_type, has_bias,
196 w_stride, h_stride,
198 init_backend(backend_type);
199 }
200
201 // move constructor
203 : Base(std::move(other))
204 , params_(std::move(other.params_))
205 , cws_(std::move(other.cws_)) {
206 init_backend(core::backend_t::internal);
207 }
208
210 serial_size_t fan_in_size() const override {
211 return params_.weight.width_ *
212 params_.weight.height_ * params_.in.depth_;
213 }
214
216 serial_size_t fan_out_size() const override {
217 return (params_.weight.width_ / params_.w_stride) *
218 (params_.weight.height_ / params_.h_stride) *
219 params_.out.depth_;
220 }
221
226 void forward_propagation(const std::vector<tensor_t*>& in_data,
227 std::vector<tensor_t*>& out_data) override {
228 // launch convolutional kernel
229 if (in_data.size() == 3) {
230 Base::backend_->conv2d_q(in_data, out_data);
231
232 // activations
233 this->forward_activation(*out_data[0], *out_data[1]);
234 } else if (in_data.size() == 6) {
235 Base::backend_->conv2d_eq(in_data, out_data);
236 }
237 }
238
246 void back_propagation(const std::vector<tensor_t*>& in_data,
247 const std::vector<tensor_t*>& out_data,
248 std::vector<tensor_t*>& out_grad,
249 std::vector<tensor_t*>& in_grad) override {
250 Base::backend_->conv2d_q(in_data, out_data, out_grad, in_grad);
251 }
252
253 std::vector<index3d<serial_size_t>> in_shape() const override {
254 if (params_.has_bias) {
255 return { params_.in, params_.weight,
256 index3d<serial_size_t>(1, 1, params_.out.depth_) };
257 } else {
258 return { params_.in, params_.weight };
259 }
260 }
261
262 std::vector<index3d<serial_size_t>>
263 out_shape() const override { return { params_.out, params_.out }; }
264
265 std::string layer_type() const override { return "q_conv"; }
266
267 image<> weight_to_image() const {
268 image<> img;
269 const serial_size_t border_width = 1;
270 const auto pitch = params_.weight.width_ + border_width;
271 const auto width = params_.out.depth_ * pitch + border_width;
272 const auto height = params_.in.depth_ * pitch + border_width;
273 const image<>::intensity_t bg_color = 255;
274 const vec_t& W = *this->get_weights()[0];
275
276 img.resize(width, height);
277 img.fill(bg_color);
278
279 auto minmax = std::minmax_element(W.begin(), W.end());
280
281 for (serial_size_t r = 0; r < params_.in.depth_; ++r) {
282 for (serial_size_t c = 0; c < params_.out.depth_; ++c) {
283 if (!params_.tbl.is_connected(c, r)) continue;
284
285 const auto top = r * pitch + border_width;
286 const auto left = c * pitch + border_width;
287
288 serial_size_t idx = 0;
289
290 for (serial_size_t y = 0; y < params_.weight.height_; ++y) {
291 for (serial_size_t x = 0; x < params_.weight.width_; ++x) {
292 idx = c * params_.in.depth_ + r;
293 idx = params_.weight.get_index(x, y, idx);
294 const float_t w = W[idx];
295
296 img.at(left + x, top + y)
297 = static_cast<image<>::intensity_t>(
298 rescale(w, *minmax.first,
299 *minmax.second, 0, 255));
300 }
301 }
302 }
303 }
304 return img;
305 }
306
307 private:
308 void conv_set_params(const shape3d& in,
309 serial_size_t w_width,
310 serial_size_t w_height,
311 serial_size_t outc,
312 padding ptype,
313 bool has_bias,
314 serial_size_t w_stride,
315 serial_size_t h_stride,
316 const connection_table& tbl = connection_table()) {
317 params_.in = in;
318 params_.in_padded = shape3d(in_length(in.width_, w_width, ptype),
319 in_length(in.height_, w_height, ptype),
320 in.depth_);
321 params_.out =
322 shape3d(conv_out_length(in.width_, w_width, w_stride, ptype),
323 conv_out_length(in.height_, w_height, h_stride, ptype),
324 outc);
325 params_.weight = shape3d(w_width, w_height, in.depth_ * outc);
326 params_.has_bias = has_bias;
327 params_.pad_type = ptype;
328 params_.w_stride = w_stride;
329 params_.h_stride = h_stride;
330 params_.tbl = tbl;
331 }
332
333 void init() {
334 if (params_.pad_type == padding::same) {
335 cws_.prev_out_buf_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
336 cws_.prev_delta_padded_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
337 }
338 else {
339 cws_.prev_out_buf_.clear();
340 }
341 }
342
343 serial_size_t in_length(serial_size_t in_length,
344 serial_size_t window_size, padding pad_type) const {
345 return pad_type == padding::same ?
346 (in_length + window_size - 1) : in_length;
347 }
348
349 static serial_size_t conv_out_length(serial_size_t in_length,
350 serial_size_t window_size,
351 serial_size_t stride, padding pad_type) {
352 float_t tmp;
353 if (pad_type == padding::same) {
354 tmp = static_cast<float_t>(in_length) / stride;
355 } else if (pad_type == padding::valid) {
356 tmp = static_cast<float_t>(in_length - window_size + 1) / stride;
357 } else {
358 throw nn_error("Not recognized pad_type.");
359 }
360 return static_cast<serial_size_t>(ceil(tmp));
361 }
362
363 static serial_size_t conv_out_dim(serial_size_t in_width,
364 serial_size_t in_height,
365 serial_size_t window_size,
366 serial_size_t w_stride,
367 serial_size_t h_stride, padding pad_type) {
368 return conv_out_length(in_width, window_size, w_stride, pad_type) *
369 conv_out_length(in_height, window_size, h_stride, pad_type);
370 }
371
372 serial_size_t conv_out_dim(serial_size_t in_width,
373 serial_size_t in_height,
374 serial_size_t window_width,
375 serial_size_t window_height,
376 serial_size_t w_stride,
377 serial_size_t h_stride, padding pad_type) const {
378 return conv_out_length(in_width, window_width, w_stride, pad_type) *
379 conv_out_length(in_height, window_height, h_stride, pad_type);
380 }
381
382 void copy_and_pad_input(const tensor_t& in) {
384
385 serial_size_t sample_count = static_cast<serial_size_t>(in.size());
386
387 cws.prev_out_padded_.resize(sample_count);
388
389 if (params_.pad_type == padding::same) {
390 cws.prev_out_buf_.resize(sample_count, cws.prev_out_buf_[0]);
391 cws.prev_delta_padded_.resize(sample_count, cws.prev_delta_padded_[0]);
392 }
393
394 for (serial_size_t sample = 0; sample < sample_count; ++sample) {
395 if (params_.pad_type == padding::valid) {
396 cws.prev_out_padded_[sample] = &(in[sample]);
397 }
398 else {
399 vec_t* dst = &cws.prev_out_buf_[sample];
400
401 // make padded version in order to avoid corner-case in fprop/bprop
402 for (serial_size_t c = 0; c < params_.in.depth_; c++) {
403 float_t *pimg = &(*dst)[params_.in_padded.get_index(params_.weight.width_ / 2, params_.weight.height_ / 2, c)];
404 const float_t *pin = &in[sample][params_.in.get_index(0, 0, c)];
405
406 for (serial_size_t y = 0; y < params_.in.height_; y++, pin += params_.in.width_, pimg += params_.in_padded.width_) {
407 std::copy(pin, pin + params_.in.width_, pimg);
408 }
409 }
410
411 cws.prev_out_padded_[sample] = &(cws.prev_out_buf_[sample]);
412 }
413 }
414 }
415
416 void copy_and_unpad_delta(const tensor_t& delta, tensor_t& delta_unpadded) {
417 if (params_.pad_type == padding::valid) {
419 }
420 else {
421 for (serial_size_t sample = 0; sample < delta.size(); sample++) {
422 serial_size_t idx = 0;
423 const vec_t& src = delta[sample];
424 vec_t& dst = delta_unpadded[sample];
425
426 for (serial_size_t c = 0; c < params_.in.depth_; c++) {
427 float_t *pdst = &dst[params_.in.get_index(0, 0, c)];
428 idx = params_.in_padded.get_index(params_.weight.width_ / 2,
429 params_.weight.height_ / 2, c);
430 const float_t *pin = &src[idx];
431
432 for (serial_size_t y = 0; y < params_.in.height_; y++) {
433 std::copy(pin, pin + params_.in.width_, pdst);
434 pdst += params_.in.width_;
435 pin += params_.in_padded.width_;
436 }
437 }
438 }
439 }
440 }
441
442 void init_backend(const backend_t backend_type) {
443 std::shared_ptr<core::backend> backend = nullptr;
444
445 // allocate new backend
446 if (backend_type == backend_t::internal) {
447 backend = std::make_shared<core::tiny_backend>(&params_,
448 [this](const tensor_t& in) {
449 return copy_and_pad_input(in);
450 },
451 [this](const tensor_t& delta, tensor_t& dst) {
452 return copy_and_unpad_delta(delta, dst);
453 },
454 [this](const tensor_t& p_delta,
455 const tensor_t& out, tensor_t& c_delta) {
456 return Base::backward_activation(p_delta, out, c_delta);
457 },
458 &cws_);
459 } else if (backend_type == backend_t::nnpack) {
460 backend = std::make_shared<core::nnp_backend>(&params_,
461 [this](const tensor_t& in) {
462 return copy_and_pad_input(in);
463 },
464 &cws_);
465 } else if (backend_type == backend_t::libdnn) {
466 backend = std::make_shared<core::dnn_backend>();
467#ifdef CNN_USE_AVX
468 } else if (backend_type == backend_t::avx) {
469 backend = std::make_shared<core::avx_backend>(&params_,
470 [this](const tensor_t& in) {
471 return copy_and_pad_input(in);
472 },
473 [this](const tensor_t& delta, tensor_t& dst) {
474 return copy_and_unpad_delta(delta, dst);
475 },
476 [this](const tensor_t& p_delta,
477 const tensor_t& out, tensor_t& c_delta) {
478 return Base::backward_activation(p_delta, out, c_delta);
479 },
480 &cws_);
481#endif
482 } else {
483 throw nn_error("Not supported backend type.");
484 }
485
486 if (backend) {
487 Base::set_backend(backend);
488 Base::backend_->set_layer(this);
489 } else {
490 throw nn_error("Could not allocate the backend.");
491 }
492 }
493
494 /* The convolution parameters */
495 conv_params params_;
496
497 /* The type of backend */
498 //backend_t backend_type_;
499
500 /* Workers buffers */
502};
503
504} // namespace tiny_dnn
Definition backend.h:68
Definition conv_params.h:92
single-input, single-output network with activation function
Definition feedforward_layer.h:37
Simple image utility class.
Definition image.h:94
serial_size_t in_channels() const
number of outgoing edges in this layer
Definition layer.h:146
error exception class for tiny-dnn
Definition nn_error.h:37
2D convolution layer
Definition quantized_convolutional_layer.h:54
quantized_convolutional_layer(quantized_convolutional_layer &&other)
number of incoming connections for each output unit
Definition quantized_convolutional_layer.h:202
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition quantized_convolutional_layer.h:143
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition quantized_convolutional_layer.h:246
serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition quantized_convolutional_layer.h:216
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition quantized_convolutional_layer.h:74
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition quantized_convolutional_layer.h:108
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition quantized_convolutional_layer.h:265
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition quantized_convolutional_layer.h:226
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition quantized_convolutional_layer.h:263
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition quantized_convolutional_layer.h:180
serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition quantized_convolutional_layer.h:210
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition quantized_convolutional_layer.h:253
Definition conv_params.h:40