tiny_dnn 1.0.0
A header only, dependency-free deep learning framework in C++11
Loading...
Searching...
No Matches
convolutional_layer.h
1/*
2 Copyright (c) 2013, Taiga Nomi
3 All rights reserved.
4
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in the
11 documentation and/or other materials provided with the distribution.
12 * Neither the name of the <organization> nor the
13 names of its contributors may be used to endorse or promote products
14 derived from this software without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
17 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26*/
27#pragma once
28
29#include <vector>
30#include <string>
31#include <algorithm>
32
33#include "tiny_dnn/core/kernels/conv2d_op.h"
34#include "tiny_dnn/core/kernels/conv2d_grad_op.h"
35#include "tiny_dnn/core/kernels/conv2d_op_opencl.h"
36#include "tiny_dnn/core/kernels/conv2d_op_libdnn.h"
37
38#include "tiny_dnn/util/util.h"
39#include "tiny_dnn/util/image.h"
40#include "tiny_dnn/activations/activation_function.h"
41
42using namespace tiny_dnn::core;
43
44namespace tiny_dnn {
45
51template<typename Activation = activation::identity>
52class convolutional_layer : public feedforward_layer<Activation> {
53 public:
55 CNN_USE_LAYER_MEMBERS;
56
74 serial_size_t in_height,
75 serial_size_t window_size,
76 serial_size_t in_channels,
77 serial_size_t out_channels,
78 padding pad_type = padding::valid,
79 bool has_bias = true,
80 serial_size_t w_stride = 1,
81 serial_size_t h_stride = 1,
82 backend_t backend_type = core::default_engine())
84 in_channels, out_channels, connection_table(),
85 pad_type, has_bias, w_stride, h_stride,
86 backend_type) {}
87
106 serial_size_t in_height,
107 serial_size_t window_width,
108 serial_size_t window_height,
109 serial_size_t in_channels,
110 serial_size_t out_channels,
111 padding pad_type = padding::valid,
112 bool has_bias = true,
113 serial_size_t w_stride = 1,
114 serial_size_t h_stride = 1,
115 backend_t backend_type = core::default_engine())
117 in_channels, out_channels, connection_table(),
118 pad_type, has_bias, w_stride, h_stride,
119 backend_type) {}
120
139 serial_size_t in_height,
140 serial_size_t window_size,
141 serial_size_t in_channels,
142 serial_size_t out_channels,
144 padding pad_type = padding::valid,
145 bool has_bias = true,
146 serial_size_t w_stride = 1,
147 serial_size_t h_stride = 1,
148 backend_t backend_type = core::default_engine())
150 in_channels, out_channels, connection_table,
151 pad_type, has_bias, w_stride, h_stride,
152 backend_type) {}
153
173 serial_size_t in_height,
174 serial_size_t window_width,
175 serial_size_t window_height,
176 serial_size_t in_channels,
177 serial_size_t out_channels,
179 padding pad_type = padding::valid,
180 bool has_bias = true,
181 serial_size_t w_stride = 1,
182 serial_size_t h_stride = 1,
183 backend_t backend_type = core::default_engine())
184 : Base(std_input_order(has_bias)) {
185 conv_set_params(shape3d(in_width, in_height, in_channels),
187 out_channels, pad_type, has_bias,
188 w_stride, h_stride,
190 init_backend(backend_type);
191 Base::set_backend_type(backend_type);
192 }
193
194 // move constructor
196 : Base(std::move(other))
197 , params_(std::move(other.params_))
198 , padding_op_(std::move(other.padding_op_))
199 , kernel_fwd_(std::move(other.kernel_fwd_))
200 , kernel_back_(std::move(other.kernel_back_))
201 , cws_(std::move(other.cws_)) {
202 init_backend(std::move(other.engine()));
203 }
204
206 serial_size_t fan_in_size() const override {
207 return params_.weight.width_ *
208 params_.weight.height_ * params_.in.depth_;
209 }
210
212 serial_size_t fan_out_size() const override {
213 return (params_.weight.width_ / params_.w_stride) *
214 (params_.weight.height_ / params_.h_stride) *
215 params_.out.depth_;
216 }
217
222 void forward_propagation(const std::vector<tensor_t*>& in_data,
223 std::vector<tensor_t*>& out_data) override {
224 // apply padding to the input tensor
225 padding_op_.copy_and_pad_input(*in_data[0], cws_.prev_out_padded_);
226
227 std::vector<tensor_t*> in_data_(in_data.size());
228 in_data_[0] = in_data_padded(in_data);
229
230 for (serial_size_t i = 1; i < in_data.size(); ++i) {
231 in_data_[i] = in_data[i];
232 }
233
234 // forward convolutional op context
235 auto ctx = OpKernelContext(in_data_, out_data);
236 ctx.setParallelize(layer::parallelize());
237 ctx.setEngine(layer::engine());
238
239 // launch convolutional kernel
240 kernel_fwd_->compute(ctx);
241
242 // activations
243 // TODO(edgar/nyanp): refactor and move activations outside
244 this->forward_activation(*out_data[0], *out_data[1]);
245 }
246
254 void back_propagation(const std::vector<tensor_t*>& in_data,
255 const std::vector<tensor_t*>& out_data,
256 std::vector<tensor_t*>& out_grad,
257 std::vector<tensor_t*>& in_grad) override {
258 // activations
259 // TODO(edgar/nyanp): refactor and move activations outside
260 this->backward_activation(*out_grad[0], *out_data[0], *out_grad[1]);
261
262 std::vector<tensor_t*> in_data_;
263 in_data_.push_back(in_data_padded(in_data));
264
265 for (serial_size_t i = 1; i < in_data.size(); ++i) {
266 in_data_.push_back(in_data[i]);
267 }
268
269 std::vector<tensor_t*> in_grad_;
270 for (serial_size_t i = 0; i < in_grad.size(); ++i) {
271 in_grad_.push_back(in_grad[i]);
272 }
273
274 if (params_.pad_type == padding::same) {
275 in_grad_[0] = &cws_.prev_delta_padded_;
276 }
277
278 auto ctx = OpKernelContext(in_data_, out_data, out_grad, in_grad_);
279 ctx.setParams(&params_);
280 ctx.setParallelize(layer::parallelize());
281 ctx.setEngine(layer::engine());
282
283 // launch convolutional kernel
284 kernel_back_->compute(ctx);
285
286 // unpad deltas
287 padding_op_.copy_and_unpad_delta(cws_.prev_delta_padded_, *in_grad[0]);
288 }
289
290 void set_sample_count(serial_size_t sample_count) override {
291 Base::set_sample_count(sample_count);
292 cws_.prev_delta_padded_.resize(
294 vec_t(params_.in_padded.size(), float_t(0)));
295 }
296
297 std::vector<index3d<serial_size_t>> in_shape() const override {
298 if (params_.has_bias) {
299 return { params_.in, params_.weight,
300 index3d<serial_size_t>(1, 1, params_.out.depth_) };
301 }
302 else {
303 return { params_.in, params_.weight };
304 }
305 }
306
307 std::vector<index3d<serial_size_t>>
308 out_shape() const override { return { params_.out, params_.out }; }
309
310 std::string layer_type() const override {
311 return std::string("conv");
312 }
313
314 //TODO(edgar): check this
315 std::string kernel_file() const override {
316 return std::string("../tiny_cnn/core/kernels/cl_kernels/conv_layer_spatial.cl");
317 }
318
319 //TODO(edgar): is it really needed?
320 std::string kernel_header() const override {
321 std::stringstream ss;
322 ss << "#define MULTI\n";
323 ss << "#define KERNEL_H " << params_.weight.height_ << "\n";
324 ss << "#define KERNEL_W " << params_.weight.width_ << "\n";
325 ss << "#define CHANNELS " << params_.weight.depth_ << "\n";
326 ss << "#define STRIDE_H " << params_.h_stride << "\n";
327 ss << "#define STRIDE_W " << params_.w_stride << "\n";
328 ss << "#define APPLY_BIAS " << params_.has_bias << "\n";
329 ss << "#define OUTPUT_Z " << params_.out.depth_ << "\n";
330 // TODO(edgar): REVISE THIS
331 ss << "#define ZPAR " << params_.out.depth_ << "\n";
332 return ss.str();
333 }
334
335 image<> weight_to_image() const {
336 image<> img;
337 const serial_size_t border_width = 1;
338 const auto pitch = params_.weight.width_ + border_width;
339 const auto width = params_.out.depth_ * pitch + border_width;
340 const auto height = params_.in.depth_ * pitch + border_width;
341 const image<>::intensity_t bg_color = 255;
342 const vec_t& W = *this->weights()[0];
343
344 img.resize(width, height);
345 img.fill(bg_color);
346
347 auto minmax = std::minmax_element(W.begin(), W.end());
348
349 for (serial_size_t r = 0; r < params_.in.depth_; ++r) {
350 for (serial_size_t c = 0; c < params_.out.depth_; ++c) {
351 if (!params_.tbl.is_connected(c, r)) continue;
352
353 const auto top = r * pitch + border_width;
354 const auto left = c * pitch + border_width;
355
356 serial_size_t idx = 0;
357
358 for (serial_size_t y = 0; y < params_.weight.height_; ++y) {
359 for (serial_size_t x = 0; x < params_.weight.width_; ++x) {
360 idx = c * params_.in.depth_ + r;
361 idx = params_.weight.get_index(x, y, idx);
362 const float_t w = W[idx];
363
364 img.at(left + x, top + y)
365 = static_cast<image<>::intensity_t>(
366 rescale(w, *minmax.first,
367 *minmax.second, 0, 255));
368 }
369 }
370 }
371 }
372 return img;
373 }
374
375
376 template <class Archive>
377 static void load_and_construct(
378 Archive & ar, cereal::construct<convolutional_layer> & construct) {
379 serial_size_t w_width, w_height, out_ch, w_stride, h_stride;
380 bool has_bias;
381 shape3d in;
382 padding pad_type;
384
385 ar(cereal::make_nvp("in_size", in),
386 cereal::make_nvp("window_width", w_width),
387 cereal::make_nvp("window_height", w_height),
388 cereal::make_nvp("out_channels", out_ch),
389 cereal::make_nvp("connection_table", tbl),
390 cereal::make_nvp("pad_type", pad_type),
391 cereal::make_nvp("has_bias", has_bias),
392 cereal::make_nvp("w_stride", w_stride),
393 cereal::make_nvp("h_stride", h_stride)
394 );
395
396 construct(in.width_, in.height_, w_width, w_height, in.depth_,
397 out_ch, tbl, pad_type, has_bias, w_stride, h_stride);
398 }
399
400 template <class Archive>
401 void serialize(Archive & ar) {
402 layer::serialize_prolog(ar);
403 ar(cereal::make_nvp("in_size", params_.in),
404 cereal::make_nvp("window_width", params_.weight.width_),
405 cereal::make_nvp("window_height", params_.weight.height_),
406 cereal::make_nvp("out_channels", params_.out.depth_),
407 cereal::make_nvp("connection_table", params_.tbl),
408 cereal::make_nvp("pad_type", params_.pad_type),
409 cereal::make_nvp("has_bias", params_.has_bias),
410 cereal::make_nvp("w_stride", params_.w_stride),
411 cereal::make_nvp("h_stride", params_.h_stride)
412 );
413 }
414
415private:
416 tensor_t* in_data_padded(const std::vector<tensor_t*>& in) {
417 return (params_.pad_type == padding::valid) ?
418 in[0] : &cws_.prev_out_padded_;
419 }
420
421 void conv_set_params(const shape3d& in,
422 serial_size_t w_width,
423 serial_size_t w_height,
424 serial_size_t outc,
425 padding ptype,
426 bool has_bias,
427 serial_size_t w_stride,
428 serial_size_t h_stride,
429 const connection_table& tbl = connection_table()) {
430 params_.in = in;
431 params_.in_padded = shape3d(in_length(in.width_, w_width, ptype),
432 in_length(in.height_, w_height, ptype),
433 in.depth_);
434 params_.out =
435 shape3d(conv_out_length(in.width_, w_width, w_stride, ptype),
436 conv_out_length(in.height_, w_height, h_stride, ptype),
437 outc);
438 params_.weight = shape3d(w_width, w_height, in.depth_ * outc);
439 params_.has_bias = has_bias;
440 params_.pad_type = ptype;
441 params_.w_stride = w_stride;
442 params_.h_stride = h_stride;
443 params_.tbl = tbl;
444
445 // init padding buffer
446 if (params_.pad_type == padding::same) {
447 cws_.prev_delta_padded_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
448 }
449
450 // set parameters to padding operation
451 padding_op_ = Conv2dPadding(params_);
452 }
453
454 serial_size_t in_length(serial_size_t in_length,
455 serial_size_t window_size, padding pad_type) const {
456 return pad_type == padding::same ?
457 (in_length + window_size - 1) : in_length;
458 }
459
460 static serial_size_t conv_out_dim(serial_size_t in_width,
461 serial_size_t in_height,
462 serial_size_t window_size,
463 serial_size_t w_stride,
464 serial_size_t h_stride, padding pad_type) {
465 return conv_out_length(in_width, window_size, w_stride, pad_type) *
466 conv_out_length(in_height, window_size, h_stride, pad_type);
467 }
468
469 serial_size_t conv_out_dim(serial_size_t in_width,
470 serial_size_t in_height,
471 serial_size_t window_width,
472 serial_size_t window_height,
473 serial_size_t w_stride,
474 serial_size_t h_stride, padding pad_type) const {
475 return conv_out_length(in_width, window_width, w_stride, pad_type) *
476 conv_out_length(in_height, window_height, h_stride, pad_type);
477 }
478
479 void createOp() override {
480 init_backend(layer::engine());
481 }
482
483 void init_backend(const backend_t backend_type) {
485 core::OpKernelConstruction(layer::device(), &params_);
486
487 if (backend_type == backend_t::internal ||
488 backend_type == backend_t::nnpack ||
489 backend_type == backend_t::avx) {
490
491 kernel_fwd_.reset(new Conv2dOp(ctx));
492 kernel_back_.reset(new Conv2dGradOp(ctx));
493 return;
494 }
495 else if (backend_type == backend_t::opencl) {
496 throw nn_error("Not implemented engine: " + to_string(backend_type));
497 /*kernel_fwd_.reset(new Conv2dOpenCLForwardOp(ctx));
498 kernel_back_.reset(new Conv2dOpenCLBackwardOp(ctx));
499 return;*/
500 }
501 else if (backend_type == backend_t::libdnn) {
502 if (layer::device() == nullptr) return;
503 kernel_fwd_.reset(new Conv2dLibDNNForwardOp(ctx));
504 kernel_back_.reset(new Conv2dLibDNNBackwardOp(ctx));
505 return;
506 }
507 else {
508 throw nn_error("Not supported engine: " + to_string(backend_type));
509 }
510
511 }
512
513 private:
514 /* The convolution parameters */
515 conv_params params_;
516
517 /* Padding operation */
518 Conv2dPadding padding_op_;
519
520 /* Forward and backward ops */
521 std::shared_ptr<core::OpKernel> kernel_fwd_;
522 std::shared_ptr<core::OpKernel> kernel_back_;
523
524 /* Buffer to store padded data */
526 tensor_t prev_out_padded_;
527 tensor_t prev_delta_padded_;
528 } cws_;
529};
530
531} // namespace tiny_dnn
Definition conv2d_grad_op.h:54
Definition conv2d_op_libdnn.h:305
Definition conv2d_op_libdnn.h:55
Definition conv2d_op.h:55
2D convolution layer
Definition convolutional_layer.h:52
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition convolutional_layer.h:297
serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition convolutional_layer.h:212
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition convolutional_layer.h:310
serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition convolutional_layer.h:206
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition convolutional_layer.h:308
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition convolutional_layer.h:222
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition convolutional_layer.h:254
convolutional_layer(convolutional_layer &&other)
number of incoming connections for each output unit
Definition convolutional_layer.h:195
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition convolutional_layer.h:73
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition convolutional_layer.h:172
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition convolutional_layer.h:138
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition convolutional_layer.h:105
Definition conv_params.h:121
Definition op_kernel.h:55
Definition op_kernel.h:72
Definition conv_params.h:92
single-input, single-output network with activation function
Definition feedforward_layer.h:37
Simple image utility class.
Definition image.h:94
serial_size_t in_channels() const
number of outgoing edges in this layer
Definition layer.h:146
error exception class for tiny-dnn
Definition nn_error.h:37
Definition conv_params.h:40