tiny_dnn 1.0.0
A header only, dependency-free deep learning framework in C++11
Loading...
Searching...
No Matches
activation_function.h
1/*
2 Copyright (c) 2013, Taiga Nomi
3 All rights reserved.
4
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in the
11 documentation and/or other materials provided with the distribution.
12 * Neither the name of the <organization> nor the
13 names of its contributors may be used to endorse or promote products
14 derived from this software without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
17 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26*/
27#pragma once
28#include "tiny_dnn/util/util.h"
29#include <algorithm>
30
31namespace tiny_dnn {
32namespace activation {
33
34class function {
35public:
36 function() = default;
37 function(const function &) = default;
38#ifndef CNN_DEFAULT_MOVE_CONSTRUCTOR_UNAVAILABLE
39 function(function &&) = default;
40#endif
41 function &operator =(const function &) = default;
42#ifndef CNN_DEFAULT_ASSIGNMENT_OPERATOR_UNAVAILABLE
43 function &operator =(function &&) = default;
44#endif
45 virtual ~function() = default;
46
47 virtual float_t f(const vec_t& v, size_t index) const = 0;
48 void itef(vec_t& out, const vec_t& in, size_t cnt) const {
49 for (size_t i = 0; i < cnt; i++) {
50 out[i] = f(in, i);
51 }
52 }
53
54 // dfi/dyi
55 virtual float_t df(float_t y) const = 0;
56
57 // dfi/dyk (k=0,1,..n)
58 virtual vec_t df(const vec_t& y, size_t i) const { vec_t v(y.size(), 0); v[i] = df(y[i]); return v; }
59
60 // return if dfi/dyk is one-hot vector
61 virtual bool one_hot() const { return true; }
62
63 // target value range for learning
64 virtual std::pair<float_t, float_t> scale() const = 0;
65};
66
67class identity : public function {
68public:
69 using function::df;
70 float_t f(const vec_t& v, size_t i) const override { return v[i]; }
71 float_t df(float_t /*y*/) const override { return float_t(1); }
72 std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
73};
74
75class sigmoid : public function {
76public:
77 using function::df;
78 float_t f(const vec_t& v, size_t i) const override { return float_t(1) / (float_t(1) + std::exp(-v[i])); }
79 float_t df(float_t y) const override { return y * (float_t(1) - y); }
80 std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
81};
82
83class relu : public function {
84public:
85 using function::df;
86 float_t f(const vec_t& v, size_t i) const override { return std::max(float_t(0), v[i]); }
87 float_t df(float_t y) const override { return y > float_t(0) ? float_t(1) : float_t(0); }
88 std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
89};
90
91typedef relu rectified_linear; // for compatibility
92
93class leaky_relu : public function {
94public:
95 using function::df;
96 float_t f(const vec_t& v, size_t i) const override { return (v[i] > float_t(0)) ? v[i] : float_t(0.01) * v[i]; }
97 float_t df(float_t y) const override { return y > float_t(0) ? float_t(1) : float_t(0.01); }
98 std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
99};
100
101class elu : public function {
102public:
103 using function::df;
104 float_t f(const vec_t& v, size_t i) const override { return (v[i]<float_t(0) ? (exp(v[i])- float_t(1)) : v[i]); }
105 float_t df(float_t y) const override { return (y > float_t(0) ? float_t(1) : (float_t(1)+y)); }
106 std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
107};
108
109class softmax : public function {
110public:
111 float_t f(const vec_t& v, size_t i) const override {
112 float_t alpha = *std::max_element(v.begin(), v.end());
113 float_t numer = std::exp(v[i] - alpha);
114 float_t denom = float_t(0);
115 for (auto x : v)
116 denom += std::exp(x - alpha);
117 return numer / denom;
118 }
119
120 float_t df(float_t y) const override {
121 return y * (float_t(1) - y);
122 }
123
124 virtual vec_t df(const vec_t& y, size_t index) const override {
125 vec_t v(y.size(), 0);
126 for (size_t i = 0; i < y.size(); i++)
127 v[i] = (i == index) ? df(y[index]) : -y[i] * y[index];
128
129 return v;
130 }
131
132 virtual bool one_hot() const override { return false; }
133
134 std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0), float_t(1)); }
135};
136
137class tan_h : public function {
138public:
139 using function::df;
140
141 float_t f(const vec_t& v, size_t i) const override {
142 return std::tanh(v[i]);
143 }
144
145 void itef(vec_t& out, const vec_t& in, size_t cnt) const {
146 for (size_t i = 0; i < cnt; i++) {
147 out[i] = std::tanh(in[i]);
148 }
149 }
150
151 // fast approximation of tanh (improve 2-3% speed in LeNet-5)
152 /*float_t f(float_t x) const {
153 const float_t x2 = x * x;
154 x *= 1.0 + x2 * (0.1653 + x2 * 0.0097);
155 return x / std::sqrt(1.0 + x * x);// invsqrt(static_cast<float>(1.0 + x * x));
156 }*/
157
158 float_t df(float_t y) const override { return float_t(1) - sqr(y); }
159 std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(-0.8), float_t(0.8)); }
160
161private:
162 /*float invsqrt(float x) const {
163 float x2 = x * 0.5f;
164 long i = *reinterpret_cast<long*>(&x);
165
166 i = 0x5f3759df - (i >> 1);
167 x = *reinterpret_cast<float*>(&i);
168 x = x * (1.5f - (x2 * x * x));
169 return x;
170 }*/
171};
172
173// s tan_h, but scaled to match the other functions
174class tan_hp1m2 : public function {
175public:
176 using function::df;
177 float_t f(const vec_t& v, size_t i) const override {
178 const float_t ep = std::exp(v[i]);
179 return ep / (ep + std::exp(-v[i]));
180 }
181
182 float_t df(float_t y) const override { return 2 * y *(float_t(1) - y); }
183 std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
184};
185
186} // namespace activation
187} // namespace tiny_dnn
Definition activation_function.h:101
Definition activation_function.h:34
Definition activation_function.h:67
Definition activation_function.h:93
Definition activation_function.h:83
Definition activation_function.h:75
Definition activation_function.h:109
Definition activation_function.h:137
Definition activation_function.h:174
Simple image utility class.
Definition image.h:94