Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
import numpy as np
from numpy.typing import ArrayLike
from .layer import Layer
class L1Norm(Layer):
# Parameters
# ‾‾‾‾‾‾‾‾‾‾
# Attributes
# ‾‾‾‾‾‾‾‾‾‾
# Methods
# ‾‾‾‾‾‾‾
def __init__(self, axis=None, epsilon: float = 1e-8) -> None:
super().__init__()
self.axis = axis
self.scales = None
self.epsilon = epsilon
def forward(self, input: ArrayLike) -> ArrayLike:
# Parameters
# ‾‾‾‾‾‾‾‾‾‾
# Returns
# ‾‾‾‾‾‾‾
norm = np.abs(input).sum(axis=self.axis, keepdims=True)
norm = 1. / (norm + self.epsilon)
output = input * norm
self.scales = -np.sign(output)
self.gradient = np.zeros_like(output, dtype=float)
return output
def backward(self, gradient: ArrayLike) -> ArrayLike:
# Parameters
# ‾‾‾‾‾‾‾‾‾‾
# Returns
# ‾‾‾‾‾‾‾
self.gradient += self.scales
gradient[:] += self.gradient
return gradient
class L2Norm(Layer):
# Parameters
# ‾‾‾‾‾‾‾‾‾‾
# Attributes
# ‾‾‾‾‾‾‾‾‾‾
# Methods
# ‾‾‾‾‾‾‾
def __init__(self, axis=None, epsilon: float = 1e-8) -> None:
super().__init__()
self.axis = axis
self.scales = None
self.epsilon = epsilon
def forward(self, input: ArrayLike) -> ArrayLike:
# Parameters
# ‾‾‾‾‾‾‾‾‾‾
# Returns
# ‾‾‾‾‾‾‾
norm = (input * input).sum(axis=self.axis, keepdims=True)
norm = 1. / np.sqrt(norm + self.epsilon)
output = input * norm
self.scales = (1. - output) * norm
self.gradient = np.zeros_like(output, dtype=float)
return output
def backward(self, gradient: ArrayLike) -> ArrayLike:
# Parameters
# ‾‾‾‾‾‾‾‾‾‾
# Returns
# ‾‾‾‾‾‾‾
self.gradient += self.scales
gradient[:] += self.delta
return gradient