Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import numpy as np
from abc import ABC, abstractmethod
from typing import Protocol
class Teacher(Protocol):
@property
def learningRate(self) -> float:
...
def cosineRange(startRate: float, endRate: float, epochs: int) -> np.ndarray:
"""
This function generates a cosine shape for a given range
"""
epoch = np.linspace(0, np.pi, epochs)
rates = (np.cos(epoch) + 1) * 0.5 * (endRate - startRate) + startRate
return rates
class Scheduler(ABC):
"""
base class of learning rate scheduler
"""
__slots__ = ['name', 'learningRate', 'teacher']
def __init__(self, teacher: Teacher) -> None:
self.name = self.__class__.__name__
self.teacher = teacher
self.learningRate = self.teacher.learningRate
@abstractmethod
def update(self) -> float:
"""
this is used for calculating the new learning rate
"""
pass
def step(self) -> None:
"""
this hands over the new learning rate
"""
self.teacher.learningRate = self.update()
class ExponentialLR(Scheduler):
"""
implementation of an exponential learning rate decay
"""
__slots__ = ['decayRate']
def __init__(self, teacher: Teacher, decayRate: float) -> None:
super().__init__(teacher)
self.decayRate = decayRate
def update(self) -> float:
"""
this decreases the learning rate by a constant factor
"""
self.learningRate = self.learningRate * self.decayRate
return self.learningRate
class SteppedLR(Scheduler):
"""
implementation of stepped learning rate
"""
__slots__ = ['decayRate', 'stepSize', '_steps']
def __init__(self, teacher: Teacher, decayRate: float, stepSize: int) -> None:
super().__init__(teacher)
self.decayRate = decayRate
self.stepSize = stepSize
self._steps = 0
def update(self) -> float:
"""
counts steps and if at a certain milestone and
decreases learning rate by a constant factor
"""
if self._steps % self.stepSize == 0 and self._steps != 0:
self.learningRate = self.learningRate * self.decayRate
self._steps += 1
return self.learningRate
class CyclicalLR(Scheduler):
"""
learningRate is the base rate
limitRate is the upper or lower learningRate limit
stepsUp/stepsDown is the time it takes to go up/down
"""
__slots__ = ['totalSteps', 'steps', 'rates']
def __init__(self, teacher: Teacher, learningRateScale: float, stepsUp: int, stepsDown: int, shape: str = 'zickzack') -> None:
super().__init__(teacher)
self.totalSteps = stepsUp + stepsDown
self.learningRateScale = learningRateScale
self.stepsUp = stepsUp
self.stepsDown = stepsDown
self.steps = 0
self.shape = shape
limitRate = self.learningRate * learningRateScale
# creating linearly increasing/decreasing learning rate
if shape == 'zickzack':
if self.learningRate > limitRate:
self.rates = np.concatenate((np.arange(self.learningRate, limitRate, (limitRate-self.learningRate)/stepsDown), np.arange(limitRate, self.learningRate, (self.learningRate-limitRate)/stepsUp)))
if self.learningRate < limitRate:
self.rates = np.concatenate((np.arange(self.learningRate, limitRate, (limitRate-self.learningRate)/stepsUp), np.arange(limitRate, self.learningRate, (self.learningRate-limitRate)/stepsDown)))
# creating a stepped style increasing/decreasing learning rate
elif shape == 'stepped':
if self.learningRate < limitRate:
self.rates = np.array([self.learningRate] * stepsUp + [limitRate] * stepsDown)
if self.learningRate > limitRate:
self.rates = np.array([self.learningRate] * stepsDown + [limitRate] * stepsUp)
# creating smoothly increasing/decreasing learning rate
elif shape == 'cosine':
if self.learningRate < limitRate:
ratesStart = cosineRange(self.learningRate, limitRate, stepsUp)
ratesEnd = cosineRange(limitRate, self.learningRate, stepsDown)
if self.learningRate > limitRate:
ratesStart = cosineRange(self.learningRate, limitRate, stepsDown)
ratesEnd = cosineRange(limitRate, self.learningRate, stepsUp)
self.rates = np.concatenate((ratesStart, ratesEnd))
else:
raise ValueError(f'{shape} is not an option for shape')
def update(self) -> float:
"""
this steps through prepared learning rate arrays
"""
learningRate = self.rates[self.steps]
if self.steps == self.totalSteps - 1:
self.steps = 0
else:
self.steps += 1
return learningRate