Skip to content
Snippets Groups Projects
Commit a09ee61c authored by johannes bilk's avatar johannes bilk
Browse files

more on PTQ, struggling with activation functions

parent c4b3f64d
No related branches found
No related tags found
No related merge requests found
......@@ -9,17 +9,35 @@ class Activation(Layer):
the main activation function class containing all the methods used for activation function
it's an abstract class, meaning it should never be used directly, but instead used a base
"""
__slots__ = ['input', 'activation']
__slots__ = ['input', 'activation', 'useQuantization', 'scale', 'bits', 'lut', 'inputRange']
def __init__(self) -> None:
super().__init__()
self.useQuantization = False
self.scale = 1
self.bits = 8
self.lut = None
self.inputRange = (0, 1)
def forward(self, input: ArrayLike) -> np.ndarray:
"""
creates the activation and introduces non-linearity to the network
Creates the activation and introduces non-linearity to the network.
Uses a lookup table (LUT) for the quantized path.
"""
self.input = input
self.activation = self._function(self.input)
if self.useQuantization:
if not self.lut:
raise ValueError("No LUT generated for this layer")
# Assuming symmetric quantization for activation functions
quantized_indices = np.clip(np.round(input).astype(np.int32), 0, 2**self.bits - 1)
# Use the indices to look up the activation values in the LUT
self.activation = self.lut[quantized_indices]
else:
# For the non-quantized path, directly compute the activation
self.activation = self._function(self.input)
return self.activation
def backward(self, gradient: ArrayLike) -> np.ndarray:
......@@ -28,6 +46,26 @@ class Activation(Layer):
"""
return self._derivative() * gradient
def quantize(self, bits: int = 8) -> None:
# Initialization steps...
self.bits = bits
self.lut = np.zeros((2 ** bits,), dtype=np.int32) # For 8-bit, size is 256
# Determine the floating-point range for inputs based on `self.inputRange`
min_val, max_val = self.inputRange
# Populate the LUT for quantized outputs
for i in range(2 ** bits):
# Map quantized input index to floating-point range
real_value = min_val + (max_val - min_val) * (i / (2 ** bits - 1))
# Apply the actual activation function
activation_output = self._function(real_value)
# Re-quantize the activation output back to quantized domain
quantized_output = np.round((activation_output - min_val) / (max_val - min_val) * (2 ** bits - 1))
self.lut[i] = quantized_output
self.useQuantization = True
@abstractmethod
def _function(self, input: ArrayLike) -> np.ndarray:
"""
......@@ -55,6 +93,7 @@ class Relu(Activation):
def __init__(self) -> None:
super().__init__()
self.inputRange = (0, 6)
def _function(self, input: ArrayLike) -> np.ndarray:
return np.maximum(0.0, input)
......@@ -73,6 +112,7 @@ class Elu(Activation):
def __init__(self, alpha: float = 1.0) -> None:
super().__init__()
self.alpha = alpha
self.inputRange = (-1, 6)
def _function(self, input: ArrayLike) -> np.ndarray:
return np.where(input <= 0., self.alpha * np.exp(input) - 1, input)
......@@ -91,6 +131,7 @@ class LeakyRelu(Activation):
def __init__(self, epislon: float = 1e-1) -> None:
super().__init__()
self.epislon = epislon
self.inputRange = (-6, 6)
def _function(self, input: ArrayLike) -> np.ndarray:
input[input <= 0.] *= self.epislon
......@@ -111,6 +152,7 @@ class Tanh(Activation):
def __init__(self) -> None:
super().__init__()
self.inputRange = (-1, 1)
def _function(self, input: ArrayLike) -> np.ndarray:
return np.tanh(input)
......@@ -166,6 +208,7 @@ class SoftPlus(Activation):
def __init__(self) -> None:
super().__init__()
self.inputRange = (0, 6)
def _function(self, input: ArrayLike) -> np.ndarray:
return np.log(1. + np.exp(input))
......@@ -187,6 +230,7 @@ class SoftSign(Activation):
def __init__(self) -> None:
super().__init__()
self.inputRange = (-1, 1)
def _function(self, input: ArrayLike) -> np.ndarray:
return input / (np.abs(input) + 1.)
......@@ -199,7 +243,7 @@ class SoftSign(Activation):
class Identity(Activation):
"""
The identity activation function.
The identity function simply returns its input without any transformation.
It is often used as the activation function for the output layer of a neural network
when the task involves regression, i.e., predicting a continuous output value.
......@@ -208,6 +252,7 @@ class Identity(Activation):
def __init__(self) -> None:
super().__init__()
self.inputRange = (-6, 6)
def _function(self, input: ArrayLike) -> np.ndarray:
return input
......
......@@ -128,4 +128,4 @@ class Dropout(Layer):
printString = self.name
printString += ' size: ' + str(self.size)
printString += ' probability: ' + str(self.probability)
return printString
\ No newline at end of file
return printString
......@@ -46,7 +46,7 @@ class Weights(object):
or quantized (and dequantized back) weight values for computation.
"""
if self._useQuantization:
return self.dequantize()
return self._quantizedValues
else:
return self._values
......@@ -123,7 +123,7 @@ class Weights(object):
self.qMax = 2 ** (bits - 1) - 1
self.qMin = - self.qMax
elif scheme == "asymmetric":
sefl.qMax = 2 ** bits - 1
self.qMax = 2 ** bits - 1
self.qMin = 0
else:
raise ValueError(f"{scheme} is not a recognized quantization scheme")
......
import numpy as np
from collections import namedtuple
from copy import deepcopy
from .module import Module
from .layer import Layer
......@@ -31,20 +32,20 @@ class Quantizer:
pass
@property
def quantizationError(self) -> QuantizationError:
def quantizationError(self, quantizedModule: Module) -> QuantizationError:
"""
this returns the two main errors of the quantization
"""
return QuantizationError(self._roundingError(), self._clippingError())
return QuantizationError(self._roundingError(quantizedModule), self._clippingError(quantizedModule))
def _roundingError(self) -> float:
def _roundingError(self, quantizedModule: Module) -> float:
"""
A private method for calculating the mean absolute rounding error.
"""
totalError = 0.
totalElements = 0
for layer in self.module:
for layer in quantizedModule:
try:
params = layer.params()
except AttributeError:
......@@ -61,11 +62,11 @@ class Quantizer:
meanError = totalError / totalElements if totalElements > 0 else 0
return meanError
def _clippingError(self) -> float:
def _clippingError(self, quantizedModule: Module) -> float:
totalClippingError = 0.
totalElements = 0
for layer in self.module:
for layer in quantizedModule:
try:
params = layer.params()
except AttributeError:
......@@ -92,43 +93,49 @@ class Quantizer:
meanClippingError = totalClippingError / totalElements if totalElements > 0 else 0
return meanClippingError
def __call__(self, module: Module) -> None:
def __call__(self, module: Module) -> Module:
"""
Applies quantization to all quantizable parameters in the module.
"""
self.module = module
for layer in self.module:
qunaitzedModule = deepcopy(module)
for layer in qunaitzedModule:
self._quantizeLayer(layer)
def dequantize(self) -> None:
return qunaitzedModule
def dequantize(self, quatizedModule: Module) -> Module:
"""
Applies dequantization to all dequantizable parameters in the module.
"""
for layer in self.module:
for layer in quatizedModule:
self._dequantizeLayer(layer)
return quatizedModule
def _quantizeLayer(self, layer: Layer) -> None:
"""
Quantizes the weights (and biases) of a single layer if applicable.
Quantizes the weights (and biases) of a single layer if applicable,
or the layer itself if it supports direct quantization.
"""
try:
if hasattr(layer, 'params'):
# For layers with parameters like weights and biases
params = layer.params()
except AttributeError:
# 'params' method not found in the layer, skip updating
return
for param in params:
param.quantize(bits=self.bits, scheme=self.scheme)
for param in params:
param.quantize(bits=self.bits, scheme=self.scheme)
elif hasattr(layer, 'quantize'):
# For layers supporting direct quantization, like activation layers with LUT
layer.quantize(bits=self.bits)
def _dequantizeLayer(self, layer: Layer) -> None:
"""
Dequantizes the weights (and biases) of a single layer if applicable.
Dequantizes the weights (and biases) of a single layer if applicable,
or the layer itself if it supports direct dequantization.
"""
try:
if hasattr(layer, 'params'):
# For layers with parameters like weights and biases
params = layer.params()
except AttributeError:
# 'params' method not found in the layer, skip updating
return
for param in params:
param.dequantize()
for param in params:
param.dequantize()
elif hasattr(layer, 'dequantize'):
# For layers supporting direct dequantization, if applicable
layer.dequantize()
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment