遗漏的部分
This commit is contained in:
qing committed
1 parent
84fb370da8
commit
0493383ee8
63 files changed
+28237
-4
No files matched your search
@@ -0,0 +1,231 @@
|
||||
#include "nn/NeuralNetwork.hpp"
|
||||
namespace qing{
|
||||
/* 构造函数 使用基本参数构造 */
|
||||
NeuralNetwork::NeuralNetwork(
|
||||
const long inputno, const long outputno,
|
||||
const std::vector<float> &weights,
|
||||
const std::vector<float> &bias,
|
||||
const float learning_rate,
|
||||
const ActivationFunc func_type
|
||||
): inputno(inputno), outputno(outputno),
|
||||
weights(inputno, outputno, weights), bias( 1, outputno, bias),
|
||||
learning_rate(learning_rate),
|
||||
f_type(func_type) {
|
||||
;/* 进行形状检查 */
|
||||
}
|
||||
|
||||
/* 获取输入层大小 */
|
||||
long NeuralNetwork::get_inputno() const {
|
||||
return this->inputno;
|
||||
}
|
||||
|
||||
/* 获取输出层大小 */
|
||||
long NeuralNetwork::get_outputno() const {
|
||||
return this->outputno;
|
||||
}
|
||||
|
||||
/* 获取权重 */
|
||||
Matrx<float>& NeuralNetwork::get_weights() {
|
||||
return this->weights;
|
||||
}
|
||||
|
||||
/* 获取偏置 */
|
||||
Matrx<float>& NeuralNetwork::get_bias() {
|
||||
return this->bias;
|
||||
}
|
||||
|
||||
/* 正态分布Xavier初始化 */
|
||||
void NeuralNetwork::xavierNormalInit(std::vector<float> &weights, long fan_in, long fan_out)
|
||||
{
|
||||
|
||||
/* 高质量的随机数生成器 */
|
||||
static std::mt19937 generator(std::random_device{}());
|
||||
|
||||
/* 计算标准差 */
|
||||
float sigma = std::sqrt(2.0f / (fan_in + fan_out));
|
||||
|
||||
/* 创建正态分布对象，均值为0，标准差为sigma */
|
||||
std::normal_distribution<float> distribution(0.0f, sigma);
|
||||
|
||||
/* 遍历权重矩阵，进行赋值 */
|
||||
long k=0;
|
||||
for (long i=0; i<fan_in; ++i)
|
||||
for (long j=0; j<fan_out; ++j)
|
||||
weights[k++] = distribution(generator);
|
||||
|
||||
}
|
||||
|
||||
/* 均匀分布Xavier初始化函数 */
|
||||
void NeuralNetwork::xavierUniformInit(std::vector<float> &weights, int fan_in, int fan_out)
|
||||
{
|
||||
/* 使用高质量的随机数生成器 */
|
||||
static std::mt19937 generator(std::random_device{}());
|
||||
|
||||
/* 计算区间范围：sqrt(2 / (fan_in + fan_out)) */
|
||||
float scale = std::sqrt(2.0f / (fan_in + fan_out));
|
||||
|
||||
/* 创建均匀分布对象，范围为[-scale, scale] */
|
||||
std::uniform_real_distribution<float> distribution(-scale, scale);
|
||||
|
||||
/* 遍历权重矩阵，进行赋值 */
|
||||
long k=0;
|
||||
for (long i=0; i<fan_in; ++i)
|
||||
for (long j=0; j<fan_out; ++j)
|
||||
weights[k++] = distribution(generator);
|
||||
}
|
||||
|
||||
/* 工厂模式 通过形状构建全连接层 */
|
||||
NeuralNetwork NeuralNetwork::Create_in_Factory(const long inputno, const long outputno, float learning_rate, ActivationFunc func_type)
|
||||
{
|
||||
std::vector<float> weights(inputno * outputno);
|
||||
xavierUniformInit(weights, inputno, outputno);
|
||||
std::vector<float> bias(outputno);
|
||||
NeuralNetwork layer(inputno, outputno, weights, bias, learning_rate, func_type);
|
||||
return layer; /* FIXME: 使用移动语义 */
|
||||
}
|
||||
|
||||
|
||||
/* 前向反馈 */
|
||||
std::vector<float> NeuralNetwork::forward(const std::vector<float>& x)
|
||||
|
||||
{
|
||||
|
||||
|
||||
/*
|
||||
* 矩阵乘法（点积）
|
||||
* wi => weights inputs(column)
|
||||
* wo => weights outputs(row)
|
||||
*
|
||||
* NOTE: 通常矩阵乘法是三层循环，这里我们用两层
|
||||
*/
|
||||
//std::vector<float> res(this->outputno); /* (1, outputs) */
|
||||
//for (long i=0; i<this->outputno; ++i) {
|
||||
// long wo = i;
|
||||
// float sum=0.0;
|
||||
// for (long j=0, wi=wo; j<this->inputno; ++j, wi+=this->outputno) {
|
||||
// sum += x[j] * this->weights[wi];
|
||||
// }
|
||||
// res[i]= sum;
|
||||
//}
|
||||
Matrx<float>mx{1, this->inputno, x};
|
||||
|
||||
|
||||
/*
|
||||
* 为输入进行备份
|
||||
*
|
||||
* FIXME: 使用移动语义传入参数，并且储存
|
||||
*/
|
||||
this->prev_inputs = mx; /* (1, inputs) */
|
||||
|
||||
|
||||
Matrx<float>r = mx * this->weights; /* (1, outputs) */
|
||||
|
||||
/* 偏置操作 */
|
||||
//for (long i=0; i<this->outputno; ++i) {
|
||||
// res[i]+=this->bias[i];
|
||||
//}
|
||||
r = r + this->bias;
|
||||
|
||||
/*
|
||||
* 为原始计算结果进行备份
|
||||
*/
|
||||
this->prev_raw_inputs = r;
|
||||
|
||||
/*
|
||||
* 使用ReLU激活函数
|
||||
*
|
||||
* FIXME: 建立一个算法类
|
||||
*/
|
||||
f_t f = getf(f_type);
|
||||
|
||||
std::vector<float> vec =r.get_data();
|
||||
for (long i=0; i<this->outputno; ++i) {
|
||||
float& v = vec[i];
|
||||
v = f(v);
|
||||
}
|
||||
|
||||
r = Matrx<float>(1, outputno, vec);
|
||||
|
||||
/*
|
||||
* 为输出进行备份
|
||||
*/
|
||||
this->prev_outputs = r;
|
||||
|
||||
|
||||
/* 返回前向反馈的结果。 (1, outputs) */
|
||||
return r.get_data();
|
||||
}
|
||||
|
||||
|
||||
/* 反向传播 */
|
||||
std::vector<float> NeuralNetwork::backward(const std::vector<float>& errors) /* errors => (outputs, 1) */
|
||||
|
||||
{
|
||||
|
||||
/* 匹配激活导数 */
|
||||
f_t fdot = getfdot(f_type);
|
||||
|
||||
std::vector<float> vec = this->prev_raw_inputs.get_data();
|
||||
/* 将原始输出转化为导数 */
|
||||
for (long i=0; i<this->outputno; ++i) {
|
||||
float& v = vec[i];
|
||||
v = errors[i] * fdot(v);
|
||||
}
|
||||
|
||||
/* 误差乘以导数 */
|
||||
//std::vector<float> first(this->outputno);
|
||||
//for (long i=0; i<this->outputno; ++i) {
|
||||
// first[i] = errors[i] * this->prev_raw_inputs[i];
|
||||
//}
|
||||
Matrx<float> first{1, this->outputno, vec};
|
||||
|
||||
/* 更新权重梯度 => (errors * 导数) @ this->inputs => (1, outputs ) @ (inputs, 1) = (outputs, inputs) */
|
||||
//this->weights_grad = std::vector<float>(this->outputno * this->inputno);
|
||||
//long k = 0;
|
||||
//for (long i=0; i<this->inputno; ++i) {
|
||||
// for (long j=0; j<this->outputno; ++j) {
|
||||
// this->weights_grad[k++] = first[j] * this->prev_inputs[i];
|
||||
// }
|
||||
//}
|
||||
Matrx<float> mx {this->inputno, 1, this->prev_inputs.get_data()}; /* (inputs, 1) */
|
||||
this->weights_grad = mx * first; /* (inputs, 1) @ (1, outputs) => (inputs, outputs) */
|
||||
|
||||
/*
|
||||
* 更新偏置梯度
|
||||
*
|
||||
* NOTE: 在多批次情况下，应该沿着每一样本纵向求平均数
|
||||
*/
|
||||
//this->bias_grad = std::vector<float>(this->outputno);
|
||||
//for (long i=0; i<this->outputno; ++i) {
|
||||
// this->bias_grad[i] = first[i];
|
||||
//}
|
||||
this->bias_grad = first;
|
||||
|
||||
|
||||
/*
|
||||
* 计算上一层的误差责任分担，然后返回
|
||||
*
|
||||
* this->weights @ first => (outputs, inputs) @ (1, outputs) = (inputs, 1)
|
||||
*/
|
||||
//std::vector<float> uplevel_errors(this->inputno);
|
||||
//for (long i=0; i<this->inputno; ++i) {
|
||||
// long wii = i * this->outputno; /* 每一列的起始点 */
|
||||
// float sum = 0.0f;
|
||||
// for (long j=0, wi=wii; j < this->outputno; ++j, ++wi) {
|
||||
// sum += this->weights[wi] * first[j];
|
||||
// }
|
||||
// uplevel_errors[i] = sum;
|
||||
//}
|
||||
Matrx<float> first1{this->outputno, 1, first.get_data()};
|
||||
Matrx<float> uplevel_errs = this->weights * first1; /* (inputs, outputs) @ (outputs, 1) */
|
||||
return uplevel_errs.get_data();
|
||||
|
||||
/* 使用移动语义返回上一层误差 */
|
||||
//return std::move(uplevel_errors);
|
||||
}
|
||||
|
||||
void NeuralNetwork::update(float discount) {
|
||||
this->weights = this->weights + (this->weights_grad * this->learning_rate * discount);
|
||||
this->bias = this->bias + (this->bias_grad * this->learning_rate * discount);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user