移植bot4cj
This commit is contained in:
qing committed
1 parent
7803843155
commit
78ad3e6d57
76 files changed
+14468
No files matched your search
@@ -0,0 +1,204 @@
|
||||
// Adam.cj - Adam 优化器实现（添加全连接层支持）
|
||||
package bot4cj.math
|
||||
|
||||
import std.math.*
|
||||
import std.collection.*
|
||||
|
||||
public class Adam {
|
||||
let learningRate: FP32
|
||||
let beta1: FP32
|
||||
let beta2: FP32
|
||||
let epsilon: FP32
|
||||
|
||||
var m: HashMap<String, Matrix1>
|
||||
var v: HashMap<String, Matrix1>
|
||||
var t: Int64
|
||||
|
||||
// 完整参数构造函数
|
||||
public init(
|
||||
lr: FP32,
|
||||
beta1: FP32,
|
||||
beta2: FP32,
|
||||
epsilon: FP32
|
||||
) {
|
||||
learningRate = lr
|
||||
this.beta1 = beta1
|
||||
this.beta2 = beta2
|
||||
this.epsilon = epsilon
|
||||
m = HashMap<String, Matrix1>()
|
||||
v = HashMap<String, Matrix1>()
|
||||
t = 0
|
||||
}
|
||||
|
||||
// 默认参数构造函数
|
||||
public init() {
|
||||
learningRate = FP32(0.001)
|
||||
beta1 = FP32(0.9)
|
||||
beta2 = FP32(0.999)
|
||||
epsilon = FP32(0.00000001)
|
||||
m = HashMap<String, Matrix1>()
|
||||
v = HashMap<String, Matrix1>()
|
||||
t = 0
|
||||
}
|
||||
|
||||
// 仅自定义学习率的构造函数
|
||||
public init(lr: FP32) {
|
||||
learningRate = lr
|
||||
beta1 = FP32(0.9)
|
||||
beta2 = FP32(0.999)
|
||||
epsilon = FP32(0.00000001)
|
||||
m = HashMap<String, Matrix1>()
|
||||
v = HashMap<String, Matrix1>()
|
||||
t = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算幂运算的辅助函数
|
||||
*/
|
||||
private func power(base: FP32, exp: Int64): FP32 {
|
||||
var result = FP32(1.0)
|
||||
for (i in 0..exp) {
|
||||
result = result * base
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算平方根的辅助函数 - 使用牛顿迭代法
|
||||
*/
|
||||
private func sqrt(val: FP32): FP32 {
|
||||
let x = val.getValue()
|
||||
if (x <= 0.0) {
|
||||
return FP32(0.0)
|
||||
}
|
||||
var guess = x
|
||||
for (i in 0..10) {
|
||||
guess = (guess + x / guess) / 2.0
|
||||
}
|
||||
return FP32(guess)
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新矩阵参数（通用方法）
|
||||
* @param param 要更新的参数矩阵
|
||||
* @param grad 梯度矩阵
|
||||
* @param key 参数在HashMap中的键名
|
||||
*/
|
||||
private func updateMatrix(
|
||||
param: Matrix1,
|
||||
grad: Matrix1,
|
||||
key: String
|
||||
): Unit {
|
||||
// 初始化动量
|
||||
let hasState = m.contains(key)
|
||||
if (!hasState) {
|
||||
let rows = param.get_row()
|
||||
let cols = param.get_col()
|
||||
m.add(key, Matrix1(rows, cols))
|
||||
v.add(key, Matrix1(rows, cols))
|
||||
}
|
||||
|
||||
let mMat = m.get(key).getOrThrow()
|
||||
let vMat = v.get(key).getOrThrow()
|
||||
|
||||
let rows = param.get_row()
|
||||
let cols = param.get_col()
|
||||
|
||||
for (r in 0..rows) {
|
||||
for (c in 0..cols) {
|
||||
let g = grad.get(r, c).getOrThrow()
|
||||
|
||||
// 更新一阶矩
|
||||
let oldM = mMat.get(r, c).getOrThrow()
|
||||
let newM = beta1 * oldM + (FP32(1.0) - beta1) * g
|
||||
mMat.set(r, c, newM)
|
||||
|
||||
// 更新二阶矩
|
||||
let oldV = vMat.get(r, c).getOrThrow()
|
||||
let newV = beta2 * oldV + (FP32(1.0) - beta2) * g * g
|
||||
vMat.set(r, c, newV)
|
||||
|
||||
// 偏差修正
|
||||
let one = FP32(1.0)
|
||||
let beta1Pow = power(beta1, t)
|
||||
let beta2Pow = power(beta2, t)
|
||||
|
||||
let mHat = newM / (one - beta1Pow)
|
||||
let vHat = newV / (one - beta2Pow)
|
||||
|
||||
// 更新参数
|
||||
let oldParam = param.get(r, c).getOrThrow()
|
||||
let update = learningRate * mHat / (sqrt(vHat) + epsilon)
|
||||
param.set(r, c, oldParam - update)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新卷积层参数
|
||||
*/
|
||||
public func updateConvLayer(layer: ConvLayer): Unit {
|
||||
t += 1
|
||||
|
||||
// 更新卷积核
|
||||
var kGrad = match (layer.kernelsGrad) {
|
||||
case Some(val) => val
|
||||
case _ => throw Exception("No kernel gradients")
|
||||
}
|
||||
|
||||
for (oc in 0..layer.nOutputChannels) {
|
||||
let key = "conv_kernel_" + oc.toString()
|
||||
updateMatrix(layer.kernels[oc], kGrad[oc], key)
|
||||
}
|
||||
|
||||
// 更新偏置
|
||||
var bGrad = match (layer.biasGrad) {
|
||||
case Some(val) => val
|
||||
case _ => throw Exception("No bias gradient")
|
||||
}
|
||||
|
||||
let biasKey = "conv_bias"
|
||||
updateMatrix(layer.biases, bGrad, biasKey)
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新全连接层参数（Nn1）
|
||||
*/
|
||||
public func updateFCLayer(layer: Nn1): Unit {
|
||||
t += 1
|
||||
|
||||
// 更新权重
|
||||
var wGrad = match (layer.weights_grad) {
|
||||
case Some(val) => val
|
||||
case _ => throw Exception("No weights gradient")
|
||||
}
|
||||
|
||||
let weightKey = "fc_weights"
|
||||
updateMatrix(layer.weights, wGrad, weightKey)
|
||||
|
||||
// 更新偏置
|
||||
var bGrad = match (layer.bias_grad) {
|
||||
case Some(val) => val
|
||||
case _ => throw Exception("No bias gradient")
|
||||
}
|
||||
|
||||
let biasKey = "fc_bias"
|
||||
updateMatrix(layer.bias, bGrad, biasKey)
|
||||
}
|
||||
|
||||
/**
|
||||
* 重置优化器状态
|
||||
*/
|
||||
public func reset(): Unit {
|
||||
m.clear()
|
||||
v.clear()
|
||||
t = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前迭代次数
|
||||
*/
|
||||
public func getStep(): Int64 {
|
||||
t
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package bot4cj.math
|
||||
import std.math.*
|
||||
|
||||
/* -------- Activation Functions 激活函数族 --------
|
||||
* TODO(20260516-qing): 增加序列化、反序列化工厂函数 */
|
||||
public enum Af {
|
||||
|
||||
None | Sigmoid | ReLU | Leaky_ReLU(FP32) | Tanh | Softmax
|
||||
|
||||
/* -------- 执行激活函数 -------- */
|
||||
public func forward(x: Matrix1) {
|
||||
match(this) {
|
||||
case None => none(x)
|
||||
case Sigmoid => sigmoid(x)
|
||||
case ReLU => relu(x)
|
||||
case Leaky_ReLU(alpha) => leaky_relu(x, alpha)
|
||||
case Tanh => tanh(x)
|
||||
case Softmax => softmax(x)
|
||||
}
|
||||
}
|
||||
|
||||
/* -------- 激活函数的导数 -------- */
|
||||
public func backward(y: Matrix1) {
|
||||
match(this) {
|
||||
case None => none_dot(y)
|
||||
case Sigmoid => sigmoid_dot(y)
|
||||
case ReLU => relu_dot(y)
|
||||
case Leaky_ReLU(alpha) => leaky_relu_dot(y, alpha)
|
||||
case Tanh => tanh_dot(y)
|
||||
case Softmax => softmax_jacobian(y)
|
||||
}
|
||||
}
|
||||
|
||||
/* -------- 序列化与反序列化 -------- */
|
||||
public func get_type(): String {
|
||||
return match(this) {
|
||||
case None => "None"
|
||||
case Sigmoid => "Sigmoid"
|
||||
case ReLU => "ReLU"
|
||||
case Leaky_ReLU(alpha) => "Leaky_ReLU"
|
||||
case Tanh => "Tanh"
|
||||
case Softmax => "Softmax"
|
||||
}
|
||||
}
|
||||
|
||||
/* */
|
||||
public func get_value(): FP32 {
|
||||
return match(this) {
|
||||
case None => FP32(0.0f32)
|
||||
case Sigmoid => FP32(0.0f32)
|
||||
case ReLU => FP32(0.0f32)
|
||||
case Leaky_ReLU(alpha) => alpha
|
||||
case Tanh => FP32(0.0f32)
|
||||
case Softmax => FP32(0.0f32)
|
||||
}
|
||||
}
|
||||
|
||||
/* -------- 空的激活函数 -------- */
|
||||
private static func none(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
N.set(r, c, FP32(x1)) }
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 空激活函数的导数 -------- */
|
||||
private static func none_dot(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
N.set(r, c, FP32(1.0f32)) }
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- S型生长曲线 -------- */
|
||||
private static func sigmoid(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 = 1.0f32 / (1.0f32 + exp(-x1))
|
||||
N.set(r, c, FP32(x2)) }
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- S型生长曲线的导数 -------- */
|
||||
private static func sigmoid_dot(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 = 1.0f32 / (1.0f32 + exp(-x1))
|
||||
let x3 = x2 * (1.0f32 - x2)
|
||||
N.set(r, c, FP32(x3)) }
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 线性整流单元 -------- */
|
||||
private static func relu(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 =if (x1 > 0.0f32) {x1} else{0.0f32}
|
||||
N.set(r, c, FP32(x2))
|
||||
}
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 线性整流单元的导数 -------- */
|
||||
private static func relu_dot(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 =if (x1 > 0.0f32) {1.0f32} else{0.0f32}
|
||||
N.set(r, c, FP32(x2))
|
||||
}
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 带泄露的线性整流单元 -------- */
|
||||
private static func leaky_relu(M: Matrix1, alpha: FP32) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 =if (x1 > 0.0f32) {x1} else{x1 * alpha.getValue()}
|
||||
N.set(r, c, FP32(x2))
|
||||
}
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 带泄露的整流单元导数 -------- */
|
||||
private static func leaky_relu_dot(M: Matrix1, alpha: FP32) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 =if (x1 > 0.0f32) { 1.0f32 } else{alpha.getValue()}
|
||||
N.set(r, c, FP32(x2)) }
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 双曲正切函数 --------*/
|
||||
private static func tanh(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 = tanh(x1)
|
||||
N.set(r, c, FP32(x2)) }
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 双曲正切函数的导数 -------- */
|
||||
private static func tanh_dot(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let y = M.get(r, c).getOrThrow()
|
||||
let y_val= y.getValue() /*---- NOTE(20260525-qing): 修复命名（x => y） ----*/
|
||||
let rea = 1.0f32 - y_val * y_val
|
||||
N.set(r, c, FP32(rea)) }
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 归一化指数函数 -------- */
|
||||
private static func softmax(M: Matrix1) {
|
||||
let row = M.get_row()
|
||||
let col = M.get_col()
|
||||
let N = Matrix1(row, col)
|
||||
var max_val = 0.0f32
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
if (x1 > max_val) {
|
||||
max_val = x1 }
|
||||
}
|
||||
}
|
||||
var sum = 0.0f32
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = M.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 = exp(x1 - max_val)
|
||||
N.set(r, c, FP32(x2))
|
||||
sum += x2
|
||||
}
|
||||
}
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let x = N.get(r, c).getOrThrow()
|
||||
let x1= x.getValue()
|
||||
let x2 = x1 / sum
|
||||
N.set(r, c, FP32(x2))
|
||||
}
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 计算Softmax的雅可比矩阵 --------
|
||||
* 输入：已计算好的softmax输出（概率分布）
|
||||
* 输出：雅可比矩阵 J[i][j] = ∂y_i/∂x_j */
|
||||
private func softmax_jacobian(softmax_output: Matrix1) {
|
||||
let row = softmax_output.get_row()
|
||||
let col = softmax_output.get_col()
|
||||
let n = row * col
|
||||
var jacobian = Matrix1(n, n)
|
||||
|
||||
for (i in 0..n) {
|
||||
for (j in 0..n) {
|
||||
|
||||
let row_i = i / col /* 需要将一维索引转换为二维索引来访问Matrix1 */
|
||||
let col_i = i % col
|
||||
let row_j = j / col
|
||||
let col_j = j % col
|
||||
|
||||
let val_i = softmax_output.get(row_i, col_i).getOrThrow().getValue()
|
||||
let val_j = softmax_output.get(row_j, col_j).getOrThrow().getValue()
|
||||
|
||||
if (i == j) { /* 对角线元素: y_i * (1 - y_i) */
|
||||
jacobian.set(i, j, FP32(val_i * (1.0f32 - val_i))) }
|
||||
else { /* 非对角线元素: -y_i * y_j */
|
||||
jacobian.set(i, j, FP32(-val_i * val_j)) }
|
||||
}
|
||||
}
|
||||
|
||||
return jacobian
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package bot4cj.math
|
||||
import std.random.*
|
||||
import std.time.*
|
||||
import std.math.*
|
||||
|
||||
/* -------- 神经网络使用的权重矩阵初始化类 --------
|
||||
* TODO(20260517-qing): 将该函数改为枚举类，统一一个接口，根据模式匹配执行初始化
|
||||
* - 这样在矩阵式中可以定义初始化方式 */
|
||||
public class Ini {
|
||||
|
||||
/* -------- xavier正态初始化 -------- */
|
||||
public static func xavierNormalInit(target: Matrix1, fan_in: Int64, fan_out: Int64) {
|
||||
let seed = DateTime.now().nanosecond
|
||||
let random = Random(UInt64(seed))
|
||||
let sigma = sqrt(2.0f32 / Float32(fan_in + fan_out)) /* 计算标准差 */
|
||||
for (r in 0..fan_in) {
|
||||
for (c in 0..fan_out) {
|
||||
let v = random.nextGaussianFloat32(mean: 0.0f32, sigma:sigma)
|
||||
target.set(r, c, FP32(v))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* -------- Glorot均匀初始化 -------- */
|
||||
public static func xavierUniformInit(target: Matrix1, fan_in: Int64, fan_out: Int64) {
|
||||
let limit = sqrt(6.0f32 / Float32(fan_in + fan_out))
|
||||
let seed = DateTime.now().nanosecond
|
||||
let random = Random(UInt64(seed))
|
||||
|
||||
for (r in 0..fan_in) {
|
||||
for (c in 0..fan_out) {
|
||||
let v = (random.nextFloat32() * 2.0f32 - 1.0f32) * limit /* 均匀分布 [-limit, limit] */
|
||||
target.set(r, c, FP32(v)) }
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
package bot4cj.math
|
||||
import std.fs.*
|
||||
import std.io.*
|
||||
import std.convert.*
|
||||
//foreign func createMatrix(row: UInt64, col: UInt64): CPointer<Unit>
|
||||
//foreign func freeMatrix(M: CPointer<Unit>): Unit
|
||||
//foreign func witchMatrix(M: CPointer<Unit>, row: UInt64, col: UInt64): CPointer<Float32>
|
||||
//foreign func getMatrix(M: CPointer<Unit>, row: UInt64, col: UInt64): Float32
|
||||
//foreign func setMatrix(M: CPointer<Unit>, row: UInt64, col: UInt64, v: Float32): Unit
|
||||
//foreign func getrowMatrix(M: CPointer<Unit>): UInt64
|
||||
//foreign func getcolMatrix(M: CPointer<Unit>): UInt64
|
||||
//foreign func sameShapeMatrix(M: CPointer<Unit>, S: CPointer<Unit>): Bool
|
||||
//foreign func dotShapeMatrix(M: CPointer<Unit>, S: CPointer<Unit>): Bool
|
||||
//foreign func plusMatrix(M: CPointer<Unit>, S:CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func plusBroadcastMatrix(M: CPointer<Unit>, T: Float32): CPointer<Unit>
|
||||
//foreign func subMatrix(M: CPointer<Unit>, S: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func subBroadcastMatrix(M: CPointer<Unit>, T: Float32): CPointer<Unit>
|
||||
//foreign func multiMatrix(M: CPointer<Unit>, S: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func multiBroadcastMatrix(M: CPointer<Unit>, T: Float32): CPointer<Unit>
|
||||
//foreign func divMatrix(M: CPointer<Unit>, S: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func divBroadcastMatrix(M: CPointer<Unit>, T: Float32): CPointer<Unit>
|
||||
//foreign func dotMatrix(M: CPointer<Unit>, S: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func transposeMatrix(M: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func xavierNormalInit(fan_in: UInt64, fan_out: UInt64): CPointer<Unit>
|
||||
//foreign func xavierUniformInit(fan_in: UInt64, fan_out: UInt64): CPointer<Unit>
|
||||
|
||||
|
||||
/* -------- 32位浮点数 -------- */
|
||||
public struct FP32 {
|
||||
private var val: Float32
|
||||
public init(t: Float32) { /* 以Float32初始化对象 */
|
||||
val = t }
|
||||
public func getValue() { /* 获取值 */
|
||||
return val }
|
||||
/* -------- 运算符重载 -------- */
|
||||
//public operator func =(t: Float32) { /*-------- this can not --------*/
|
||||
// val = t }
|
||||
public operator func +(t: Float32) {
|
||||
return FP32(val + t)}
|
||||
public operator func +(S: FP32) {
|
||||
return this + S.getValue() }
|
||||
public operator func -(t: Float32) {
|
||||
return FP32(val - t) }
|
||||
public operator func -(S: FP32) {
|
||||
return this - S.getValue() }
|
||||
public operator func *(t: Float32) {
|
||||
return FP32(val * t) }
|
||||
public operator func *(S: FP32) {
|
||||
return this * S.getValue() }
|
||||
public operator func /(t: Float32) {
|
||||
return FP32(val / t) }
|
||||
public operator func /(S: FP32) {
|
||||
return this / S.getValue() }
|
||||
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* 新版的矩阵对象，仓颉原生实现
|
||||
* TODO(20260517-qing): 设计原生的多维数组类
|
||||
*
|
||||
|
||||
___col___
|
||||
/ \
|
||||
+---------+ \
|
||||
| + + + + | | row - 第一维度
|
||||
| + + + + | |
|
||||
+---------+ /
|
||||
|
||||
|
||||
* TODO(20260523-qing): 以某种方式统一矩阵式的字符串矩阵和这里的数值矩阵
|
||||
*/
|
||||
public class Matrix1 {
|
||||
|
||||
private var row: Int64 /* ---- 行数 ---- */
|
||||
private var col: Int64 /* ---- 列数 ---- */
|
||||
private var arr: Array<FP32> /* ---- 数据 ---- */
|
||||
|
||||
/* -------- 构造器，默认值为0 -------- */
|
||||
public Matrix1(row: Int64, col: Int64) {
|
||||
this.row = row
|
||||
this.col = col
|
||||
this.arr = Array<FP32>(row * col, {_ => FP32(0.0f32)})
|
||||
}
|
||||
|
||||
/* --------- 这几个方法是访问器 -------- */
|
||||
/* ------------------------------------- */
|
||||
/* -------- TODO(20260524-qing): 使用prop成员 -------- */
|
||||
|
||||
public func get(r: Int64, c: Int64) {
|
||||
return arr.get(r * this.col + c)
|
||||
}
|
||||
|
||||
public func set(r: Int64, c: Int64, v: FP32) {
|
||||
arr[r * this.col + c] = v
|
||||
}
|
||||
|
||||
public func get_row() { return row }
|
||||
|
||||
public func get_col() { return col }
|
||||
|
||||
/* -------- 这两个函数是比较器 -------- */
|
||||
/* ------------------------------------ */
|
||||
public func has_same_shape(other: Matrix1) {
|
||||
let a = this.get_row() == other.get_row()
|
||||
let b = this.get_col() == other.get_col()
|
||||
return a && b
|
||||
}
|
||||
|
||||
public func has_dot_shape(other: Matrix1) {
|
||||
return this.get_col() == other.get_row()
|
||||
}
|
||||
|
||||
/* -------- 这些是运算符重载 -------- */
|
||||
/* ---------------------------------- */
|
||||
public operator func +(a: FP32) { /* ---- 矩阵广播加法 ---- */
|
||||
let r =get_row()
|
||||
let c = get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c) {
|
||||
let v = this.get(ir, ic).getOrThrow()
|
||||
M.set(ir, ic, v + a)
|
||||
}
|
||||
}
|
||||
|
||||
return M
|
||||
}
|
||||
|
||||
public operator func +(S: Matrix1) { /* ---- 矩阵加法 ---- */
|
||||
if (!has_same_shape(S)) {
|
||||
throw IllegalArgumentException("Matrixes doesn't match")
|
||||
}
|
||||
let r =get_row()
|
||||
let c = get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c) {
|
||||
let v = this.get(ir, ic).getOrThrow()
|
||||
let vs = S.get(ir, ic).getOrThrow()
|
||||
M.set(ir, ic, v + vs)
|
||||
}
|
||||
}
|
||||
|
||||
return M
|
||||
}
|
||||
|
||||
public operator func -(a: FP32) { /* ---- 矩阵广播减法 ----*/
|
||||
let r = get_row()
|
||||
let c = get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c){
|
||||
let v = this.get(ir, ic).getOrThrow()
|
||||
M.set(ir, ic, v - a)
|
||||
}
|
||||
}
|
||||
return M
|
||||
}
|
||||
|
||||
public operator func -(S: Matrix1) { /* ---- 矩阵减法 ---- */
|
||||
if (!has_same_shape(S)) {
|
||||
throw IllegalArgumentException("Matrixes doesn't match")
|
||||
}
|
||||
let r =get_row()
|
||||
let c = get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c) {
|
||||
let v = this.get(ir, ic).getOrThrow()
|
||||
let vs = S.get(ir, ic).getOrThrow()
|
||||
M.set(ir, ic, v - vs)
|
||||
}
|
||||
}
|
||||
|
||||
return M
|
||||
}
|
||||
|
||||
public operator func *(a: FP32) { /* ---- 矩阵广播乘法 ---- */
|
||||
let r = get_row()
|
||||
let c = get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c){
|
||||
let v = this.get(ir, ic).getOrThrow()
|
||||
M.set(ir, ic, v * a)
|
||||
}
|
||||
}
|
||||
return M
|
||||
}
|
||||
|
||||
public operator func *(S: Matrix1) { /* ---- 逐元素相乘 ---- */
|
||||
if (!has_same_shape(S)) {
|
||||
throw IllegalArgumentException("Matrixes doesn't match")
|
||||
}
|
||||
let r =get_row()
|
||||
let c = get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c) {
|
||||
let v = this.get(ir, ic).getOrThrow()
|
||||
let vs = S.get(ir, ic).getOrThrow()
|
||||
M.set(ir, ic, v * vs)
|
||||
}
|
||||
}
|
||||
|
||||
return M
|
||||
}
|
||||
|
||||
public operator func /(a: FP32) { /* ---- 矩阵广播除法 ---- */
|
||||
let r = get_row()
|
||||
let c = get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c){
|
||||
let v = this.get(ir, ic).getOrThrow()
|
||||
M.set(ir, ic, v / a)
|
||||
}
|
||||
}
|
||||
return M
|
||||
}
|
||||
|
||||
public operator func /(S: Matrix1) { /* ---- 逐元素除法 ---- */
|
||||
if (!has_same_shape(S)) {
|
||||
throw IllegalArgumentException("Matrixes doesn't match")
|
||||
}
|
||||
let r =get_row()
|
||||
let c = get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c) {
|
||||
let v = this.get(ir, ic).getOrThrow()
|
||||
let vs = S.get(ir, ic).getOrThrow()
|
||||
M.set(ir, ic, v / vs)
|
||||
}
|
||||
}
|
||||
|
||||
return M
|
||||
}
|
||||
|
||||
/* ------------------------------ */
|
||||
/* -------- 矩阵点积乘法 -------- */
|
||||
public func dot(S: Matrix1) {
|
||||
if (!has_dot_shape(S)) {
|
||||
throw IllegalArgumentException("Matrixes doesn't match")
|
||||
}
|
||||
let r =get_row()
|
||||
let c = S.get_col()
|
||||
let M = Matrix1(r, c)
|
||||
for (ir in 0..get_row()) {
|
||||
for (ic in 0..S.get_col()) {
|
||||
var sum = FP32(0.0f32)
|
||||
for (ik in 0..get_col()) {
|
||||
let a = get(ir, ik).getOrThrow()
|
||||
let b = S.get(ik, ic).getOrThrow()
|
||||
sum += a * b
|
||||
}
|
||||
M.set(ir, ic, sum)
|
||||
}
|
||||
}
|
||||
return M
|
||||
}
|
||||
|
||||
/* -------- 克隆一个一模一样的矩阵 -------- */
|
||||
public func clone() {
|
||||
let r = get_row()
|
||||
let c = get_col()
|
||||
let N = Matrix1(r, c)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c) {
|
||||
let v = get(ir, ic).getOrThrow()
|
||||
N.set(ir, ic, v)
|
||||
}
|
||||
}
|
||||
return N
|
||||
}
|
||||
|
||||
/* -------- 矩阵转置，返回转置后的矩阵 -------- */
|
||||
public func T() {
|
||||
let r = get_row()
|
||||
let c = get_col()
|
||||
let N = Matrix1(c, r)
|
||||
for (ir in 0..r) {
|
||||
for (ic in 0..c) {
|
||||
let v = get(ir, ic).getOrThrow()
|
||||
N.set(ic, ir, v)
|
||||
}
|
||||
}
|
||||
return N
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* -------- 旧版的矩阵类，依赖c++编译的共享动态库 --------
|
||||
* TODO: 将矩阵元数据存储在仓颉部分(done at 202605xx ) */
|
||||
//public class Matrix {
|
||||
//
|
||||
// /* -------- 主初始化器，初始化并获取资源 -------- */
|
||||
// public init(row: UInt64, col: UInt64) {
|
||||
// unsafe { M = createMatrix(row, col) }
|
||||
// }
|
||||
//
|
||||
// /* -------- 初始化器，利用已经存在的元数据进行构建 -------- */
|
||||
// public Matrix(m: CPointer<Unit>) { M = m }
|
||||
//
|
||||
// /* -------- Xavier正态分布初始化 -------- */
|
||||
// public static func initWithXavierNormal(fan_in: UInt64, fan_out: UInt64) {
|
||||
// unsafe {
|
||||
// let m = xavierNormalInit(fan_in, fan_out);
|
||||
// return Matrix(m)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /* -------- Glorot均匀初始化 -------- */
|
||||
// public static func initWithXavierUniform(fan_in: UInt64, fan_out: UInt64) {
|
||||
// unsafe {
|
||||
// let m = xavierUniformInit(fan_in, fan_out);
|
||||
// return Matrix(m)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /* ======== 在析构函数中释放资源 ======== */
|
||||
// ~init() {
|
||||
// unsafe { freeMatrix(M) }
|
||||
// }
|
||||
//
|
||||
// /* -------- 取下标，由于不能返回值类型的引用被弃用 -------- */
|
||||
// //public operator func [](r: UInt64, c: UInt64): CPointer<Float32> {
|
||||
// // unsafe { return witchMatrix(M, r, c) }
|
||||
// //}
|
||||
//
|
||||
// /* -------- 获得某个矩阵单元的值 -------- */
|
||||
// public func get(r: UInt64, c: UInt64): Float32 {
|
||||
// unsafe { return getMatrix(M, r, c) }
|
||||
// }
|
||||
//
|
||||
// /* -------- 设置某个矩阵单元的值 --------*/
|
||||
// public func set(r: UInt64, c: UInt64, v: Float32) {
|
||||
// unsafe { setMatrix(M, r, c, v) }
|
||||
// }
|
||||
//
|
||||
// /* ------- 两个矩阵形状是否相同 --------
|
||||
// * FIXME: 这个函数所重载的运算符不贴切 */
|
||||
// public operator func ==(s: Matrix): Bool {
|
||||
// unsafe { return sameShapeMatrix(M, s.getData()) }
|
||||
// }
|
||||
//
|
||||
// /* -------- 获得行数 -------- */
|
||||
// public func getRow(): UInt64 {
|
||||
// unsafe { return getrowMatrix(M) }
|
||||
// }
|
||||
//
|
||||
// /* -------- 获取列数 --------*/
|
||||
// public func getCol(): UInt64 {
|
||||
// unsafe { return getcolMatrix(M) }
|
||||
// }
|
||||
//
|
||||
// /* -------- 获取指向元数据的C风格指针 -------- */
|
||||
// public func getData(): CPointer<Unit> { return M; }
|
||||
//
|
||||
// /* -------- 是否两个矩阵具有点积和的形状 -------- */
|
||||
// public func hasDotShapeWith(s: Matrix): Bool {
|
||||
// unsafe { return dotShapeMatrix(M, s.getData()) }
|
||||
// }
|
||||
//
|
||||
// /* -------- 逐元素算数运算 -------- */
|
||||
// /* -------------------------------- */
|
||||
// public operator func +(s: Matrix): Matrix {
|
||||
// unsafe {
|
||||
// if (!sameShapeMatrix(M, s.getData())) {
|
||||
// println('------------')
|
||||
// print_shape()
|
||||
// s.print_shape()
|
||||
// println('------------')
|
||||
// throw IllegalArgumentException("not same shape")
|
||||
// }
|
||||
// let r = plusMatrix(M, s.getData())
|
||||
// return Matrix(r)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// public operator func +(t: Float32): Matrix {
|
||||
// unsafe {
|
||||
// let r = plusBroadcastMatrix(M, t)
|
||||
// return Matrix(r);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// public operator func -(s: Matrix): Matrix {
|
||||
// unsafe {
|
||||
// if (!(this == s)) {
|
||||
// println('------------')
|
||||
// print_shape()
|
||||
// s.print_shape()
|
||||
// println('------------')
|
||||
// throw IllegalArgumentException("not same shape")
|
||||
// }
|
||||
// let r = subMatrix(M, s.getData())
|
||||
// return Matrix(r)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// public operator func -(t: Float32): Matrix {
|
||||
// unsafe {
|
||||
// let r = subBroadcastMatrix(M, t)
|
||||
// return Matrix(r);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// public operator func *(s: Matrix): Matrix {
|
||||
// unsafe {
|
||||
// if (!(this == s)) {
|
||||
// println('------------')
|
||||
// print_shape()
|
||||
// s.print_shape()
|
||||
// println('------------')
|
||||
// throw IllegalArgumentException("not same shape")
|
||||
// }
|
||||
// let r = multiMatrix(M, s.getData())
|
||||
// return Matrix(r)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// public operator func *(t: Float32): Matrix {
|
||||
// unsafe {
|
||||
// let r = multiBroadcastMatrix(M, t)
|
||||
// return Matrix(r)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// public operator func /(s: Matrix): Matrix {
|
||||
// unsafe {
|
||||
// if (!(this == s)) {
|
||||
// println('------------')
|
||||
// print_shape()
|
||||
// s.print_shape()
|
||||
// println('------------')
|
||||
// throw IllegalArgumentException("not same shape")
|
||||
// }
|
||||
// let r = divMatrix(M, s.getData())
|
||||
// return Matrix(r)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// public operator func /(t: Float32): Matrix {
|
||||
// unsafe {
|
||||
// let r = divBroadcastMatrix(M, t)
|
||||
// return Matrix(r)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /* -------------------------------- */
|
||||
// /* -------------------------------- */
|
||||
//
|
||||
//
|
||||
// /* -------- 矩阵点积和算法 -------- */
|
||||
// public func dot (s: Matrix): Matrix {
|
||||
// unsafe {
|
||||
// if (!hasDotShapeWith(s)) {
|
||||
// println('------------')
|
||||
// print_shape()
|
||||
// s.print_shape()
|
||||
// println('------------')
|
||||
// throw IllegalArgumentException("not dot shape")
|
||||
// }
|
||||
// let r = dotMatrix(M, s.getData())
|
||||
// return Matrix(r)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /* -------- 矩阵转置 -------- */
|
||||
// public func T(): Matrix {
|
||||
// unsafe {
|
||||
// let r = transposeMatrix(M)
|
||||
// return Matrix(r)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /* -------- 打印矩阵 -------- */
|
||||
// public func print(): Unit {
|
||||
// unsafe {
|
||||
// for (i in 0..getRow()) {
|
||||
// for (j in 0..getCol()) {
|
||||
// let a = this.get(i, j)
|
||||
// print(a)
|
||||
// print(" ")
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// println()
|
||||
// }
|
||||
//
|
||||
// /* -------- 打印形状 -------- */
|
||||
// public func print_shape() {
|
||||
// println("${getRow()}, ${getCol()}")
|
||||
// }
|
||||
//
|
||||
// /* -------- 保存矩阵 -------- */
|
||||
// public func save(file: File) {
|
||||
// unsafe {
|
||||
// for (i in 0..getRow()) {
|
||||
// for (j in 0..getCol()) {
|
||||
// file.write(this.get(i, j).toString().toArray())
|
||||
// if (j < getCol()-1) {
|
||||
// file.write(" ".toArray()) }
|
||||
// else { file.write("\n".toArray()) }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /* -------- 静态方法，加载单独项 -------- */
|
||||
// private static func loadItem(sr: StringReader<InputStream>) {
|
||||
// let s = sr.readUntil({r => r==r' ' || r ==r'\n'})
|
||||
// return Float32.parse(s.getOrThrow())
|
||||
// }
|
||||
//
|
||||
// /* -------- 静态方法，加载矩阵 -------- */
|
||||
// public static func load(sr: StringReader<InputStream>, row: UInt64, col: UInt64): Matrix {
|
||||
// let m = Matrix(row, col)
|
||||
// unsafe {
|
||||
// for (i in 0..m.getRow()) {
|
||||
// for (j in 0..m.getCol()) {
|
||||
// let f = loadItem(sr)
|
||||
// m.set(i, j, f)
|
||||
// }
|
||||
// }
|
||||
// m
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// /* -------- 矩阵的C风格指针 -------- */
|
||||
// private var M: CPointer<Unit>
|
||||
//}
|
||||
@@ -0,0 +1,56 @@
|
||||
package bot4cj.math
|
||||
import std.collection.*
|
||||
|
||||
/* -------- 张量 --------
|
||||
* INFO(20260524-qing): 输入一个维度列表 */
|
||||
class Tensor<T> {
|
||||
private let data: Array<T>
|
||||
private let dims: Array<Int64>
|
||||
private let size: Int64
|
||||
public init(dims: Array<Int64>, val: T) {
|
||||
var size = 1
|
||||
for (item in dims) { /* ---- 计算总量 ---- */
|
||||
size *= item
|
||||
}
|
||||
this.data = Array<T>(size, {_ => val})
|
||||
this.dims = dims
|
||||
this.size = size
|
||||
}
|
||||
|
||||
/* -------- 私有构造函数，用于内部创建张量 -------- */
|
||||
private init(dims: Array<Int64>, data: Array<T>) {
|
||||
this.dims = dims
|
||||
this.data = data
|
||||
var size = 1
|
||||
for (item in dims) {
|
||||
size *= item
|
||||
}
|
||||
this.size = size
|
||||
}
|
||||
|
||||
/* -------- 获取维度列表（形状） -------- */
|
||||
public func shape(): Array<Int64> {
|
||||
return dims
|
||||
}
|
||||
|
||||
/* -------- 逐元素进行某种运算 -------- */
|
||||
public func oneByOne(S: Tensor<T>, f: (T, T) -> T): Tensor<T> {
|
||||
let s1 = S.shape()
|
||||
|
||||
/* ---- 比较维度数量 ---- */
|
||||
if (s1 != dims) {
|
||||
throw IllegalArgumentException("Tensors' shape are not same")
|
||||
}
|
||||
|
||||
/* ---- 比较维度形状 ----*/
|
||||
for (i in 0..dims.size) {
|
||||
if (s1[i] != dims[i]) {
|
||||
throw IllegalArgumentException("Tensors' shape are not same")
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- 逐个进行计算 ---- */
|
||||
let res_data = Array<T>(size, {i => f(data[i], S.data[i])})
|
||||
return Tensor<T>(dims, res_data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
// cnn.cj - 修正 forward 中的偏置加法
|
||||
package bot4cj.math
|
||||
|
||||
import std.math.*
|
||||
import std.random.*
|
||||
import std.collection.*
|
||||
|
||||
public class ConvLayer {
|
||||
|
||||
let nInputChannels: Int64
|
||||
let nOutputChannels: Int64
|
||||
let kernelHeight: Int64
|
||||
let kernelWidth: Int64
|
||||
let stride: Int64
|
||||
let padding: Int64
|
||||
let learningRate: FP32
|
||||
let f: Af
|
||||
|
||||
var connectingMatrix: Matrix1
|
||||
var kernels: Array<Matrix1>
|
||||
var biases: Matrix1
|
||||
|
||||
var prevInput: Option<Matrix1> = Option.None
|
||||
var prevInputShape: Option<(Int64, Int64, Int64, Int64)> = Option.None
|
||||
var prevCombination: Option<Matrix1> = Option.None
|
||||
var prevOutput: Option<Matrix1> = Option.None
|
||||
var prevOutputShape: Option<(Int64, Int64, Int64, Int64)> = Option.None
|
||||
|
||||
var kernelsGrad: Option<Array<Matrix1>> = Option.None
|
||||
var biasGrad: Option<Matrix1> = Option.None
|
||||
|
||||
public init(
|
||||
inCh: Int64,
|
||||
outCh: Int64,
|
||||
kernelSize: (Int64, Int64),
|
||||
stride: Int64,
|
||||
padding: Int64,
|
||||
lr: FP32,
|
||||
activation: Af,
|
||||
connMatrix: ?Matrix1
|
||||
) {
|
||||
nInputChannels = inCh
|
||||
nOutputChannels = outCh
|
||||
kernelHeight = kernelSize[0]
|
||||
kernelWidth = kernelSize[1]
|
||||
this.stride = stride
|
||||
this.padding = padding
|
||||
learningRate = lr
|
||||
this.f = activation
|
||||
|
||||
// 初始化 connectingMatrix
|
||||
connectingMatrix = match (connMatrix) {
|
||||
case m: Matrix1 => m
|
||||
case _ => Matrix1(nInputChannels, nOutputChannels)
|
||||
}
|
||||
let isNone = match (connMatrix) {
|
||||
case _: Matrix1 => false
|
||||
case _ => true
|
||||
}
|
||||
if (isNone) {
|
||||
for (r in 0..nInputChannels) {
|
||||
for (c in 0..nOutputChannels) {
|
||||
connectingMatrix.set(r, c, FP32(1.0))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化 kernels
|
||||
kernels = Array(nOutputChannels, { _ => Matrix1(1, 1) })
|
||||
for (oc in 0..nOutputChannels) {
|
||||
var nConnections = 0
|
||||
for (ic in 0..nInputChannels) {
|
||||
let val = connectingMatrix.get(ic, oc).getOrThrow()
|
||||
if (val.getValue() > 0.0) {
|
||||
nConnections += 1
|
||||
}
|
||||
}
|
||||
let kernelLen = nConnections * kernelHeight * kernelWidth
|
||||
let kernel = Matrix1(1, kernelLen)
|
||||
|
||||
Ini.xavierUniformInit(kernel, 1, kernelLen)
|
||||
|
||||
kernels[oc] = kernel
|
||||
}
|
||||
|
||||
// 初始化 biases
|
||||
biases = Matrix1(1, nOutputChannels)
|
||||
Ini.xavierUniformInit(biases, 1, nOutputChannels)
|
||||
}
|
||||
|
||||
// ==================== 前向传播 ====================
|
||||
public func forward(x: Matrix1, inputShape: (Int64, Int64, Int64, Int64)): Matrix1 {
|
||||
prevInput = Option.Some(x)
|
||||
prevInputShape = Option.Some(inputShape)
|
||||
|
||||
let batch = inputShape[0]
|
||||
let inH = inputShape[1]
|
||||
let inW = inputShape[2]
|
||||
|
||||
let outH = (inH + 2 * padding - kernelHeight) / stride + 1
|
||||
let outW = (inW + 2 * padding - kernelWidth) / stride + 1
|
||||
let outSize = batch * outH * outW
|
||||
|
||||
var combination = Matrix1(outSize, nOutputChannels)
|
||||
var output = Matrix1(outSize, nOutputChannels)
|
||||
|
||||
for (b in 0..batch) {
|
||||
for (oh in 0..outH) {
|
||||
for (ow in 0..outW) {
|
||||
let outRow = b * outH * outW + oh * outW + ow
|
||||
|
||||
for (oc in 0..nOutputChannels) {
|
||||
var sum = FP32(0.0)
|
||||
var kernelIdx = 0
|
||||
|
||||
for (ic in 0..nInputChannels) {
|
||||
let connVal = connectingMatrix.get(ic, oc).getOrThrow()
|
||||
if (connVal.getValue() > 0.0) {
|
||||
for (kh in 0..kernelHeight) {
|
||||
for (kw in 0..kernelWidth) {
|
||||
let ih = oh * stride + kh - padding
|
||||
let iw = ow * stride + kw - padding
|
||||
|
||||
if (ih >= 0 && ih < inH && iw >= 0 && iw < inW) {
|
||||
let inRow = b * inH * inW + ih * inW + iw
|
||||
let xVal = x.get(inRow, ic).getOrThrow()
|
||||
let kVal = kernels[oc].get(0, kernelIdx).getOrThrow()
|
||||
sum = sum + xVal * kVal
|
||||
}
|
||||
kernelIdx += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
combination.set(outRow, oc, sum)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 手动广播偏置加法（因为 Matrix1 的 + 要求形状相同）
|
||||
for (r in 0..outSize) {
|
||||
for (c in 0..nOutputChannels) {
|
||||
let val = combination.get(r, c).getOrThrow()
|
||||
let biasVal = biases.get(0, c).getOrThrow()
|
||||
combination.set(r, c, val + biasVal)
|
||||
}
|
||||
}
|
||||
|
||||
prevCombination = Option.Some(combination)
|
||||
|
||||
output = f.forward(combination)
|
||||
prevOutput = Option.Some(output)
|
||||
prevOutputShape = Option.Some((batch, outH, outW, nOutputChannels))
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
// ==================== 反向传播 ====================
|
||||
public func backward(errs: Matrix1): Matrix1 {
|
||||
var x = Matrix1(1, 1)
|
||||
match (prevInput) {
|
||||
case Some(T) => x = T
|
||||
case _ => throw Exception("No input data")
|
||||
}
|
||||
|
||||
var shape = (0, 0, 0, 0)
|
||||
match (prevInputShape) {
|
||||
case Some(T) => shape = T
|
||||
case _ => throw Exception("No input shape")
|
||||
}
|
||||
|
||||
var combination = Matrix1(1, 1)
|
||||
match (prevCombination) {
|
||||
case Some(val) => combination = val
|
||||
case _ => throw Exception("No combination")
|
||||
}
|
||||
|
||||
var outShape = (0, 0, 0, 0)
|
||||
match (prevOutputShape) {
|
||||
case Some(val) => outShape = val
|
||||
case _ => throw Exception("No output shape")
|
||||
}
|
||||
|
||||
let batch = shape[0]
|
||||
let inH = shape[1]
|
||||
let inW = shape[2]
|
||||
let outH = outShape[1]
|
||||
let outW = outShape[2]
|
||||
|
||||
let delta = errs * f.backward(combination)
|
||||
|
||||
calcKernelGradient(x, delta, batch, inH, inW, outH, outW)
|
||||
calcBiasGradient(delta)
|
||||
|
||||
let errsUp = calcUpsampledError(delta, batch, inH, inW, outH, outW)
|
||||
|
||||
return errsUp
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算卷积核梯度
|
||||
*/
|
||||
private func calcKernelGradient(
|
||||
x: Matrix1,
|
||||
delta: Matrix1,
|
||||
batch: Int64,
|
||||
inH: Int64,
|
||||
inW: Int64,
|
||||
outH: Int64,
|
||||
outW: Int64
|
||||
): Unit {
|
||||
var grads = Array(nOutputChannels, { _ => Matrix1(1, 1) })
|
||||
|
||||
for (oc in 0..nOutputChannels) {
|
||||
let kernelCols = kernels[oc].get_col()
|
||||
var grad = Matrix1(1, kernelCols)
|
||||
|
||||
for (b in 0..batch) {
|
||||
for (oh in 0..outH) {
|
||||
for (ow in 0..outW) {
|
||||
let deltaRow = b * outH * outW + oh * outW + ow
|
||||
let deltaVal = delta.get(deltaRow, oc).getOrThrow()
|
||||
|
||||
var kernelIdx = 0
|
||||
for (ic in 0..nInputChannels) {
|
||||
let connVal = connectingMatrix.get(ic, oc).getOrThrow()
|
||||
if (connVal.getValue() > 0.0) {
|
||||
for (kh in 0..kernelHeight) {
|
||||
for (kw in 0..kernelWidth) {
|
||||
let ih = oh * stride + kh - padding
|
||||
let iw = ow * stride + kw - padding
|
||||
|
||||
if (ih >= 0 && ih < inH && iw >= 0 && iw < inW) {
|
||||
let inRow = b * inH * inW + ih * inW + iw
|
||||
let xVal = x.get(inRow, ic).getOrThrow()
|
||||
let oldGrad = grad.get(0, kernelIdx).getOrThrow()
|
||||
grad.set(0, kernelIdx, oldGrad + xVal * deltaVal)
|
||||
}
|
||||
kernelIdx += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
grads[oc] = grad
|
||||
}
|
||||
|
||||
kernelsGrad = Option.Some(grads)
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算偏置梯度
|
||||
*/
|
||||
private func calcBiasGradient(delta: Matrix1): Unit {
|
||||
let rows = delta.get_row()
|
||||
var grad = Matrix1(1, nOutputChannels)
|
||||
|
||||
for (oc in 0..nOutputChannels) {
|
||||
var sum = FP32(0.0)
|
||||
for (r in 0..rows) {
|
||||
sum = sum + delta.get(r, oc).getOrThrow()
|
||||
}
|
||||
grad.set(0, oc, sum)
|
||||
}
|
||||
|
||||
biasGrad = Option.Some(grad)
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算传递到上一层的误差
|
||||
*/
|
||||
private func calcUpsampledError(
|
||||
delta: Matrix1,
|
||||
batch: Int64,
|
||||
inH: Int64,
|
||||
inW: Int64,
|
||||
outH: Int64,
|
||||
outW: Int64
|
||||
): Matrix1 {
|
||||
let inSize = batch * inH * inW
|
||||
var errsUp = Matrix1(inSize, nInputChannels)
|
||||
|
||||
for (b in 0..batch) {
|
||||
for (oh in 0..outH) {
|
||||
for (ow in 0..outW) {
|
||||
let deltaRow = b * outH * outW + oh * outW + ow
|
||||
|
||||
for (oc in 0..nOutputChannels) {
|
||||
let deltaVal = delta.get(deltaRow, oc).getOrThrow()
|
||||
|
||||
var kernelIdx = 0
|
||||
for (ic in 0..nInputChannels) {
|
||||
let connVal = connectingMatrix.get(ic, oc).getOrThrow()
|
||||
if (connVal.getValue() > 0.0) {
|
||||
for (kh in 0..kernelHeight) {
|
||||
for (kw in 0..kernelWidth) {
|
||||
let ih = oh * stride + kh - padding
|
||||
let iw = ow * stride + kw - padding
|
||||
|
||||
if (ih >= 0 && ih < inH && iw >= 0 && iw < inW) {
|
||||
let inRow = b * inH * inW + ih * inW + iw
|
||||
let kVal = kernels[oc].get(0, kernelIdx).getOrThrow()
|
||||
let oldErr = errsUp.get(inRow, ic).getOrThrow()
|
||||
errsUp.set(inRow, ic, oldErr + deltaVal * kVal)
|
||||
}
|
||||
kernelIdx += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errsUp
|
||||
}
|
||||
|
||||
// ==================== 参数更新 ====================
|
||||
public func update(discount: Float32): Unit {
|
||||
var kGrad = Array(0, { _ => Matrix1(1, 1) })
|
||||
match (kernelsGrad) {
|
||||
case Some(val) => kGrad = val
|
||||
case _ => throw Exception("No kernel gradients")
|
||||
}
|
||||
|
||||
var bGrad = Matrix1(1, 1)
|
||||
match (biasGrad) {
|
||||
case Some(val) => bGrad = val
|
||||
case _ => throw Exception("No bias gradient")
|
||||
}
|
||||
|
||||
for (oc in 0..nOutputChannels) {
|
||||
let rows = kernels[oc].get_row()
|
||||
let cols = kernels[oc].get_col()
|
||||
for (r in 0..rows) {
|
||||
for (c in 0..cols) {
|
||||
let oldVal = kernels[oc].get(r, c).getOrThrow()
|
||||
let gradVal = kGrad[oc].get(r, c).getOrThrow()
|
||||
kernels[oc].set(r, c, oldVal - gradVal * learningRate * FP32(discount))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (oc in 0..nOutputChannels) {
|
||||
let oldBias = biases.get(0, oc).getOrThrow()
|
||||
let gradBias = bGrad.get(0, oc).getOrThrow()
|
||||
biases.set(0, oc, oldBias - gradBias * learningRate * FP32(discount))
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 形状访问器 ====================
|
||||
public func getInShape(): (Int64, Int64) {
|
||||
(nInputChannels, kernelHeight * kernelWidth)
|
||||
}
|
||||
|
||||
public func getOutShape(): Int64 {
|
||||
nOutputChannels
|
||||
}
|
||||
|
||||
public func getLR(): FP32 {
|
||||
learningRate
|
||||
}
|
||||
|
||||
public func getFType(): String {
|
||||
f.get_type()
|
||||
}
|
||||
|
||||
public func calculateOutputSize(inputHeight: Int64, inputWidth: Int64): (Int64, Int64) {
|
||||
let outH = (inputHeight + 2 * padding - kernelHeight) / stride + 1
|
||||
let outW = (inputWidth + 2 * padding - kernelWidth) / stride + 1
|
||||
(outH, outW)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package bot4cj.math
|
||||
//foreign func start_fft(M: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func gen_wave(): CPointer<Unit>
|
||||
/*-------- 生成测试用的正弦波 --------*/
|
||||
//public func genWave(): Matrix {
|
||||
// unsafe { return Matrix(gen_wave()) }
|
||||
//}
|
||||
/*-------- 执行fft算法 --------*/
|
||||
//public func start_fft(M: Matrix): Matrix {
|
||||
// unsafe { return Matrix(start_fft(M.getData())) }
|
||||
//}
|
||||
@@ -0,0 +1,136 @@
|
||||
// flatten.cj - 移除调试代码
|
||||
package bot4cj.math
|
||||
|
||||
import std.math.*
|
||||
import std.collection.*
|
||||
|
||||
public class FlattenLayer {
|
||||
|
||||
let nInputChannels: Int64
|
||||
let inputHeight: Int64
|
||||
let inputWidth: Int64
|
||||
let outputSize: Int64
|
||||
|
||||
var prevInput: Option<Matrix1>
|
||||
var prevInputShape: Option<(Int64, Int64, Int64, Int64)>
|
||||
var prevOutput: Option<Matrix1>
|
||||
var prevOutputShape: Option<(Int64, Int64)>
|
||||
|
||||
public init(inputChannels: Int64, inputHeight: Int64, inputWidth: Int64) {
|
||||
nInputChannels = inputChannels
|
||||
this.inputHeight = inputHeight
|
||||
this.inputWidth = inputWidth
|
||||
outputSize = inputChannels * inputHeight * inputWidth
|
||||
|
||||
prevInput = None
|
||||
prevInputShape = None
|
||||
prevOutput = None
|
||||
prevOutputShape = None
|
||||
}
|
||||
|
||||
public init(inputShape: (Int64, Int64, Int64)) {
|
||||
nInputChannels = inputShape[0]
|
||||
inputHeight = inputShape[1]
|
||||
inputWidth = inputShape[2]
|
||||
outputSize = nInputChannels * inputHeight * inputWidth
|
||||
|
||||
prevInput = None
|
||||
prevInputShape = None
|
||||
prevOutput = None
|
||||
prevOutputShape = None
|
||||
}
|
||||
|
||||
public func forward(x: Matrix1, inputShape: (Int64, Int64, Int64, Int64)): Matrix1 {
|
||||
prevInput = x
|
||||
prevInputShape = inputShape
|
||||
|
||||
let batch = inputShape[0]
|
||||
let inH = inputShape[1]
|
||||
let inW = inputShape[2]
|
||||
let inCh = inputShape[3]
|
||||
|
||||
let totalRows = batch * outputSize
|
||||
var output = Matrix1(totalRows, 1)
|
||||
|
||||
for (b in 0..batch) {
|
||||
for (h in 0..inH) {
|
||||
for (w in 0..inW) {
|
||||
for (ch in 0..inCh) {
|
||||
let inRow = b * inH * inW + h * inW + w
|
||||
let outRow = b * outputSize + (ch * inH * inW + h * inW + w)
|
||||
if (outRow < totalRows) {
|
||||
let val = x.get(inRow, ch).getOrThrow()
|
||||
output.set(outRow, 0, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prevOutput = output
|
||||
prevOutputShape = (totalRows, 1)
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
public func backward(errs: Matrix1): Matrix1 {
|
||||
var shape = (0, 0, 0, 0)
|
||||
match (prevInputShape) {
|
||||
case Some(T) => shape = T
|
||||
case _ => throw Exception("FlattenLayer.backward: No input shape. Call forward() first.")
|
||||
}
|
||||
|
||||
let batch = shape[0]
|
||||
let inH = shape[1]
|
||||
let inW = shape[2]
|
||||
let inCh = shape[3]
|
||||
|
||||
let inSize = batch * inH * inW
|
||||
var errsUp = Matrix1(inSize, nInputChannels)
|
||||
|
||||
let errsCols = errs.get_col()
|
||||
|
||||
for (b in 0..batch) {
|
||||
for (h in 0..inH) {
|
||||
for (w in 0..inW) {
|
||||
let inRow = b * inH * inW + h * inW + w
|
||||
|
||||
for (ch in 0..inCh) {
|
||||
let outRow = b * outputSize + (ch * inH * inW + h * inW + w)
|
||||
|
||||
if (outRow < errs.get_row()) {
|
||||
if (errsCols == 1) {
|
||||
let val = errs.get(outRow, 0).getOrThrow()
|
||||
errsUp.set(inRow, ch, val)
|
||||
} else {
|
||||
var sum = FP32(0.0)
|
||||
for (col in 0..errsCols) {
|
||||
sum = sum + errs.get(outRow, col).getOrThrow()
|
||||
}
|
||||
errsUp.set(inRow, ch, sum)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errsUp
|
||||
}
|
||||
|
||||
public func update(discount: Float32): Unit {
|
||||
// 无参数
|
||||
}
|
||||
|
||||
public func getInShape(): (Int64, Int64, Int64) {
|
||||
(nInputChannels, inputHeight, inputWidth)
|
||||
}
|
||||
|
||||
public func getOutShape(): Int64 {
|
||||
outputSize
|
||||
}
|
||||
|
||||
public func getOutputSize(): Int64 {
|
||||
outputSize
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package bot4cj.math
|
||||
|
||||
/**
|
||||
* 一维卡尔曼滤波器
|
||||
* 使用时先定义一个kalman指针，然后调用kalmanCreate()创建一个滤波器
|
||||
* 每次读取到传感器数据后即可调用KalmanFilter()来对数据进行滤波
|
||||
* 使用示例
|
||||
* extKalman_t p; //定义一个卡尔曼滤波器结构体
|
||||
* float SersorData; //需要进行滤波的数据
|
||||
* KalmanCreate(&p,20,200); //初始化该滤波器的Q=20 R=200参数
|
||||
* while(1)
|
||||
* {
|
||||
* SersorData = sersor(); //获取数据
|
||||
* SersorData = KalmanFilter(&p,SersorData); //对数据进行滤波
|
||||
* }
|
||||
*/
|
||||
@C
|
||||
public struct extKalman {
|
||||
var X_last: Float32 // 上一时刻的最优结果 X(k-|k-1)
|
||||
var P_last: Float32 // 上一时刻最优结果的协方差 P(k-1|k-1)
|
||||
var A: Float32 // 系统参数
|
||||
var B: Float32
|
||||
var Q: Float32
|
||||
var R: Float32
|
||||
var H: Float32
|
||||
|
||||
/**
|
||||
* @name kalmanCreate
|
||||
* @brief 创建一个卡尔曼滤波器
|
||||
* @param p: 滤波器
|
||||
* T_Q:系统噪声协方差
|
||||
* T_R:测量噪声协方差
|
||||
*
|
||||
* @retval none
|
||||
* @attention R固定，Q越大，代表越信任侧量值，Q无穷代表只用测量值
|
||||
* 反之，Q越小代表越信任模型预测值，Q为零则是只用模型预测
|
||||
*/
|
||||
public init(T_Q: Float32, T_R: Float32) {
|
||||
X_last = 0.0f32
|
||||
P_last = 0.0f32
|
||||
Q = T_Q
|
||||
R = T_R
|
||||
A = 1.0f32
|
||||
B = 0.0f32
|
||||
H = 1.0f32
|
||||
}
|
||||
|
||||
/**
|
||||
* @name KalmanFilter
|
||||
* @brief 卡尔曼滤波器
|
||||
* @param p: 滤波器
|
||||
* dat:待滤波数据
|
||||
* @retval 滤波后的数据
|
||||
* @attention Z(k)是系统输入,即测量值 X(k|k)是卡尔曼滤波后的值,即最终输出
|
||||
* A=1 B=0 H=1 I=1 W(K) V(k)是高斯白噪声,叠加在测量值上了,可以不用管
|
||||
* 以下是卡尔曼的5个核心公式
|
||||
* 一阶H'即为它本身,否则为转置矩阵
|
||||
*/
|
||||
public mut func filter(dat: Float32): Float32 {
|
||||
let X_mid = A * X_last //百度对应公式(1) x(k|k-1) = A*X(k-1|k-1)+B*U(k)+W(K)
|
||||
let P_mid = A * P_last + Q //百度对应公式(2) p(k|k-1) = A*p(k-1|k-1)*A'+Q
|
||||
let kg = P_mid / (P_mid + R) //百度对应公式(4) kg(k) = p(k|k-1)*H'/(H*p(k|k-1)*H'+R)
|
||||
let X_now = X_mid + kg * (dat - X_mid) //百度对应公式(3) x(k|k) = X(k|k-1)+kg(k)*(Z(k)-H*X(k|k-1))
|
||||
let P_now = (1.0f32 - kg) * P_mid //百度对应公式(5) p(k|k) = (I-kg(k)*H)*P(k|k-1)
|
||||
P_last = P_now
|
||||
X_last = X_now
|
||||
return X_now //输出预测结果x(k|k)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
package bot4cj.math
|
||||
import std.fs.*
|
||||
import std.io.*
|
||||
import std.convert.*
|
||||
//foreign func Sigmoid_do(m: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func Sigmoid_dot(m: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func ReLU_do(m: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func ReLU_dot(m: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func Leaky_ReLU_do(m: CPointer<Unit>, alpha: Float32): CPointer<Unit>
|
||||
//foreign func Leaky_ReLU_dot(m: CPointer<Unit>, alpha: Float32): CPointer<Unit>
|
||||
//foreign func Tanh_do(m: CPointer<Unit>): CPointer<Unit>
|
||||
//foreign func Tanh_dot(m: CPointer<Unit>): CPointer<Unit>
|
||||
|
||||
|
||||
/* -------- 新版的神经网络，使用原生矩阵对象实现 --------
|
||||
* TODO(20260517-qing): 增加batch批次处理，使用ArrayList<Matrix1>
|
||||
* TODO*(20260519-qing): 将神经网络转换为MatrixObject矩阵式对象
|
||||
* TODO(20260524-qing): 增加Adam优化器 */
|
||||
|
||||
public class Nn1 {
|
||||
|
||||
let inputno: Int64 /* -------- 输入 -------- */
|
||||
let outputno: Int64 /* -------- 输出 -------- */
|
||||
var weights: Matrix1 /* -- (inputs, outputs) - */
|
||||
var bias: Matrix1 /* ----- (1, output) ---- */
|
||||
let learning_rate: FP32 /* ------- 学习率 ------- */
|
||||
let f: Af /* ------ 激活函数 ------ */
|
||||
|
||||
/*------------------*/
|
||||
var prev_inputs: Option<Matrix1> = None /* ---- 之前的输入 ---- */
|
||||
var prev_raw_inputs: Option<Matrix1> = None /* ---- 之前的原始输出 ---- */
|
||||
var prev_outputs: Option<Matrix1> = None /* ---- 之前的输出 ---- */
|
||||
/*------------------*/
|
||||
var weights_grad: Option<Matrix1> = None /* ---- 权重梯度 ---- */
|
||||
var bias_grad: Option<Matrix1> = None /* ---- 偏置梯度 ---- */
|
||||
|
||||
|
||||
/* -------- 初始化器 --------
|
||||
* TODO(20260516-qing): 不要把初始化方法写死，使用枚举类 */
|
||||
public init(i: Int64, o: Int64, lr: Float32, f: Af) {
|
||||
inputno = i
|
||||
outputno =o
|
||||
learning_rate=FP32(lr)
|
||||
this.f = f
|
||||
weights = Matrix1(inputno, outputno)
|
||||
bias = Matrix1(1, outputno)
|
||||
|
||||
/* -------- 初始化权重 -------- */
|
||||
Ini.xavierUniformInit(weights, inputno, outputno)
|
||||
}
|
||||
|
||||
/* -------- 神经网络推理 -------- */
|
||||
public func forward(x: Matrix1) {
|
||||
prev_inputs = x
|
||||
var r = x.dot(weights)
|
||||
r = r + bias
|
||||
prev_raw_inputs = r
|
||||
let r1 = f.forward(r)
|
||||
prev_outputs = r1
|
||||
return r1
|
||||
}
|
||||
|
||||
/* -------- 反向传播算法 -------- */
|
||||
public func backward(errs: Matrix1)
|
||||
{
|
||||
if (is_softmax(f)) {
|
||||
return softmax_backward(errs)
|
||||
}
|
||||
let fbak = f.backward(prev_raw_inputs.getOrThrow())
|
||||
let first = errs * fbak /* 逐元素相乘 */
|
||||
let mx = prev_inputs.getOrThrow().T()
|
||||
weights_grad = mx.dot(first)
|
||||
bias_grad = first
|
||||
let errs_up = first.dot(weights.T())
|
||||
return errs_up
|
||||
}
|
||||
|
||||
/* -------- 更新权重值、偏爱因子 --------
|
||||
* NOTE: discount（折扣）是为了方便深度强化学习给出奖励值 */
|
||||
public func update(discount: Float32) {
|
||||
weights = weights - weights_grad.getOrThrow() * learning_rate * FP32(discount)
|
||||
bias = bias - bias_grad.getOrThrow() * learning_rate * FP32(discount)
|
||||
}
|
||||
|
||||
/**/
|
||||
private static func is_softmax(f: Af) {
|
||||
return match (f) {
|
||||
case Af.Softmax => true
|
||||
case _ => false
|
||||
}
|
||||
}
|
||||
|
||||
private func softmax_backward(errs: Matrix1) {
|
||||
/* 使用雅可比矩阵的方式 */
|
||||
let y = prev_outputs.getOrThrow() /* softmax 输出 */
|
||||
let dy = errs /* 上游梯度 ∂L/∂y */
|
||||
let jacobian = f.backward(y) /* n×n 雅可比矩阵 */
|
||||
|
||||
/* 将 dy 展平成列向量 n×1 */
|
||||
let row = dy.get_row()
|
||||
let col = dy.get_col()
|
||||
let n = row * col
|
||||
var dy_vec = Matrix1(n, 1)
|
||||
var idx = 0
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let val = dy.get(r, c).getOrThrow()
|
||||
dy_vec.set(idx, 0, val)
|
||||
idx = idx + 1
|
||||
}
|
||||
}
|
||||
|
||||
/* dx_vec = J · dy_vec (n×n × n×1 = n×1) */
|
||||
var dx_vec = Matrix1(n, 1)
|
||||
for (i in 0..n) {
|
||||
var sum = 0.0f32
|
||||
for (j in 0..n) {
|
||||
let j_val = jacobian.get(i, j).getOrThrow().getValue()
|
||||
let dy_val = dy_vec.get(j, 0).getOrThrow().getValue()
|
||||
sum = sum + j_val * dy_val
|
||||
}
|
||||
dx_vec.set(i, 0, FP32(sum))
|
||||
}
|
||||
|
||||
/* 恢复为原始形状，作为 first */
|
||||
var first = Matrix1(row, col)
|
||||
idx = 0
|
||||
for (r in 0..row) {
|
||||
for (c in 0..col) {
|
||||
let val = dx_vec.get(idx, 0).getOrThrow()
|
||||
first.set(r, c, val)
|
||||
idx = idx + 1
|
||||
}
|
||||
}
|
||||
|
||||
let mx = prev_inputs.getOrThrow().T()
|
||||
weights_grad = mx.dot(first)
|
||||
bias_grad = first
|
||||
let errs_up = first.dot(weights.T())
|
||||
return errs_up
|
||||
}
|
||||
|
||||
|
||||
/* -------- 形状访问器 -------- */
|
||||
public func get_in_shape() {
|
||||
return inputno }
|
||||
|
||||
public func get_out_shape() {
|
||||
return outputno }
|
||||
|
||||
public func get_lr() {
|
||||
return learning_rate }
|
||||
|
||||
public func get_f_type() {
|
||||
return f.get_type() }
|
||||
|
||||
public func get_f_val() {
|
||||
return f.get_value() }
|
||||
/* ---------------------------- */
|
||||
|
||||
}
|
||||
|
||||
|
||||
///* -------- 旧版激活函数 --------*/
|
||||
//public enum ActivationFuncs {
|
||||
// Sigmoid | ReLU | Leaky_ReLU | Tanh
|
||||
// func fdo(m: Matrix): Matrix { // 暂时只设置ReLU
|
||||
// unsafe {
|
||||
// match(this) {
|
||||
// case Sigmoid => Matrix(Sigmoid_do(m.getData()))
|
||||
// case ReLU => Matrix(ReLU_do(m.getData()))
|
||||
// case Leaky_ReLU => Matrix(Leaky_ReLU_do(m.getData(), 0.1f32))
|
||||
// case Tanh => Matrix(Tanh_do(m.getData()))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// func fdot(m: Matrix): Matrix {
|
||||
// unsafe {
|
||||
// match(this) {
|
||||
// case Sigmoid => Matrix(Sigmoid_dot(m.getData()))
|
||||
// case ReLU => Matrix(ReLU_dot(m.getData()))
|
||||
// case Leaky_ReLU => Matrix(Leaky_ReLU_dot(m.getData(), 0.1f32))
|
||||
// case Tanh => Matrix(Tanh_dot(m.getData()))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// func toString(): String {
|
||||
// match(this) {
|
||||
// case Sigmoid => "Sigmoid"
|
||||
// case ReLU => "ReLU"
|
||||
// case Leaky_ReLU => "Leaky_ReLU"
|
||||
// case Tanh => "Tanh"
|
||||
// }
|
||||
// }
|
||||
// static func parse(s: String) {
|
||||
// match(s) {
|
||||
// case "Sigmoid" => ActivationFuncs.Sigmoid
|
||||
// case "ReLU" => ActivationFuncs.ReLU
|
||||
// case "Leaky_ReLU" => ActivationFuncs.Leaky_ReLU
|
||||
// case "Tanh" => ActivationFuncs.Tanh
|
||||
// case _ => throw IllegalArgumentException()
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
///* -------- 旧版的神经网络类 --------
|
||||
// * 这个实现依赖C++语言的动态分享库，特点是速度快
|
||||
// * 这个实现可能是有问题的 */
|
||||
//public class Nn {
|
||||
//
|
||||
// /* -------- 主初始化器 -------- */
|
||||
// public init(
|
||||
// inputno: UInt64, outputno: UInt64, f: ActivationFuncs, learning_rate: Float32) {
|
||||
// this.inputno = inputno
|
||||
// this.outputno = outputno
|
||||
// weights = Matrix.initWithXavierUniform(inputno, outputno)
|
||||
// bias = Matrix(1, outputno)
|
||||
// this.f = f
|
||||
// this.learning_rate = learning_rate
|
||||
// }
|
||||
//
|
||||
// /* -------- 提供了权重矩阵的初始化器 -------- */
|
||||
// public init(
|
||||
// inputno: UInt64, outputno: UInt64, f: ActivationFuncs, learning_rate: Float32, weights: Matrix, bias: Matrix) {
|
||||
// this.inputno = inputno
|
||||
// this.outputno = outputno
|
||||
// this.weights = weights
|
||||
// this.bias = bias
|
||||
// this.f = f
|
||||
// this.learning_rate = learning_rate
|
||||
// }
|
||||
//
|
||||
// /* -------- 神经网络推理 -------- */
|
||||
// func forward(x: Matrix): Matrix {
|
||||
// /* x => (1, inputs) */
|
||||
// let r = x.dot(weights) + bias /* (1, outputno) */
|
||||
// return r
|
||||
// }
|
||||
//
|
||||
// /* -------- 实施激活函数 -------- */
|
||||
// func fdo(x: Matrix): Matrix {
|
||||
// return f.fdo(x) }
|
||||
//
|
||||
// /* -------- 激活函数导数 -------- */
|
||||
// func fdot(x: Matrix): Matrix {
|
||||
// return f.fdot(x) }
|
||||
//
|
||||
// /* This is Wrong */
|
||||
// //func bak_bsgrad(prev_raw_outputs_dotf: Matrix, errs: Matrix): Matrix {
|
||||
// /* prev_raw_outputs_dotf => (1, outputno)
|
||||
// * errs => (1, outputno) */
|
||||
//
|
||||
// // return errs * prev_raw_outputs_dotf
|
||||
// //}
|
||||
//
|
||||
// /* -------- 权重梯度的传播 -------- */
|
||||
// func bak_wsgrad(prev_inputs: Matrix, delta: Matrix) {
|
||||
// /* prev_inputs => (1, inputs) */
|
||||
// return prev_inputs.T().dot(delta) /* (inputs, outputs) */
|
||||
// }
|
||||
//
|
||||
// /* -------- 计算上一层的误差 -------- */
|
||||
// func bak_uperrs(delta: Matrix, prev_dotf: Matrix) {
|
||||
// let first = weights.dot(delta.T()).T() // (1, inputs)
|
||||
//
|
||||
// var uperrs = first * prev_dotf
|
||||
// uperrs
|
||||
// }
|
||||
//
|
||||
// /* -------- 更新权重、偏爱因子 -------- */
|
||||
// func update(weights_grad: Matrix, bias_grad: Matrix, discount: Float32) {
|
||||
// weights -= weights_grad * learning_rate * discount
|
||||
// bias -= bias_grad * learning_rate * discount
|
||||
// }
|
||||
//
|
||||
// /* -------- 打印权重、偏爱因子 -------- */
|
||||
// func print() {
|
||||
// weights.print()
|
||||
// bias.print()
|
||||
// }
|
||||
//
|
||||
// /* -------- 打印神经网络全连接层的形状 -------- */
|
||||
// func print_shape() {
|
||||
// println("${inputno}, ${outputno}") }
|
||||
//
|
||||
// /* -------- 储存神经网络 -------- */
|
||||
// func save(file: File) {
|
||||
// file.write("FullConnectLayer ".toArray())
|
||||
// file.write(inputno.toString().toArray())
|
||||
// file.write(" ".toArray())
|
||||
// file.write(outputno.toString().toArray())
|
||||
// file.write(" ".toArray())
|
||||
// file.write(learning_rate.toString().toArray())
|
||||
// file.write(" ".toArray())
|
||||
// file.write(f.toString().toArray())
|
||||
// file.write("\n".toArray())
|
||||
// weights.save(file)
|
||||
// //file.write("\n".toArray())
|
||||
// bias.save(file)
|
||||
// //file.write("\n".toArray())
|
||||
// }
|
||||
//
|
||||
// /* -------- 静态方法，加载单个项目 -------- */
|
||||
// static func loadItem<T>(sr: StringReader<InputStream>) where T <: Parsable<T>
|
||||
// {
|
||||
// let s = sr.readUntil({r => r == r" " || r == r"\n"})
|
||||
// let ss = s.getOrThrow().trimAscii()
|
||||
// println(ss)
|
||||
// return T.parse(ss)
|
||||
// }
|
||||
//
|
||||
// /* -------- 静态方法，加载神经网络 -------- */
|
||||
// static func load(sr: StringReader<InputStream>)
|
||||
// {
|
||||
// let inputno = loadItem<UInt64>(sr)
|
||||
// let outputno = loadItem<UInt64>(sr)
|
||||
// let lr = loadItem<Float32>(sr)
|
||||
// let fna = sr.readUntil(r'\n')
|
||||
// let f = ActivationFuncs.parse(fna.getOrThrow().trimAscii())
|
||||
// let weights = Matrix.load(sr, inputno, outputno)
|
||||
// let bias = Matrix.load(sr, 1, outputno)
|
||||
// Nn(inputno, outputno, f, lr, weights, bias)
|
||||
// }
|
||||
//
|
||||
// let inputno: UInt64
|
||||
// let outputno: UInt64
|
||||
// var weights: Matrix /* (inputs, outputs) */
|
||||
// var bias: Matrix /* (1, outputs) */
|
||||
// var learning_rate: Float32
|
||||
// let f: ActivationFuncs
|
||||
//}
|
||||
@@ -0,0 +1,164 @@
|
||||
package bot4cj.math
|
||||
import std.io.*
|
||||
import std.fs.*
|
||||
import std.math.*
|
||||
import std.time.*
|
||||
import std.random.*
|
||||
import std.collection.*
|
||||
|
||||
/* -------- 准备与门的测试用例 --------
|
||||
* INFO(20260524-qing): 输入{2}，输出{1} */
|
||||
public func prepare_andDoor(m: Matrix1, t: Matrix1) {
|
||||
let random = Random(UInt64(DateTime.now().nanosecond))
|
||||
let a = random.nextInt8(2)
|
||||
let b = random.nextInt8(2)
|
||||
let t0 = if ((a == 0 && b == 0) || (a ==1 && b == 1)) { 1.0f32 } else { 0.0f32 }
|
||||
//let t1 = if (a != b) { 1.0f32 } else { 0.0f32 }
|
||||
m.set(0, 0, FP32(Float32(a)))
|
||||
m.set(0, 1, FP32(Float32(b)))
|
||||
t.set(0, 0, FP32(t0))
|
||||
//t.set(0, 1, FP32(t1))
|
||||
}
|
||||
|
||||
/* -------- 新版神经网络推理，使用原生矩阵 -------- */
|
||||
public func forward1(nns: ArrayList<Nn1>, m: Matrix1) {
|
||||
var out_dof = Matrix1(1, 1) /* ---- 给一个值占位 ----*/
|
||||
for (idx in 0..nns.size) {
|
||||
let x = if (idx == 0) { m } else { out_dof }
|
||||
out_dof= nns[idx].forward(x)
|
||||
}
|
||||
return out_dof
|
||||
}
|
||||
|
||||
/* 【神经网络训练】
|
||||
* description: 输入一组原始数据和教师数据，进行神经网络训练
|
||||
* @param nns: 输入神经网络结构
|
||||
* @param m: 输入一组原始数据
|
||||
* @param t: 输入一组教师数据
|
||||
* @param discount: 输入本次训练的折扣系数
|
||||
*/
|
||||
public func train1(nns: ArrayList<Nn1>, m: Matrix1, t: Matrix1, discount: Float32) {
|
||||
|
||||
/* ======== 这个对象用于暂时占位 ======== */
|
||||
var res = Matrix1(1, 1)
|
||||
|
||||
/* ======== 正向传播 ======== */
|
||||
for (idx in 0..nns.size) {
|
||||
let x = if (idx == 0) { m } else { res }
|
||||
res = nns[idx].forward(x)
|
||||
}
|
||||
|
||||
/* ======== 计算误差 ======== */
|
||||
var errs = t - res
|
||||
|
||||
/* ======== 反向传播 ======== */
|
||||
for (idx in (nns.size-1)..=0:-1) {
|
||||
errs = nns[idx].backward(errs)
|
||||
}
|
||||
|
||||
/* ======== 梯度更新 ======== */
|
||||
for (idx in 0..nns.size) {
|
||||
nns[idx].update(discount)
|
||||
}
|
||||
|
||||
/* ======== 计算MSE损失 ========
|
||||
FIXME*(20260517-qing): 对于多个结果计算误差（目前只计算了第一个结果） */
|
||||
|
||||
let t0 = t.get(0, 0).getOrThrow().getValue()
|
||||
//let t1 = t.get(0, 1).getOrThrow().getValue()
|
||||
let r0 = res.get(0, 0).getOrThrow().getValue()
|
||||
//let r1 = res.get(0, 1).getOrThrow().getValue()
|
||||
let loss0 = 0.5 * (t0 - r0) * (t0 - r0)
|
||||
//let loss1 = 0.5 * (t1 - r1) * (t1 - r1)
|
||||
//return (loss0 + loss1) / 2.0
|
||||
return loss0
|
||||
}
|
||||
|
||||
/* -------- （旧版）加载多层神经网络 --------
|
||||
TODO: 也许用类封装
|
||||
INFO(20260517-qing): 新版使用仓颉原生矩阵，然后与MatrixObject类互转 */
|
||||
//public func load_nns(filename: String): ArrayList<Nn> {
|
||||
// let nns = ArrayList<Nn>()
|
||||
// let file = File(filename, OpenMode.Read)
|
||||
// let sr = StringReader<InputStream>(file)
|
||||
// while (true) {
|
||||
// if (let Some(S) <- sr.readUntil(r' ')) {
|
||||
// var S2 = S.trimAscii()
|
||||
// if (S2 == "FullConnectLayer") {
|
||||
// nns.add(Nn.load(sr)) }
|
||||
// else if (S2 == "") { break }
|
||||
// else { throw IllegalArgumentException() }
|
||||
// }
|
||||
// else { break }
|
||||
// }
|
||||
|
||||
// file.close()
|
||||
// return nns
|
||||
//}
|
||||
|
||||
/* -------- （旧版）储存多层神经网络 -------- */
|
||||
//public func save_nns(nns: ArrayList<Nn>, filename: String) {
|
||||
//
|
||||
// let file: File
|
||||
// if (exists(filename)) {
|
||||
// file = File(filename, OpenMode.Write) }
|
||||
// else { file= File.create(filename) }
|
||||
// for (nn in nns) { /* 存储全部的层 */
|
||||
// nn.save(file) }
|
||||
// file.close()
|
||||
//}
|
||||
|
||||
/* -------- 旧版神经网络推理 -------- */
|
||||
//public func forward(nns: ArrayList<Nn>, m: Matrix) {
|
||||
// var out_dof = Matrix(1, 1)
|
||||
// for (idx in 0..nns.size) {
|
||||
// let x = if (idx == 0) { m } else { out_dof }
|
||||
// let res = nns[idx].forward(x)
|
||||
// out_dof = nns[idx].fdo(res)
|
||||
// }
|
||||
// return out_dof
|
||||
//}
|
||||
|
||||
/* -------- 旧版的神经网络训练 -------- */
|
||||
//public func train(nns: ArrayList<Nn>, m: Matrix, t: Matrix, discount: Float32) {
|
||||
//
|
||||
// /* ----------------- */
|
||||
// let outs_dof = ArrayList<Matrix> ()
|
||||
// let outs_dotf = ArrayList<Matrix> ()
|
||||
//
|
||||
// /* -------- 正向传播 -------- */
|
||||
// for (idx in 0..nns.size) {
|
||||
// let x = if (idx == 0) { m } else { outs_dof[idx-1] }
|
||||
// let res = nns[idx].forward(x)
|
||||
// let out_dof = nns[idx].fdo(res)
|
||||
// let out_dotf = nns[idx].fdot(out_dof)
|
||||
// outs_dof.add(out_dof)
|
||||
// outs_dotf.add(out_dotf)
|
||||
// }
|
||||
//
|
||||
// /* -------- 计算误差 -------- */
|
||||
// var errs = outs_dof[nns.size-1] - t
|
||||
//
|
||||
// /* -------- 反向传播 -------- */
|
||||
// for (idx in (nns.size-1)..=0:-1) {
|
||||
// let up = if (idx == 0) { m} else { outs_dof[idx-1] }
|
||||
// let bias_grad = errs
|
||||
// //let delta = nns[idx].bak_bsgrad(errs, outs_dotf[idx])
|
||||
// let weights_grad = nns[idx].bak_wsgrad(up, bias_grad)
|
||||
// if (idx > 0) {
|
||||
// errs = nns[idx].bak_uperrs(bias_grad, outs_dotf[idx-1]) }
|
||||
// nns[idx].update(weights_grad, bias_grad, discount)
|
||||
// }
|
||||
//
|
||||
// /* -------- 输出损失 -------- */
|
||||
// unsafe {
|
||||
// let t0 = t.get(0, 0)
|
||||
// let t1 = t.get(0, 1)
|
||||
// let r0 = outs_dof[nns.size-1].get(0, 0)
|
||||
// let r1 = outs_dof[nns.size-1].get(0, 1)
|
||||
// let loss0 = -t0 * log(r0) - (1.0f32-t0) * log(1.0f32-r0);
|
||||
// let loss1 = -t1 * log(r1) - (1.0f32-t1) * log(1.0f32-r1);
|
||||
// return 0.5 * (loss0 * loss0 + loss1 * loss1)
|
||||
// }
|
||||
//
|
||||
//}
|
||||
@@ -0,0 +1,418 @@
|
||||
// pooling_layer.cj
|
||||
package bot4cj.math
|
||||
|
||||
import std.math.*
|
||||
import std.collection.*
|
||||
|
||||
/**
|
||||
* 池化层类
|
||||
* 设计风格与 Nn1 和 ConvLayer 保持一致
|
||||
*/
|
||||
public class PoolingLayer {
|
||||
|
||||
// ==================== 层配置参数 ====================
|
||||
let nChannels: Int64
|
||||
let poolHeight: Int64
|
||||
let poolWidth: Int64
|
||||
let stride: Int64
|
||||
let padding: Int64
|
||||
let poolType: String
|
||||
let f: Af
|
||||
let learningRateWeight: FP32
|
||||
let learningRateBias: FP32
|
||||
|
||||
// ==================== 可学习参数 ====================
|
||||
var weights: Matrix1
|
||||
var biases: Matrix1
|
||||
|
||||
// ==================== 运行时缓存 ====================
|
||||
var prevInput: Option<Matrix1> = Option.None
|
||||
var prevInputShape: Option<(Int64, Int64, Int64, Int64)> = Option.None
|
||||
var prevCombination: Option<Matrix1> = Option.None
|
||||
var prevOutput: Option<Matrix1> = Option.None
|
||||
var prevOutputShape: Option<(Int64, Int64, Int64, Int64)> = Option.None
|
||||
var prevMaxIndices: Option<Matrix1> = Option.None
|
||||
|
||||
var weightsGrad: Option<Matrix1> = Option.None
|
||||
var biasGrad: Option<Matrix1> = Option.None
|
||||
|
||||
// ==================== 构造函数 ====================
|
||||
public init(
|
||||
nChannels: Int64,
|
||||
poolSize: (Int64, Int64),
|
||||
stride: Int64,
|
||||
padding: Int64,
|
||||
poolType: String,
|
||||
activation: Af,
|
||||
lrWeight: FP32,
|
||||
lrBias: FP32
|
||||
) {
|
||||
this.nChannels = nChannels
|
||||
poolHeight = poolSize[0]
|
||||
poolWidth = poolSize[1]
|
||||
this.stride = stride
|
||||
this.padding = padding
|
||||
this.poolType = poolType
|
||||
this.f = activation
|
||||
learningRateWeight = lrWeight
|
||||
learningRateBias = lrBias
|
||||
|
||||
// 初始化权值
|
||||
weights = Matrix1(1, nChannels)
|
||||
if (poolType == "average") {
|
||||
let avgWeight = FP32(1.0) / FP32(Float32(poolHeight * poolWidth))
|
||||
for (c in 0..nChannels) {
|
||||
weights.set(0, c, avgWeight)
|
||||
}
|
||||
} else {
|
||||
for (c in 0..nChannels) {
|
||||
weights.set(0, c, FP32(1.0))
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化偏置为 0
|
||||
biases = Matrix1(1, nChannels)
|
||||
for (c in 0..nChannels) {
|
||||
biases.set(0, c, FP32(0.0))
|
||||
}
|
||||
}
|
||||
|
||||
public init(
|
||||
nChannels: Int64,
|
||||
poolSize: (Int64, Int64),
|
||||
poolType: String,
|
||||
activation: Af
|
||||
) {
|
||||
this.nChannels = nChannels
|
||||
poolHeight = poolSize[0]
|
||||
poolWidth = poolSize[1]
|
||||
stride = poolHeight
|
||||
padding = 0
|
||||
this.poolType = poolType
|
||||
this.f = activation
|
||||
learningRateWeight = FP32(0.1)
|
||||
learningRateBias = FP32(0.1)
|
||||
|
||||
weights = Matrix1(1, nChannels)
|
||||
if (poolType == "average") {
|
||||
let avgWeight = FP32(1.0) / FP32(Float32(poolHeight * poolWidth))
|
||||
for (c in 0..nChannels) {
|
||||
weights.set(0, c, avgWeight)
|
||||
}
|
||||
} else {
|
||||
for (c in 0..nChannels) {
|
||||
weights.set(0, c, FP32(1.0))
|
||||
}
|
||||
}
|
||||
|
||||
biases = Matrix1(1, nChannels)
|
||||
for (c in 0..nChannels) {
|
||||
biases.set(0, c, FP32(0.0))
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 前向传播 ====================
|
||||
public func forward(x: Matrix1, inputShape: (Int64, Int64, Int64, Int64)): Matrix1 {
|
||||
prevInput = Option.Some(x)
|
||||
prevInputShape = Option.Some(inputShape)
|
||||
|
||||
let batch = inputShape[0]
|
||||
let inH = inputShape[1]
|
||||
let inW = inputShape[2]
|
||||
|
||||
let outH = (inH + 2 * padding - poolHeight) / stride + 1
|
||||
let outW = (inW + 2 * padding - poolWidth) / stride + 1
|
||||
let outSize = batch * outH * outW
|
||||
|
||||
var combination = Matrix1(outSize, nChannels)
|
||||
var output = Matrix1(outSize, nChannels)
|
||||
var maxIndices = Matrix1(outSize, nChannels)
|
||||
|
||||
for (b in 0..batch) {
|
||||
for (oh in 0..outH) {
|
||||
for (ow in 0..outW) {
|
||||
let outRow = b * outH * outW + oh * outW + ow
|
||||
|
||||
for (ch in 0..nChannels) {
|
||||
var pooledVal = FP32(0.0)
|
||||
var maxIdx = 0
|
||||
|
||||
if (poolType == "max") {
|
||||
var maxVal = FP32(-1.0e30)
|
||||
var kernelIdx = 0
|
||||
for (kh in 0..poolHeight) {
|
||||
for (kw in 0..poolWidth) {
|
||||
let ih = oh * stride + kh - padding
|
||||
let iw = ow * stride + kw - padding
|
||||
|
||||
if (ih >= 0 && ih < inH && iw >= 0 && iw < inW) {
|
||||
let inRow = b * inH * inW + ih * inW + iw
|
||||
let val = x.get(inRow, ch).getOrThrow()
|
||||
if (val.getValue() > maxVal.getValue()) {
|
||||
maxVal = val
|
||||
maxIdx = kernelIdx
|
||||
}
|
||||
}
|
||||
kernelIdx += 1
|
||||
}
|
||||
}
|
||||
pooledVal = maxVal
|
||||
} else {
|
||||
var sum = FP32(0.0)
|
||||
var count = 0
|
||||
for (kh in 0..poolHeight) {
|
||||
for (kw in 0..poolWidth) {
|
||||
let ih = oh * stride + kh - padding
|
||||
let iw = ow * stride + kw - padding
|
||||
|
||||
if (ih >= 0 && ih < inH && iw >= 0 && iw < inW) {
|
||||
let inRow = b * inH * inW + ih * inW + iw
|
||||
sum = sum + x.get(inRow, ch).getOrThrow()
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
if (count > 0) {
|
||||
pooledVal = sum / FP32(Float32(count))
|
||||
}
|
||||
}
|
||||
|
||||
combination.set(outRow, ch, pooledVal)
|
||||
maxIndices.set(outRow, ch, FP32(Float32(maxIdx)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 逐通道应用权值和偏置
|
||||
for (r in 0..outSize) {
|
||||
for (ch in 0..nChannels) {
|
||||
let val = combination.get(r, ch).getOrThrow()
|
||||
let w = weights.get(0, ch).getOrThrow()
|
||||
let b = biases.get(0, ch).getOrThrow()
|
||||
combination.set(r, ch, val * w + b)
|
||||
}
|
||||
}
|
||||
|
||||
prevCombination = Option.Some(combination)
|
||||
|
||||
output = f.forward(combination)
|
||||
prevOutput = Option.Some(output)
|
||||
prevOutputShape = Option.Some((batch, outH, outW, nChannels))
|
||||
prevMaxIndices = Option.Some(maxIndices)
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
// ==================== 反向传播 ====================
|
||||
public func backward(errs: Matrix1): Matrix1 {
|
||||
var x = Matrix1(1, 1)
|
||||
match (prevInput) {
|
||||
case Some(val) => x = val
|
||||
case _ => throw Exception("No input data")
|
||||
}
|
||||
|
||||
var shape = (0, 0, 0, 0)
|
||||
match (prevInputShape) {
|
||||
case Some(val) => shape = val
|
||||
case _ => throw Exception("No input shape")
|
||||
}
|
||||
|
||||
var combination = Matrix1(1, 1)
|
||||
match (prevCombination) {
|
||||
case Some(val) => combination = val
|
||||
case _ => throw Exception("No combination")
|
||||
}
|
||||
|
||||
var outShape = (0, 0, 0, 0)
|
||||
match (prevOutputShape) {
|
||||
case Some(val) => outShape = val
|
||||
case _ => throw Exception("No output shape")
|
||||
}
|
||||
|
||||
var maxIndices = Matrix1(1, 1)
|
||||
let hasMaxIndices = match (prevMaxIndices) {
|
||||
case Some(_) => true
|
||||
case _ => false
|
||||
}
|
||||
if (hasMaxIndices) {
|
||||
maxIndices = match (prevMaxIndices) {
|
||||
case Some(val) => val
|
||||
case _ => Matrix1(1, 1)
|
||||
}
|
||||
}
|
||||
|
||||
let batch = shape[0]
|
||||
let inH = shape[1]
|
||||
let inW = shape[2]
|
||||
let outH = outShape[1]
|
||||
let outW = outShape[2]
|
||||
|
||||
let delta = errs * f.backward(combination)
|
||||
|
||||
calcWeightBiasGradient(delta, combination)
|
||||
|
||||
let errsUp = calcUpsampledError(delta, maxIndices, batch, inH, inW, outH, outW)
|
||||
|
||||
return errsUp
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算权值和偏置梯度
|
||||
*/
|
||||
private func calcWeightBiasGradient(delta: Matrix1, combination: Matrix1): Unit {
|
||||
let outSize = delta.get_row()
|
||||
|
||||
var wGrad = Matrix1(1, nChannels)
|
||||
var bGrad = Matrix1(1, nChannels)
|
||||
|
||||
for (ch in 0..nChannels) {
|
||||
var wSum = FP32(0.0)
|
||||
var bSum = FP32(0.0)
|
||||
for (r in 0..outSize) {
|
||||
let d = delta.get(r, ch).getOrThrow()
|
||||
bSum = bSum + d
|
||||
let comb = combination.get(r, ch).getOrThrow()
|
||||
wSum = wSum + d * comb
|
||||
}
|
||||
wGrad.set(0, ch, wSum)
|
||||
bGrad.set(0, ch, bSum)
|
||||
}
|
||||
|
||||
weightsGrad = Option.Some(wGrad)
|
||||
biasGrad = Option.Some(bGrad)
|
||||
}
|
||||
|
||||
/**
|
||||
* 上采样误差（池化层的反向传播核心）
|
||||
*/
|
||||
private func calcUpsampledError(
|
||||
delta: Matrix1,
|
||||
maxIndices: Matrix1,
|
||||
batch: Int64,
|
||||
inH: Int64,
|
||||
inW: Int64,
|
||||
outH: Int64,
|
||||
outW: Int64
|
||||
): Matrix1 {
|
||||
let inSize = batch * inH * inW
|
||||
var errsUp = Matrix1(inSize, nChannels)
|
||||
|
||||
for (b in 0..batch) {
|
||||
for (oh in 0..outH) {
|
||||
for (ow in 0..outW) {
|
||||
let deltaRow = b * outH * outW + oh * outW + ow
|
||||
|
||||
for (ch in 0..nChannels) {
|
||||
let deltaVal = delta.get(deltaRow, ch).getOrThrow()
|
||||
let w = weights.get(0, ch).getOrThrow()
|
||||
|
||||
if (poolType == "max") {
|
||||
let maxIdx = Int64(maxIndices.get(deltaRow, ch).getOrThrow().getValue())
|
||||
|
||||
var kernelIdx = 0
|
||||
var found = false
|
||||
for (kh in 0..poolHeight) {
|
||||
for (kw in 0..poolWidth) {
|
||||
if (kernelIdx == maxIdx && !found) {
|
||||
let ih = oh * stride + kh - padding
|
||||
let iw = ow * stride + kw - padding
|
||||
|
||||
if (ih >= 0 && ih < inH && iw >= 0 && iw < inW) {
|
||||
let inRow = b * inH * inW + ih * inW + iw
|
||||
errsUp.set(inRow, ch, deltaVal * w)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
kernelIdx += 1
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var count = 0
|
||||
for (kh in 0..poolHeight) {
|
||||
for (kw in 0..poolWidth) {
|
||||
let ih = oh * stride + kh - padding
|
||||
let iw = ow * stride + kw - padding
|
||||
if (ih >= 0 && ih < inH && iw >= 0 && iw < inW) {
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (count > 0) {
|
||||
let avgDelta = deltaVal * w / FP32(Float32(count))
|
||||
for (kh in 0..poolHeight) {
|
||||
for (kw in 0..poolWidth) {
|
||||
let ih = oh * stride + kh - padding
|
||||
let iw = ow * stride + kw - padding
|
||||
if (ih >= 0 && ih < inH && iw >= 0 && iw < inW) {
|
||||
let inRow = b * inH * inW + ih * inW + iw
|
||||
let oldVal = errsUp.get(inRow, ch).getOrThrow()
|
||||
errsUp.set(inRow, ch, oldVal + avgDelta)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errsUp
|
||||
}
|
||||
|
||||
// ==================== 参数更新 ====================
|
||||
public func update(discount: Float32): Unit {
|
||||
var wGrad = Matrix1(1, 1)
|
||||
match (weightsGrad) {
|
||||
case Some(val) => wGrad = val
|
||||
case _ => throw Exception("No weight gradients")
|
||||
}
|
||||
|
||||
var bGrad = Matrix1(1, 1)
|
||||
match (biasGrad) {
|
||||
case Some(val) => bGrad = val
|
||||
case _ => throw Exception("No bias gradients")
|
||||
}
|
||||
|
||||
for (ch in 0..nChannels) {
|
||||
let oldW = weights.get(0, ch).getOrThrow()
|
||||
let gW = wGrad.get(0, ch).getOrThrow()
|
||||
weights.set(0, ch, oldW + gW * learningRateWeight * FP32(discount))
|
||||
}
|
||||
|
||||
for (ch in 0..nChannels) {
|
||||
let oldB = biases.get(0, ch).getOrThrow()
|
||||
let gB = bGrad.get(0, ch).getOrThrow()
|
||||
biases.set(0, ch, oldB + gB * learningRateBias * FP32(discount))
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 形状访问器 ====================
|
||||
public func getInShape(): (Int64, Int64) {
|
||||
(nChannels, poolHeight * poolWidth)
|
||||
}
|
||||
|
||||
public func getOutShape(): Int64 {
|
||||
nChannels
|
||||
}
|
||||
|
||||
public func getPoolType(): String {
|
||||
poolType
|
||||
}
|
||||
|
||||
public func getLR(): FP32 {
|
||||
learningRateWeight
|
||||
}
|
||||
|
||||
public func getFType(): String {
|
||||
f.get_type()
|
||||
}
|
||||
|
||||
public func calculateOutputSize(inputHeight: Int64, inputWidth: Int64): (Int64, Int64) {
|
||||
let outH = (inputHeight + 2 * padding - poolHeight) / stride + 1
|
||||
let outW = (inputWidth + 2 * padding - poolWidth) / stride + 1
|
||||
(outH, outW)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user