```pythondef sigmoid_forward(X):out = util.sigmoid(X)cache = outreturn out, cachedef sigmoid_backward(dout, cache):return cache * (1. - cache) * doutdef relu_forward(X):out = np.maximum(X, 0)cache = Xreturn out, cachedef relu_backward(dout, cache):dX = dout.copy()dX[cache <= 0] = 0return dXdef lrelu_forward(X, a=1e-3):out = np.maximum(a * X, X)cache = (X, a)return out, cachedef lrelu_backward(dout, cache):X, a = cachedX = dout.copy()dX[X < 0] *= areturn dXdef tanh_forward(X):out = np.tanh(X)cache = outreturn out, cachedef tanh_backward(dout, cache):dX = (1 - cache**2) * doutreturn dX```
def dropout_forward(X, p_dropout):u = np.random.binomial(1, p_dropout, size=X.shape) / p_dropoutout = X * ucache = ureturn out, cachedef dropout_backward(dout, cache):dX = dout * cachereturn dX

def bn_forward(X, gamma, beta, cache, momentum=.9, train=True):running_mean, running_var = cacheif train:mu = np.mean(X, axis=0)var = np.var(X, axis=0)X_norm = (X - mu) / np.sqrt(var + c.eps)out = gamma * X_norm + betacache = (X, X_norm, mu, var, gamma, beta)running_mean = util.exp_running_avg(running_mean, mu, momentum)running_var = util.exp_running_avg(running_var, var, momentum)else:X_norm = (X - running_mean) / np.sqrt(running_var + c.eps)out = gamma * X_norm + betacache = Nonereturn out, cache, running_mean, running_vardef bn_backward(dout, cache):X, X_norm, mu, var, gamma, beta = cacheN, D = X.shapeX_mu = X - mustd_inv = 1. / np.sqrt(var + c.eps)dX_norm = dout * gammadvar = np.sum(dX_norm * X_mu, axis=0) * -.5 * std_inv**3dmu = np.sum(dX_norm * -std_inv, axis=0) + dvar * np.mean(-2. * X_mu, axis=0)dX = (dX_norm * std_inv) + (dvar * 2 * X_mu / N) + (dmu / N)dgamma = np.sum(dout * X_norm, axis=0)dbeta = np.sum(dout, axis=0)return dX, dgamma, dbeta
def fc_forward(X, W, b):out = X @ W + bcache = (W, X)return out, cachedef fc_backward(dout, cache):W, h = cachedW = h.T @ doutdb = np.sum(dout, axis=0)dX = dout @ W.Treturn dX, dW, db
【柠檬少年|卷积神经网络各层反向传播公式推导】def maxpool_forward(X, size=2, stride=2):def maxpool(X_col):max_idx = np.argmax(X_col, axis=0)out = X_col[max_idx, range(max_idx.size)]return out, max_idxreturn _pool_forward(X, maxpool, size, stride)def maxpool_backward(dout, cache):def dmaxpool(dX_col, dout_col, pool_cache):dX_col[pool_cache, range(dout_col.size)] = dout_colreturn dX_colreturn _pool_backward(dout, dmaxpool, cache)def avgpool_forward(X, size=2, stride=2):def avgpool(X_col):out = np.mean(X_col, axis=0)cache = Nonereturn out, cachereturn _pool_forward(X, avgpool, size, stride)def avgpool_backward(dout, cache):def davgpool(dX_col, dout_col, pool_cache):dX_col[:, range(dout_col.size)] = 1. / dX_col.shape[0] * dout_colreturn dX_colreturn _pool_backward(dout, davgpool, cache)def _pool_forward(X, pool_fun, size=2, stride=2):n, d, h, w = X.shapeh_out = (h - size) / stride + 1w_out = (w - size) / stride + 1if not w_out.is_integer() or not h_out.is_integer():raise Exception('Invalid output dimension!')h_out, w_out = int(h_out), int(w_out)X_reshaped = X.reshape(n * d, 1, h, w)X_col = im2col_indices(X_reshaped, size, size, padding=0, stride=stride)out, pool_cache = pool_fun(X_col)out = out.reshape(h_out, w_out, n, d)out = out.transpose(2, 3, 0, 1)cache = (X, size, stride, X_col, pool_cache)return out, cachedef _pool_backward(dout, dpool_fun, cache):X, size, stride, X_col, pool_cache = cachen, d, w, h = X.shapedX_col = np.zeros_like(X_col)dout_col = dout.transpose(2, 3, 0, 1).ravel()dX = dpool_fun(dX_col, dout_col, pool_cache)dX = col2im_indices(dX_col, (n * d, 1, h, w), size, size, padding=0, stride=stride)dX = dX.reshape(X.shape)return dX