1. 强化学习基础从马尔可夫决策过程MDP开始想象一下你在玩一款迷宫游戏。每走一步都会改变你的位置状态而你的目标是找到出口最大化奖励。这就是强化学习的核心场景——智能体通过与环境交互学习最优策略。马尔可夫决策过程MDP是描述这类问题的数学框架它包含五个关键要素状态S比如迷宫中的坐标位置动作A上下左右移动转移概率P执行动作后到达新状态的概率奖励R到达新状态获得的即时反馈折扣因子γ衡量未来奖励的当前价值在实际编码时我们可以用Python字典表示这些要素。比如定义一个简单的网格世界# 定义状态空间和动作空间 states [(0,0), (0,1), (1,0), (1,1)] actions [up, down, left, right] # 定义转移概率和奖励函数 transitions { (0,0): { up: {(0,1): 0.9, (0,0): 0.1}, right: {(1,0): 0.8, (0,0): 0.2} }, # 其他状态的转移规则... } rewards {(1,1): 10} # 只有到达(1,1)才有奖励2. 动态规划与蒙特卡洛经典求解方法对比2.1 动态规划DP全知全能的完美主义者动态规划要求完全了解环境模型知道所有P和R通过贝尔曼方程迭代求解。以值迭代算法为例def value_iteration(states, actions, transitions, rewards, gamma0.9, theta1e-6): V {s: 0 for s in states} while True: delta 0 for s in states: v V[s] # 计算每个状态的最大价值 V[s] max( sum(p*(rewards.get(s1,0) gamma*V[s1]) for s1, p in transitions[s][a].items()) for a in actions ) delta max(delta, abs(v - V[s])) if delta theta: break return VDP的优点是计算精确但现实问题往往无法获得完整的环境模型。2.2 蒙特卡洛MC经验主义的实践者蒙特卡洛方法直接从经验中学习不需要环境模型。比如首次访问型MC预测def mc_prediction(policy, env, num_episodes1000, gamma0.9): returns_sum defaultdict(float) returns_count defaultdict(float) V defaultdict(float) for _ in range(num_episodes): episode [] state env.reset() while True: action policy(state) next_state, reward, done, _ env.step(action) episode.append((state, action, reward)) if done: break state next_state G 0 for t in reversed(range(len(episode))): state, _, reward episode[t] G gamma * G reward if state not in [x[0] for x in episode[:t]]: returns_sum[state] G returns_count[state] 1.0 V[state] returns_sum[state] / returns_count[state] return VMC的优势是不需要环境模型但需要完整轨迹且方差较大。3. 时序差分与Q学习平衡偏差与方差3.1 时序差分TD折中的智慧TD方法结合了DP的自举思想和MC的采样思想。TD(0)更新规则为def td_learning(policy, env, num_episodes1000, alpha0.1, gamma0.9): V defaultdict(float) for _ in range(num_episodes): state env.reset() while True: action policy(state) next_state, reward, done, _ env.step(action) V[state] alpha * (reward gamma*V[next_state] - V[state]) if done: break state next_state return V3.2 Q学习异策略的经典算法Q学习通过分离行为策略和目标策略实现高效学习def q_learning(env, num_episodes1000, alpha0.1, gamma0.9, epsilon0.1): Q defaultdict(lambda: np.zeros(env.action_space.n)) for _ in range(num_episodes): state env.reset() while True: # ε-greedy行为策略 if np.random.random() epsilon: action env.action_space.sample() else: action np.argmax(Q[state]) next_state, reward, done, _ env.step(action) # 贪心目标策略 best_next_action np.argmax(Q[next_state]) td_target reward gamma * Q[next_state][best_next_action] Q[state][action] alpha * (td_target - Q[state][action]) if done: break state next_state return Q4. 策略梯度与PPO直接优化策略4.1 策略梯度从参数到动作策略梯度方法直接参数化策略并沿梯度方向更新def policy_gradient(env, policy_fn, num_episodes1000, alpha0.01, gamma0.99): optimizer torch.optim.Adam(policy_fn.parameters(), lralpha) for _ in range(num_episodes): log_probs [] rewards [] state env.reset() while True: state torch.FloatTensor(state) action_dist policy_fn(state) action action_dist.sample() log_prob action_dist.log_prob(action) next_state, reward, done, _ env.step(action.item()) log_probs.append(log_prob) rewards.append(reward) if done: break state next_state # 计算折扣回报 returns [] R 0 for r in reversed(rewards): R r gamma * R returns.insert(0, R) returns torch.tensor(returns) # 计算策略梯度 policy_loss [] for log_prob, R in zip(log_probs, returns): policy_loss.append(-log_prob * R) policy_loss torch.cat(policy_loss).sum() optimizer.zero_grad() policy_loss.backward() optimizer.step()4.2 PPO算法带约束的策略优化PPO通过限制策略更新幅度确保稳定性class PPOTrainer: def __init__(self, policy, clip_param0.2, lr3e-4): self.policy policy self.optimizer torch.optim.Adam(policy.parameters(), lrlr) self.clip_param clip_param def update(self, samples): states, actions, old_log_probs, returns, advantages samples # 计算新策略的概率比 dist self.policy(states) new_log_probs dist.log_prob(actions) ratio (new_log_probs - old_log_probs).exp() # PPO裁剪目标函数 surr1 ratio * advantages surr2 torch.clamp(ratio, 1.0 - self.clip_param, 1.0 self.clip_param) * advantages policy_loss -torch.min(surr1, surr2).mean() # 价值函数损失 value_loss (returns - self.policy.value(states)).pow(2).mean() # 总损失 loss policy_loss 0.5 * value_loss self.optimizer.zero_grad() loss.backward() self.optimizer.step()5. 实战用PPO训练CartPole智能体让我们用Stable-Baselines3库实现PPO算法import gym from stable_baselines3 import PPO from stable_baselines3.common.env_util import make_vec_env # 创建并行环境 env make_vec_env(CartPole-v1, n_envs4) # 定义PPO模型 model PPO( MlpPolicy, env, verbose1, learning_rate3e-4, n_steps2048, batch_size64, n_epochs10, gamma0.99, gae_lambda0.95, clip_range0.2, ent_coef0.0 ) # 训练模型 model.learn(total_timesteps100000) # 测试训练好的模型 obs env.reset() for _ in range(1000): action, _ model.predict(obs) obs, _, done, _ env.step(action) if done: obs env.reset()训练过程中有几个关键技巧使用广义优势估计GAE降低方差适当调整clip_range参数平衡探索与利用监控episode长度和奖励曲线判断收敛我在实际训练中发现当平均episode长度接近环境最大值CartPole为500时说明策略已经收敛。如果出现性能波动可以尝试减小学习率或增加batch_size。