强化学习与Q-Learning算法原理及Python迷宫导航实战：从MDP到Deep Q-Learning的完整指南 | 附代码与教程文档

研究流程脉络图
│
├─ 问题定义 → 智能体如何在迷宫中通过试错学习找到终点？
├─ 环境建模 → 定义状态(位置)、动作(上下左右)、奖励(撞墙-10，终点+50，每步-1)
├─ 算法选择 → Q-Learning (离线策略、时序差分)
├─ 代码实现 → Python + NumPy，ε-贪婪策略，Q表更新
├─ 结果分析 → 最优路径可视化、奖励曲线收敛性
└─ 优化扩展 → 超参数调优、Deep Q-Learning应对高维状态

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap

# 迷宫地图 (0=通路, 1=墙壁)
maze_grid = np.array([
    [0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
    [0, 0, 0, 0, 1, 0, 0, 0, 0, 1],
    # ......(省略部分迷宫行以简化篇幅，完整代码见社群)
    [1, 1, 1, 0, 1, 1, 1, 1, 0, 0]
])
start_pos = (0, 0)
goal_pos = (9, 9)

# 超参数设置
total_episodes = 5000
alpha = 0.1          # 学习率
gamma = 0.9          # 折扣因子
epsilon = 0.5        # 初始探索率
reward_wall = -10
reward_goal = 50
reward_step = -1

actions = [(0,-1), (0,1), (-1,0), (1,0)]  # 左、右、上、下
Q_table = np.zeros(maze_grid.shape + (len(actions),))

def is_valid_cell(pos, grid):
    r, c = pos
    if r < 0 or r >= grid.shape[0]: return False
    if c < 0 or c >= grid.shape[1]: return False
    if grid[r, c] == 1: return False
    return True

def choose_action(state, q_tbl, eps, num_acts):
    if np.random.random() < eps:
        return np.random.randint(num_acts)
    else:
        return np.argmax(q_tbl[state])

rewards_per_episode = []

for ep in range(total_episodes):
    state = start_pos
    total_reward = 0
    done = False
    while not done:
        act_idx = choose_action(state, Q_table, epsilon, len(actions))
        move = actions[act_idx]
        next_state = (state[0] + move[0], state[1] + move[1])

        if not is_valid_cell(next_state, maze_grid):
            reward = reward_wall
            done = True
        elif next_state == goal_pos:
            reward = reward_goal
            done = True
        else:
            reward = reward_step

        old_val = Q_table[state][act_idx]
        # 计算下一状态的最大Q值 (如果回合结束且未达目标，则为0)
        if done and next_state != goal_pos:
            max_next_q = 0
        else:
            max_next_q = np.max(Q_table[next_state])

        new_val = old_val + alpha * (reward + gamma * max_next_q - old_val)
        Q_table[state][act_idx] = new_val

        if not done:
            state = next_state
        total_reward += reward

    epsilon = max(0.01, epsilon * 0.995)
    rewards_per_episode.append(total_reward)

def get_best_path(q_tbl, start, goal, acts, grid, max_steps=200):
    path = [start]
    cur = start
    visited = set()
    for _ in range(max_steps):
        if cur == goal: break
        visited.add(cur)
        best_a = None
        best_val = -float('inf')
        for i, mv in enumerate(acts):
            nxt = (cur[0]+mv[0], cur[1]+mv[1])
            if (0 <= nxt[0] < grid.shape[0] and 0 <= nxt[1] < grid.shape[1] 
                and grid[nxt] == 0 and nxt not in visited):
                if q_tbl[cur][i] > best_val:
                    best_val = q_tbl[cur][i]
                    best_a = i
        if best_a is None: break
        move = acts[best_a]
        cur = (cur[0]+move[0], cur[1]+move[1])
        path.append(cur)
    return path

optimal_path = get_best_path(Q_table, start_pos, goal_pos, actions, maze_grid)

def plot_maze_path(path, grid):
    cmap = ListedColormap(['#eef8ea', '#a8c79c'])
    plt.figure(figsize=(8,8))
    plt.imshow(grid, cmap=cmap)
    plt.scatter(start_pos[1], start_pos[0], marker='o', color='#81c784', s=200, label='起点')
    plt.scatter(goal_pos[1], goal_pos[0], marker='*', color='#388e3c', s=300, label='终点')
    if path:
        rows, cols = zip(*path)
        plt.plot(cols, rows, color='#60b37a', linewidth=4, label='最优路径')
    plt.title('强化学习：机器人迷宫导航')
    plt.gca().invert_yaxis()
    plt.legend()
    plt.show()

plot_maze_path(optimal_path, maze_grid)

def plot_rewards_curve(rewards):
    plt.figure(figsize=(10,5))
    plt.plot(rewards)
    plt.title('每轮总奖励变化')
    plt.xlabel('训练轮次')
    plt.ylabel('总奖励')
    plt.grid(True)
    plt.show()

plot_rewards_curve(rewards_per_episode)