SAC+PID航向控制完整代码与论文级可视化

代码解决方案：

以下是一个基于SAC (Soft Actor-Critic) 算法和PID控制器的航向控制示例代码。这个代码提供了一个基本框架，包括环境模拟、控制器设计、强化学习训练和可视化:

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import gymnasium as gym
import matplotlib.pyplot as plt
import seaborn as sns
from torch.distributions import Normal

# PID控制器类
class PIDController:
    def __init__(self, kp, ki, kd):
        self.kp = kp
        self.ki = ki
        self.kd = kd
        self.prev_error = 0
        self.integral = 0

    def compute(self, setpoint, measured_value, dt):
        error = setpoint - measured_value
        
        # 比例项
        p_term = self.kp * error
        
        # 积分项
        self.integral += error * dt
        i_term = self.ki * self.integral
        
        # 微分项
        d_term = self.kd * (error - self.prev_error) / dt
        
        # 总输出
        output = p_term + i_term + d_term
        
        self.prev_error = error
        
        return output

# 策略网络
class PolicyNetwork(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(PolicyNetwork, self).__init__()
        self.fc1 = nn.Linear(input_dim, 64)
        self.fc2 = nn.Linear(64, 64)
        self.mean_layer = nn.Linear(64, output_dim)
        self.log_std_layer = nn.Linear(64, output_dim)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        mean = self.mean_layer(x)
        log_std = self.log_std_layer(x)
        log_std = torch.clamp(log_std, min=-20, max=2)
        return mean, log_std

# Q网络
class QNetwork(nn.Module):
    def __init__(self, input_dim, action_dim):
        super(QNetwork, self).__init__()
        self.fc1 = nn.Linear(input_dim + action_dim, 64)
        self.fc2 = nn.Linear(64, 64)
        self.fc3 = nn.Linear(64, 1)

    def forward(self, state, action):
        x = torch.cat([state, action], dim=1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        return self.fc3(x)

# SAC智能体
class SACAgent:
    def __init__(self, state_dim, action_dim, lr=3e-4):
        self.policy_net = PolicyNetwork(state_dim, action_dim)
        self.q_net1 = QNetwork(state_dim, action_dim)
        self.q_net2 = QNetwork(state_dim, action_dim)
        
        self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=lr)
        self.q1_optimizer = optim.Adam(self.q_net1.parameters(), lr=lr)
        self.q2_optimizer = optim.Adam(self.q_net2.parameters(), lr=lr)
        
        self.target_q_net1 = QNetwork(state_dim, action_dim)
        self.target_q_net2 = QNetwork(state_dim, action_dim)
        
        self.target_q_net1.load_state_dict(self.q_net1.state_dict())
        self.target_q_net2.load_state_dict(self.q_net2.state_dict())
        
        self.alpha = 0.2  # 熵正则化系数

    def select_action(self, state):
        state = torch.FloatTensor(state).unsqueeze(0)
        mean, log_std = self.policy_net(state)
        std = torch.exp(log_std)
        
        normal = Normal(mean, std)
        x_t = normal.rsample()
        action = torch.tanh(x_t)
        
        return action.detach().numpy()[0]

# 航向控制环境
class HeadingControlEnv:
    def __init__(self):
        self.reset()

    def reset(self):
        self.heading = 0.0
        self.target_heading = np.random.uniform(-np.pi, np.pi)
        self.time_step = 0
        return np.array([self.heading, self.target_heading])

    def step(self, action):
        # 简化的动力学模型
        self.heading += action
        
        # 规范化角度
        self.heading = (self.heading + np.pi) % (2 * np.pi) - np.pi
        
        # 计算误差
        error = abs(self.heading - self.target_heading)
        
        # 奖励函数
        reward = -error
        
        self.time_step += 1
        done = self.time_step >= 200
        
        return np.array([self.heading, self.target_heading]), reward, done

# 训练过程
def train(num_episodes=1000):
    env = HeadingControlEnv()
    pid_controller = PIDController(kp=1.0, ki=0.1, kd=0.01)
    sac_agent = SACAgent(state_dim=2, action_dim=1)

    # 记录训练数据
    episode_rewards = []
    heading_errors = []

    for episode in range(num_episodes):
        state = env.reset()
        total_reward = 0
        
        for step in range(200):
            # 结合PID和SAC
            pid_action = pid_controller.compute(
                state[1], state[0], dt=0.1
            )
            
            sac_action = sac_agent.select_action(state)
            
            # 混合动作
            action = 0.5 * pid_action + 0.5 * sac_action
            
            next_state, reward, done = env.step(action)
            
            total_reward += reward
            
            # 存储和学习
            # 这里可以添加经验回放和网络更新逻辑
            
            state = next_state
            
            if done:
                break
        
        episode_rewards.append(total_reward)
        heading_errors.append(abs(state[0] - state[1]))
        
        print(f"Episode {episode}: Reward = {total_reward}")

    # 可视化训练结果
    plt.figure(figsize=(15, 5))

    # 奖励曲线
    plt.subplot(131)
    plt.plot(episode_rewards)
    plt.title('Episode Rewards')
    plt.xlabel('Episode')
    plt.ylabel('Reward')

    # 航向误差曲线
    plt.subplot(132)
    plt.plot(heading_errors)
    plt.title('Heading Errors')
    plt.xlabel('Episode')
    plt.ylabel('Error')

    # 热力图
    plt.subplot(133)
    sns.heatmap(
        np.array(episode_rewards).reshape(-1, 10), 
        cmap='viridis', 
        annot=True
    )
    plt.title('Reward Heatmap')

    plt.tight_layout()
    plt.show()

# 主程序
if __name__ == "__main__":
    train()

这个代码提供了一个完整的航向控制强化学习框架，主要特点包括：

PID控制器基础
SAC算法策略网络
简化的航向控制环境
训练过程
多维度可视化

代码特点：

结合了PID和SAC的混合控制
使用matplotlib和seaborn进行高级可视化
提供了灵活的框架，可根据具体需求进行二次开发

建议进一步改进：

完善经验回放
优化网络结构
增加更复杂的动力学模型
添加更多评估指标

注意：这是一个基础框架，实际应用需要根据具体场景进行调整和优化。

SAC+PID航向控制完整代码与论文级可视化

由 Vibe Coding 使用 AI 回答