51hhh/mobileNet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[CF]完成钢琴小游戏

Browse Source
This commit is contained in:
51hhh 2025-08-20 16:41:21 +08:00
parent 24f9ff2428
commit 6b4d03876e
12 changed files with 1767 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

828
game/贪吃蛇/snake_game.html Normal file
View File

@ -0,0 +1,828 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AI 姿态控制贪吃蛇</title>
<style>
body {
font-family: Arial, sans-serif;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
min-height: 100vh;
background: linear-gradient(135deg, #a8c0ff, #3f2b96);
color: #fff;
margin: 0;
padding: 20px;
}
h1 {
margin-bottom: 20px;
color: #fff;
text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.3);
}
#game-area {
display: flex;
gap: 20px;
flex-wrap: wrap;
justify-content: center;
}
#snake-canvas-container {
position: relative;
border: 5px solid #00f2fe;
box-shadow: 0 0 20px #00f2fe, 0 0 40px #00f2fe;
background-color: #333;
border-radius: 10px;
overflow: hidden;
}
#snakeCanvas {
background-color: #333;
display: block;
}
#game-info-panel {
background: rgba(0, 0, 0, 0.6);
padding: 25px;
border-radius: 10px;
box-shadow: 0 0 15px rgba(0, 0, 0, 0.5);
min-width: 300px;
display: flex;
flex-direction: column;
gap: 15px;
}
.info-item {
font-size: 1.1em;
display: flex;
justify-content: space-between;
}
.info-item span:first-child {
font-weight: bold;
color: #a8c0ff;
}
#status-display {
background: #222;
padding: 10px;
border-radius: 5px;
font-size: 0.9em;
min-height: 30px;
color: #00f2fe;
text-align: center;
}
.gesture-indicator {
font-size: 1.2em;
color: #00ff00;
margin-top: 10px;
padding: 5px;
background: #2c2c2c;
border-radius: 5px;
text-align: center;
min-height: 25px;
}
.control-buttons {
display: flex;
gap: 10px;
justify-content: center;
margin-top: 20px;
}
.control-buttons button {
padding: 12px 25px;
font-size: 1.1em;
border: none;
border-radius: 5px;
cursor: pointer;
transition: all 0.3s ease;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.4);
color: white;
}
#startBtn {
background: linear-gradient(45deg, #2196F3, #21CBF3);
}
#startBtn:hover {
background: linear-gradient(45deg, #1976D2, #1AC8EF);
transform: translateY(-2px);
}
#startBtn:disabled {
background: #607d8b;
cursor: not-allowed;
opacity: 0.7;
}
#importModelBtn {
background: linear-gradient(45deg, #FFC107, #FFEB3B);
}
#importModelBtn:hover {
background: linear-gradient(45deg, #FFA000, #FFD700);
transform: translateY(-2px);
}
#importModelBtn:disabled {
background: #9E9E9E;
cursor: not-allowed;
opacity: 0.7;
}
#video-feed-container {
position: absolute;
top: 20px;
left: 20px;
width: 320px;
height: 240px;
border: 3px solid #6a1b9a;
border-radius: 10px;
background: #000;
overflow: hidden;
box-shadow: 0 5px 20px rgba(0, 0, 0, 0.6);
z-index: 100; /* 确保在游戏画布之上 */
}
#video-feed-container video {
width: 100%;
height: 100%;
object-fit: cover;
border-radius: 7px;
}
#video-feed-container canvas {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
border-radius: 7px;
}
.overlay {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.85);
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
z-index: 1000;
color: white;
font-size: 2em;
text-align: center;
opacity: 0;
visibility: hidden;
transition: opacity 0.3s ease, visibility 0.3s ease;
}
.overlay.active {
opacity: 1;
visibility: visible;
}
.overlay h2 {
font-size: 3em;
margin-bottom: 20px;
text-shadow: 3px 3px 6px rgba(0, 0, 0, 0.5);
}
.overlay button {
margin-top: 30px;
font-size: 1.5em;
padding: 15px 30px;
background: linear-gradient(45deg, #28a745, #20c997);
color: white;
border: none;
border-radius: 50px;
cursor: pointer;
box-shadow: 0 6px 15px rgba(0, 0, 0, 0.3);
transition: transform 0.2s, box-shadow 0.2s;
}
.overlay button:hover {
transform: translateY(-3px);
box-shadow: 0 8px 20px rgba(0, 0, 0, 0.4);
}
.overlay button:active {
transform: translateY(0);
}
.instructions {
background: rgba(0, 0, 0, 0.6);
padding: 15px;
border-radius: 8px;
margin-top: 20px;
font-size: 0.9em;
line-height: 1.6;
width: 80%;
max-width: 500px;
text-align: left;
box-shadow: 0 0 10px rgba(0, 0, 0, 0.3);
animation: fadeIn 1s ease-out;
}
.instructions h3 {
color: #00f2fe;
margin-bottom: 10px;
text-align: center;
}
.instructions ul {
list-style: none;
padding-left: 0;
}
.instructions li {
margin-bottom: 5px;
display: flex;
align-items: center;
}
.instructions li strong {
color: #a8c0ff;
min-width: 80px;
}
.instructions li span {
flex-grow: 1;
}
@keyframes fadeIn {
from { opacity: 0; transform: translateY(20px); }
to { opacity: 1; transform: translateY(0); }
}
</style>
<!-- 引入 TensorFlow.js 核心库 -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@4.20.0/dist/tf.min.js"></script>
<!-- 引入 pose-detection 库 (包含 MoveNet) -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/pose-detection@2.1.3/dist/pose-detection.min.js"></script>
<!-- 引入 KNN 分类器 -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/knn-classifier@1.2.2/dist/knn-classifier.min.js"></script>
</head>
<body>
<h1>AI 姿态控制贪吃蛇</h1>
<div id="game-area">
<div id="video-feed-container">
<video id="videoFeed" autoplay muted playsinline></video>
<canvas id="poseCanvas"></canvas>
</div>
<div id="snake-canvas-container">
<canvas id="snakeCanvas"></canvas>
</div>
<div id="game-info-panel">
<div class="info-item"><span>得分:</span> <span id="score">0</span></div>
<div class="info-item"><span>姿态识别:</span> <span id="currentGesture">未识别</span></div>
<div class="info-item"><span>控制指令:</span> <span id="controlCommand">静止</span></div>
<div id="status-display">等待模型导入...</div>
<div class="gesture-indicator" id="gestureConfidence"></div>
<div class="control-buttons">
<button id="startBtn" disabled>开始游戏 (Enter)</button>
<input type="file" id="fileImporter" accept=".json" style="display: none;">
<button id="importModelBtn">导入模型</button>
</div>
<div class="instructions">
<h3>姿态控制说明</h3>
<ul>
<li><strong>向上:</strong> <span>双手举过头顶</span></li>
<li><strong>向下:</strong> <span>双手放在身体两侧或下垂</span></li>
<li><strong>向左:</strong> <span>左手平举</span></li>
<li><strong>向右:</strong> <span>右手平举</span></li>
<li><strong>静止:</strong> <span>保持站立姿势 (无特定动作)</span></li>
</ul>
<p style="font-size:0.8em;text-align:center;margin-top:10px;color:#ccc;">(确保在摄像头画面中能清晰识别全身)</p>
</div>
</div>
</div>
<!-- 游戏结束/开始 Overlay -->
<div id="gameOverOverlay" class="overlay">
<h2 id="overlayText">游戏结束！</h2>
<p>最终得分: <span id="finalScore">0</span></p>
<button id="restartBtn">重新开始 (Enter)</button>
</div>
<script>
// ==========================================================
// 全局变量和 DOM 引用
// ==========================================================
const videoElement = document.getElementById('videoFeed');
const poseCanvas = document.getElementById('poseCanvas');
const poseCtx = poseCanvas.getContext('2d');
const snakeCanvas = document.getElementById('snakeCanvas');
const snakeCtx = snakeCanvas.getContext('2d');
const scoreDisplay = document.getElementById('score');
const currentGestureDisplay = document.getElementById('currentGesture');
const controlCommandDisplay = document.getElementById('controlCommand');
const statusDisplay = document.getElementById('status-display');
const gestureConfidenceDisplay = document.getElementById('gestureConfidence');
const startBtn = document.getElementById('startBtn');
const importModelBtn = document.getElementById('importModelBtn');
const fileImporter = document.getElementById('fileImporter');
const gameOverOverlay = document.getElementById('gameOverOverlay');
const overlayText = document.getElementById('overlayText');
const finalScoreDisplay = document.getElementById('finalScore');
const restartBtn = document.getElementById('restartBtn');
let detector; // MoveNet 姿态检测器
let classifier; // KNN 分类器
let isPoseDetectionReady = false; // 姿态检测系统是否就绪
let isModelLoaded = false; // KNN 模型是否已加载
let animationFrameId; // 用于姿态检测的 requestAnimationFrame ID
let gameLoopId; // 用于贪吃蛇游戏循环的 setInterval ID
let gameStatus = 'initial'; // 'initial', 'loading', 'ready', 'playing', 'paused', 'gameOver'
let currentDetectedClassId = null; // 当前检测到的姿态分类ID
let currentConfidence = 0; // 当前检测到的置信度
// 用于映射姿态分类ID到游戏方向
// 请确保这些ID与您训练模型时使用的类别ID一致
const gestureClassToGameDirection = {
'0': 'UP', // 类别0 -> 向上
'1': 'DOWN', // 类别1 -> 向下
'2': 'LEFT', // 类别2 -> 向左
'3': 'RIGHT' // 类别3 -> 向右
};
// 用于显示给用户的类别名称（加载模型时从JSON中读取）
let importedClassNames = {};
let snakeDirection = 'RIGHT'; // 贪吃蛇当前方向
// ==========================================================
// 贪吃蛇游戏设置
// ==========================================================
const GRID_SIZE = 20; // 网格大小（像素）
const CANVAS_WIDTH = 600;
const CANVAS_HEIGHT = 400;
snakeCanvas.width = CANVAS_WIDTH;
snakeCanvas.height = CANVAS_HEIGHT;
let snake = [{ x: 10, y: 10 }]; // 蛇的身体 (以网格坐标表示)
let food = {}; // 食物位置
let score = 0;
let gameSpeed = 300; // 游戏速度（毫秒）
// ==========================================================
// 初始化函数
// ==========================================================
document.addEventListener('DOMContentLoaded', initApp);
async function initApp() {
updateGameStatus('loading');
statusDisplay.textContent = '正在加载 MoveNet 模型和摄像头...';
startBtn.disabled = true;
importModelBtn.disabled = true;
try {
// 初始化 KNN 分类器
classifier = knnClassifier.create();
// 初始化 MoveNet 检测器
const detectorConfig = { modelType: poseDetection.movenet.modelType.SINGLEPOSE_LIGHTNING };
detector = await poseDetection.createDetector(poseDetection.SupportedModels.MoveNet, detectorConfig);
// 设置摄像头
await setupCamera();
isPoseDetectionReady = true;
statusDisplay.textContent = 'MoveNet 和摄像头已就绪。请导入姿态模型。';
importModelBtn.disabled = false; // 启用导入按钮
// 启动姿态检测循环（只进行检测和绘制，不预测，直到模型导入）
startPoseDetectionLoop();
// 绑定按钮事件
startBtn.addEventListener('click', startGame);
importModelBtn.addEventListener('click', () => fileImporter.click());
fileImporter.addEventListener('change', handleModelImport);
restartBtn.addEventListener('click', resetGame);
// 键盘快捷键
document.addEventListener('keydown', (e) => {
if (e.key === 'Enter') {
if (gameStatus === 'ready' || gameStatus === 'gameOver') {
startBtn.click(); // 模拟点击开始按钮
} else if (gameStatus === 'playing') {
// 可选：添加暂停功能
}
}
});
updateGameUI();
} catch (error) {
console.error("应用初始化失败:", error);
statusDisplay.textContent = `初始化失败: ${error.message}`;
alert(`应用初始化失败: ${error.message}\n请检查摄像头权限或网络连接。`);
}
}
// ==========================================================
// 摄像头和姿态检测相关
// ==========================================================
async function setupCamera() {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: {
width: { ideal: videoElement.width }, // 使用videoFeed的width/height
height: { ideal: videoElement.height },
facingMode: 'user'
}
});
videoElement.srcObject = stream;
return new Promise((resolve, reject) => {
videoElement.onloadedmetadata = () => {
videoElement.play().then(() => {
// 确保 poseCanvas 和 videoFeed 尺寸匹配
poseCanvas.width = videoElement.videoWidth;
poseCanvas.height = videoElement.videoHeight;
resolve();
}).catch(reject);
};
setTimeout(() => reject(new Error('摄像头元数据加载或播放超时')), 10000);
});
} catch (error) {
if (error.name === 'NotAllowedError') {
throw new Error('用户拒绝了摄像头权限。');
} else if (error.name === 'NotFoundError') {
throw new Error('未找到摄像头设备。');
} else {
throw error;
}
}
}
async function startPoseDetectionLoop() {
if (!isPoseDetectionReady) return;
async function detectAndPredict() {
try {
const poses = await detector.estimatePoses(videoElement, { flipHorizontal: true }); // 镜像摄像头
poseCtx.clearRect(0, 0, poseCanvas.width, poseCanvas.height);
if (poses && poses.length > 0) {
drawPose(poses[0]); // 绘制检测到的姿态
if (isModelLoaded && gameStatus === 'playing') { // 仅在游戏进行中时才预测并控制贪吃蛇
const poseTensor = flattenPose(poses[0]);
if (classifier.getNumClasses() > 0) {
const prediction = await classifier.predictClass(poseTensor);
poseTensor.dispose(); // 释放张量内存
const predictedClassId = prediction.label;
const confidence = prediction.confidences[predictedClassId];
currentConfidence = (confidence * 100).toFixed(1);
// 设定一个置信度阈值，例如 70%
if (confidence > 0.70) {
currentDetectedClassId = predictedClassId;
const gameDirection = gestureClassToGameDirection[predictedClassId];
if (gameDirection) {
changeSnakeDirection(gameDirection); // 控制贪吃蛇移动
controlCommandDisplay.textContent = gameDirection;
} else {
controlCommandDisplay.textContent = '未知方向';
}
} else {
currentDetectedClassId = null; // 置信度不足，视为未识别
currentConfidence = 0;
controlCommandDisplay.textContent = '静止 (置信度不足)';
}
} else {
poseTensor.dispose(); // 确保张量被释放
currentDetectedClassId = null;
currentConfidence = 0;
controlCommandDisplay.textContent = '静止 (模型无数据)';
}
} else if (!isModelLoaded) {
currentDetectedClassId = null;
currentConfidence = 0;
controlCommandDisplay.textContent = '静止 (等待模型)';
} else if (gameStatus !== 'playing') {
currentDetectedClassId = null;
currentConfidence = 0;
controlCommandDisplay.textContent = '静止';
}
} else {
currentDetectedClassId = null;
currentConfidence = 0;
currentGestureDisplay.textContent = '未检测到姿态';
controlCommandDisplay.textContent = '静止';
}
updateGameUI(); // 更新显示
} catch (error) {
console.error('姿态检测或预测出错:', error);
statusDisplay.textContent = `检测错误: ${error.message}`;
// 如果错误持续发生，考虑停止循环或提供更多反馈
} finally {
animationFrameId = requestAnimationFrame(detectAndPredict);
}
}
animationFrameId = requestAnimationFrame(detectAndPredict); // 启动循环
}
// 展平姿态关键点并归一化
function flattenPose(pose) {
// 将所有关键点的 (x, y) 坐标展平为一个一维数组
// 并进行归一化 (除以视频的宽度和高度)，使得特征独立于图像大小
const keypoints = pose.keypoints.map(p => [p.x / videoElement.videoWidth, p.y / videoElement.videoHeight]).flat();
return tf.tensor(keypoints); // 转换为 TensorFlow 张量
}
// 绘制 MoveNet 姿态骨骼图
function drawPose(pose) {
if (pose.keypoints) {
// 绘制关键点
for (const keypoint of pose.keypoints) {
if (keypoint.score > 0.3) { // 仅绘制置信度高的关键点
poseCtx.beginPath();
poseCtx.arc(keypoint.x, keypoint.y, 5, 0, 2 * Math.PI);
poseCtx.fillStyle = '#00f2fe'; // 浅蓝色
poseCtx.fill();
}
}
// 绘制骨骼连接线
const adjacentPairs = poseDetection.util.getAdjacentPairs(poseDetection.SupportedModels.MoveNet);
adjacentPairs.forEach(([i, j]) => {
const kp1 = pose.keypoints[i];
const kp2 = pose.keypoints[j];
if (kp1.score > 0.3 && kp2.score > 0.3) { // 仅连接置信度高的关键点
poseCtx.beginPath();
poseCtx.moveTo(kp1.x, kp1.y);
poseCtx.lineTo(kp2.x, kp2.y);
poseCtx.strokeStyle = '#a8c0ff'; // 紫蓝色
poseCtx.lineWidth = 2;
poseCtx.stroke();
}
});
}
}
// ==========================================================
// 模型导入导出
// ==========================================================
async function handleModelImport(event) {
const file = event.target.files[0];
if (!file) return;
updateGameStatus('loading');
statusDisplay.textContent = '正在导入模型...';
startBtn.disabled = true;
importModelBtn.disabled = true;
try {
await loadModelFromFile(file);
statusDisplay.textContent = '姿态模型导入成功！';
isModelLoaded = true;
startBtn.disabled = false; // 启用开始按钮
importModelBtn.disabled = true; // 禁用导入按钮
updateGameStatus('ready');
} catch (error) {
console.error('模型导入失败:', error);
statusDisplay.textContent = `模型导入失败: ${error.message}`;
alert(`模型导入失败: ${error.message}\n请确保文件是正确的模型JSON文件。`);
startBtn.disabled = true; // 导入失败则不能开始
importModelBtn.disabled = false; // 可以再试一次导入
isModelLoaded = false;
updateGameStatus('initial');
} finally {
fileImporter.value = ''; // 清空文件输入
}
}
async function loadModelFromFile(file) {
// 这段逻辑与您之前在 script.js 中 `importModel` 的核心逻辑一致
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = async (e) => {
try {
const loadedModelData = JSON.parse(e.target.result);
if (!loadedModelData || !loadedModelData.dataset || !loadedModelData.classMap) {
throw new Error('模型数据结构不正确 (缺少 classMap 或 dataset)。');
}
classifier.clearAllClasses();
const dataset = {};
let totalExamples = 0;
for (const classId in loadedModelData.dataset) {
const classData = loadedModelData.dataset[classId];
if (classData && classData.length > 0) {
if (classData.some(sample => !Array.isArray(sample) || sample.some(val => typeof val !== 'number'))) {
throw new Error(`类别 ${classId} 包含无效的样本数据 (不是数字数组)。`);
}
const tensors = classData.map(data => tf.tensor1d(data));
const stacked = tf.stack(tensors);
dataset[classId] = stacked;
totalExamples += classData.length;
tensors.forEach(t => t.dispose());
} else {
console.warn(`类别 ${classId} 没有样本数据。`);
}
}
classifier.setClassifierDataset(dataset);
importedClassNames = loadedModelData.classMap; // 更新类别名称映射
console.log(`模型加载成功！共加载 ${totalExamples} 个样本。`);
console.log('类别映射 (导入):', importedClassNames);
resolve();
} catch (error) {
reject(error);
}
};
reader.onerror = (error) => {
reject(new Error('文件读取失败。'));
};
reader.readAsText(file);
});
}
// ===================================
// 贪吃蛇游戏逻辑
// ===================================
function startGame() {
if (!isPoseDetectionReady) {
alert('姿态检测系统未准备就绪，请稍候。');
return;
}
if (!isModelLoaded) {
alert('请先导入姿态模型！');
return;
}
if (!classifier || classifier.getNumClasses() === 0) {
alert('已导入的模型中没有训练数据，请导入一个有效的模型文件。');
return;
}
gameOverOverlay.classList.remove('active');
updateGameStatus('playing');
resetGameState();
generateFood();
gameLoopId = setInterval(drawGame, gameSpeed); // 开始游戏循环
startBtn.disabled = true;
importModelBtn.disabled = true;
}
function resetGame() {
gameOverOverlay.classList.remove('active');
updateGameStatus('ready');
resetGameState(); // 重置游戏状态
snakeDirection = 'RIGHT'; // 重新开始时默认为向右
startBtn.disabled = false;
importModelBtn.disabled = true; // 模型加载后禁用导入按钮
updateGameUI();
}
function resetGameState() {
clearInterval(gameLoopId);
snake = [{ x: 10, y: 10 }]; // 蛇回到起始位置，长度为1
food = {};
score = 0;
updateScoreDisplay();
}
function generateFood() {
food = {
x: Math.floor(Math.random() * (CANVAS_WIDTH / GRID_SIZE)),
y: Math.floor(Math.random() * (CANVAS_HEIGHT / GRID_SIZE))
};
// 确保食物不生成在蛇的身体上
for (let i = 0; i < snake.length; i++) {
if (food.x === snake[i].x && food.y === snake[i].y) {
generateFood(); // 重新生成
}
}
}
function drawGame() {
// 清空 snakeCanvas
snakeCtx.clearRect(0, 0, CANVAS_WIDTH, CANVAS_HEIGHT);
// 绘制食物
snakeCtx.fillStyle = '#ff3333'; // 红色
snakeCtx.fillRect(food.x * GRID_SIZE, food.y * GRID_SIZE, GRID_SIZE, GRID_SIZE);
// 绘制蛇
for (let i = 0; i < snake.length; i++) {
snakeCtx.fillStyle = (i === 0) ? '#00cc00' : '#009900'; // 蛇头绿色，身体深绿
snakeCtx.fillRect(snake[i].x * GRID_SIZE, snake[i].y * GRID_SIZE, GRID_SIZE, GRID_SIZE);
snakeCtx.strokeStyle = '#222'; // 边框
snakeCtx.strokeRect(snake[i].x * GRID_SIZE, snake[i].y * GRID_SIZE, GRID_SIZE, GRID_SIZE);
}
// 移动蛇头
let headX = snake[0].x;
let headY = snake[0].y;
switch (snakeDirection) {
case 'LEFT': headX--; break;
case 'UP': headY--; break;
case 'RIGHT': headX++; break;
case 'DOWN': headY++; break;
}
// 碰撞检测
// 撞墙
if (headX < 0 || headX >= CANVAS_WIDTH / GRID_SIZE ||
headY < 0 || headY >= CANVAS_HEIGHT / GRID_SIZE) {
endGame();
return;
}
// 撞自己
for (let i = 1; i < snake.length; i++) {
if (headX === snake[i].x && headY === snake[i].y) {
endGame();
return;
}
}
// 吃到食物
if (headX === food.x && headY === food.y) {
score++;
updateScoreDisplay();
generateFood();
// 暂时不增加速度，保持稳定
} else {
// 如果没吃到食物，移除蛇尾，实现移动效果
snake.pop();
}
// 添加新蛇头
snake.unshift({ x: headX, y: headY });
}
function changeSnakeDirection(newDirection) {
// 防止蛇立即反向掉头
if (newDirection === 'LEFT' && snakeDirection !== 'RIGHT') snakeDirection = 'LEFT';
else if (newDirection === 'UP' && snakeDirection !== 'DOWN') snakeDirection = 'UP';
else if (newDirection === 'RIGHT' && snakeDirection !== 'LEFT') snakeDirection = 'RIGHT';
else if (newDirection === 'DOWN' && snakeDirection !== 'UP') snakeDirection = 'DOWN';
}
function endGame() {
updateGameStatus('gameOver');
clearInterval(gameLoopId);
overlayText.textContent = '游戏结束！';
finalScoreDisplay.textContent = score;
gameOverOverlay.classList.add('active');
startBtn.disabled = true; // 游戏结束后开始按钮不可用，直到点击重玩
importModelBtn.disabled = true; // 保持禁用
}
// ==========================================================
// UI 更新
// ==========================================================
function updateGameStatus(status) {
gameStatus = status;
console.log("Game status updated to:", gameStatus);
}
function updateScoreDisplay() {
scoreDisplay.textContent = score;
}
function updateGameUI() {
// 更新姿态识别显示
if (currentDetectedClassId !== null) {
const className = importedClassNames[currentDetectedClassId] || `未知类别 ${currentDetectedClassId}`;
currentGestureDisplay.textContent = `${className} (${currentConfidence}%)`;
gestureConfidenceDisplay.textContent = `置信度: ${currentConfidence}%`;
gestureConfidenceDisplay.style.color = currentConfidence > 70 ? '#00ff00' : '#ffaa00';
} else {
currentGestureDisplay.textContent = '未识别';
gestureConfidenceDisplay.textContent = '';
}
// 更新状态信息
if (gameStatus === 'initial') {
statusDisplay.textContent = '等待模型导入...';
} else if (gameStatus === 'loading') {
// 状态文本已经在 initApp 或 handleModelImport 中设置
} else if (gameStatus === 'ready') {
statusDisplay.textContent = '模型已加载，点击“开始游戏”或按“Enter”键。';
} else if (gameStatus === 'playing') {
statusDisplay.textContent = '游戏进行中...';
} else if (gameStatus === 'gameOver') {
statusDisplay.textContent = '游戏结束！点击“重新开始”或按“Enter”键。';
}
}
</script>
</body>
</html>

792
game/钢琴/index.html Normal file
View File

@ -0,0 +1,792 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AI 空气钢琴 - 手势控制</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: 'Arial', sans-serif;
height: 100vh;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
/* 钢琴主题背景色 */
background: linear-gradient(135deg, #2c3e50 0%, #34495e 100%);
color: white;
overflow: hidden;
}
h1 {
font-size: 3.5em;
margin-bottom: 5px;
color: #ecf0f1; /* 亮灰色 */
text-shadow: 3px 3px 6px rgba(0, 0, 0, 0.4);
}
.subtitle {
font-size: 1.2em;
margin-bottom: 30px;
color: #bdc3c7; /* 浅灰色 */
}
#main-app {
display: flex;
gap: 30px;
align-items: flex-start;
flex-wrap: wrap;
justify-content: center;
padding: 20px;
}
#video-feed-container {
position: relative;
width: 480px;
height: 360px;
background: rgba(0, 0, 0, 0.5);
border: 5px solid #bdc3c7; /* 浅灰色边框 */
border-radius: 15px;
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.5);
overflow: hidden;
}
#video-feed-container video {
width: 100%;
height: 100%;
object-fit: cover;
border-radius: 10px;
}
#video-feed-container canvas {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
border-radius: 10px;
}
.status-display {
position: absolute;
bottom: 0px;
left: 0;
right: 0;
height: 40px;
background: rgba(0, 0, 0, 0.7);
color: #2ecc71; /* 绿色状态文本 */
display: flex;
align-items: center;
justify-content: center;
font-size: 0.95em;
border-bottom-left-radius: 10px;
border-bottom-right-radius: 10px;
font-weight: bold;
text-shadow: 1px 1px 2px rgba(0,0,0,0.3);
}
#control-panel {
background: rgba(0, 0, 0, 0.6);
padding: 25px;
border-radius: 15px;
box-shadow: 0 5px 20px rgba(0, 0, 0, 0.4);
display: flex;
flex-direction: column;
gap: 20px;
min-width: 350px;
}
.panel-section {
padding: 15px;
border-radius: 10px;
background: rgba(255, 255, 255, 0.1);
text-align: center;
}
.panel-section h3 {
color: #ADD8E6; /* 浅蓝色标题 */
margin-bottom: 15px;
font-size: 1.5em;
}
.panel-section p {
font-size: 1.1em;
margin-bottom: 10px;
color: #e0f2f7;
}
.action-button {
padding: 12px 25px;
font-size: 1.1em;
border: none;
border-radius: 8px;
cursor: pointer;
transition: all 0.3s ease;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3);
color: white;
font-weight: bold;
min-width: 150px;
}
#importModelBtn {
background: linear-gradient(45deg, #f39c12, #f1c40f); /* 橙黄渐变 */
}
#importModelBtn:hover {
background: linear-gradient(45deg, #e67e22, #f39c12);
transform: translateY(-2px);
}
#importModelBtn:disabled {
background: #607d8b;
cursor: not-allowed;
opacity: 0.7;
}
#startStopBtn {
background: linear-gradient(45deg, #27ae60, #2ecc71); /* 绿色渐变 */
}
#startStopBtn.playing {
background: linear-gradient(45deg, #c0392b, #e74c3c); /* 红色渐变 */
}
#startStopBtn:hover:not(:disabled) {
transform: translateY(-2px);
}
#startStopBtn:disabled {
background: #607d8b;
cursor: not-allowed;
opacity: 0.7;
}
.info-item {
font-size: 1.1em;
padding: 8px 0;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.info-item:last-child {
border-bottom: none;
}
.info-label {
font-weight: bold;
color: #dbe4ee;
}
.info-value {
float: right;
color: #e0f2f7;
}
#currentPlayingAction {
font-size: 1.8em;
font-weight: bold;
color: #f1c40f; /* 鲜黄色 */
text-shadow: 0 0 10px rgba(241, 196, 15, 0.6);
margin-top: 15px;
animation: pulseText 1s infinite alternate;
}
@keyframes pulseText {
from { transform: scale(1); opacity: 1; }
to { transform: scale(1.05); opacity: 0.9; }
}
/* 手部关键点和连接线样式 */
.keypoint {
fill: #f1c40f; /* 关键点颜色 */
stroke: #f1c40f;
stroke-width: 2px;
}
.connection {
stroke: #3498db; /* 连接线颜色 蓝色 */
stroke-width: 3px;
}
.mapping-list {
list-style: none;
padding: 0;
text-align: left;
margin-top: 10px;
}
.mapping-list li {
margin-bottom: 5px;
font-size: 0.95em;
color: #e0f2f7;
}
.mapping-list li strong {
color: #ADD8E6;
min-width: 70px;
display: inline-block;
}
.mapping-list li span.action-name {
color: #f1c40f;
}
</style>
<!-- 引入 TensorFlow.js 核心库 -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@4.20.0/dist/tf.min.js"></script>
<!-- KNN 分类器 -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/knn-classifier@1.2.2/dist/knn-classifier.min.js"></script>
<!-- !!! 重点：MediaPipe Hands 解决方案文件 -->
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/hands"></script>
<!-- TensorFlow Models - Hand Pose Detection 库 -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/hand-pose-detection"></script>
</head>
<body>
<h1>AI 空气钢琴</h1>
<p class="subtitle">通过手势弹奏虚拟钢琴！</p>
<div id="main-app">
<!-- 视频流区域 -->
<div id="video-feed-container">
<video id="videoFeed" autoplay muted playsinline></video>
<canvas id="poseCanvas"></canvas>
<div class="status-display" id="globalStatus">正在加载模型，请稍候...</div>
</div>
<!-- 控制面板 -->
<div id="control-panel">
<div class="panel-section">
<h3>模型管理</h3>
<input type="file" id="fileImporter" accept=".json" style="display: none;">
<button id="importModelBtn" class="action-button">导入手势模型</button>
</div>
<div class="panel-section">
<h3>演奏控制</h3>
<button id="startStopBtn" class="action-button" disabled>开始演奏</button>
<div class="info-item" style="margin-top: 15px;">
<span class="info-label">实时手势:</span>
<span class="info-value" id="currentGestureDisplay">未识别</span>
</div>
<div class="info-item">
<span class="info-label">置信度:</span>
<span class="info-value" id="confidenceDisplay">0%</span>
</div>
<div class="info-item">
<span class="info-label">当前演奏音符:</span> <!-- 文本修改为音符 -->
<span class="info-value" id="currentPlayingAction">无</span>
</div>
</div>
<div class="panel-section">
<h3>音符映射</h3> <!-- 文本修改为音符 -->
<p>请确保您的手势分类与音符对应：</p>
<ul class="mapping-list" id="gestureMappingList">
<!-- JS会动态填充此列表 -->
<li><strong>ID 0:</strong> <span class="action-name">中央C (C4)</span> → 音段 1</li>
<li><strong>ID 1:</strong> <span class="action-name">D4</span> → 音段 2</li>
<li><strong>ID 2:</strong> <span class="action-name">E4</span> → 音段 3</li>
<li><strong>ID 3:</strong> <span class="action-name">F4</span> → 音段 4</li>
<li><strong>ID 4:</strong> <span class="action-name">G4</span> → 音段 5</li>
<li><strong>ID 5:</strong> <span class="action-name">A4</span> → 音段 6</li>
<li><strong>ID 6:</strong> <span class="action-name">B4</span> → 音段 7</li>
<li><strong>ID 7:</strong> <span class="action-name">高音C (C5)</span> → 音段 8</li>
</ul>
</div>
</div>
</div>
<script>
// ==========================================================
// 全局变量和 DOM 引用
// ==========================================================
const videoElement = document.getElementById('videoFeed');
const poseCanvas = document.getElementById('poseCanvas');
const poseCtx = poseCanvas.getContext('2d');
const globalStatusDisplay = document.getElementById('globalStatus');
const currentGestureDisplay = document.getElementById('currentGestureDisplay');
const confidenceDisplay = document.getElementById('confidenceDisplay');
const currentPlayingActionDisplay = document.getElementById('currentPlayingAction');
const gestureMappingList = document.getElementById('gestureMappingList');
const importModelBtn = document.getElementById('importModelBtn');
const fileImporter = document.getElementById('fileImporter');
const startStopBtn = document.getElementById('startStopBtn');
let detector; // Hand Pose Detection Detector
let classifier; // KNN 分类器
let isHandDetectionReady = false;
let isModelLoaded = false;
let isPlaying = false;
let animationFrameId;
let currentDetectedClassId = null;
let currentPlayingActionId = null;
// 用于映射手势分类ID到音频文件和音符名称
// !!! IMPORTANT: 请确保这些音频文件存在于 'sounds/' 文件夹中，并且名称正确 !!!
const gestureClassToAudioMap = {
'0': { name: '中央C (C2)', audio: new Audio('sounds/C2.mp3') },
'1': { name: 'D2', audio: new Audio('sounds/D2.mp3') },
'2': { name: 'E2', audio: new Audio('sounds/E2.mp3') },
'3': { name: 'F2', audio: new Audio('sounds/F2.mp3') },
'4': { name: 'G2', audio: new Audio('sounds/G2.mp3') },
'5': { name: 'A2', audio: new Audio('sounds/A2.mp3') },
'6': { name: 'B2', audio: new Audio('sounds/B2.mp3') },
'7': { name: '高音C (C3)', audio: new Audio('sounds/C3.mp3') },
'8': { name: '空', audio: new Audio('sounds/rest.mp3') }
// 您需要在 index.html 中训练 8 个不同的手势，分别对应这些类别ID (0-7)。
// 如果不训练足够多的手势，KNN 分类器将无法预测这些类别。
};
// 可配置项
const MIN_CONFIDENCE_THRESHOLD = 0.30; // 最低置信度阈值 (70%)
// MediaPipe Hands 模型连接点，用于绘制骨骼
const HAND_CONNECTIONS = [
[0, 1], [1, 2], [2, 3], [3, 4], // Thumb (大拇指)
[0, 5], [5, 6], [6, 7], [7, 8], // Index (食指)
[0, 9], [9, 10], [10, 11], [11, 12], // Middle (中指)
[0, 13], [13, 14], [14, 15], [15, 16], // Ring (无名指)
[0, 17], [17, 18], [18, 19], [19, 20], // Pinky (小指)
// 掌心连接 (连接腕部到手指基部，形成手掌轮廓)
[0, 5], [5, 9], [9, 13], [13, 17], [17, 0] // 闭合手掌
];
// ==========================================================
// 初始化函数
// ==========================================================
document.addEventListener('DOMContentLoaded', initApp);
async function initApp() {
updateGlobalStatus('正在加载手势模型和摄像头...');
lockControls(true);
try {
// 初始化 KNN 分类器
classifier = knnClassifier.create();
// --- 切换为手部检测器 ---
const model = handPoseDetection.SupportedModels.MediaPipeHands;
const detectorConfig = {
runtime: 'mediapipe',
solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/hands'
};
detector = await handPoseDetection.createDetector(model, detectorConfig);
await setupCamera();
isHandDetectionReady = true;
updateGlobalStatus('手部检测器和摄像头已就绪。请导入您的手势模型。', 'ready');
lockControls(false);
startStopBtn.disabled = true;
// 启动手势检测循环
startDetectionLoop();
// 绑定按钮事件
importModelBtn.addEventListener('click', () => fileImporter.click());
fileImporter.addEventListener('change', handleModelImport);
startStopBtn.addEventListener('click', togglePlaying);
// 预加载所有音频
preloadAudios();
// 初始更新映射UI，显示所有8个音符的映射
updateGestureMappingUI();
} catch (error) {
console.error("应用初始化失败:", error);
updateGlobalStatus(`初始化失败: ${error.message}`, 'error');
alert(`应用初始化失败: ${error.message}\n请检查摄像头权限、网络连接或刷新页面。`);
}
}
// ==========================================================
// 摄像头和手部检测相关
// ==========================================================
async function setupCamera() {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: {
width: { ideal: videoElement.width || 480 },
height: { ideal: videoElement.height || 360 },
facingMode: 'user'
}
});
videoElement.srcObject = stream;
return new Promise((resolve, reject) => {
videoElement.onloadedmetadata = () => {
videoElement.play().then(() => {
poseCanvas.width = videoElement.videoWidth;
poseCanvas.height = videoElement.videoHeight;
resolve();
}).catch(reject);
};
setTimeout(() => reject(new Error('摄像头元数据加载或播放超时')), 10000);
});
} catch (error) {
if (error.name === 'NotAllowedError') {
throw new Error('用户拒绝了摄像头权限。');
} else if (error.name === 'NotFoundError') {
throw new Error('未找到摄像头设备。');
} else {
throw error;
}
}
}
async function startDetectionLoop() {
if (!isHandDetectionReady) return;
async function detectAndPredict() {
try {
const hands = await detector.estimateHands(videoElement, { flipHorizontal: false });
poseCtx.clearRect(0, 0, poseCanvas.width, poseCanvas.height);
// 声明并初始化 currentConfidencePercentage
let currentConfidencePercentage = "0"; // 默认值为字符串 "0"，避免未定义错误
if (hands && hands.length > 0) {
drawHand(hands[0]);
if (isModelLoaded && isPlaying) {
const handTensor = flattenHand(hands[0]);
if (classifier.getNumClasses() > 0) {
const prediction = await classifier.predictClass(handTensor);
handTensor.dispose();
const predictedClassId = prediction.label;
const confidence = prediction.confidences[predictedClassId];
// 始终计算并更新 currentConfidencePercentage
currentConfidencePercentage = (confidence * 100).toFixed(1);
if (confidence > MIN_CONFIDENCE_THRESHOLD) {
currentDetectedClassId = predictedClassId;
const noteInfo = gestureClassToAudioMap[predictedClassId];
if (noteInfo) {
// 这里使用 currentConfidencePercentage
currentGestureDisplay.textContent = `${noteInfo.name} (C:${currentConfidencePercentage}%)`;
if (currentPlayingActionId !== predictedClassId) {
playAudioForNote(predictedClassId, noteInfo.audio, noteInfo.name);
currentPlayingActionId = predictedClassId;
}
} else {
currentGestureDisplay.textContent = `未知音符 (ID:${predictedClassId})`;
currentPlayingActionId = null;
currentPlayingActionDisplay.textContent = '无';
}
} else {
// 置信度不足
currentDetectedClassId = null;
// 这里也使用 currentConfidencePercentage
currentGestureDisplay.textContent = `未识别 (C:${currentConfidencePercentage}%)`;
if(currentPlayingActionId !== null) {
stopAllPlayingAudios();
currentPlayingActionId = null;
currentPlayingActionDisplay.textContent = '无';
}
}
} else {
// 分类器没有数据
handTensor.dispose();
currentDetectedClassId = null;
currentGestureDisplay.textContent = '模型无数据';
currentPlayingActionId = null;
currentPlayingActionDisplay.textContent = '无';
stopAllPlayingAudios();
}
} else {
// 非演奏模式或未加载模型
currentDetectedClassId = null;
currentGestureDisplay.textContent = '静止';
currentPlayingActionId = null;
currentPlayingActionDisplay.textContent = '无';
stopAllPlayingAudios();
}
} else {
// 未检测到手部
currentDetectedClassId = null;
currentGestureDisplay.textContent = '请将手放入画面';
currentPlayingActionId = null;
currentPlayingActionDisplay.textContent = '无';
stopAllPlayingAudios();
}
// 无论如何，确保 confidenceDisplay 总是更新
confidenceDisplay.textContent = `${currentConfidencePercentage}%`;
} catch (error) {
console.error('手势检测或预测出错:', error);
updateGlobalStatus(`检测错误: ${error.message}`, 'error');
} finally {
animationFrameId = requestAnimationFrame(detectAndPredict);
}
}
animationFrameId = requestAnimationFrame(detectAndPredict);
}
function flattenHand(hand) {
const keypoints = hand.keypoints.map(p => [p.x / videoElement.videoWidth, p.y / videoElement.videoHeight]).flat();
return tf.tensor(keypoints);
}
function drawHand(hand) {
if (hand.keypoints) {
const keypoints = hand.keypoints;
poseCtx.strokeStyle = '#3498db'; /* 蓝色连接线 */
poseCtx.lineWidth = 3;
poseCtx.shadowColor = '#3498db';
poseCtx.shadowBlur = 5;
for (const connection of HAND_CONNECTIONS) {
const start = keypoints[connection[0]];
const end = keypoints[connection[1]];
if (start && end) {
poseCtx.beginPath();
poseCtx.moveTo(start.x, start.y);
poseCtx.lineTo(end.x, end.y);
poseCtx.stroke();
}
}
poseCtx.fillStyle = '#f1c40f'; /* 黄色关键点 */
poseCtx.shadowColor = '#f1c40f';
poseCtx.shadowBlur = 8;
for (const keypoint of keypoints) {
if (keypoint) {
poseCtx.beginPath();
poseCtx.arc(keypoint.x, keypoint.y, 5, 0, 2 * Math.PI);
poseCtx.fill();
}
}
poseCtx.shadowBlur = 0;
poseCtx.shadowColor = 'transparent';
}
}
// ==========================================================
// 模型导入
// ==========================================================
async function handleModelImport(event) {
const file = event.target.files[0];
if (!file) return;
updateGlobalStatus('正在导入模型...', 'loading');
lockControls(true);
try {
await loadModelFromFile(file);
updateGlobalStatus('手势模型导入成功！', 'success');
isModelLoaded = true;
lockControls(false);
importModelBtn.disabled = true;
startStopBtn.disabled = false;
} catch (error) {
console.error('模型导入失败:', error);
updateGlobalStatus(`模型导入失败: ${error.message}`, 'error');
alert(`模型导入失败: ${error.message}\n请确保文件是正确的模型JSON文件。`);
isModelLoaded = false;
lockControls(false);
startStopBtn.disabled = true;
importModelBtn.disabled = false;
} finally {
fileImporter.value = '';
}
}
async function loadModelFromFile(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = async (e) => {
try {
const loadedModelData = JSON.parse(e.target.result);
// 确保模型包含 classMap 和 dataset
if (!loadedModelData || !loadedModelData.dataset || !loadedModelData.classMap) {
throw new Error('模型数据结构不正确 (缺少 classMap 或 dataset)。');
}
classifier.clearAllClasses();
const dataset = {};
let totalExamples = 0;
for (const classId in loadedModelData.dataset) {
const classData = loadedModelData.dataset[classId];
if (classData && classData.length > 0) {
if (classData.some(sample => !Array.isArray(sample) || sample.some(val => typeof val !== 'number'))) {
throw new Error(`类别 ${classId} 包含无效的样本数据 (不是数字数组)。`);
}
const tensors = classData.map(data => tf.tensor1d(data));
const stacked = tf.stack(tensors);
dataset[classId] = stacked;
totalExamples += classData.length;
tensors.forEach(t => t.dispose());
} else {
console.warn(`类别 ${classId} 没有样本数据。`);
}
}
classifier.setClassifierDataset(dataset);
// 您可以在这里遍历 loadedModelData.classMap 来更新 `gestureClassToAudioMap` 中的 `name` 字段
// 以便 UI 上的音符名称直接从训练模型中获取。
// 当前代码是直接使用 `gestureClassToAudioMap` 中预设的音符名称。
updateGestureMappingUI(); // 再次调用以确保UI更新
console.log(`模型加载成功！共加载 ${totalExamples} 个样本。`);
console.log('导入类别映射:', loadedModelData.classMap);
resolve();
} catch (error) {
reject(error);
}
};
reader.onerror = (error) => {
reject(new Error('文件读取失败。'));
};
reader.readAsText(file);
});
}
// ==========================================================
// 音频播放逻辑
// ==========================================================
function preloadAudios() {
for (const key in gestureClassToAudioMap) {
const audioObj = gestureClassToAudioMap[key].audio;
if (audioObj) {
audioObj.load();
console.log(`预加载音频: ${audioObj.src}`);
}
}
}
// 播放单个音符
function playAudioForNote(noteId, audioObj, noteName) {
// 每次播放新音符时，停止之前所有可能正在播放的音符
stopAllPlayingAudios();
// 确保音频从头开始播放
if (audioObj.readyState >= 2) {
audioObj.currentTime = 0;
}
audioObj.play().catch(e => {
console.warn('音频播放失败 (可能需要用户交互):', e);
// 可以在这里提示用户点击任意地方进行交互，以便后续播放音频
// 例如：document.body.addEventListener('click', () => audio.play(), { once: true });
});
currentPlayingActionDisplay.textContent = noteName;
}
// 停止所有正在播放的音频
function stopAllPlayingAudios() {
for (const key in gestureClassToAudioMap) {
const audioObj = gestureClassToAudioMap[key].audio;
if (audioObj && !audioObj.paused) {
audioObj.pause();
audioObj.currentTime = 0; // 重置到开始
}
}
}
// ==========================================================
// UI 更新和控制
// ==========================================================
function updateGlobalStatus(message, type = 'info') {
globalStatusDisplay.textContent = message;
if (type === 'error') {
globalStatusDisplay.style.color = '#e74c3c'; /* 红色 */
} else if (type === 'success') {
globalStatusDisplay.style.color = '#2ecc71'; /* 绿色 */
} else {
globalStatusDisplay.style.color = '#bdc3c7'; /* 默认灰色 */
}
}
function lockControls(lock) {
importModelBtn.disabled = lock;
startStopBtn.disabled = lock;
}
function togglePlaying() {
if (!isModelLoaded) {
alert('请先导入手势模型！');
return;
}
if (classifier.getNumClasses() === 0) {
alert('已导入的模型中没有训练数据，请导入一个有效的模型文件。');
return;
}
if (classifier.getNumClasses() < Object.keys(gestureClassToAudioMap).length) {
alert(`警告：导入的模型只包含 ${classifier.getNumClasses()} 个类别，但需要 ${Object.keys(gestureClassToAudioMap).length} 个音符手势。请确保导入完整的模型！`);
// 允许继续，但用户会发现部分音符无法弹奏
}
isPlaying = !isPlaying;
if (isPlaying) {
startStopBtn.textContent = '停止演奏';
startStopBtn.classList.add('playing');
importModelBtn.disabled = true;
updateGlobalStatus('开始演奏，请摆出您的钢琴手势！', 'info'); // 文本修改
currentPlayingActionDisplay.textContent = '无';
} else {
startStopBtn.textContent = '开始演奏';
startStopBtn.classList.remove('playing');
importModelBtn.disabled = false;
updateGlobalStatus('已停止演奏，等待您开始。', 'ready');
currentPlayingActionDisplay.textContent = '无';
currentPlayingActionId = null; // 停止演奏时重置当前播放音符ID
stopAllPlayingAudios(); // 停止所有正在播放的音频
}
currentDetectedClassId = null;
// currentConfidence 在 detectAndPredict 循环中更新
currentGestureDisplay.textContent = '静止';
confidenceDisplay.textContent = '0%';
}
function updateGestureMappingUI() {
gestureMappingList.innerHTML = '';
// 这种情况下，我们假设 mapping 列表应该显示所有预设的8个音符
const orderedNoteNames = [
'中央C (C4)', 'D4', 'E4', 'F4', 'G4', 'A4', 'B4', '高音C (C5)'
];
for (let i = 0; i < orderedNoteNames.length; i++) {
const classId = String(i); // 确保 classId 是字符串以匹配 map keys
const noteName = orderedNoteNames[i];
const listItem = document.createElement('li');
listItem.innerHTML = `<strong>ID ${classId}:</strong> <span class="action-name">${noteName}</span> → 音段 ${i + 1}`;
gestureMappingList.appendChild(listItem);
}
}
// --- 应用启动和清理 ---
window.onbeforeunload = () => {
if (animationFrameId) {
cancelAnimationFrame(animationFrameId);
}
if (detector) {
detector.dispose();
}
if (classifier) {
classifier.clearAllClasses();
}
tf.disposeAll();
stopAllPlayingAudios(); // 页面关闭时停止所有音频
console.log('Resources cleaned up.');
};
</script>
</body>
</html>

BIN
game/钢琴/sounds/A2.mp3 Normal file
View File

Binary file not shown.

BIN
game/钢琴/sounds/B2.mp3 Normal file
View File

Binary file not shown.

BIN
game/钢琴/sounds/C2.mp3 Normal file
View File

Binary file not shown.

BIN
game/钢琴/sounds/C3.mp3 Normal file
View File

Binary file not shown.

BIN
game/钢琴/sounds/D2.mp3 Normal file
View File

Binary file not shown.

BIN
game/钢琴/sounds/E2.mp3 Normal file
View File

Binary file not shown.

BIN
game/钢琴/sounds/F2.mp3 Normal file
View File

Binary file not shown.

BIN
game/钢琴/sounds/G2.mp3 Normal file
View File

Binary file not shown.

137
姿态分类/script.js
View File

@ -1,15 +1,17 @@
/** /**
* ============================================================================= * =============================================================================
* 动态版 - 姿态识别与模型管理脚本 (v2.0) * 动态版 - 姿态识别与模型管理脚本 (v2.1)
* - 新增自动采集样本功能
* ============================================================================= * =============================================================================
* 功能列表: * 功能列表:
* - 实时姿态检测 (MoveNet) * - 实时姿态检测 (MoveNet)
* - KNN 分类器训练 * - KNN 分类器训练
* - 实时姿态预测 * - 实时姿态预测
* - 坐标完美对齐 (Canvas与Video重叠) * - 坐标完美对齐 (Canvas与Video重叠)
* - ✅ 动态添加/删除/重命名姿态类别 * - 动态添加/删除/重命名姿态类别
* - ✅ 模型导出为包含类别信息的 JSON 文件 * - 模型导出为包含类别信息的 JSON 文件
* - ✅ 从 JSON 文件导入模型并恢复类别状态 * - 从 JSON 文件导入模型并恢复类别状态
* - ✅ 新增：自动采集10次样本，间隔0.3秒
* ============================================================================= * =============================================================================
*/ */
@ -32,6 +34,7 @@ const fileImporter = document.getElementById('file-importer');
let detector, classifier, animationFrameId; let detector, classifier, animationFrameId;
let isPredicting = false; let isPredicting = false;
let isAutoCollecting = false; // 新增：标记是否正在进行自动采集
// 📌 核心状态管理: 使用一个对象来管理所有动态状态 // 📌 核心状态管理: 使用一个对象来管理所有动态状态
const appState = { const appState = {
@ -105,12 +108,14 @@ function createClassUI(classId, className) {
poseClassDiv.className = 'pose-class'; poseClassDiv.className = 'pose-class';
poseClassDiv.dataset.classId = classId; poseClassDiv.dataset.classId = classId;
// 📌 修改这里：添加 btn-auto-sample 按钮
poseClassDiv.innerHTML = ` poseClassDiv.innerHTML = `
<div class="class-info"> <div class="class-info">
<input type="text" class="class-name-input" value="${className}" data-class-id="${classId}"> <input type="text" class="class-name-input" value="${className}" data-class-id="${classId}">
<span class="sample-count">(0 样本)</span> <span class="sample-count">(0 样本)</span>
</div> </div>
<div class="class-actions"> <div class="class-actions">
<button class="btn-auto-sample" data-class-id="${classId}">自动采集</button>
<button class="btn-sample" data-class-id="${classId}">采集样本</button> <button class="btn-sample" data-class-id="${classId}">采集样本</button>
<button class="btn-delete-class" title="删除类别" data-class-id="${classId}">×</button> <button class="btn-delete-class" title="删除类别" data-class-id="${classId}">×</button>
</div> </div>
@ -124,10 +129,17 @@ function createClassUI(classId, className) {
appState.classMap[classId] = e.target.value; appState.classMap[classId] = e.target.value;
}); });
const autoSampleButton = poseClassDiv.querySelector('.btn-auto-sample'); // 新增
autoSampleButton.addEventListener('click', () => toggleAutoCollection(classId, autoSampleButton)); // 新增
const sampleButton = poseClassDiv.querySelector('.btn-sample'); const sampleButton = poseClassDiv.querySelector('.btn-sample');
sampleButton.addEventListener('click', () => addExample(classId)); sampleButton.addEventListener('click', () => addExample(classId));
if (isPredicting) sampleButton.disabled = true; // 如果在预测中，禁用新按钮 // 初始化时根据预测状态禁用按钮
if (isPredicting) {
sampleButton.disabled = true;
autoSampleButton.disabled = true; // 新增
}
const deleteButton = poseClassDiv.querySelector('.btn-delete-class'); const deleteButton = poseClassDiv.querySelector('.btn-delete-class');
deleteButton.addEventListener('click', () => deleteClass(classId)); deleteButton.addEventListener('click', () => deleteClass(classId));
@ -177,11 +189,81 @@ async function addExample(classId) {
updateSampleCounts(); updateSampleCounts();
checkExportAbility(); checkExportAbility();
console.log(`为类别 ${appState.classMap[classId]} 采集1个样本。`);
return true; // 表示采集成功
} else { } else {
console.warn(`为类别 ${appState.classMap[classId]} 采集样本失败，未检测到姿态。`); console.warn(`为类别 ${appState.classMap[classId]} 采集样本失败，未检测到姿态。`);
return false; // 表示采集失败
} }
} }
// --- 新增：自动采集逻辑 ---
let autoCollectionIntervalId = null; // 用于存储 setInterval ID
let autoCollectionCount = 0; // 计数器
const AUTO_COLLECTION_TOTAL = 10; // 总共采集次数
const AUTO_COLLECTION_INTERVAL = 300; // 间隔时间 0.3 秒
async function toggleAutoCollection(classId, buttonElement) {
if (isAutoCollecting) {
// 如果正在自动采集，则停止
stopAutoCollection(buttonElement);
} else {
// 否则，开始自动采集
startAutoCollection(classId, buttonElement);
}
}
async function startAutoCollection(classId, buttonElement) {
isAutoCollecting = true;
autoCollectionCount = 0;
// 禁用其他采集和预测按钮
predictButton.disabled = true;
exportButton.disabled = true;
importButton.disabled = true;
addClassButton.disabled = true;
document.querySelectorAll('.btn-sample, .btn-auto-sample, .btn-delete-class, .class-name-input').forEach(btn => {
if (btn !== buttonElement) { // 不禁用当前自动采集按钮
btn.disabled = true;
}
if (btn.classList.contains('class-name-input')) btn.disabled = true;
});
buttonElement.innerText = `停止采集 (0/${AUTO_COLLECTION_TOTAL})`;
buttonElement.classList.add('stop'); // 添加停止样式
const performCollection = async () => {
if (autoCollectionCount < AUTO_COLLECTION_TOTAL) {
const success = await addExample(classId); // 调用手动采集功能
if (success) {
autoCollectionCount++;
}
buttonElement.innerText = `停止采集 (${autoCollectionCount}/${AUTO_COLLECTION_TOTAL})`;
} else {
stopAutoCollection(buttonElement);
alert(`类别 "${appState.classMap[classId]}" 自动采集完成！`);
}
};
// 立即执行一次，然后设置定时器
await performCollection();
if (autoCollectionCount < AUTO_COLLECTION_TOTAL) {
autoCollectionIntervalId = setInterval(performCollection, AUTO_COLLECTION_INTERVAL);
}
}
function stopAutoCollection(buttonElement) {
clearInterval(autoCollectionIntervalId);
autoCollectionIntervalId = null;
isAutoCollecting = false;
buttonElement.innerText = '自动采集';
buttonElement.classList.remove('stop'); // 移除停止样式
// 重新启用按钮（根据应用状态）
updatePredictionUI(); // 根据预测状态重新启用/禁用相关按钮
enableControls(); // 重新启用添加类别、导出、导入按钮
}
// --- 模型与预测逻辑 --- // --- 模型与预测逻辑 ---
/** /**
@ -205,16 +287,20 @@ async function mainLoop() {
if (poses && poses.length > 0) { if (poses && poses.length > 0) {
drawPose(poses[0]); drawPose(poses[0]);
if (isPredicting && classifier.getNumClasses() > 0) { // 只有当不在自动采集状态时才进行预测
if (isPredicting && classifier.getNumClasses() > 0 && !isAutoCollecting) {
const poseTensor = flattenPose(poses[0]); const poseTensor = flattenPose(poses[0]);
const result = await classifier.predictClass(poseTensor, 3); const result = await classifier.predictClass(poseTensor, 3);
poseTensor.dispose(); poseTensor.dispose();
const confidence = Math.round(result.confidences[result.label] * 100); const confidence = Math.round(result.confidences[result.label] * 100);
// 📌 动态获取类别名称
const predictedClassName = appState.classMap[result.label] || '未知类别'; const predictedClassName = appState.classMap[result.label] || '未知类别';
resultElement.innerText = `姿态: ${predictedClassName} (${confidence}%)`; resultElement.innerText = `姿态: ${predictedClassName} (${confidence}%)`;
} else if (isAutoCollecting) {
resultElement.innerText = "自动采集中...";
} }
} else {
resultElement.innerText = "未检测到姿态";
} }
animationFrameId = requestAnimationFrame(mainLoop); animationFrameId = requestAnimationFrame(mainLoop);
} }
@ -237,7 +323,6 @@ function exportModel() {
datasetObj[key] = data.arraySync(); datasetObj[key] = data.arraySync();
}); });
// 📌 导出格式大更新: 同时保存 classMap 和 dataset
const modelData = { const modelData = {
classMap: appState.classMap, classMap: appState.classMap,
dataset: datasetObj dataset: datasetObj
@ -268,7 +353,6 @@ function importModel(event) {
try { try {
const modelData = JSON.parse(e.target.result); const modelData = JSON.parse(e.target.result);
// 📌 导入格式验证
if (!modelData.classMap || !modelData.dataset) { if (!modelData.classMap || !modelData.dataset) {
throw new Error("无效的模型文件格式。"); throw new Error("无效的模型文件格式。");
} }
@ -364,40 +448,53 @@ function updateSampleCounts() {
* 根据状态更新UI * 根据状态更新UI
*/ */
function updatePredictionUI() { function updatePredictionUI() {
const allActionButtons = document.querySelectorAll('.btn-sample, .btn-delete-class, .btn-add-class, #btn-import'); // 禁用所有采集按钮（包括手动和自动）和删除按钮
document.querySelectorAll('.btn-sample, .btn-auto-sample, .btn-delete-class').forEach(btn => btn.disabled = isPredicting || isAutoCollecting);
// 禁用添加类别和导入模型的按钮
addClassButton.disabled = isPredicting || isAutoCollecting;
importButton.disabled = isPredicting || isAutoCollecting;
// 禁用类别名称输入框
document.querySelectorAll('.class-name-input').forEach(input => input.disabled = isPredicting || isAutoCollecting);
if (isPredicting) { if (isPredicting) {
predictButton.innerText = "停止预测"; predictButton.innerText = "停止预测";
predictButton.classList.add('stop'); predictButton.classList.add('stop');
resultElement.innerText = "正在分析..."; resultElement.innerText = "正在分析...";
allActionButtons.forEach(btn => btn.disabled = true);
document.querySelectorAll('.class-name-input').forEach(input => input.disabled = true);
checkExportAbility();
} else { } else {
predictButton.innerText = "开始预测"; predictButton.innerText = "开始预测";
predictButton.classList.remove('stop'); predictButton.classList.remove('stop');
resultElement.innerText = "已停止"; resultElement.innerText = "已停止";
allActionButtons.forEach(btn => btn.disabled = false);
document.querySelectorAll('.class-name-input').forEach(input => input.disabled = false);
checkExportAbility();
} }
// 只有在有类别且有样本时才能预测 // 只有在有类别且有样本时才能预测
predictButton.disabled = isPredicting ? false : classifier.getNumClasses() === 0; predictButton.disabled = isPredicting ? false : classifier.getNumClasses() === 0 || isAutoCollecting;
checkExportAbility();
} }
/**
* 通用启用/禁用控件 (在自动采集停止后调用)
*/
function enableControls() { function enableControls() {
[predictButton, importButton, exportButton, addClassButton].forEach(btn => btn.disabled = false); // 重新评估所有按钮的状态
checkExportAbility(); // 自动采集按钮的状态由其自身管理
predictButton.disabled = classifier.getNumClasses() === 0;
importButton.disabled = false; // 导入按钮总是可以手动启用
addClassButton.disabled = false;
checkExportAbility(); // 重新检查导出按钮
updatePredictionUI(); // 再次调用，确保其他按钮状态正确
} }
/** 检查是否可以导出模型并更新按钮状态 */ /** 检查是否可以导出模型并更新按钮状态 */
function checkExportAbility() { function checkExportAbility() {
exportButton.disabled = isPredicting || classifier.getNumClasses() === 0; exportButton.disabled = isPredicting || classifier.getNumClasses() === 0 || isAutoCollecting;
} }
function cleanup() { function cleanup() {
if (detector) detector.dispose(); if (detector) detector.dispose();
if (classifier) classifier.clearAllClasses(); if (classifier) classifier.clearAllClasses();
if (animationFrameId) cancelAnimationFrame(animationFrameId); if (animationFrameId) cancelAnimationFrame(animationFrameId);
if (autoCollectionIntervalId) clearInterval(autoCollectionIntervalId); // 清理自动采集定时器
} }
// --- 启动应用 --- // --- 启动应用 ---

31
姿态分类/style.css
View File

@ -169,7 +169,7 @@ h3 {
.class-actions { .class-actions {
display: flex; display: flex;
align-items: center; align-items: center;
gap: 0.5rem; /* 按钮间距 */ gap: 5px; /* 按钮间距 */
} }
/* 📌 新增: 删除按钮样式 */ /* 📌 新增: 删除按钮样式 */
@ -254,3 +254,32 @@ h3 {
display: flex; display: flex;
gap: 1rem; gap: 1rem;
} }
.btn-auto-sample {
padding: 8px 15px;
font-size: 0.9em;
background-color: #5cb85c; /* 绿色 */
color: white;
border: none;
border-radius: 4px;
cursor: pointer;
transition: background-color 0.2s ease;
margin-right: 8px; /* 与手动采集按钮之间留出间距 */
}
.btn-auto-sample:hover {
background-color: #4cae4c;
}
.btn-auto-sample:disabled {
background-color: #cccccc;
cursor: not-allowed;
}
/* 添加一个停止按钮样式 */
.btn-auto-sample.stop {
background-color: #d9534f; /* 红色 */
}
.btn-auto-sample.stop:hover {
background-color: #c9302c;
}