887 lines
31 KiB
HTML
887 lines
31 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>Goood Space - AI 空气钢琴</title>
|
||
<style>
|
||
* {
|
||
margin: 0;
|
||
padding: 0;
|
||
box-sizing: border-box;
|
||
}
|
||
|
||
body {
|
||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||
min-height: 100vh;
|
||
background: linear-gradient(135deg, #0a0e27 0%, #1a1f3a 50%, #2d1b69 100%);
|
||
color: #ffffff;
|
||
display: flex;
|
||
flex-direction: column;
|
||
align-items: center;
|
||
padding: 2rem;
|
||
overflow-x: hidden;
|
||
}
|
||
|
||
/* Background Animation */
|
||
body::before {
|
||
content: '';
|
||
position: fixed;
|
||
width: 200%;
|
||
height: 200%;
|
||
top: -50%;
|
||
left: -50%;
|
||
background: radial-gradient(circle, rgba(255, 170, 0, 0.1) 0%, transparent 70%);
|
||
animation: rotate 40s linear infinite;
|
||
pointer-events: none;
|
||
z-index: -1;
|
||
}
|
||
|
||
body::after {
|
||
content: '';
|
||
position: fixed;
|
||
width: 150%;
|
||
height: 150%;
|
||
top: -25%;
|
||
left: -25%;
|
||
background: radial-gradient(circle, rgba(241, 196, 15, 0.05) 0%, transparent 60%);
|
||
animation: rotate-reverse 35s linear infinite;
|
||
pointer-events: none;
|
||
z-index: -1;
|
||
}
|
||
|
||
@keyframes rotate {
|
||
from { transform: rotate(0deg); }
|
||
to { transform: rotate(360deg); }
|
||
}
|
||
|
||
@keyframes rotate-reverse {
|
||
from { transform: rotate(360deg); }
|
||
to { transform: rotate(0deg); }
|
||
}
|
||
|
||
/* Header */
|
||
.header {
|
||
text-align: center;
|
||
margin-bottom: 3rem;
|
||
animation: fadeInDown 0.8s ease-out;
|
||
}
|
||
|
||
.brand-title {
|
||
font-size: 3.5rem;
|
||
background: linear-gradient(90deg, #ffaa00, #f1c40f, #00ff88);
|
||
background-clip: text;
|
||
-webkit-background-clip: text;
|
||
-webkit-text-fill-color: transparent;
|
||
text-shadow: 0 0 60px rgba(241, 196, 15, 0.5);
|
||
margin-bottom: 0.5rem;
|
||
}
|
||
|
||
.subtitle {
|
||
font-size: 1.5rem;
|
||
color: #f1c40f;
|
||
text-shadow: 0 0 20px rgba(241, 196, 15, 0.5);
|
||
}
|
||
|
||
.tagline {
|
||
font-size: 1rem;
|
||
color: #a8c0ff;
|
||
margin-top: 0.5rem;
|
||
}
|
||
|
||
@keyframes fadeInDown {
|
||
from {
|
||
opacity: 0;
|
||
transform: translateY(-30px);
|
||
}
|
||
to {
|
||
opacity: 1;
|
||
transform: translateY(0);
|
||
}
|
||
}
|
||
|
||
/* Main Container */
|
||
.main-container {
|
||
display: flex;
|
||
gap: 3rem;
|
||
flex-wrap: wrap;
|
||
justify-content: center;
|
||
max-width: 1400px;
|
||
width: 100%;
|
||
}
|
||
|
||
/* Video Section */
|
||
.video-section {
|
||
position: relative;
|
||
background: linear-gradient(135deg, rgba(255, 255, 255, 0.1) 0%, rgba(255, 255, 255, 0.05) 100%);
|
||
border: 3px solid #f1c40f;
|
||
border-radius: 20px;
|
||
padding: 2rem;
|
||
backdrop-filter: blur(10px);
|
||
box-shadow: 0 0 30px rgba(241, 196, 15, 0.5);
|
||
}
|
||
|
||
#video-container {
|
||
position: relative;
|
||
width: 480px;
|
||
height: 360px;
|
||
border-radius: 15px;
|
||
overflow: hidden;
|
||
background: #000;
|
||
}
|
||
|
||
#videoFeed, #poseCanvas {
|
||
position: absolute;
|
||
top: 0;
|
||
left: 0;
|
||
width: 100%;
|
||
height: 100%;
|
||
border-radius: 15px;
|
||
}
|
||
|
||
#videoFeed {
|
||
z-index: 1;
|
||
}
|
||
|
||
#poseCanvas {
|
||
z-index: 2;
|
||
}
|
||
|
||
.status-display {
|
||
position: absolute;
|
||
bottom: 0;
|
||
left: 0;
|
||
right: 0;
|
||
background: rgba(0, 0, 0, 0.8);
|
||
padding: 1rem;
|
||
text-align: center;
|
||
color: #00ff88;
|
||
border-bottom-left-radius: 15px;
|
||
border-bottom-right-radius: 15px;
|
||
z-index: 3;
|
||
}
|
||
|
||
/* Control Panel */
|
||
.control-panel {
|
||
background: linear-gradient(135deg, rgba(255, 255, 255, 0.1) 0%, rgba(255, 255, 255, 0.05) 100%);
|
||
border: 2px solid rgba(255, 255, 255, 0.1);
|
||
border-radius: 20px;
|
||
padding: 2rem;
|
||
backdrop-filter: blur(10px);
|
||
min-width: 400px;
|
||
}
|
||
|
||
.panel-section {
|
||
background: rgba(0, 0, 0, 0.3);
|
||
border-radius: 15px;
|
||
padding: 1.5rem;
|
||
margin-bottom: 2rem;
|
||
}
|
||
|
||
.panel-section h3 {
|
||
color: #f1c40f;
|
||
margin-bottom: 1rem;
|
||
text-align: center;
|
||
font-size: 1.3rem;
|
||
text-shadow: 0 0 10px rgba(241, 196, 15, 0.5);
|
||
}
|
||
|
||
/* Piano Keys Visual */
|
||
.piano-keys {
|
||
display: flex;
|
||
justify-content: center;
|
||
margin: 1.5rem 0;
|
||
padding: 1rem;
|
||
background: #222;
|
||
border-radius: 10px;
|
||
}
|
||
|
||
.piano-key {
|
||
width: 40px;
|
||
height: 120px;
|
||
background: linear-gradient(to bottom, #fff, #f0f0f0);
|
||
border: 1px solid #ccc;
|
||
border-radius: 0 0 5px 5px;
|
||
margin: 0 2px;
|
||
transition: all 0.1s ease;
|
||
position: relative;
|
||
}
|
||
|
||
.piano-key.active {
|
||
background: linear-gradient(to bottom, #f1c40f, #f39c12);
|
||
transform: translateY(2px);
|
||
box-shadow: 0 0 20px rgba(241, 196, 15, 0.8);
|
||
}
|
||
|
||
.piano-key .note-label {
|
||
position: absolute;
|
||
bottom: 10px;
|
||
left: 50%;
|
||
transform: translateX(-50%);
|
||
color: #333;
|
||
font-size: 0.8rem;
|
||
font-weight: bold;
|
||
}
|
||
|
||
/* Action Button */
|
||
.action-button {
|
||
width: 100%;
|
||
padding: 1.2rem;
|
||
font-size: 1.3rem;
|
||
border: none;
|
||
border-radius: 50px;
|
||
cursor: pointer;
|
||
transition: all 0.3s ease;
|
||
font-weight: bold;
|
||
margin-bottom: 1rem;
|
||
}
|
||
|
||
#startStopBtn {
|
||
background: linear-gradient(45deg, #27ae60, #2ecc71);
|
||
color: white;
|
||
box-shadow: 0 4px 20px rgba(46, 204, 113, 0.3);
|
||
}
|
||
|
||
#startStopBtn.playing {
|
||
background: linear-gradient(45deg, #c0392b, #e74c3c);
|
||
box-shadow: 0 4px 20px rgba(231, 76, 60, 0.3);
|
||
}
|
||
|
||
#startStopBtn:hover:not(:disabled) {
|
||
transform: scale(1.05);
|
||
box-shadow: 0 6px 30px rgba(46, 204, 113, 0.5);
|
||
}
|
||
|
||
#startStopBtn:disabled {
|
||
background: #555;
|
||
color: #999;
|
||
cursor: not-allowed;
|
||
transform: none;
|
||
box-shadow: none;
|
||
}
|
||
|
||
#importModelBtn {
|
||
background: linear-gradient(45deg, #f39c12, #f1c40f);
|
||
color: #333;
|
||
box-shadow: 0 4px 20px rgba(241, 196, 15, 0.3);
|
||
}
|
||
|
||
#importModelBtn:hover:not(:disabled) {
|
||
transform: scale(1.05);
|
||
box-shadow: 0 6px 30px rgba(241, 196, 15, 0.5);
|
||
}
|
||
|
||
#importModelBtn:disabled {
|
||
background: #555;
|
||
color: #999;
|
||
cursor: not-allowed;
|
||
}
|
||
|
||
/* Info Display */
|
||
.info-display {
|
||
display: grid;
|
||
gap: 1rem;
|
||
}
|
||
|
||
.info-item {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
padding: 0.8rem;
|
||
background: rgba(255, 255, 255, 0.05);
|
||
border-radius: 8px;
|
||
border-left: 3px solid #f1c40f;
|
||
}
|
||
|
||
.info-label {
|
||
color: #a8c0ff;
|
||
font-weight: bold;
|
||
}
|
||
|
||
.info-value {
|
||
color: #00ff88;
|
||
font-weight: bold;
|
||
}
|
||
|
||
#currentPlayingNote {
|
||
font-size: 2rem;
|
||
color: #f1c40f;
|
||
text-shadow: 0 0 20px rgba(241, 196, 15, 0.8);
|
||
text-align: center;
|
||
padding: 1rem;
|
||
background: rgba(241, 196, 15, 0.1);
|
||
border-radius: 10px;
|
||
margin-top: 1rem;
|
||
animation: pulseNote 1s infinite alternate;
|
||
}
|
||
|
||
@keyframes pulseNote {
|
||
from { transform: scale(1); opacity: 1; }
|
||
to { transform: scale(1.05); opacity: 0.9; }
|
||
}
|
||
|
||
/* Note Mapping */
|
||
.note-mapping {
|
||
background: rgba(0, 0, 0, 0.5);
|
||
border-radius: 10px;
|
||
padding: 1rem;
|
||
margin-top: 1rem;
|
||
}
|
||
|
||
.note-mapping ul {
|
||
list-style: none;
|
||
padding: 0;
|
||
}
|
||
|
||
.note-mapping li {
|
||
padding: 0.5rem;
|
||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
||
color: #e0f2f7;
|
||
display: flex;
|
||
justify-content: space-between;
|
||
}
|
||
|
||
.note-mapping li:last-child {
|
||
border-bottom: none;
|
||
}
|
||
|
||
.note-mapping .note-id {
|
||
color: #f1c40f;
|
||
font-weight: bold;
|
||
}
|
||
|
||
/* Responsive Design */
|
||
@media (max-width: 1200px) {
|
||
.main-container {
|
||
flex-direction: column;
|
||
align-items: center;
|
||
}
|
||
|
||
#video-container {
|
||
width: 100%;
|
||
max-width: 480px;
|
||
height: auto;
|
||
aspect-ratio: 4/3;
|
||
}
|
||
|
||
.control-panel {
|
||
width: 100%;
|
||
min-width: unset;
|
||
}
|
||
}
|
||
|
||
@media (max-width: 768px) {
|
||
.brand-title {
|
||
font-size: 2.5rem;
|
||
}
|
||
|
||
.subtitle {
|
||
font-size: 1.2rem;
|
||
}
|
||
|
||
.piano-keys {
|
||
overflow-x: auto;
|
||
}
|
||
|
||
.piano-key {
|
||
width: 35px;
|
||
height: 100px;
|
||
}
|
||
}
|
||
</style>
|
||
|
||
<!-- TensorFlow.js 核心库 -->
|
||
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@4.20.0/dist/tf.min.js"></script>
|
||
<!-- KNN 分类器 -->
|
||
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/knn-classifier@1.2.2/dist/knn-classifier.min.js"></script>
|
||
<!-- MediaPipe Hands -->
|
||
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/hands"></script>
|
||
<!-- Hand Pose Detection -->
|
||
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/hand-pose-detection"></script>
|
||
</head>
|
||
<body>
|
||
<div class="header">
|
||
<h1 class="brand-title">Goood Space</h1>
|
||
<p class="subtitle">AI 空气钢琴</p>
|
||
<p class="tagline">通过手势弹奏虚拟钢琴！</p>
|
||
</div>
|
||
|
||
<div class="main-container">
|
||
<div class="video-section">
|
||
<div id="video-container">
|
||
<video id="videoFeed" autoplay muted playsinline></video>
|
||
<canvas id="poseCanvas"></canvas>
|
||
<div class="status-display" id="globalStatus">正在加载模型，请稍候...</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="control-panel">
|
||
<div class="panel-section">
|
||
<h3>🎹 钢琴键盘</h3>
|
||
<div class="piano-keys" id="pianoKeys">
|
||
<div class="piano-key" data-note="0">
|
||
<span class="note-label">C</span>
|
||
</div>
|
||
<div class="piano-key" data-note="1">
|
||
<span class="note-label">D</span>
|
||
</div>
|
||
<div class="piano-key" data-note="2">
|
||
<span class="note-label">E</span>
|
||
</div>
|
||
<div class="piano-key" data-note="3">
|
||
<span class="note-label">F</span>
|
||
</div>
|
||
<div class="piano-key" data-note="4">
|
||
<span class="note-label">G</span>
|
||
</div>
|
||
<div class="piano-key" data-note="5">
|
||
<span class="note-label">A</span>
|
||
</div>
|
||
<div class="piano-key" data-note="6">
|
||
<span class="note-label">B</span>
|
||
</div>
|
||
<div class="piano-key" data-note="7">
|
||
<span class="note-label">C′</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="panel-section">
|
||
<h3>🎮 演奏控制</h3>
|
||
<input type="file" id="fileImporter" accept=".json" style="display: none;">
|
||
<button id="importModelBtn" class="action-button">导入手势模型</button>
|
||
<button id="startStopBtn" class="action-button" disabled>开始演奏</button>
|
||
|
||
<div class="info-display">
|
||
<div class="info-item">
|
||
<span class="info-label">实时手势:</span>
|
||
<span class="info-value" id="currentGestureDisplay">未识别</span>
|
||
</div>
|
||
<div class="info-item">
|
||
<span class="info-label">置信度:</span>
|
||
<span class="info-value" id="confidenceDisplay">0%</span>
|
||
</div>
|
||
</div>
|
||
|
||
<div id="currentPlayingNote">🎵 无</div>
|
||
</div>
|
||
|
||
<div class="panel-section">
|
||
<h3>📋 音符映射</h3>
|
||
<div class="note-mapping">
|
||
<ul id="gestureMappingList">
|
||
<li><span class="note-id">ID 0:</span> 中央C (C2)</li>
|
||
<li><span class="note-id">ID 1:</span> D2</li>
|
||
<li><span class="note-id">ID 2:</span> E2</li>
|
||
<li><span class="note-id">ID 3:</span> F2</li>
|
||
<li><span class="note-id">ID 4:</span> G2</li>
|
||
<li><span class="note-id">ID 5:</span> A2</li>
|
||
<li><span class="note-id">ID 6:</span> B2</li>
|
||
<li><span class="note-id">ID 7:</span> 高音C (C3)</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
// DOM 元素
|
||
const videoElement = document.getElementById('videoFeed');
|
||
const poseCanvas = document.getElementById('poseCanvas');
|
||
const poseCtx = poseCanvas.getContext('2d');
|
||
const globalStatusDisplay = document.getElementById('globalStatus');
|
||
const currentGestureDisplay = document.getElementById('currentGestureDisplay');
|
||
const confidenceDisplay = document.getElementById('confidenceDisplay');
|
||
const currentPlayingNoteDisplay = document.getElementById('currentPlayingNote');
|
||
const pianoKeysContainer = document.getElementById('pianoKeys');
|
||
const pianoKeys = pianoKeysContainer.querySelectorAll('.piano-key');
|
||
const importModelBtn = document.getElementById('importModelBtn');
|
||
const fileImporter = document.getElementById('fileImporter');
|
||
const startStopBtn = document.getElementById('startStopBtn');
|
||
|
||
// 全局变量
|
||
let detector;
|
||
let classifier;
|
||
let isHandDetectionReady = false;
|
||
let isModelLoaded = false;
|
||
let isPlaying = false;
|
||
let animationFrameId;
|
||
let currentPlayingNoteId = null;
|
||
|
||
// 音符映射
|
||
const gestureClassToAudioMap = {
|
||
'0': { name: '中央C (C2)', audio: null },
|
||
'1': { name: 'D2', audio: null },
|
||
'2': { name: 'E2', audio: null },
|
||
'3': { name: 'F2', audio: null },
|
||
'4': { name: 'G2', audio: null },
|
||
'5': { name: 'A2', audio: null },
|
||
'6': { name: 'B2', audio: null },
|
||
'7': { name: '高音C (C3)', audio: null }
|
||
};
|
||
|
||
// 创建音频对象（使用Web Audio API生成音符）
|
||
const audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
||
const noteFrequencies = {
|
||
'0': 130.81, // C2
|
||
'1': 146.83, // D2
|
||
'2': 164.81, // E2
|
||
'3': 174.61, // F2
|
||
'4': 196.00, // G2
|
||
'5': 220.00, // A2
|
||
'6': 246.94, // B2
|
||
'7': 261.63 // C3
|
||
};
|
||
|
||
const MIN_CONFIDENCE_THRESHOLD = 0.30;
|
||
|
||
// MediaPipe Hands 连接点
|
||
const HAND_CONNECTIONS = [
|
||
[0, 1], [1, 2], [2, 3], [3, 4],
|
||
[0, 5], [5, 6], [6, 7], [7, 8],
|
||
[0, 9], [9, 10], [10, 11], [11, 12],
|
||
[0, 13], [13, 14], [14, 15], [15, 16],
|
||
[0, 17], [17, 18], [18, 19], [19, 20],
|
||
[0, 5], [5, 9], [9, 13], [13, 17], [17, 0]
|
||
];
|
||
|
||
// 初始化应用
|
||
async function initApp() {
|
||
updateGlobalStatus('正在加载手势模型和摄像头...');
|
||
|
||
try {
|
||
// 初始化 KNN 分类器
|
||
classifier = knnClassifier.create();
|
||
|
||
// 初始化手部检测器
|
||
const model = handPoseDetection.SupportedModels.MediaPipeHands;
|
||
const detectorConfig = {
|
||
runtime: 'mediapipe',
|
||
solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/hands'
|
||
};
|
||
detector = await handPoseDetection.createDetector(model, detectorConfig);
|
||
|
||
await setupCamera();
|
||
isHandDetectionReady = true;
|
||
|
||
updateGlobalStatus('手部检测器和摄像头已就绪。');
|
||
|
||
// 启动检测循环
|
||
startDetectionLoop();
|
||
|
||
// 绑定事件
|
||
importModelBtn.addEventListener('click', () => fileImporter.click());
|
||
fileImporter.addEventListener('change', (event) => loadKNNModelData(event.target.files[0]));
|
||
startStopBtn.addEventListener('click', togglePlaying);
|
||
|
||
// // 尝试从 CDN 加载模型
|
||
// try {
|
||
// const cdnJsonUrl = 'https://goood-space-assets.oss-cn-beijing.aliyuncs.com/public/models/hand-knn-model.json';
|
||
// await loadKNNModelData(null, cdnJsonUrl);
|
||
// updateGlobalStatus('CDN 手势识别模型加载成功！');
|
||
// isModelLoaded = true;
|
||
// importModelBtn.disabled = true;
|
||
// startStopBtn.disabled = false;
|
||
// } catch (cdnError) {
|
||
// console.warn('CDN 模型加载失败:', cdnError);
|
||
// updateGlobalStatus('请手动导入模型文件');
|
||
// importModelBtn.disabled = false;
|
||
// startStopBtn.disabled = true;
|
||
// }
|
||
|
||
} catch (error) {
|
||
console.error('应用初始化失败:', error);
|
||
updateGlobalStatus('初始化失败: ' + error.message);
|
||
}
|
||
}
|
||
|
||
// 设置摄像头
|
||
async function setupCamera() {
|
||
const stream = await navigator.mediaDevices.getUserMedia({
|
||
video: { width: 480, height: 360, facingMode: 'user' }
|
||
});
|
||
|
||
videoElement.srcObject = stream;
|
||
|
||
return new Promise((resolve) => {
|
||
videoElement.onloadedmetadata = () => {
|
||
videoElement.play();
|
||
poseCanvas.width = videoElement.videoWidth;
|
||
poseCanvas.height = videoElement.videoHeight;
|
||
resolve();
|
||
};
|
||
});
|
||
}
|
||
|
||
// 加载 KNN 模型
|
||
async function loadKNNModelData(file = null, cdnUrl = null) {
|
||
updateGlobalStatus('正在加载模型...');
|
||
|
||
try {
|
||
let loadedModelData;
|
||
|
||
if (file) {
|
||
const reader = new FileReader();
|
||
loadedModelData = await new Promise((resolve, reject) => {
|
||
reader.onload = e => resolve(JSON.parse(e.target.result));
|
||
reader.onerror = () => reject(new Error('文件读取失败'));
|
||
reader.readAsText(file);
|
||
});
|
||
} else if (cdnUrl) {
|
||
const response = await fetch(cdnUrl);
|
||
if (!response.ok) {
|
||
throw new Error('无法从 CDN 加载模型');
|
||
}
|
||
loadedModelData = await response.json();
|
||
} else {
|
||
throw new Error('未提供模型文件或 URL');
|
||
}
|
||
|
||
if (!loadedModelData || !loadedModelData.dataset) {
|
||
throw new Error('模型数据格式不正确');
|
||
}
|
||
|
||
classifier.clearAllClasses();
|
||
|
||
const dataset = {};
|
||
for (const classId in loadedModelData.dataset) {
|
||
const classData = loadedModelData.dataset[classId];
|
||
if (classData && classData.length > 0) {
|
||
const tensors = classData.map(data => tf.tensor1d(data));
|
||
dataset[classId] = tf.stack(tensors);
|
||
tensors.forEach(t => t.dispose());
|
||
}
|
||
}
|
||
|
||
classifier.setClassifierDataset(dataset);
|
||
|
||
isModelLoaded = true;
|
||
importModelBtn.disabled = true;
|
||
startStopBtn.disabled = false;
|
||
updateGlobalStatus('手势模型加载成功！');
|
||
|
||
} catch (error) {
|
||
console.error('模型加载失败:', error);
|
||
updateGlobalStatus('模型加载失败: ' + error.message);
|
||
isModelLoaded = false;
|
||
importModelBtn.disabled = false;
|
||
startStopBtn.disabled = true;
|
||
throw error;
|
||
} finally {
|
||
fileImporter.value = '';
|
||
}
|
||
}
|
||
|
||
// 检测循环
|
||
async function startDetectionLoop() {
|
||
if (!isHandDetectionReady) return;
|
||
|
||
async function detectAndPredict() {
|
||
try {
|
||
const hands = await detector.estimateHands(videoElement, { flipHorizontal: false });
|
||
poseCtx.clearRect(0, 0, poseCanvas.width, poseCanvas.height);
|
||
|
||
let currentConfidencePercentage = "0";
|
||
|
||
if (hands && hands.length > 0) {
|
||
drawHand(hands[0]);
|
||
|
||
if (isModelLoaded && isPlaying) {
|
||
const handTensor = flattenHand(hands[0]);
|
||
if (classifier.getNumClasses() > 0) {
|
||
const prediction = await classifier.predictClass(handTensor);
|
||
handTensor.dispose();
|
||
|
||
const predictedClassId = prediction.label;
|
||
const confidence = prediction.confidences[predictedClassId];
|
||
currentConfidencePercentage = (confidence * 100).toFixed(1);
|
||
|
||
if (confidence > MIN_CONFIDENCE_THRESHOLD) {
|
||
const noteInfo = gestureClassToAudioMap[predictedClassId];
|
||
if (noteInfo) {
|
||
currentGestureDisplay.textContent = noteInfo.name;
|
||
if (currentPlayingNoteId !== predictedClassId) {
|
||
playNote(predictedClassId, noteInfo.name);
|
||
currentPlayingNoteId = predictedClassId;
|
||
}
|
||
} else {
|
||
currentGestureDisplay.textContent = '未知音符';
|
||
stopAllNotes();
|
||
}
|
||
} else {
|
||
currentGestureDisplay.textContent = '未识别';
|
||
if (currentPlayingNoteId !== null) {
|
||
stopAllNotes();
|
||
}
|
||
}
|
||
} else {
|
||
handTensor.dispose();
|
||
currentGestureDisplay.textContent = '模型无数据';
|
||
stopAllNotes();
|
||
}
|
||
} else {
|
||
currentGestureDisplay.textContent = '静止';
|
||
stopAllNotes();
|
||
}
|
||
} else {
|
||
currentGestureDisplay.textContent = '请将手放入画面';
|
||
stopAllNotes();
|
||
}
|
||
|
||
confidenceDisplay.textContent = currentConfidencePercentage + '%';
|
||
} catch (error) {
|
||
console.error('检测错误:', error);
|
||
}
|
||
|
||
animationFrameId = requestAnimationFrame(detectAndPredict);
|
||
}
|
||
|
||
animationFrameId = requestAnimationFrame(detectAndPredict);
|
||
}
|
||
|
||
// 展平手部数据
|
||
function flattenHand(hand) {
|
||
const keypoints = hand.keypoints.map(p => [
|
||
p.x / videoElement.videoWidth,
|
||
p.y / videoElement.videoHeight
|
||
]).flat();
|
||
return tf.tensor(keypoints);
|
||
}
|
||
|
||
// 绘制手部
|
||
function drawHand(hand) {
|
||
// 绘制连接线
|
||
poseCtx.strokeStyle = '#3498db';
|
||
poseCtx.lineWidth = 3;
|
||
poseCtx.shadowColor = '#3498db';
|
||
poseCtx.shadowBlur = 5;
|
||
|
||
for (const connection of HAND_CONNECTIONS) {
|
||
const start = hand.keypoints[connection[0]];
|
||
const end = hand.keypoints[connection[1]];
|
||
|
||
poseCtx.beginPath();
|
||
poseCtx.moveTo(start.x, start.y);
|
||
poseCtx.lineTo(end.x, end.y);
|
||
poseCtx.stroke();
|
||
}
|
||
|
||
// 绘制关键点
|
||
poseCtx.fillStyle = '#f1c40f';
|
||
poseCtx.shadowColor = '#f1c40f';
|
||
poseCtx.shadowBlur = 8;
|
||
|
||
for (const keypoint of hand.keypoints) {
|
||
poseCtx.beginPath();
|
||
poseCtx.arc(keypoint.x, keypoint.y, 5, 0, 2 * Math.PI);
|
||
poseCtx.fill();
|
||
}
|
||
|
||
poseCtx.shadowBlur = 0;
|
||
}
|
||
|
||
// 播放音符（使用 Web Audio API）
|
||
let currentOscillator = null;
|
||
let currentGainNode = null;
|
||
|
||
function playNote(noteId, noteName) {
|
||
// 停止之前的音符
|
||
stopAllNotes();
|
||
|
||
// 创建振荡器
|
||
currentOscillator = audioContext.createOscillator();
|
||
currentGainNode = audioContext.createGain();
|
||
|
||
// 设置频率
|
||
currentOscillator.frequency.value = noteFrequencies[noteId] || 440;
|
||
currentOscillator.type = 'sine';
|
||
|
||
// 设置音量包络
|
||
currentGainNode.gain.setValueAtTime(0, audioContext.currentTime);
|
||
currentGainNode.gain.linearRampToValueAtTime(0.3, audioContext.currentTime + 0.01);
|
||
currentGainNode.gain.exponentialRampToValueAtTime(0.01, audioContext.currentTime + 1);
|
||
|
||
// 连接节点
|
||
currentOscillator.connect(currentGainNode);
|
||
currentGainNode.connect(audioContext.destination);
|
||
|
||
// 开始播放
|
||
currentOscillator.start(audioContext.currentTime);
|
||
currentOscillator.stop(audioContext.currentTime + 1);
|
||
|
||
// 更新UI
|
||
currentPlayingNoteDisplay.textContent = '🎵 ' + noteName;
|
||
|
||
// 高亮钢琴键
|
||
pianoKeys.forEach(key => key.classList.remove('active'));
|
||
const activeKey = pianoKeysContainer.querySelector(`[data-note="${noteId}"]`);
|
||
if (activeKey) {
|
||
activeKey.classList.add('active');
|
||
}
|
||
}
|
||
|
||
// 停止所有音符
|
||
function stopAllNotes() {
|
||
if (currentOscillator) {
|
||
try {
|
||
currentOscillator.stop();
|
||
} catch (e) {
|
||
// 已经停止
|
||
}
|
||
currentOscillator = null;
|
||
}
|
||
|
||
currentPlayingNoteId = null;
|
||
currentPlayingNoteDisplay.textContent = '🎵 无';
|
||
pianoKeys.forEach(key => key.classList.remove('active'));
|
||
}
|
||
|
||
// 切换演奏状态
|
||
function togglePlaying() {
|
||
if (!isModelLoaded) {
|
||
alert('请先导入手势模型！');
|
||
return;
|
||
}
|
||
|
||
isPlaying = !isPlaying;
|
||
if (isPlaying) {
|
||
startStopBtn.textContent = '停止演奏';
|
||
startStopBtn.classList.add('playing');
|
||
importModelBtn.disabled = true;
|
||
updateGlobalStatus('开始演奏，请摆出您的钢琴手势！');
|
||
} else {
|
||
startStopBtn.textContent = '开始演奏';
|
||
startStopBtn.classList.remove('playing');
|
||
importModelBtn.disabled = false;
|
||
updateGlobalStatus('已停止演奏');
|
||
stopAllNotes();
|
||
}
|
||
|
||
currentGestureDisplay.textContent = '静止';
|
||
confidenceDisplay.textContent = '0%';
|
||
}
|
||
|
||
// 更新状态
|
||
function updateGlobalStatus(message) {
|
||
globalStatusDisplay.textContent = message;
|
||
}
|
||
|
||
// 初始化
|
||
window.addEventListener('DOMContentLoaded', initApp);
|
||
|
||
// 清理资源
|
||
window.addEventListener('beforeunload', () => {
|
||
if (animationFrameId) {
|
||
cancelAnimationFrame(animationFrameId);
|
||
}
|
||
if (detector) {
|
||
detector.dispose();
|
||
}
|
||
if (classifier) {
|
||
classifier.clearAllClasses();
|
||
}
|
||
tf.disposeAll();
|
||
stopAllNotes();
|
||
});
|
||
</script>
|
||
</body>
|
||
</html> |