931 lines
37 KiB
JavaScript
931 lines
37 KiB
JavaScript
import * as THREE from 'three';
|
|
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
|
|
import { TransformControls } from 'three/examples/jsm/controls/TransformControls.js';
|
|
import { createSkater } from '../character/skater.js';
|
|
import { BONEDEF } from '../character/skeleton.js';
|
|
import {
|
|
applyTiltAnimation,
|
|
applyTiltStickPose,
|
|
captureSkeletonKeyframe,
|
|
clipAsJson,
|
|
clipAsModule,
|
|
createTiltClip,
|
|
deleteClipKeyframe,
|
|
sanitizeTiltClip,
|
|
sampleTiltStick,
|
|
setClipKeyframe,
|
|
smoothTiltClip,
|
|
} from '../anim/clip.js';
|
|
import {
|
|
bakeStickLandmarks,
|
|
detectVideoPose,
|
|
drawPoseOverlay,
|
|
drawStickOverlay,
|
|
loadPoseLandmarker,
|
|
retargetPoseToSkeleton,
|
|
} from './mediapipePose.js';
|
|
import { createSeededStickTracker, stickGripFromWrists } from './stickTracker.js';
|
|
import { parseFreeMoCapCsv } from './freemocapImport.js';
|
|
|
|
const $ = (id) => document.getElementById(id);
|
|
const el = {
|
|
stage: $('stage'), video: $('referenceVideo'), overlay: $('poseOverlay'), videoEmpty: $('videoEmpty'),
|
|
fileInput: $('fileInput'), clipInput: $('clipInput'), chooseVideo: $('chooseVideo'), clipName: $('clipName'),
|
|
mocapSource: $('mocapSource'), browserMocapControls: $('browserMocapControls'),
|
|
freeMocapControls: $('freeMocapControls'), freeMocapFps: $('freeMocapFps'),
|
|
importFreeMocap: $('importFreeMocap'), importFreeMocapCsv: $('importFreeMocapCsv'),
|
|
freeMocapInput: $('freeMocapInput'), freeMocapState: $('freeMocapState'),
|
|
captureFps: $('captureFps'), trimIn: $('trimIn'), trimOut: $('trimOut'), mirror: $('mirrorPose'),
|
|
extract: $('extract'), progress: $('progress'), status: $('status'), confidence: $('confidence'),
|
|
scrubber: $('scrubber'), markers: $('markers'), timelineEnd: $('timelineEnd'), timecode: $('timecode'),
|
|
playPause: $('playPause'), prevKey: $('prevKey'), nextKey: $('nextKey'), timelineSource: $('timelineSource'),
|
|
boneSelect: $('boneSelect'), setKey: $('setKey'), deleteKey: $('deleteKey'), resetBone: $('resetBone'),
|
|
smoothKeys: $('smoothKeys'), keyCount: $('keyCount'), durationLabel: $('durationLabel'), loop: $('loopClip'),
|
|
trackStick: $('trackStick'), seedStick: $('seedStick'), correctStick: $('correctStick'), clearStick: $('clearStick'),
|
|
applyStick: $('applyStick'), flipStick: $('flipStick'), shotSide: $('shotSide'), stickStatus: $('stickStatus'),
|
|
saveState: $('saveState'), saveProject: $('saveProject'), loadProject: $('loadProject'), importClip: $('importClip'),
|
|
exportJson: $('exportJson'), exportModule: $('exportModule'), newClip: $('newClip'),
|
|
};
|
|
|
|
let clip = createTiltClip();
|
|
let currentTime = 0;
|
|
let videoUrl = null;
|
|
let videoFile = null;
|
|
let videoIn = 0;
|
|
let playing = false;
|
|
let localPlayStarted = 0;
|
|
let localPlayOffset = 0;
|
|
let selectedBone = 'pelvis';
|
|
let lastLandmarks = null;
|
|
let dirty = false;
|
|
let extracting = false;
|
|
let markersSignature = '';
|
|
let activeMarker = null;
|
|
let stickSeed = null;
|
|
let stickTracker = null;
|
|
let currentStick = null;
|
|
let pickingStick = null;
|
|
let stickPickPurpose = null;
|
|
const sampledStick = {};
|
|
|
|
// ---- Three.js rig preview -------------------------------------------------
|
|
const renderer = new THREE.WebGLRenderer({ canvas: el.stage, antialias: true, alpha: false });
|
|
renderer.setPixelRatio(Math.min(devicePixelRatio, 2));
|
|
renderer.shadowMap.enabled = true;
|
|
renderer.toneMapping = THREE.ACESFilmicToneMapping;
|
|
renderer.toneMappingExposure = 1.05;
|
|
const scene = new THREE.Scene();
|
|
scene.background = new THREE.Color(0x0a0f16);
|
|
scene.fog = new THREE.Fog(0x0a0f16, 7, 18);
|
|
scene.add(new THREE.HemisphereLight(0xd9ecff, 0x111827, 1.8));
|
|
const key = new THREE.DirectionalLight(0xffffff, 2.4);
|
|
key.position.set(3, 7, 5);
|
|
key.castShadow = true;
|
|
key.shadow.mapSize.set(1024, 1024);
|
|
scene.add(key);
|
|
const rim = new THREE.DirectionalLight(0x68e0c2, 1.1);
|
|
rim.position.set(-4, 3, -4);
|
|
scene.add(rim);
|
|
|
|
const floor = new THREE.Mesh(
|
|
new THREE.CircleGeometry(3.4, 64),
|
|
new THREE.MeshStandardMaterial({ color: 0x141f2c, roughness: 0.94, metalness: 0.03 }),
|
|
);
|
|
floor.rotation.x = -Math.PI / 2;
|
|
floor.receiveShadow = true;
|
|
scene.add(floor);
|
|
const grid = new THREE.GridHelper(6, 24, 0x35506a, 0x1b2a3a);
|
|
grid.position.y = 0.002;
|
|
scene.add(grid);
|
|
|
|
const camera = new THREE.PerspectiveCamera(36, 1, 0.03, 30);
|
|
camera.position.set(2.8, 1.7, 3.8);
|
|
const orbit = new OrbitControls(camera, el.stage);
|
|
orbit.target.set(0, 0.92, 0);
|
|
orbit.enableDamping = true;
|
|
orbit.dampingFactor = 0.08;
|
|
orbit.minDistance = 1.6;
|
|
orbit.maxDistance = 9;
|
|
orbit.update();
|
|
|
|
const skater = createSkater({ seed: 2201, scene, physics: null, index: 0, team: 0 });
|
|
skater.animator.moveSpeed = 0;
|
|
skater.animator.bladeSpeed = 0;
|
|
skater.animator.effort = 0;
|
|
for (let i = 0; i < 30; i++) skater.animator.update(1 / 60);
|
|
skater.mover.updateMatrixWorld(true);
|
|
|
|
function setStudioShotSide(side, { updateClip = true } = {}) {
|
|
const normalized = side === 'left' ? 'left' : 'right';
|
|
const topHand = normalized === 'left' ? 'L' : 'R';
|
|
skater.skelData.bones[`hand${topHand}`].add(skater.stick.group);
|
|
skater.shotSide = normalized;
|
|
skater.topHand = topHand;
|
|
skater.lowerHand = topHand === 'L' ? 'R' : 'L';
|
|
skater.animator.setShotSide(normalized);
|
|
el.shotSide.value = normalized;
|
|
if (updateClip) clip.shotSide = normalized;
|
|
skater.mover.updateMatrixWorld(true);
|
|
}
|
|
|
|
function stickBakeOptions() {
|
|
return {
|
|
mirror: el.mirror.checked,
|
|
socketHand: skater.stick.group.parent === skater.skelData.bones.handL ? 'L' : 'R',
|
|
};
|
|
}
|
|
|
|
const skeletonHelper = new THREE.SkeletonHelper(skater.skelData.rootBone);
|
|
skeletonHelper.material.color.set(0x68e0c2);
|
|
skeletonHelper.material.transparent = true;
|
|
skeletonHelper.material.opacity = 0.48;
|
|
skeletonHelper.material.depthTest = false;
|
|
scene.add(skeletonHelper);
|
|
|
|
const transform = new TransformControls(camera, el.stage);
|
|
transform.setMode('rotate');
|
|
transform.setSpace('local');
|
|
transform.setSize(0.62);
|
|
scene.add(transform.getHelper());
|
|
transform.addEventListener('dragging-changed', (event) => {
|
|
orbit.enabled = !event.value;
|
|
if (event.value) stopPlayback();
|
|
});
|
|
transform.addEventListener('objectChange', () => {
|
|
skater.mover.updateMatrixWorld(true);
|
|
updateRotationInputs();
|
|
markDirty('pose changed — set key to keep it');
|
|
});
|
|
|
|
const handleGeo = new THREE.SphereGeometry(0.025, 10, 8);
|
|
const handleMat = new THREE.MeshBasicMaterial({ color: 0xf6c85f, depthTest: false, transparent: true, opacity: 0.82 });
|
|
const selectedMat = new THREE.MeshBasicMaterial({ color: 0x68e0c2, depthTest: false });
|
|
const handles = BONEDEF.filter(([name]) => name !== 'root').map(([name]) => {
|
|
const mesh = new THREE.Mesh(handleGeo, handleMat);
|
|
mesh.userData.boneName = name;
|
|
mesh.renderOrder = 20;
|
|
scene.add(mesh);
|
|
return mesh;
|
|
});
|
|
|
|
function updateHandles() {
|
|
for (const handle of handles) {
|
|
skater.skelData.bones[handle.userData.boneName].getWorldPosition(handle.position);
|
|
handle.material = handle.userData.boneName === selectedBone ? selectedMat : handleMat;
|
|
}
|
|
}
|
|
|
|
function resize() {
|
|
const rect = el.stage.getBoundingClientRect();
|
|
if (!rect.width || !rect.height) return;
|
|
renderer.setSize(rect.width, rect.height, false);
|
|
camera.aspect = rect.width / rect.height;
|
|
camera.updateProjectionMatrix();
|
|
}
|
|
new ResizeObserver(resize).observe(el.stage);
|
|
resize();
|
|
|
|
function layoutVideoOverlay() {
|
|
const wrap = el.video.parentElement.getBoundingClientRect();
|
|
if (!wrap.width || !wrap.height || !el.video.videoWidth || !el.video.videoHeight) return;
|
|
const scale = Math.min(wrap.width / el.video.videoWidth, wrap.height / el.video.videoHeight);
|
|
const width = el.video.videoWidth * scale;
|
|
const height = el.video.videoHeight * scale;
|
|
Object.assign(el.overlay.style, {
|
|
width: `${width}px`, height: `${height}px`,
|
|
left: `${(wrap.width - width) * 0.5}px`, top: `${(wrap.height - height) * 0.5}px`,
|
|
});
|
|
}
|
|
|
|
function renderReferenceOverlay() {
|
|
drawPoseOverlay(el.overlay, el.video, lastLandmarks, { mirror: el.mirror.checked });
|
|
drawStickOverlay(el.overlay, currentStick, { picking: pickingStick });
|
|
}
|
|
new ResizeObserver(() => { layoutVideoOverlay(); renderReferenceOverlay(); }).observe(el.video.parentElement);
|
|
|
|
const raycaster = new THREE.Raycaster();
|
|
const pointer = new THREE.Vector2();
|
|
el.stage.addEventListener('pointerup', (event) => {
|
|
if (transform.dragging) return;
|
|
const rect = el.stage.getBoundingClientRect();
|
|
pointer.set(((event.clientX - rect.left) / rect.width) * 2 - 1, -((event.clientY - rect.top) / rect.height) * 2 + 1);
|
|
raycaster.setFromCamera(pointer, camera);
|
|
const hit = raycaster.intersectObjects(handles, false)[0];
|
|
if (hit) selectBone(hit.object.userData.boneName);
|
|
});
|
|
|
|
// ---- bone inspector -------------------------------------------------------
|
|
const boneGroups = [
|
|
['Core', ['root', 'pelvis', 'spine1', 'spine2', 'spine3', 'neck', 'head']],
|
|
['Left arm', ['clavicleL', 'upperArmL', 'forearmL', 'handL']],
|
|
['Right arm', ['clavicleR', 'upperArmR', 'forearmR', 'handR']],
|
|
['Left leg', ['thighL', 'shinL', 'footL', 'toeL']],
|
|
['Right leg', ['thighR', 'shinR', 'footR', 'toeR']],
|
|
];
|
|
for (const [label, names] of boneGroups) {
|
|
const group = document.createElement('optgroup');
|
|
group.label = label;
|
|
for (const name of names) group.append(new Option(name, name));
|
|
el.boneSelect.append(group);
|
|
}
|
|
const euler = new THREE.Euler(0, 0, 0, 'XYZ');
|
|
const deg = THREE.MathUtils.radToDeg;
|
|
const rad = THREE.MathUtils.degToRad;
|
|
|
|
function selectBone(name) {
|
|
selectedBone = name;
|
|
el.boneSelect.value = name;
|
|
transform.attach(skater.skelData.bones[name]);
|
|
updateRotationInputs();
|
|
updateHandles();
|
|
}
|
|
|
|
function updateRotationInputs() {
|
|
euler.setFromQuaternion(skater.skelData.bones[selectedBone].quaternion, 'XYZ');
|
|
for (const input of document.querySelectorAll('.rot,.rot-num')) {
|
|
input.value = Math.round(deg(euler[input.dataset.axis]));
|
|
}
|
|
}
|
|
|
|
function setRotationAxis(axis, degrees) {
|
|
stopPlayback();
|
|
euler.setFromQuaternion(skater.skelData.bones[selectedBone].quaternion, 'XYZ');
|
|
euler[axis] = rad(Number(degrees) || 0);
|
|
skater.skelData.bones[selectedBone].quaternion.setFromEuler(euler);
|
|
skater.mover.updateMatrixWorld(true);
|
|
updateRotationInputs();
|
|
markDirty('pose changed — set key to keep it');
|
|
}
|
|
for (const input of document.querySelectorAll('.rot,.rot-num')) {
|
|
input.addEventListener('input', () => setRotationAxis(input.dataset.axis, input.value));
|
|
}
|
|
el.boneSelect.addEventListener('change', () => selectBone(el.boneSelect.value));
|
|
el.resetBone.addEventListener('click', () => {
|
|
stopPlayback();
|
|
skater.skelData.bones[selectedBone].quaternion.identity();
|
|
skater.mover.updateMatrixWorld(true);
|
|
updateRotationInputs();
|
|
markDirty('bone reset — set key to keep it');
|
|
});
|
|
window.addEventListener('keydown', (event) => {
|
|
if (event.target.matches('input,select')) return;
|
|
if (event.key.toLowerCase() === 'q' || event.key.toLowerCase() === 'w') transform.setMode('rotate');
|
|
});
|
|
|
|
// ---- timeline + clips -----------------------------------------------------
|
|
function timelineDuration() {
|
|
const videoRange = el.video.readyState >= 1 ? Math.max(0, Number(el.trimOut.value) - Number(el.trimIn.value)) : 0;
|
|
return Math.max(clip.duration || 0, videoRange, 0.1);
|
|
}
|
|
|
|
function formatTime(time) {
|
|
const mins = Math.floor(time / 60).toString().padStart(2, '0');
|
|
const secs = Math.floor(time % 60).toString().padStart(2, '0');
|
|
const ms = Math.floor((time % 1) * 1000).toString().padStart(3, '0');
|
|
return `${mins}:${secs}.${ms}`;
|
|
}
|
|
|
|
function markDirty(message = 'unsaved') {
|
|
dirty = true;
|
|
el.saveState.textContent = message;
|
|
}
|
|
|
|
function editKeyTime() {
|
|
const tolerance = 0.5 / Math.max(1, Number(clip.fps) || 12) + 1e-5;
|
|
let nearest = null;
|
|
let distance = Infinity;
|
|
for (const frame of clip.keyframes) {
|
|
const nextDistance = Math.abs(frame.time - currentTime);
|
|
if (nextDistance < distance) { nearest = frame; distance = nextDistance; }
|
|
}
|
|
return nearest && distance <= tolerance ? nearest.time : currentTime;
|
|
}
|
|
|
|
function updateTimeline() {
|
|
const duration = timelineDuration();
|
|
el.scrubber.max = duration;
|
|
el.scrubber.value = Math.min(currentTime, duration);
|
|
el.timelineEnd.textContent = `${duration.toFixed(2)} s`;
|
|
el.timecode.textContent = formatTime(currentTime);
|
|
el.keyCount.textContent = clip.keyframes.length;
|
|
el.durationLabel.textContent = `${(clip.duration || 0).toFixed(2)} s`;
|
|
const signature = `${duration.toFixed(4)}|${clip.keyframes.map((frame) => frame.time.toFixed(4)).join(',')}`;
|
|
if (signature !== markersSignature) {
|
|
markersSignature = signature;
|
|
activeMarker = null;
|
|
el.markers.replaceChildren();
|
|
for (const frame of clip.keyframes) {
|
|
const marker = document.createElement('button');
|
|
marker.className = 'key-marker';
|
|
marker.dataset.time = frame.time;
|
|
marker.style.left = `${(frame.time / duration) * 100}%`;
|
|
marker.title = `key ${frame.time.toFixed(3)}s · ${Math.round(frame.confidence * 100)}% body${frame.stick ? ` · ${Math.round(frame.stick.confidence * 100)}% stick` : ''}`;
|
|
marker.addEventListener('click', () => setCurrentTime(frame.time, { seekVideo: true }));
|
|
el.markers.append(marker);
|
|
}
|
|
}
|
|
const exact = [...el.markers.children].find((marker) => Math.abs(Number(marker.dataset.time) - currentTime) < 1 / 240) ?? null;
|
|
if (exact !== activeMarker) {
|
|
activeMarker?.classList.remove('current');
|
|
exact?.classList.add('current');
|
|
activeMarker = exact;
|
|
}
|
|
}
|
|
|
|
function setCurrentTime(time, { seekVideo = false } = {}) {
|
|
currentTime = Math.max(0, Math.min(timelineDuration(), Number(time) || 0));
|
|
if (clip.keyframes.length) applyTiltAnimation(skater, clip, currentTime);
|
|
const stick = sampleTiltStick(clip, currentTime, sampledStick);
|
|
currentStick = stick ? structuredClone(stick)
|
|
: stickSeed ? { ...stickSeed, grip: stickSeed.grip ?? stickSeed.butt, confidence: 1 } : null;
|
|
if (stick && !pickingStick) el.stickStatus.textContent = `Baked stick key · ${Math.round(stick.confidence * 100)}% tracking confidence.`;
|
|
skater.mover.updateMatrixWorld(true);
|
|
updateRotationInputs();
|
|
updateHandles();
|
|
if (seekVideo && el.video.readyState >= 1) el.video.currentTime = Math.min(el.video.duration, videoIn + currentTime);
|
|
renderReferenceOverlay();
|
|
updateTimeline();
|
|
}
|
|
|
|
el.scrubber.addEventListener('input', () => {
|
|
if (playing) stopPlayback();
|
|
setCurrentTime(el.scrubber.value, { seekVideo: true });
|
|
});
|
|
el.setKey.addEventListener('click', () => {
|
|
stopPlayback();
|
|
const keyTime = editKeyTime();
|
|
const frame = captureSkeletonKeyframe(skater.skelData, keyTime);
|
|
const stick = sampleTiltStick(clip, currentTime, {});
|
|
if (stick) frame.stick = structuredClone(stick);
|
|
setClipKeyframe(clip, frame, 0.5 / Math.max(1, Number(clip.fps) || 12) + 1e-5);
|
|
markDirty('key saved in clip · project unsaved');
|
|
setCurrentTime(keyTime);
|
|
});
|
|
el.deleteKey.addEventListener('click', () => {
|
|
if (deleteClipKeyframe(clip, currentTime, 1 / Math.max(30, clip.fps * 2))) {
|
|
markDirty();
|
|
setCurrentTime(currentTime);
|
|
}
|
|
});
|
|
el.smoothKeys.addEventListener('click', () => {
|
|
smoothTiltClip(clip, 0.35);
|
|
markDirty('smoothed');
|
|
setCurrentTime(currentTime);
|
|
});
|
|
el.loop.addEventListener('change', () => { clip.loop = el.loop.value === 'loop'; markDirty(); });
|
|
el.clipName.addEventListener('input', () => { clip.name = el.clipName.value.trim() || 'reference-motion'; markDirty(); });
|
|
el.prevKey.addEventListener('click', () => {
|
|
const frame = [...clip.keyframes].reverse().find((item) => item.time < currentTime - 1e-4) ?? clip.keyframes.at(-1);
|
|
if (frame) setCurrentTime(frame.time, { seekVideo: true });
|
|
});
|
|
el.nextKey.addEventListener('click', () => {
|
|
const frame = clip.keyframes.find((item) => item.time > currentTime + 1e-4) ?? clip.keyframes[0];
|
|
if (frame) setCurrentTime(frame.time, { seekVideo: true });
|
|
});
|
|
|
|
function stopPlayback() {
|
|
playing = false;
|
|
el.video.pause();
|
|
el.playPause.textContent = '▶ Play';
|
|
}
|
|
|
|
function startPlayback() {
|
|
if (timelineDuration() <= 0) return;
|
|
playing = true;
|
|
el.playPause.textContent = '❚❚ Pause';
|
|
if (el.video.readyState >= 2) {
|
|
if (currentTime >= timelineDuration() - 0.01) setCurrentTime(0, { seekVideo: true });
|
|
el.video.currentTime = Math.min(el.video.duration, videoIn + currentTime);
|
|
el.video.play().catch(() => stopPlayback());
|
|
el.timelineSource.textContent = 'video + clip';
|
|
} else {
|
|
localPlayStarted = performance.now() / 1000;
|
|
localPlayOffset = currentTime;
|
|
el.timelineSource.textContent = 'clip';
|
|
}
|
|
}
|
|
el.playPause.addEventListener('click', () => playing ? stopPlayback() : startPlayback());
|
|
el.video.addEventListener('ended', () => {
|
|
if (clip.loop) { setCurrentTime(0, { seekVideo: true }); startPlayback(); }
|
|
else stopPlayback();
|
|
});
|
|
|
|
// ---- video + MediaPipe extraction ----------------------------------------
|
|
function clearStickTracking() {
|
|
stickSeed = null;
|
|
stickTracker = null;
|
|
currentStick = null;
|
|
pickingStick = null;
|
|
stickPickPurpose = null;
|
|
el.trackStick.checked = false;
|
|
el.overlay.classList.remove('picking');
|
|
el.stickStatus.textContent = 'Choose the top-hand socket, then mark both shaft ends in either order.';
|
|
renderReferenceOverlay();
|
|
}
|
|
|
|
el.seedStick.addEventListener('click', async () => {
|
|
if (el.video.readyState < 1) { el.stickStatus.textContent = 'Choose a video first.'; return; }
|
|
stopPlayback();
|
|
videoIn = Math.max(0, Number(el.trimIn.value) || 0);
|
|
await seekVideo(videoIn);
|
|
currentTime = 0;
|
|
pickingStick = {};
|
|
stickPickPurpose = 'seed';
|
|
currentStick = null;
|
|
el.overlay.classList.add('picking');
|
|
el.stickStatus.textContent = 'Click either end of the stick.';
|
|
renderReferenceOverlay();
|
|
});
|
|
|
|
el.correctStick.addEventListener('click', async () => {
|
|
if (el.video.readyState < 1) { el.stickStatus.textContent = 'Choose a video first.'; return; }
|
|
stopPlayback();
|
|
await seekVideo(Math.min(el.video.duration, videoIn + currentTime));
|
|
pickingStick = {};
|
|
stickPickPurpose = 'correct';
|
|
el.overlay.classList.add('picking');
|
|
el.stickStatus.textContent = 'Correction: click either end of the stick.';
|
|
renderReferenceOverlay();
|
|
});
|
|
|
|
el.overlay.addEventListener('pointerdown', async (event) => {
|
|
if (!pickingStick) return;
|
|
event.preventDefault();
|
|
const rect = el.overlay.getBoundingClientRect();
|
|
const point = [
|
|
Math.max(0, Math.min(1, (event.clientX - rect.left) / rect.width)),
|
|
Math.max(0, Math.min(1, (event.clientY - rect.top) / rect.height)),
|
|
];
|
|
if (!pickingStick.butt) {
|
|
pickingStick.butt = point;
|
|
el.stickStatus.textContent = 'Now click the other end.';
|
|
} else {
|
|
pickingStick.blade = point;
|
|
const picked = { butt: [...pickingStick.butt], blade: [...point], confidence: 1 };
|
|
if (stickPickPurpose === 'seed') {
|
|
const dx = point[0] - pickingStick.butt[0];
|
|
const dy = point[1] - pickingStick.butt[1];
|
|
stickSeed = {
|
|
butt: picked.butt, blade: picked.blade,
|
|
grip: [pickingStick.butt[0] + dx * 0.3, pickingStick.butt[1] + dy * 0.3],
|
|
};
|
|
currentStick = { ...stickSeed, confidence: 1 };
|
|
stickTracker = createSeededStickTracker(el.video, stickSeed);
|
|
el.trackStick.checked = true;
|
|
el.stickStatus.textContent = 'Seeded. Generate will track butt, grip, and blade on every sampled frame.';
|
|
} else {
|
|
el.stickStatus.textContent = 'Re-detecting the body frame and baking the correction…';
|
|
const pose = await detectVideoPose(el.video, performance.now());
|
|
if (pose) {
|
|
picked.grip = stickGripFromWrists(picked, pose.normalized);
|
|
const baked = bakeStickLandmarks(
|
|
skater.skelData, skater.mover, pose.normalized, picked, stickBakeOptions(),
|
|
);
|
|
if (baked) {
|
|
const frame = captureSkeletonKeyframe(skater.skelData, currentTime);
|
|
frame.stick = baked;
|
|
setClipKeyframe(clip, frame);
|
|
currentStick = baked;
|
|
markDirty('stick key corrected · unsaved');
|
|
updateTimeline();
|
|
el.stickStatus.textContent = `Corrected stick landmarks at ${currentTime.toFixed(3)} s.`;
|
|
}
|
|
} else el.stickStatus.textContent = 'Could not detect the body in this correction frame.';
|
|
}
|
|
pickingStick = null;
|
|
stickPickPurpose = null;
|
|
el.overlay.classList.remove('picking');
|
|
}
|
|
renderReferenceOverlay();
|
|
});
|
|
el.clearStick.addEventListener('click', clearStickTracking);
|
|
el.applyStick.addEventListener('click', () => {
|
|
stopPlayback();
|
|
if (!currentStick?.butt || !currentStick?.blade) {
|
|
el.stickStatus.textContent = 'Draw or correct the stick guide first.';
|
|
return;
|
|
}
|
|
|
|
const keyTime = editKeyTime();
|
|
const dx = currentStick.blade[0] - currentStick.butt[0];
|
|
const dy = currentStick.blade[1] - currentStick.butt[1];
|
|
const applied = {
|
|
butt: [...currentStick.butt],
|
|
grip: [...(currentStick.grip ?? currentStick.butt)],
|
|
blade: [...currentStick.blade],
|
|
target: currentStick.target ? [...currentStick.target] : [0, 0, 0],
|
|
angle: Number.isFinite(currentStick.angle) ? currentStick.angle : Math.atan2(-dy, dx),
|
|
roll: Number(currentStick.roll) || 0,
|
|
alignHands: true,
|
|
confidence: Number.isFinite(currentStick.confidence) ? currentStick.confidence : 1,
|
|
};
|
|
const frame = captureSkeletonKeyframe(skater.skelData, keyTime);
|
|
frame.stick = applied;
|
|
setClipKeyframe(clip, frame, 0.5 / Math.max(1, Number(clip.fps) || 12) + 1e-5);
|
|
currentTime = keyTime;
|
|
currentStick = structuredClone(applied);
|
|
applyTiltStickPose(skater, applied);
|
|
skater.mover.updateMatrixWorld(true);
|
|
updateHandles();
|
|
updateRotationInputs();
|
|
markDirty('stick + pose saved in clip · project unsaved');
|
|
updateTimeline();
|
|
renderReferenceOverlay();
|
|
el.stickStatus.textContent = `Applied at ${keyTime.toFixed(3)} s · shaft constrained through both hands.`;
|
|
});
|
|
el.flipStick.addEventListener('click', () => {
|
|
let flipped = 0;
|
|
for (const frame of clip.keyframes) {
|
|
if (!frame.stick) continue;
|
|
const angle = Number.isFinite(frame.stick.angle)
|
|
? frame.stick.angle
|
|
: Math.atan2(
|
|
-(frame.stick.blade[1] - frame.stick.butt[1]),
|
|
frame.stick.blade[0] - frame.stick.butt[0],
|
|
);
|
|
[frame.stick.butt, frame.stick.blade] = [frame.stick.blade, frame.stick.butt];
|
|
frame.stick.angle = Math.atan2(Math.sin(angle + Math.PI), Math.cos(angle + Math.PI));
|
|
flipped++;
|
|
}
|
|
if (!flipped) {
|
|
el.stickStatus.textContent = 'No baked stick keys to flip.';
|
|
return;
|
|
}
|
|
markDirty(`flipped ${flipped} stick keys · unsaved`);
|
|
setCurrentTime(currentTime);
|
|
el.stickStatus.textContent = `Flipped the shaft direction on ${flipped} baked keys.`;
|
|
});
|
|
|
|
el.shotSide.addEventListener('change', () => {
|
|
setStudioShotSide(el.shotSide.value);
|
|
markDirty(`${el.shotSide.value} shot · unsaved`);
|
|
setCurrentTime(currentTime);
|
|
el.stickStatus.textContent = `${el.shotSide.value === 'left' ? 'Left' : 'Right'} hand is now the stick socket.`;
|
|
});
|
|
|
|
function useVideo(file) {
|
|
stopPlayback();
|
|
clearStickTracking();
|
|
lastLandmarks = null;
|
|
if (videoUrl) URL.revokeObjectURL(videoUrl);
|
|
videoUrl = URL.createObjectURL(file);
|
|
videoFile = file;
|
|
el.video.src = videoUrl;
|
|
el.video.load();
|
|
el.videoEmpty.hidden = true;
|
|
el.status.textContent = `${file.name} loaded. Set the capture range, then generate.`;
|
|
}
|
|
el.chooseVideo.addEventListener('click', () => el.fileInput.click());
|
|
el.fileInput.addEventListener('change', () => { if (el.fileInput.files[0]) useVideo(el.fileInput.files[0]); });
|
|
|
|
async function refreshFreeMoCapStatus() {
|
|
el.freeMocapState.textContent = 'Checking local worker…';
|
|
try {
|
|
const response = await fetch('/api/freemocap/status');
|
|
const status = await response.json();
|
|
if (!response.ok || !status.available) throw new Error(status.message || 'worker unavailable');
|
|
el.freeMocapState.textContent = `Ready · FreeMoCap ${status.version}`;
|
|
el.importFreeMocap.disabled = false;
|
|
} catch (error) {
|
|
el.freeMocapState.textContent = `Worker unavailable · run npm run freemocap:setup (${error.message})`;
|
|
el.importFreeMocap.disabled = true;
|
|
}
|
|
}
|
|
|
|
el.mocapSource.addEventListener('change', () => {
|
|
const useFreeMoCap = el.mocapSource.value === 'freemocap';
|
|
el.browserMocapControls.hidden = useFreeMoCap;
|
|
el.freeMocapControls.hidden = !useFreeMoCap;
|
|
el.progress.value = 0;
|
|
el.status.textContent = useFreeMoCap
|
|
? 'Choose a video, then process it through the local FreeMoCap worker.'
|
|
: 'Choose a video, set a short in/out range, then generate.';
|
|
if (useFreeMoCap) refreshFreeMoCapStatus();
|
|
});
|
|
|
|
async function loadFreeMoCapResult(text, filename, sourceFps) {
|
|
const imported = parseFreeMoCapCsv(text);
|
|
const stride = Math.max(1, Math.ceil(imported.frames.length / 3000));
|
|
const frames = imported.frames.filter((_frame, index) => index % stride === 0);
|
|
const firstFrame = frames[0].frame;
|
|
const generatedName = filename.replace(/(?:_freemocap_data_by_frame|_body_3d_xyz)?\.(?:csv|mp4|mov|webm|m4v)$/i, '') || 'freemocap-motion';
|
|
if (!el.clipName.value.trim() || el.clipName.value === 'reference-motion') el.clipName.value = generatedName;
|
|
const next = createTiltClip({
|
|
name: el.clipName.value.trim() || generatedName,
|
|
fps: sourceFps / stride,
|
|
loop: el.loop.value === 'loop',
|
|
shotSide: el.shotSide.value,
|
|
});
|
|
for (let i = 0; i < frames.length; i++) {
|
|
const source = frames[i];
|
|
const time = (source.frame - firstFrame) / sourceFps;
|
|
const confidence = retargetPoseToSkeleton(skater.skelData, source.landmarks, { mirror: el.mirror.checked });
|
|
setClipKeyframe(next, captureSkeletonKeyframe(skater.skelData, time, { confidence }));
|
|
el.progress.value = (i + 1) / frames.length;
|
|
if (i % 60 === 0) await new Promise(requestAnimationFrame);
|
|
}
|
|
next.duration = next.keyframes.at(-1)?.time ?? 0;
|
|
smoothTiltClip(next, 0.16);
|
|
clip = next;
|
|
lastLandmarks = null;
|
|
currentStick = null;
|
|
markersSignature = '';
|
|
markDirty(`${next.keyframes.length} FreeMoCap keys · unsaved`);
|
|
el.timelineSource.textContent = `FreeMoCap ${imported.tracker}`;
|
|
el.confidence.textContent = '3D IMPORT';
|
|
el.status.textContent = `Imported ${next.keyframes.length} ${imported.trajectory} keys from ${imported.tracker} at ${sourceFps} fps${stride > 1 ? ` (sampled every ${stride} frames)` : ''}.`;
|
|
setCurrentTime(0);
|
|
}
|
|
|
|
el.importFreeMocap.addEventListener('click', async () => {
|
|
if (!videoFile || extracting) {
|
|
if (!videoFile) el.status.textContent = 'Choose or drop a reference video first.';
|
|
return;
|
|
}
|
|
extracting = true;
|
|
stopPlayback();
|
|
el.importFreeMocap.disabled = true;
|
|
el.progress.removeAttribute('value');
|
|
const sourceFps = Math.max(1, Math.min(60, Number(el.freeMocapFps.value) || 30));
|
|
try {
|
|
el.status.textContent = `Uploading ${videoFile.name}; FreeMoCap processing can take several minutes…`;
|
|
const query = new URLSearchParams({ filename: videoFile.name, fps: String(sourceFps) });
|
|
const response = await fetch(`/api/freemocap/process?${query}`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': videoFile.type || 'application/octet-stream' },
|
|
body: videoFile,
|
|
});
|
|
const result = await response.text();
|
|
if (!response.ok) throw new Error(result || `worker returned ${response.status}`);
|
|
const detectedFps = Number(response.headers.get('X-Tilt-Source-Fps')) || sourceFps;
|
|
el.freeMocapFps.value = String(Number(detectedFps.toFixed(3)));
|
|
el.progress.value = 0.85;
|
|
await loadFreeMoCapResult(result, videoFile.name, detectedFps);
|
|
} catch (error) {
|
|
console.error(error);
|
|
el.status.textContent = `FreeMoCap processing failed: ${error.message}`;
|
|
el.confidence.textContent = 'IMPORT ERROR';
|
|
} finally {
|
|
extracting = false;
|
|
el.progress.value = Number(el.progress.value) || 0;
|
|
refreshFreeMoCapStatus();
|
|
}
|
|
});
|
|
|
|
el.importFreeMocapCsv.addEventListener('click', () => el.freeMocapInput.click());
|
|
el.freeMocapInput.addEventListener('change', async () => {
|
|
const file = el.freeMocapInput.files[0];
|
|
if (!file || extracting) return;
|
|
const sourceFps = Math.max(1, Math.min(60, Number(el.freeMocapFps.value) || 30));
|
|
try {
|
|
el.status.textContent = `Reading ${file.name}…`;
|
|
await loadFreeMoCapResult(await file.text(), file.name, sourceFps);
|
|
} catch (error) {
|
|
console.error(error);
|
|
el.status.textContent = `FreeMoCap CSV import failed: ${error.message}`;
|
|
el.confidence.textContent = 'IMPORT ERROR';
|
|
} finally {
|
|
el.freeMocapInput.value = '';
|
|
}
|
|
});
|
|
document.addEventListener('dragover', (event) => event.preventDefault());
|
|
document.addEventListener('drop', (event) => {
|
|
event.preventDefault();
|
|
const file = [...event.dataTransfer.files].find((item) => item.type.startsWith('video/'));
|
|
if (file) useVideo(file);
|
|
});
|
|
el.video.addEventListener('loadedmetadata', () => {
|
|
el.trimIn.max = el.video.duration;
|
|
el.trimOut.max = el.video.duration;
|
|
el.trimOut.value = el.video.duration.toFixed(3);
|
|
videoIn = 0;
|
|
currentTime = 0;
|
|
layoutVideoOverlay();
|
|
updateTimeline();
|
|
});
|
|
for (const input of [el.trimIn, el.trimOut]) input.addEventListener('change', () => {
|
|
videoIn = Math.max(0, Number(el.trimIn.value) || 0);
|
|
if (Number(el.trimOut.value) <= videoIn) el.trimOut.value = Math.min(el.video.duration || videoIn + 1, videoIn + 1).toFixed(3);
|
|
setCurrentTime(0, { seekVideo: true });
|
|
});
|
|
|
|
function seekVideo(time) {
|
|
return new Promise((resolve, reject) => {
|
|
const done = () => { cleanup(); resolve(); };
|
|
const fail = () => { cleanup(); reject(new Error('Could not seek reference video')); };
|
|
const cleanup = () => {
|
|
el.video.removeEventListener('seeked', done);
|
|
el.video.removeEventListener('canplay', done);
|
|
el.video.removeEventListener('error', fail);
|
|
};
|
|
el.video.addEventListener('seeked', done, { once: true });
|
|
el.video.addEventListener('canplay', done, { once: true });
|
|
el.video.addEventListener('error', fail, { once: true });
|
|
if (Math.abs(el.video.currentTime - time) < 1e-4 && !el.video.seeking && el.video.readyState >= 2) {
|
|
cleanup();
|
|
requestAnimationFrame(resolve);
|
|
} else el.video.currentTime = time;
|
|
});
|
|
}
|
|
|
|
el.extract.addEventListener('click', async () => {
|
|
if (extracting || el.video.readyState < 1) {
|
|
if (el.video.readyState < 1) el.status.textContent = 'Choose a video first.';
|
|
return;
|
|
}
|
|
extracting = true;
|
|
stopPlayback();
|
|
el.extract.disabled = true;
|
|
el.progress.value = 0;
|
|
try {
|
|
const fps = Math.max(1, Math.min(30, Number(el.captureFps.value) || 12));
|
|
const start = Math.max(0, Number(el.trimIn.value) || 0);
|
|
const end = Math.min(el.video.duration, Math.max(start + 1 / fps, Number(el.trimOut.value) || el.video.duration));
|
|
const count = Math.max(2, Math.floor((end - start) * fps) + 1);
|
|
if (count > 3000) throw new Error('Capture range is too long; trim it below 3,000 sampled frames');
|
|
el.status.textContent = 'Loading MediaPipe pose model…';
|
|
await loadPoseLandmarker();
|
|
const next = createTiltClip({
|
|
name: el.clipName.value.trim() || 'reference-motion',
|
|
fps,
|
|
loop: el.loop.value === 'loop',
|
|
shotSide: el.shotSide.value,
|
|
});
|
|
next.duration = end - start;
|
|
let found = 0;
|
|
let sticksFound = 0;
|
|
if (el.trackStick.checked && !stickTracker) throw new Error('Mark the stick butt and blade before enabling stick tracking');
|
|
stickTracker?.reset();
|
|
for (let i = 0; i < count; i++) {
|
|
const sourceTime = Math.min(end, start + i / fps);
|
|
const clipTime = sourceTime - start;
|
|
el.status.textContent = `Tracking frame ${i + 1} / ${count}`;
|
|
await seekVideo(sourceTime);
|
|
const pose = await detectVideoPose(el.video, performance.now());
|
|
if (pose) {
|
|
const confidence = retargetPoseToSkeleton(skater.skelData, pose.world, { mirror: el.mirror.checked });
|
|
const frame = captureSkeletonKeyframe(skater.skelData, clipTime, { confidence });
|
|
if (el.trackStick.checked && stickTracker) {
|
|
const tracked = stickTracker.track();
|
|
tracked.grip = stickGripFromWrists(tracked, pose.normalized);
|
|
const baked = bakeStickLandmarks(
|
|
skater.skelData, skater.mover, pose.normalized, tracked, stickBakeOptions(),
|
|
);
|
|
if (baked) {
|
|
frame.stick = baked;
|
|
currentStick = baked;
|
|
sticksFound++;
|
|
}
|
|
}
|
|
setClipKeyframe(next, frame);
|
|
lastLandmarks = pose.normalized;
|
|
renderReferenceOverlay();
|
|
el.confidence.textContent = `${Math.round(confidence * 100)}% TRACK`;
|
|
found++;
|
|
}
|
|
el.progress.value = (i + 1) / count;
|
|
if (i % 3 === 0) await new Promise(requestAnimationFrame);
|
|
}
|
|
if (!found) throw new Error('No full-body pose was detected in the selected range');
|
|
clip = next;
|
|
videoIn = start;
|
|
smoothTiltClip(clip, 0.22);
|
|
markDirty(`${found} tracked keys · unsaved`);
|
|
el.status.textContent = `Generated ${found} editable keys at ${fps} fps${sticksFound ? ` with ${sticksFound} baked stick tracks` : ''}. Low-confidence frames can be corrected in the studio.`;
|
|
setCurrentTime(0, { seekVideo: true });
|
|
} catch (error) {
|
|
console.error(error);
|
|
el.status.textContent = `Capture failed: ${error.message}`;
|
|
el.confidence.textContent = 'TRACK ERROR';
|
|
} finally {
|
|
extracting = false;
|
|
el.extract.disabled = false;
|
|
}
|
|
});
|
|
|
|
el.mirror.addEventListener('change', () => {
|
|
renderReferenceOverlay();
|
|
});
|
|
|
|
// ---- save/import/export ---------------------------------------------------
|
|
const STORAGE_KEY = 'tilt-animation-projects-v1';
|
|
function projects() {
|
|
try { return JSON.parse(localStorage.getItem(STORAGE_KEY)) || {}; }
|
|
catch { return {}; }
|
|
}
|
|
function storeProjects(value) { localStorage.setItem(STORAGE_KEY, JSON.stringify(value)); }
|
|
|
|
el.saveProject.addEventListener('click', () => {
|
|
clip.name = el.clipName.value.trim() || 'reference-motion';
|
|
try {
|
|
const all = projects();
|
|
all[clip.name] = sanitizeTiltClip(clip);
|
|
storeProjects(all);
|
|
dirty = false;
|
|
el.saveState.textContent = 'saved locally';
|
|
} catch (error) {
|
|
el.status.textContent = `Local save failed (${error.message}). Export JSON to keep this clip.`;
|
|
}
|
|
});
|
|
el.loadProject.addEventListener('click', () => {
|
|
const all = projects();
|
|
const names = Object.keys(all).sort();
|
|
if (!names.length) { el.status.textContent = 'No locally saved animation projects yet.'; return; }
|
|
const name = prompt(`Open saved project:\n${names.join('\n')}`, names[0]);
|
|
if (name && all[name]) loadClip(all[name], 'saved project');
|
|
});
|
|
|
|
function loadClip(data, source = 'file') {
|
|
stopPlayback();
|
|
clip = sanitizeTiltClip(data);
|
|
setStudioShotSide(clip.shotSide, { updateClip: false });
|
|
el.clipName.value = clip.name;
|
|
el.loop.value = clip.loop ? 'loop' : 'once';
|
|
currentTime = 0;
|
|
dirty = false;
|
|
el.saveState.textContent = source;
|
|
const stickKeys = clip.keyframes.filter((frame) => frame.stick).length;
|
|
el.status.textContent = `Loaded ${clip.keyframes.length} keys${stickKeys ? ` including ${stickKeys} stick tracks` : ''} from ${source}.`;
|
|
setCurrentTime(0);
|
|
}
|
|
el.importClip.addEventListener('click', () => el.clipInput.click());
|
|
el.clipInput.addEventListener('change', async () => {
|
|
const file = el.clipInput.files[0];
|
|
if (!file) return;
|
|
try { loadClip(JSON.parse(await file.text()), file.name); }
|
|
catch (error) { el.status.textContent = `Import failed: ${error.message}`; }
|
|
el.clipInput.value = '';
|
|
});
|
|
|
|
function safeFilename(name) {
|
|
return String(name || 'reference-motion').trim().toLowerCase().replace(/[^a-z0-9_-]+/g, '-').replace(/^-|-$/g, '') || 'reference-motion';
|
|
}
|
|
function download(contents, filename, type) {
|
|
const url = URL.createObjectURL(new Blob([contents], { type }));
|
|
const anchor = document.createElement('a');
|
|
anchor.href = url;
|
|
anchor.download = filename;
|
|
anchor.click();
|
|
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
|
}
|
|
el.exportJson.addEventListener('click', () => {
|
|
clip.name = el.clipName.value.trim() || 'reference-motion';
|
|
download(clipAsJson(clip), `${safeFilename(clip.name)}.tiltanim.json`, 'application/json');
|
|
el.status.textContent = 'Exported a reloadable Tilt animation JSON clip.';
|
|
});
|
|
el.exportModule.addEventListener('click', () => {
|
|
clip.name = el.clipName.value.trim() || 'reference-motion';
|
|
download(clipAsModule(clip), `${safeFilename(clip.name)}.js`, 'text/javascript');
|
|
el.status.textContent = `Exported a pose module. Put it in src/anim/poses/generated/ and import it where the runtime action is driven.`;
|
|
});
|
|
el.newClip.addEventListener('click', () => {
|
|
if (dirty && !confirm('Discard the unsaved animation and start a new clip?')) return;
|
|
clip = createTiltClip({ name: 'reference-motion', shotSide: el.shotSide.value });
|
|
el.clipName.value = clip.name;
|
|
dirty = false;
|
|
el.saveState.textContent = 'new clip';
|
|
currentTime = 0;
|
|
updateTimeline();
|
|
});
|
|
window.addEventListener('beforeunload', (event) => {
|
|
if (!dirty) return;
|
|
event.preventDefault();
|
|
});
|
|
|
|
// ---- render loop ----------------------------------------------------------
|
|
function frame() {
|
|
requestAnimationFrame(frame);
|
|
if (playing) {
|
|
const duration = timelineDuration();
|
|
if (el.video.readyState >= 2 && !el.video.paused) {
|
|
const time = el.video.currentTime - videoIn;
|
|
if (time >= duration - 1e-3) {
|
|
if (clip.loop) { el.video.currentTime = videoIn; currentTime = 0; }
|
|
else stopPlayback();
|
|
} else setCurrentTime(time);
|
|
} else if (el.video.readyState < 2) {
|
|
let time = localPlayOffset + performance.now() / 1000 - localPlayStarted;
|
|
if (time >= duration) {
|
|
if (clip.loop) { localPlayStarted = performance.now() / 1000; localPlayOffset = 0; time %= duration; }
|
|
else { time = duration; stopPlayback(); }
|
|
}
|
|
setCurrentTime(time);
|
|
}
|
|
}
|
|
orbit.update();
|
|
skater.mover.updateMatrixWorld(true);
|
|
updateHandles();
|
|
renderer.render(scene, camera);
|
|
}
|
|
|
|
selectBone(selectedBone);
|
|
updateTimeline();
|
|
frame();
|
|
|
|
window.tiltAnimationStudio = {
|
|
get clip() { return clip; },
|
|
get skater() { return skater; },
|
|
setTime: setCurrentTime,
|
|
loadClip,
|
|
};
|