import * as THREE from 'three'; import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js'; import { TransformControls } from 'three/examples/jsm/controls/TransformControls.js'; import { createSkater } from '../character/skater.js'; import { BONEDEF } from '../character/skeleton.js'; import { applyTiltAnimation, applyTiltStickPose, captureSkeletonKeyframe, clipAsJson, clipAsModule, createTiltClip, deleteClipKeyframe, sanitizeTiltClip, sampleTiltStick, setClipKeyframe, smoothTiltClip, } from '../anim/clip.js'; import { bakeStickLandmarks, detectVideoPose, drawPoseOverlay, drawStickOverlay, loadPoseLandmarker, retargetPoseToSkeleton, } from './mediapipePose.js'; import { createSeededStickTracker, stickGripFromWrists } from './stickTracker.js'; import { parseFreeMoCapCsv } from './freemocapImport.js'; const $ = (id) => document.getElementById(id); const el = { stage: $('stage'), video: $('referenceVideo'), overlay: $('poseOverlay'), videoEmpty: $('videoEmpty'), fileInput: $('fileInput'), clipInput: $('clipInput'), chooseVideo: $('chooseVideo'), clipName: $('clipName'), mocapSource: $('mocapSource'), browserMocapControls: $('browserMocapControls'), freeMocapControls: $('freeMocapControls'), freeMocapFps: $('freeMocapFps'), importFreeMocap: $('importFreeMocap'), importFreeMocapCsv: $('importFreeMocapCsv'), freeMocapInput: $('freeMocapInput'), freeMocapState: $('freeMocapState'), captureFps: $('captureFps'), trimIn: $('trimIn'), trimOut: $('trimOut'), mirror: $('mirrorPose'), extract: $('extract'), progress: $('progress'), status: $('status'), confidence: $('confidence'), scrubber: $('scrubber'), markers: $('markers'), timelineEnd: $('timelineEnd'), timecode: $('timecode'), playPause: $('playPause'), prevKey: $('prevKey'), nextKey: $('nextKey'), timelineSource: $('timelineSource'), boneSelect: $('boneSelect'), setKey: $('setKey'), deleteKey: $('deleteKey'), resetBone: $('resetBone'), smoothKeys: $('smoothKeys'), keyCount: $('keyCount'), durationLabel: $('durationLabel'), loop: $('loopClip'), trackStick: $('trackStick'), seedStick: $('seedStick'), correctStick: $('correctStick'), clearStick: $('clearStick'), applyStick: $('applyStick'), flipStick: $('flipStick'), shotSide: $('shotSide'), stickStatus: $('stickStatus'), saveState: $('saveState'), saveProject: $('saveProject'), loadProject: $('loadProject'), importClip: $('importClip'), exportJson: $('exportJson'), exportModule: $('exportModule'), newClip: $('newClip'), }; let clip = createTiltClip(); let currentTime = 0; let videoUrl = null; let videoFile = null; let videoIn = 0; let playing = false; let localPlayStarted = 0; let localPlayOffset = 0; let selectedBone = 'pelvis'; let lastLandmarks = null; let dirty = false; let extracting = false; let markersSignature = ''; let activeMarker = null; let stickSeed = null; let stickTracker = null; let currentStick = null; let pickingStick = null; let stickPickPurpose = null; const sampledStick = {}; // ---- Three.js rig preview ------------------------------------------------- const renderer = new THREE.WebGLRenderer({ canvas: el.stage, antialias: true, alpha: false }); renderer.setPixelRatio(Math.min(devicePixelRatio, 2)); renderer.shadowMap.enabled = true; renderer.toneMapping = THREE.ACESFilmicToneMapping; renderer.toneMappingExposure = 1.05; const scene = new THREE.Scene(); scene.background = new THREE.Color(0x0a0f16); scene.fog = new THREE.Fog(0x0a0f16, 7, 18); scene.add(new THREE.HemisphereLight(0xd9ecff, 0x111827, 1.8)); const key = new THREE.DirectionalLight(0xffffff, 2.4); key.position.set(3, 7, 5); key.castShadow = true; key.shadow.mapSize.set(1024, 1024); scene.add(key); const rim = new THREE.DirectionalLight(0x68e0c2, 1.1); rim.position.set(-4, 3, -4); scene.add(rim); const floor = new THREE.Mesh( new THREE.CircleGeometry(3.4, 64), new THREE.MeshStandardMaterial({ color: 0x141f2c, roughness: 0.94, metalness: 0.03 }), ); floor.rotation.x = -Math.PI / 2; floor.receiveShadow = true; scene.add(floor); const grid = new THREE.GridHelper(6, 24, 0x35506a, 0x1b2a3a); grid.position.y = 0.002; scene.add(grid); const camera = new THREE.PerspectiveCamera(36, 1, 0.03, 30); camera.position.set(2.8, 1.7, 3.8); const orbit = new OrbitControls(camera, el.stage); orbit.target.set(0, 0.92, 0); orbit.enableDamping = true; orbit.dampingFactor = 0.08; orbit.minDistance = 1.6; orbit.maxDistance = 9; orbit.update(); const skater = createSkater({ seed: 2201, scene, physics: null, index: 0, team: 0 }); skater.animator.moveSpeed = 0; skater.animator.bladeSpeed = 0; skater.animator.effort = 0; for (let i = 0; i < 30; i++) skater.animator.update(1 / 60); skater.mover.updateMatrixWorld(true); function setStudioShotSide(side, { updateClip = true } = {}) { const normalized = side === 'left' ? 'left' : 'right'; const topHand = normalized === 'left' ? 'L' : 'R'; skater.skelData.bones[`hand${topHand}`].add(skater.stick.group); skater.shotSide = normalized; skater.topHand = topHand; skater.lowerHand = topHand === 'L' ? 'R' : 'L'; skater.animator.setShotSide(normalized); el.shotSide.value = normalized; if (updateClip) clip.shotSide = normalized; skater.mover.updateMatrixWorld(true); } function stickBakeOptions() { return { mirror: el.mirror.checked, socketHand: skater.stick.group.parent === skater.skelData.bones.handL ? 'L' : 'R', }; } const skeletonHelper = new THREE.SkeletonHelper(skater.skelData.rootBone); skeletonHelper.material.color.set(0x68e0c2); skeletonHelper.material.transparent = true; skeletonHelper.material.opacity = 0.48; skeletonHelper.material.depthTest = false; scene.add(skeletonHelper); const transform = new TransformControls(camera, el.stage); transform.setMode('rotate'); transform.setSpace('local'); transform.setSize(0.62); scene.add(transform.getHelper()); transform.addEventListener('dragging-changed', (event) => { orbit.enabled = !event.value; if (event.value) stopPlayback(); }); transform.addEventListener('objectChange', () => { skater.mover.updateMatrixWorld(true); updateRotationInputs(); markDirty('pose changed — set key to keep it'); }); const handleGeo = new THREE.SphereGeometry(0.025, 10, 8); const handleMat = new THREE.MeshBasicMaterial({ color: 0xf6c85f, depthTest: false, transparent: true, opacity: 0.82 }); const selectedMat = new THREE.MeshBasicMaterial({ color: 0x68e0c2, depthTest: false }); const handles = BONEDEF.filter(([name]) => name !== 'root').map(([name]) => { const mesh = new THREE.Mesh(handleGeo, handleMat); mesh.userData.boneName = name; mesh.renderOrder = 20; scene.add(mesh); return mesh; }); function updateHandles() { for (const handle of handles) { skater.skelData.bones[handle.userData.boneName].getWorldPosition(handle.position); handle.material = handle.userData.boneName === selectedBone ? selectedMat : handleMat; } } function resize() { const rect = el.stage.getBoundingClientRect(); if (!rect.width || !rect.height) return; renderer.setSize(rect.width, rect.height, false); camera.aspect = rect.width / rect.height; camera.updateProjectionMatrix(); } new ResizeObserver(resize).observe(el.stage); resize(); function layoutVideoOverlay() { const wrap = el.video.parentElement.getBoundingClientRect(); if (!wrap.width || !wrap.height || !el.video.videoWidth || !el.video.videoHeight) return; const scale = Math.min(wrap.width / el.video.videoWidth, wrap.height / el.video.videoHeight); const width = el.video.videoWidth * scale; const height = el.video.videoHeight * scale; Object.assign(el.overlay.style, { width: `${width}px`, height: `${height}px`, left: `${(wrap.width - width) * 0.5}px`, top: `${(wrap.height - height) * 0.5}px`, }); } function renderReferenceOverlay() { drawPoseOverlay(el.overlay, el.video, lastLandmarks, { mirror: el.mirror.checked }); drawStickOverlay(el.overlay, currentStick, { picking: pickingStick }); } new ResizeObserver(() => { layoutVideoOverlay(); renderReferenceOverlay(); }).observe(el.video.parentElement); const raycaster = new THREE.Raycaster(); const pointer = new THREE.Vector2(); el.stage.addEventListener('pointerup', (event) => { if (transform.dragging) return; const rect = el.stage.getBoundingClientRect(); pointer.set(((event.clientX - rect.left) / rect.width) * 2 - 1, -((event.clientY - rect.top) / rect.height) * 2 + 1); raycaster.setFromCamera(pointer, camera); const hit = raycaster.intersectObjects(handles, false)[0]; if (hit) selectBone(hit.object.userData.boneName); }); // ---- bone inspector ------------------------------------------------------- const boneGroups = [ ['Core', ['root', 'pelvis', 'spine1', 'spine2', 'spine3', 'neck', 'head']], ['Left arm', ['clavicleL', 'upperArmL', 'forearmL', 'handL']], ['Right arm', ['clavicleR', 'upperArmR', 'forearmR', 'handR']], ['Left leg', ['thighL', 'shinL', 'footL', 'toeL']], ['Right leg', ['thighR', 'shinR', 'footR', 'toeR']], ]; for (const [label, names] of boneGroups) { const group = document.createElement('optgroup'); group.label = label; for (const name of names) group.append(new Option(name, name)); el.boneSelect.append(group); } const euler = new THREE.Euler(0, 0, 0, 'XYZ'); const deg = THREE.MathUtils.radToDeg; const rad = THREE.MathUtils.degToRad; function selectBone(name) { selectedBone = name; el.boneSelect.value = name; transform.attach(skater.skelData.bones[name]); updateRotationInputs(); updateHandles(); } function updateRotationInputs() { euler.setFromQuaternion(skater.skelData.bones[selectedBone].quaternion, 'XYZ'); for (const input of document.querySelectorAll('.rot,.rot-num')) { input.value = Math.round(deg(euler[input.dataset.axis])); } } function setRotationAxis(axis, degrees) { stopPlayback(); euler.setFromQuaternion(skater.skelData.bones[selectedBone].quaternion, 'XYZ'); euler[axis] = rad(Number(degrees) || 0); skater.skelData.bones[selectedBone].quaternion.setFromEuler(euler); skater.mover.updateMatrixWorld(true); updateRotationInputs(); markDirty('pose changed — set key to keep it'); } for (const input of document.querySelectorAll('.rot,.rot-num')) { input.addEventListener('input', () => setRotationAxis(input.dataset.axis, input.value)); } el.boneSelect.addEventListener('change', () => selectBone(el.boneSelect.value)); el.resetBone.addEventListener('click', () => { stopPlayback(); skater.skelData.bones[selectedBone].quaternion.identity(); skater.mover.updateMatrixWorld(true); updateRotationInputs(); markDirty('bone reset — set key to keep it'); }); window.addEventListener('keydown', (event) => { if (event.target.matches('input,select')) return; if (event.key.toLowerCase() === 'q' || event.key.toLowerCase() === 'w') transform.setMode('rotate'); }); // ---- timeline + clips ----------------------------------------------------- function timelineDuration() { const videoRange = el.video.readyState >= 1 ? Math.max(0, Number(el.trimOut.value) - Number(el.trimIn.value)) : 0; return Math.max(clip.duration || 0, videoRange, 0.1); } function formatTime(time) { const mins = Math.floor(time / 60).toString().padStart(2, '0'); const secs = Math.floor(time % 60).toString().padStart(2, '0'); const ms = Math.floor((time % 1) * 1000).toString().padStart(3, '0'); return `${mins}:${secs}.${ms}`; } function markDirty(message = 'unsaved') { dirty = true; el.saveState.textContent = message; } function editKeyTime() { const tolerance = 0.5 / Math.max(1, Number(clip.fps) || 12) + 1e-5; let nearest = null; let distance = Infinity; for (const frame of clip.keyframes) { const nextDistance = Math.abs(frame.time - currentTime); if (nextDistance < distance) { nearest = frame; distance = nextDistance; } } return nearest && distance <= tolerance ? nearest.time : currentTime; } function updateTimeline() { const duration = timelineDuration(); el.scrubber.max = duration; el.scrubber.value = Math.min(currentTime, duration); el.timelineEnd.textContent = `${duration.toFixed(2)} s`; el.timecode.textContent = formatTime(currentTime); el.keyCount.textContent = clip.keyframes.length; el.durationLabel.textContent = `${(clip.duration || 0).toFixed(2)} s`; const signature = `${duration.toFixed(4)}|${clip.keyframes.map((frame) => frame.time.toFixed(4)).join(',')}`; if (signature !== markersSignature) { markersSignature = signature; activeMarker = null; el.markers.replaceChildren(); for (const frame of clip.keyframes) { const marker = document.createElement('button'); marker.className = 'key-marker'; marker.dataset.time = frame.time; marker.style.left = `${(frame.time / duration) * 100}%`; marker.title = `key ${frame.time.toFixed(3)}s · ${Math.round(frame.confidence * 100)}% body${frame.stick ? ` · ${Math.round(frame.stick.confidence * 100)}% stick` : ''}`; marker.addEventListener('click', () => setCurrentTime(frame.time, { seekVideo: true })); el.markers.append(marker); } } const exact = [...el.markers.children].find((marker) => Math.abs(Number(marker.dataset.time) - currentTime) < 1 / 240) ?? null; if (exact !== activeMarker) { activeMarker?.classList.remove('current'); exact?.classList.add('current'); activeMarker = exact; } } function setCurrentTime(time, { seekVideo = false } = {}) { currentTime = Math.max(0, Math.min(timelineDuration(), Number(time) || 0)); if (clip.keyframes.length) applyTiltAnimation(skater, clip, currentTime); const stick = sampleTiltStick(clip, currentTime, sampledStick); currentStick = stick ? structuredClone(stick) : stickSeed ? { ...stickSeed, grip: stickSeed.grip ?? stickSeed.butt, confidence: 1 } : null; if (stick && !pickingStick) el.stickStatus.textContent = `Baked stick key · ${Math.round(stick.confidence * 100)}% tracking confidence.`; skater.mover.updateMatrixWorld(true); updateRotationInputs(); updateHandles(); if (seekVideo && el.video.readyState >= 1) el.video.currentTime = Math.min(el.video.duration, videoIn + currentTime); renderReferenceOverlay(); updateTimeline(); } el.scrubber.addEventListener('input', () => { if (playing) stopPlayback(); setCurrentTime(el.scrubber.value, { seekVideo: true }); }); el.setKey.addEventListener('click', () => { stopPlayback(); const keyTime = editKeyTime(); const frame = captureSkeletonKeyframe(skater.skelData, keyTime); const stick = sampleTiltStick(clip, currentTime, {}); if (stick) frame.stick = structuredClone(stick); setClipKeyframe(clip, frame, 0.5 / Math.max(1, Number(clip.fps) || 12) + 1e-5); markDirty('key saved in clip · project unsaved'); setCurrentTime(keyTime); }); el.deleteKey.addEventListener('click', () => { if (deleteClipKeyframe(clip, currentTime, 1 / Math.max(30, clip.fps * 2))) { markDirty(); setCurrentTime(currentTime); } }); el.smoothKeys.addEventListener('click', () => { smoothTiltClip(clip, 0.35); markDirty('smoothed'); setCurrentTime(currentTime); }); el.loop.addEventListener('change', () => { clip.loop = el.loop.value === 'loop'; markDirty(); }); el.clipName.addEventListener('input', () => { clip.name = el.clipName.value.trim() || 'reference-motion'; markDirty(); }); el.prevKey.addEventListener('click', () => { const frame = [...clip.keyframes].reverse().find((item) => item.time < currentTime - 1e-4) ?? clip.keyframes.at(-1); if (frame) setCurrentTime(frame.time, { seekVideo: true }); }); el.nextKey.addEventListener('click', () => { const frame = clip.keyframes.find((item) => item.time > currentTime + 1e-4) ?? clip.keyframes[0]; if (frame) setCurrentTime(frame.time, { seekVideo: true }); }); function stopPlayback() { playing = false; el.video.pause(); el.playPause.textContent = '▶ Play'; } function startPlayback() { if (timelineDuration() <= 0) return; playing = true; el.playPause.textContent = '❚❚ Pause'; if (el.video.readyState >= 2) { if (currentTime >= timelineDuration() - 0.01) setCurrentTime(0, { seekVideo: true }); el.video.currentTime = Math.min(el.video.duration, videoIn + currentTime); el.video.play().catch(() => stopPlayback()); el.timelineSource.textContent = 'video + clip'; } else { localPlayStarted = performance.now() / 1000; localPlayOffset = currentTime; el.timelineSource.textContent = 'clip'; } } el.playPause.addEventListener('click', () => playing ? stopPlayback() : startPlayback()); el.video.addEventListener('ended', () => { if (clip.loop) { setCurrentTime(0, { seekVideo: true }); startPlayback(); } else stopPlayback(); }); // ---- video + MediaPipe extraction ---------------------------------------- function clearStickTracking() { stickSeed = null; stickTracker = null; currentStick = null; pickingStick = null; stickPickPurpose = null; el.trackStick.checked = false; el.overlay.classList.remove('picking'); el.stickStatus.textContent = 'Choose the top-hand socket, then mark both shaft ends in either order.'; renderReferenceOverlay(); } el.seedStick.addEventListener('click', async () => { if (el.video.readyState < 1) { el.stickStatus.textContent = 'Choose a video first.'; return; } stopPlayback(); videoIn = Math.max(0, Number(el.trimIn.value) || 0); await seekVideo(videoIn); currentTime = 0; pickingStick = {}; stickPickPurpose = 'seed'; currentStick = null; el.overlay.classList.add('picking'); el.stickStatus.textContent = 'Click either end of the stick.'; renderReferenceOverlay(); }); el.correctStick.addEventListener('click', async () => { if (el.video.readyState < 1) { el.stickStatus.textContent = 'Choose a video first.'; return; } stopPlayback(); await seekVideo(Math.min(el.video.duration, videoIn + currentTime)); pickingStick = {}; stickPickPurpose = 'correct'; el.overlay.classList.add('picking'); el.stickStatus.textContent = 'Correction: click either end of the stick.'; renderReferenceOverlay(); }); el.overlay.addEventListener('pointerdown', async (event) => { if (!pickingStick) return; event.preventDefault(); const rect = el.overlay.getBoundingClientRect(); const point = [ Math.max(0, Math.min(1, (event.clientX - rect.left) / rect.width)), Math.max(0, Math.min(1, (event.clientY - rect.top) / rect.height)), ]; if (!pickingStick.butt) { pickingStick.butt = point; el.stickStatus.textContent = 'Now click the other end.'; } else { pickingStick.blade = point; const picked = { butt: [...pickingStick.butt], blade: [...point], confidence: 1 }; if (stickPickPurpose === 'seed') { const dx = point[0] - pickingStick.butt[0]; const dy = point[1] - pickingStick.butt[1]; stickSeed = { butt: picked.butt, blade: picked.blade, grip: [pickingStick.butt[0] + dx * 0.3, pickingStick.butt[1] + dy * 0.3], }; currentStick = { ...stickSeed, confidence: 1 }; stickTracker = createSeededStickTracker(el.video, stickSeed); el.trackStick.checked = true; el.stickStatus.textContent = 'Seeded. Generate will track butt, grip, and blade on every sampled frame.'; } else { el.stickStatus.textContent = 'Re-detecting the body frame and baking the correction…'; const pose = await detectVideoPose(el.video, performance.now()); if (pose) { picked.grip = stickGripFromWrists(picked, pose.normalized); const baked = bakeStickLandmarks( skater.skelData, skater.mover, pose.normalized, picked, stickBakeOptions(), ); if (baked) { const frame = captureSkeletonKeyframe(skater.skelData, currentTime); frame.stick = baked; setClipKeyframe(clip, frame); currentStick = baked; markDirty('stick key corrected · unsaved'); updateTimeline(); el.stickStatus.textContent = `Corrected stick landmarks at ${currentTime.toFixed(3)} s.`; } } else el.stickStatus.textContent = 'Could not detect the body in this correction frame.'; } pickingStick = null; stickPickPurpose = null; el.overlay.classList.remove('picking'); } renderReferenceOverlay(); }); el.clearStick.addEventListener('click', clearStickTracking); el.applyStick.addEventListener('click', () => { stopPlayback(); if (!currentStick?.butt || !currentStick?.blade) { el.stickStatus.textContent = 'Draw or correct the stick guide first.'; return; } const keyTime = editKeyTime(); const dx = currentStick.blade[0] - currentStick.butt[0]; const dy = currentStick.blade[1] - currentStick.butt[1]; const applied = { butt: [...currentStick.butt], grip: [...(currentStick.grip ?? currentStick.butt)], blade: [...currentStick.blade], target: currentStick.target ? [...currentStick.target] : [0, 0, 0], angle: Number.isFinite(currentStick.angle) ? currentStick.angle : Math.atan2(-dy, dx), roll: Number(currentStick.roll) || 0, alignHands: true, confidence: Number.isFinite(currentStick.confidence) ? currentStick.confidence : 1, }; const frame = captureSkeletonKeyframe(skater.skelData, keyTime); frame.stick = applied; setClipKeyframe(clip, frame, 0.5 / Math.max(1, Number(clip.fps) || 12) + 1e-5); currentTime = keyTime; currentStick = structuredClone(applied); applyTiltStickPose(skater, applied); skater.mover.updateMatrixWorld(true); updateHandles(); updateRotationInputs(); markDirty('stick + pose saved in clip · project unsaved'); updateTimeline(); renderReferenceOverlay(); el.stickStatus.textContent = `Applied at ${keyTime.toFixed(3)} s · shaft constrained through both hands.`; }); el.flipStick.addEventListener('click', () => { let flipped = 0; for (const frame of clip.keyframes) { if (!frame.stick) continue; const angle = Number.isFinite(frame.stick.angle) ? frame.stick.angle : Math.atan2( -(frame.stick.blade[1] - frame.stick.butt[1]), frame.stick.blade[0] - frame.stick.butt[0], ); [frame.stick.butt, frame.stick.blade] = [frame.stick.blade, frame.stick.butt]; frame.stick.angle = Math.atan2(Math.sin(angle + Math.PI), Math.cos(angle + Math.PI)); flipped++; } if (!flipped) { el.stickStatus.textContent = 'No baked stick keys to flip.'; return; } markDirty(`flipped ${flipped} stick keys · unsaved`); setCurrentTime(currentTime); el.stickStatus.textContent = `Flipped the shaft direction on ${flipped} baked keys.`; }); el.shotSide.addEventListener('change', () => { setStudioShotSide(el.shotSide.value); markDirty(`${el.shotSide.value} shot · unsaved`); setCurrentTime(currentTime); el.stickStatus.textContent = `${el.shotSide.value === 'left' ? 'Left' : 'Right'} hand is now the stick socket.`; }); function useVideo(file) { stopPlayback(); clearStickTracking(); lastLandmarks = null; if (videoUrl) URL.revokeObjectURL(videoUrl); videoUrl = URL.createObjectURL(file); videoFile = file; el.video.src = videoUrl; el.video.load(); el.videoEmpty.hidden = true; el.status.textContent = `${file.name} loaded. Set the capture range, then generate.`; } el.chooseVideo.addEventListener('click', () => el.fileInput.click()); el.fileInput.addEventListener('change', () => { if (el.fileInput.files[0]) useVideo(el.fileInput.files[0]); }); async function refreshFreeMoCapStatus() { el.freeMocapState.textContent = 'Checking local worker…'; try { const response = await fetch('/api/freemocap/status'); const status = await response.json(); if (!response.ok || !status.available) throw new Error(status.message || 'worker unavailable'); el.freeMocapState.textContent = `Ready · FreeMoCap ${status.version}`; el.importFreeMocap.disabled = false; } catch (error) { el.freeMocapState.textContent = `Worker unavailable · run npm run freemocap:setup (${error.message})`; el.importFreeMocap.disabled = true; } } el.mocapSource.addEventListener('change', () => { const useFreeMoCap = el.mocapSource.value === 'freemocap'; el.browserMocapControls.hidden = useFreeMoCap; el.freeMocapControls.hidden = !useFreeMoCap; el.progress.value = 0; el.status.textContent = useFreeMoCap ? 'Choose a video, then process it through the local FreeMoCap worker.' : 'Choose a video, set a short in/out range, then generate.'; if (useFreeMoCap) refreshFreeMoCapStatus(); }); async function loadFreeMoCapResult(text, filename, sourceFps) { const imported = parseFreeMoCapCsv(text); const stride = Math.max(1, Math.ceil(imported.frames.length / 3000)); const frames = imported.frames.filter((_frame, index) => index % stride === 0); const firstFrame = frames[0].frame; const generatedName = filename.replace(/(?:_freemocap_data_by_frame|_body_3d_xyz)?\.(?:csv|mp4|mov|webm|m4v)$/i, '') || 'freemocap-motion'; if (!el.clipName.value.trim() || el.clipName.value === 'reference-motion') el.clipName.value = generatedName; const next = createTiltClip({ name: el.clipName.value.trim() || generatedName, fps: sourceFps / stride, loop: el.loop.value === 'loop', shotSide: el.shotSide.value, }); for (let i = 0; i < frames.length; i++) { const source = frames[i]; const time = (source.frame - firstFrame) / sourceFps; const confidence = retargetPoseToSkeleton(skater.skelData, source.landmarks, { mirror: el.mirror.checked }); setClipKeyframe(next, captureSkeletonKeyframe(skater.skelData, time, { confidence })); el.progress.value = (i + 1) / frames.length; if (i % 60 === 0) await new Promise(requestAnimationFrame); } next.duration = next.keyframes.at(-1)?.time ?? 0; smoothTiltClip(next, 0.16); clip = next; lastLandmarks = null; currentStick = null; markersSignature = ''; markDirty(`${next.keyframes.length} FreeMoCap keys · unsaved`); el.timelineSource.textContent = `FreeMoCap ${imported.tracker}`; el.confidence.textContent = '3D IMPORT'; el.status.textContent = `Imported ${next.keyframes.length} ${imported.trajectory} keys from ${imported.tracker} at ${sourceFps} fps${stride > 1 ? ` (sampled every ${stride} frames)` : ''}.`; setCurrentTime(0); } el.importFreeMocap.addEventListener('click', async () => { if (!videoFile || extracting) { if (!videoFile) el.status.textContent = 'Choose or drop a reference video first.'; return; } extracting = true; stopPlayback(); el.importFreeMocap.disabled = true; el.progress.removeAttribute('value'); const sourceFps = Math.max(1, Math.min(60, Number(el.freeMocapFps.value) || 30)); try { el.status.textContent = `Uploading ${videoFile.name}; FreeMoCap processing can take several minutes…`; const query = new URLSearchParams({ filename: videoFile.name, fps: String(sourceFps) }); const response = await fetch(`/api/freemocap/process?${query}`, { method: 'POST', headers: { 'Content-Type': videoFile.type || 'application/octet-stream' }, body: videoFile, }); const result = await response.text(); if (!response.ok) throw new Error(result || `worker returned ${response.status}`); const detectedFps = Number(response.headers.get('X-Tilt-Source-Fps')) || sourceFps; el.freeMocapFps.value = String(Number(detectedFps.toFixed(3))); el.progress.value = 0.85; await loadFreeMoCapResult(result, videoFile.name, detectedFps); } catch (error) { console.error(error); el.status.textContent = `FreeMoCap processing failed: ${error.message}`; el.confidence.textContent = 'IMPORT ERROR'; } finally { extracting = false; el.progress.value = Number(el.progress.value) || 0; refreshFreeMoCapStatus(); } }); el.importFreeMocapCsv.addEventListener('click', () => el.freeMocapInput.click()); el.freeMocapInput.addEventListener('change', async () => { const file = el.freeMocapInput.files[0]; if (!file || extracting) return; const sourceFps = Math.max(1, Math.min(60, Number(el.freeMocapFps.value) || 30)); try { el.status.textContent = `Reading ${file.name}…`; await loadFreeMoCapResult(await file.text(), file.name, sourceFps); } catch (error) { console.error(error); el.status.textContent = `FreeMoCap CSV import failed: ${error.message}`; el.confidence.textContent = 'IMPORT ERROR'; } finally { el.freeMocapInput.value = ''; } }); document.addEventListener('dragover', (event) => event.preventDefault()); document.addEventListener('drop', (event) => { event.preventDefault(); const file = [...event.dataTransfer.files].find((item) => item.type.startsWith('video/')); if (file) useVideo(file); }); el.video.addEventListener('loadedmetadata', () => { el.trimIn.max = el.video.duration; el.trimOut.max = el.video.duration; el.trimOut.value = el.video.duration.toFixed(3); videoIn = 0; currentTime = 0; layoutVideoOverlay(); updateTimeline(); }); for (const input of [el.trimIn, el.trimOut]) input.addEventListener('change', () => { videoIn = Math.max(0, Number(el.trimIn.value) || 0); if (Number(el.trimOut.value) <= videoIn) el.trimOut.value = Math.min(el.video.duration || videoIn + 1, videoIn + 1).toFixed(3); setCurrentTime(0, { seekVideo: true }); }); function seekVideo(time) { return new Promise((resolve, reject) => { const done = () => { cleanup(); resolve(); }; const fail = () => { cleanup(); reject(new Error('Could not seek reference video')); }; const cleanup = () => { el.video.removeEventListener('seeked', done); el.video.removeEventListener('canplay', done); el.video.removeEventListener('error', fail); }; el.video.addEventListener('seeked', done, { once: true }); el.video.addEventListener('canplay', done, { once: true }); el.video.addEventListener('error', fail, { once: true }); if (Math.abs(el.video.currentTime - time) < 1e-4 && !el.video.seeking && el.video.readyState >= 2) { cleanup(); requestAnimationFrame(resolve); } else el.video.currentTime = time; }); } el.extract.addEventListener('click', async () => { if (extracting || el.video.readyState < 1) { if (el.video.readyState < 1) el.status.textContent = 'Choose a video first.'; return; } extracting = true; stopPlayback(); el.extract.disabled = true; el.progress.value = 0; try { const fps = Math.max(1, Math.min(30, Number(el.captureFps.value) || 12)); const start = Math.max(0, Number(el.trimIn.value) || 0); const end = Math.min(el.video.duration, Math.max(start + 1 / fps, Number(el.trimOut.value) || el.video.duration)); const count = Math.max(2, Math.floor((end - start) * fps) + 1); if (count > 3000) throw new Error('Capture range is too long; trim it below 3,000 sampled frames'); el.status.textContent = 'Loading MediaPipe pose model…'; await loadPoseLandmarker(); const next = createTiltClip({ name: el.clipName.value.trim() || 'reference-motion', fps, loop: el.loop.value === 'loop', shotSide: el.shotSide.value, }); next.duration = end - start; let found = 0; let sticksFound = 0; if (el.trackStick.checked && !stickTracker) throw new Error('Mark the stick butt and blade before enabling stick tracking'); stickTracker?.reset(); for (let i = 0; i < count; i++) { const sourceTime = Math.min(end, start + i / fps); const clipTime = sourceTime - start; el.status.textContent = `Tracking frame ${i + 1} / ${count}`; await seekVideo(sourceTime); const pose = await detectVideoPose(el.video, performance.now()); if (pose) { const confidence = retargetPoseToSkeleton(skater.skelData, pose.world, { mirror: el.mirror.checked }); const frame = captureSkeletonKeyframe(skater.skelData, clipTime, { confidence }); if (el.trackStick.checked && stickTracker) { const tracked = stickTracker.track(); tracked.grip = stickGripFromWrists(tracked, pose.normalized); const baked = bakeStickLandmarks( skater.skelData, skater.mover, pose.normalized, tracked, stickBakeOptions(), ); if (baked) { frame.stick = baked; currentStick = baked; sticksFound++; } } setClipKeyframe(next, frame); lastLandmarks = pose.normalized; renderReferenceOverlay(); el.confidence.textContent = `${Math.round(confidence * 100)}% TRACK`; found++; } el.progress.value = (i + 1) / count; if (i % 3 === 0) await new Promise(requestAnimationFrame); } if (!found) throw new Error('No full-body pose was detected in the selected range'); clip = next; videoIn = start; smoothTiltClip(clip, 0.22); markDirty(`${found} tracked keys · unsaved`); el.status.textContent = `Generated ${found} editable keys at ${fps} fps${sticksFound ? ` with ${sticksFound} baked stick tracks` : ''}. Low-confidence frames can be corrected in the studio.`; setCurrentTime(0, { seekVideo: true }); } catch (error) { console.error(error); el.status.textContent = `Capture failed: ${error.message}`; el.confidence.textContent = 'TRACK ERROR'; } finally { extracting = false; el.extract.disabled = false; } }); el.mirror.addEventListener('change', () => { renderReferenceOverlay(); }); // ---- save/import/export --------------------------------------------------- const STORAGE_KEY = 'tilt-animation-projects-v1'; function projects() { try { return JSON.parse(localStorage.getItem(STORAGE_KEY)) || {}; } catch { return {}; } } function storeProjects(value) { localStorage.setItem(STORAGE_KEY, JSON.stringify(value)); } el.saveProject.addEventListener('click', () => { clip.name = el.clipName.value.trim() || 'reference-motion'; try { const all = projects(); all[clip.name] = sanitizeTiltClip(clip); storeProjects(all); dirty = false; el.saveState.textContent = 'saved locally'; } catch (error) { el.status.textContent = `Local save failed (${error.message}). Export JSON to keep this clip.`; } }); el.loadProject.addEventListener('click', () => { const all = projects(); const names = Object.keys(all).sort(); if (!names.length) { el.status.textContent = 'No locally saved animation projects yet.'; return; } const name = prompt(`Open saved project:\n${names.join('\n')}`, names[0]); if (name && all[name]) loadClip(all[name], 'saved project'); }); function loadClip(data, source = 'file') { stopPlayback(); clip = sanitizeTiltClip(data); setStudioShotSide(clip.shotSide, { updateClip: false }); el.clipName.value = clip.name; el.loop.value = clip.loop ? 'loop' : 'once'; currentTime = 0; dirty = false; el.saveState.textContent = source; const stickKeys = clip.keyframes.filter((frame) => frame.stick).length; el.status.textContent = `Loaded ${clip.keyframes.length} keys${stickKeys ? ` including ${stickKeys} stick tracks` : ''} from ${source}.`; setCurrentTime(0); } el.importClip.addEventListener('click', () => el.clipInput.click()); el.clipInput.addEventListener('change', async () => { const file = el.clipInput.files[0]; if (!file) return; try { loadClip(JSON.parse(await file.text()), file.name); } catch (error) { el.status.textContent = `Import failed: ${error.message}`; } el.clipInput.value = ''; }); function safeFilename(name) { return String(name || 'reference-motion').trim().toLowerCase().replace(/[^a-z0-9_-]+/g, '-').replace(/^-|-$/g, '') || 'reference-motion'; } function download(contents, filename, type) { const url = URL.createObjectURL(new Blob([contents], { type })); const anchor = document.createElement('a'); anchor.href = url; anchor.download = filename; anchor.click(); setTimeout(() => URL.revokeObjectURL(url), 1000); } el.exportJson.addEventListener('click', () => { clip.name = el.clipName.value.trim() || 'reference-motion'; download(clipAsJson(clip), `${safeFilename(clip.name)}.tiltanim.json`, 'application/json'); el.status.textContent = 'Exported a reloadable Tilt animation JSON clip.'; }); el.exportModule.addEventListener('click', () => { clip.name = el.clipName.value.trim() || 'reference-motion'; download(clipAsModule(clip), `${safeFilename(clip.name)}.js`, 'text/javascript'); el.status.textContent = `Exported a pose module. Put it in src/anim/poses/generated/ and import it where the runtime action is driven.`; }); el.newClip.addEventListener('click', () => { if (dirty && !confirm('Discard the unsaved animation and start a new clip?')) return; clip = createTiltClip({ name: 'reference-motion', shotSide: el.shotSide.value }); el.clipName.value = clip.name; dirty = false; el.saveState.textContent = 'new clip'; currentTime = 0; updateTimeline(); }); window.addEventListener('beforeunload', (event) => { if (!dirty) return; event.preventDefault(); }); // ---- render loop ---------------------------------------------------------- function frame() { requestAnimationFrame(frame); if (playing) { const duration = timelineDuration(); if (el.video.readyState >= 2 && !el.video.paused) { const time = el.video.currentTime - videoIn; if (time >= duration - 1e-3) { if (clip.loop) { el.video.currentTime = videoIn; currentTime = 0; } else stopPlayback(); } else setCurrentTime(time); } else if (el.video.readyState < 2) { let time = localPlayOffset + performance.now() / 1000 - localPlayStarted; if (time >= duration) { if (clip.loop) { localPlayStarted = performance.now() / 1000; localPlayOffset = 0; time %= duration; } else { time = duration; stopPlayback(); } } setCurrentTime(time); } } orbit.update(); skater.mover.updateMatrixWorld(true); updateHandles(); renderer.render(scene, camera); } selectBone(selectedBone); updateTimeline(); frame(); window.tiltAnimationStudio = { get clip() { return clip; }, get skater() { return skater; }, setTime: setCurrentTime, loadClip, };