diff --git a/packages/player/package.json b/packages/player/package.json index 7b36d7d4a817005c0f8fae5a7df331fadea59411..37c42f7b7c26354295a0f6ceb9e0734421e09c72 100644 --- a/packages/player/package.json +++ b/packages/player/package.json @@ -8,6 +8,7 @@ "@types/node": "^16.3.0", "@types/react": "^17.0.15", "@types/three": "^0.130.1", + "@types/webxr": "^0.2.3", "jest": "^27.0.6", "typescript": "^4.3.5" }, diff --git a/packages/player/src/player.tsx b/packages/player/src/player.tsx index 8cc614c902546000b3a09a6cd84d4898b0951d9a..f2e1693a1c0c5a00a45b8fe74c62c153f75ebb84 100644 --- a/packages/player/src/player.tsx +++ b/packages/player/src/player.tsx @@ -3,6 +3,7 @@ import {FTLMSE} from './mse'; import ee, {Emitter} from 'event-emitter'; import {createCircleTexture} from './texture'; import * as rematrix from 'rematrix'; +import type { Navigator, XRSession, XRReferenceSpace, XRBoundedReferenceSpace } from 'webxr'; const MAX_POINTS = 100; @@ -50,6 +51,8 @@ export class FTLPlayer { private onPointerDownLon = 0; private onPointerDownLat = 0; + private xrImmersiveRefSpace: XRReferenceSpace | XRBoundedReferenceSpace; + private pointsBuffer: THREE.BufferGeometry; constructor(element: HTMLElement) { @@ -74,6 +77,13 @@ export class FTLPlayer { const width = element.clientWidth; const height = width * (9 / 16); + this.renderer = new THREE.WebGLRenderer({alpha: true}); + this.renderer.setPixelRatio( window.devicePixelRatio ); + this.renderer.setSize( width, height ); + this.outer.appendChild( this.renderer.domElement ); + this.renderer.domElement.setAttribute('tabindex', '-1'); + this.renderer.domElement.style.outline = 'none'; + if (false) { //this.camera = new THREE.PerspectiveCamera( 75, window.innerWidth / window.innerHeight, 1, 1100 ); } else { @@ -89,74 +99,6 @@ export class FTLPlayer { this.scene = new THREE.Scene(); - this.renderer = new THREE.WebGLRenderer(); - this.renderer.setPixelRatio( window.devicePixelRatio ); - this.renderer.setSize( width, height ); - this.outer.appendChild( this.renderer.domElement ); - this.renderer.domElement.setAttribute('tabindex', '-1'); - this.renderer.domElement.style.outline = 'none'; - - this.renderer.domElement.addEventListener('click', e => { - this.renderer.domElement.focus(); - if (this.enableSelection) { - const target = e.target as any; - const width = target.clientWidth; - const height = target.clientHeight; - const offX = ((this.frameWidth - this.videoWidth) / this.frameWidth * width) / 2.0; - const offY = ((this.frameHeight - this.videoHeight) / this.frameHeight * height) / 2.0; - const x = Math.max(0, Math.min(1, (e.offsetX - offX) / (width - 2 * offX))); - const y = Math.max(0, Math.min(1, (e.offsetY - offY) / (height - 2 * offY))); - this.emit('select', Math.floor(x * this.videoWidth), Math.floor(y * this.videoHeight)); - } - }); - - this.renderer.domElement.addEventListener('wheel', e => { - if (this.enableZoom) { - this.camera.zoom -= 0.01 * e.deltaY; - this.camera.zoom = Math.max(0.2, Math.min(5, this.camera.zoom)); - this.camera.updateProjectionMatrix(); - e.preventDefault(); - } - }); - - this.renderer.domElement.addEventListener('mousedown', (event) => { - if (this.enableMovement) { - event.preventDefault(); - this.isUserInteracting = true; - this.onPointerDownPointerX = event.clientX; - this.onPointerDownPointerY = event.clientY; - // this.onPointerDownLon = this.lon; - // this.onPointerDownLat = this.lat; - } - }); - - this.renderer.domElement.addEventListener('mousemove', (event) => { - if ( this.isUserInteracting === true ) { - //this.lon = ( this.onPointerDownPointerX - event.clientX ) * 0.1 + this.onPointerDownLon; - //this.lat = ( this.onPointerDownPointerY - event.clientY ) * 0.1 + this.onPointerDownLat; - - this.cameraObject.rotationX += event.movementY * (1/25) * 5.0; - this.cameraObject.rotationY -= event.movementX * (1/25) * 5.0; - this.updatePose(); - } - }); - - this.renderer.domElement.addEventListener('mouseup', (event) => { - this.isUserInteracting = false; - }); - - this.renderer.domElement.addEventListener('keydown', (event) => { - if (this.enableMovement) { - console.log(event); - switch(event.code) { - case "KeyW" : this.cameraObject.translateZ += 0.05; this.updatePose(); break; - case "KeyS" : this.cameraObject.translateZ -= 0.05; this.updatePose(); break; - case "KeyA" : this.cameraObject.translateX -= 0.05; this.updatePose(); break; - case "KeyD" : this.cameraObject.translateX += 0.05; this.updatePose(); break; - } - } - }); - this.pointsBuffer = new THREE.BufferGeometry(); this.pointsBuffer.setDrawRange( 0, 0 ); const positions = new Float32Array( MAX_POINTS * 3 ); @@ -171,33 +113,168 @@ export class FTLPlayer { const pointCloud = new THREE.Points(this.pointsBuffer, pointMaterial); this.scene.add(pointCloud); - const update = () => { - /*me.lat = Math.max( - 85, Math.min( 85, me.lat ) ); - let phi = THREE.MathUtils.degToRad( 90 - me.lat ); - let theta = THREE.MathUtils.degToRad( me.lon );*/ + this.hasXR().then(xr => { + if (xr) { + this.initXR(); + } else { + this.initNonXR(); + } + }); + } - const now = Date.now(); - if (now < this.lastRender + (1000 / this.fps)) { - return; - } + async hasXR() { + const nav = (navigator as unknown) as Navigator; + if (!nav.xr) { + return false; + } else { + return nav.xr.isSessionSupported('immersive-ar'); + } + } - this.lastRender = now; - - this.camera.position.x = 0; - this.camera.position.y = 0; - this.camera.position.z = -2; + async initXR() { + console.log('INIT XR'); + const nav = (navigator as unknown) as Navigator; + + const onRequestSession = () => { + // Requests an 'immersive-ar' session, which ensures that the users + // environment will be visible either via video passthrough or a + // transparent display. This may be presented either in a headset or + // fullscreen on a mobile device. + return nav.xr.requestSession('immersive-ar', { optionalFeatures: ['local-floor']}) + .then((session: unknown) => { + this.renderer.xr.setSession(session as THREE.XRSession); + this.renderer.xr.enabled = true; + // @ts-ignore + this.renderer.xr.cameraAutoUpdate = false; + onSessionStarted(session as XRSession); + }); + } + + const onSessionStarted = (session: XRSession) => { + session.addEventListener('end', onSessionEnded); + + session.requestReferenceSpace('local').then((refSpace) => { + this.xrImmersiveRefSpace = refSpace; + //session.requestAnimationFrame(onXRFrame); + this.renderer.xr.setAnimationLoop(onXRFrame); + this.renderer.xr.getCamera = () => this.camera; + }); + } + + const onSessionEnded = (event) => { + this.renderer.xr.enabled = false; + } + + const onXRFrame = (t, frame) => { + let session = frame.session; + let pose = frame.getViewerPose(this.renderer.xr.getReferenceSpace()); + //console.log('POSES', pose.views.length); + const transform = pose.views[1].transform; + const position = transform.position; + const orientation = transform.orientation; + this.cameraObject.translateX = position.x; + this.cameraObject.translateY = position.y; + this.cameraObject.translateZ = -position.z; + this.cameraObject.rotationX = orientation.x * 57.3; + this.cameraObject.rotationY = orientation.y * 57.3; + this.cameraObject.rotationZ = orientation.z * 57.3; + this.updatePose(); + //session.requestAnimationFrame(onXRFrame); + this._update(); + } + + onRequestSession(); + } + + initNonXR() { + this.renderer.domElement.addEventListener('click', e => { + this.renderer.domElement.focus(); + if (this.enableSelection) { + const target = e.target as any; + const width = target.clientWidth; + const height = target.clientHeight; + const offX = ((this.frameWidth - this.videoWidth) / this.frameWidth * width) / 2.0; + const offY = ((this.frameHeight - this.videoHeight) / this.frameHeight * height) / 2.0; + const x = Math.max(0, Math.min(1, (e.offsetX - offX) / (width - 2 * offX))); + const y = Math.max(0, Math.min(1, (e.offsetY - offY) / (height - 2 * offY))); + this.emit('select', Math.floor(x * this.videoWidth), Math.floor(y * this.videoHeight)); + } + }); + + this.renderer.domElement.addEventListener('wheel', e => { + if (this.enableZoom) { + this.camera.zoom -= 0.01 * e.deltaY; + this.camera.zoom = Math.max(0.2, Math.min(5, this.camera.zoom)); + this.camera.updateProjectionMatrix(); + e.preventDefault(); + } + }); + + this.renderer.domElement.addEventListener('mousedown', (event) => { + if (this.enableMovement) { + event.preventDefault(); + this.isUserInteracting = true; + this.onPointerDownPointerX = event.clientX; + this.onPointerDownPointerY = event.clientY; + // this.onPointerDownLon = this.lon; + // this.onPointerDownLat = this.lat; + } + }); - this.camera.lookAt(0, 0, 0); + this.renderer.domElement.addEventListener('mousemove', (event) => { + if ( this.isUserInteracting === true ) { + //this.lon = ( this.onPointerDownPointerX - event.clientX ) * 0.1 + this.onPointerDownLon; + //this.lat = ( this.onPointerDownPointerY - event.clientY ) * 0.1 + this.onPointerDownLat; - this.renderer.render( this.scene, this.camera ); + this.cameraObject.rotationX += event.movementY * (1/25) * 5.0; + this.cameraObject.rotationY -= event.movementX * (1/25) * 5.0; + this.updatePose(); } + }); - function animate() { - requestAnimationFrame( animate ); - update(); + this.renderer.domElement.addEventListener('mouseup', (event) => { + this.isUserInteracting = false; + }); + + this.renderer.domElement.addEventListener('keydown', (event) => { + if (this.enableMovement) { + console.log(event); + switch(event.code) { + case "KeyW" : this.cameraObject.translateZ += 0.05; this.updatePose(); break; + case "KeyS" : this.cameraObject.translateZ -= 0.05; this.updatePose(); break; + case "KeyA" : this.cameraObject.translateX -= 0.05; this.updatePose(); break; + case "KeyD" : this.cameraObject.translateX += 0.05; this.updatePose(); break; + } } - - animate(); + }); + + const animate = () => { + requestAnimationFrame( animate ); + this._update(); + } + + animate(); + } + + private _update() { + /*me.lat = Math.max( - 85, Math.min( 85, me.lat ) ); + let phi = THREE.MathUtils.degToRad( 90 - me.lat ); + let theta = THREE.MathUtils.degToRad( me.lon );*/ + + //const now = Date.now(); + //if (now < this.lastRender + (1000 / this.fps)) { + // return; + //} + + //this.lastRender = now; + + this.camera.position.x = 0; + this.camera.position.y = 0; + this.camera.position.z = -2; + + this.camera.lookAt(0, 0, 0); + + this.renderer.render( this.scene, this.camera ); } setPoints(points: [number, number][]) { diff --git a/yarn.lock b/yarn.lock index c679a97b3d3ef4d33413ef38ace6ea0938bcbfb6..b00273d89f77ac5c9dd3e41917a739bd38ed47f2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2013,6 +2013,11 @@ dependencies: "@types/node" "*" +"@types/webxr@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@types/webxr/-/webxr-0.2.3.tgz#6e04a6c4aff9a6e9ae4265ad3eae6545dcfdcf0a" + integrity sha512-MCxbyG4KIcx8+g1RwX9uRYsEfRncWLKtGhzSFh+j8fNOgnuzuBAabGJZz45iU4JvPeqaKaLRqQt6o92NMprmwQ== + "@types/whatwg-url@^8.2.1": version "8.2.1" resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-8.2.1.tgz#f1aac222dab7c59e011663a0cb0a3117b2ef05d4"