Environment facing camera displayed full screen

Summary: A simple component that when dropped into the scene displaces the environment facing camera

Reviewed By: amberroy

Differential Revision: D5144803

fbshipit-source-id: 2175809

Author: mikearmstrong001

Libraries/Camera/LiveEnvCamera.js

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @providesModule LiveEnvCamera
+ */
+'use strict';
+
+const NativeMethodsMixin = require('NativeMethodsMixin');
+const React = require('React');
+const ReactNativeViewAttributes = require('ReactNativeViewAttributes');
+const View = require('View');
+const requireNativeComponent = require('requireNativeComponent');
+const StyleSheetPropType = require('StyleSheetPropType');
+const LayoutAndTransformPropTypes = require('LayoutAndTransformPropTypes');
+
+/**
+ * Displays the environment facing camera. By default the camera is position:'absolute'
+ * `<LiveEnvCamera />`
+ * The camera image is displayed on geometry that is 1000m away from the viewer
+ */
+const LiveEnvCamera = React.createClass({
+  mixins: [NativeMethodsMixin],
+
+  propTypes: {
+    ...View.propTypes,
+    style: StyleSheetPropType(LayoutAndTransformPropTypes),
+  },
+
+  viewConfig: {
+    uiViewClassName: 'LiveEnvCamera',
+    validAttributes: {
+      ...ReactNativeViewAttributes.RCTView,
+    },
+  },
+
+  getDefaultProps: function() {
+    return {};
+  },
+
+  render: function() {
+    const props = {...this.props} || {};
+    props.style = props.style || {};
+    if (!props.style.position) {
+      props.style.position = 'absolute';
+    }
+    // default panos to being a render group
+    if (!props.style.renderGroup) {
+      props.style.renderGroup = true;
+    }
+
+    return (
+      <RKLiveEnvCamera
+        {...props}
+        testID={this.props.testID}
+        onStartShouldSetResponder={() => true}
+        onResponderTerminationRequest={() => false}>
+        {this.props.children}
+      </RKLiveEnvCamera>
+    );
+  },
+});
+
+const RKLiveEnvCamera = requireNativeComponent('LiveEnvCamera', LiveEnvCamera, {
+  nativeOnly: {},
+});
+
+module.exports = LiveEnvCamera;

Libraries/react-vr.js

@@ -28,6 +28,9 @@ const ReactVR = {
   get Box() {
     return require('Box');
   },
+  get LiveEnvCamera() {
+    return require('LiveEnvCamera');
+  },
   get Cylinder() {
     return require('Cylinder');
   },

ReactVR/js/Modules/UIManager.js

@@ -16,6 +16,7 @@ import RCTSphere from '../Views/Sphere';
 import RCTImage from '../Views/Image';
 import RCTView from '../Views/View';
 import RCTPano from '../Views/Pano';
+import RCTLiveEnvCamera from '../Views/LiveEnvCamera';
 import RCTModel from '../Views/Model';
 import RCTScene from '../Views/Scene';
 import RCTSound from '../Views/Sound';
@@ -177,6 +178,9 @@ export default class UIManager extends Module {
     this.registerViewType('RCTView', RCTView.describe(), function() {
       return new RCTView(guiSys);
     });
+    this.registerViewType('LiveEnvCamera', RCTLiveEnvCamera.describe(), function() {
+      return new RCTLiveEnvCamera(guiSys);
+    });
     this.registerViewType('RCTImageView', RCTImage.describe(), function() {
       return new RCTImage(guiSys, rnctx);
     });

ReactVR/js/Views/LiveEnvCamera.js

@@ -0,0 +1,161 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/**
+ * RCTLiveEnvCamera
+ * Displays the environment facing camera on a sphere
+ * @class RCTLiveEnvCamera
+ * @extends RCTBaseView
+ */
+
+import RCTBaseView from './BaseView';
+import merge from '../Utils/merge';
+import * as OVRUI from 'ovrui';
+import * as THREE from 'three';
+import * as Yoga from '../Utils/Yoga.bundle';
+
+// display texture always infront of the camera
+const basic_vert = `
+      varying highp vec4 vUv;
+      void main()
+      {
+          vUv = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
+          vUv.xy = (vUv.xy + vec2(vUv.w)) * 0.5;
+          gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
+      }
+  `;
+
+const basic_frag = `
+      uniform sampler2D map;
+      varying highp vec4 vUv;
+      void main()
+      {
+        gl_FragColor = texture2DProj( map, vUv );
+      }
+  `;
+
+const SPHERE_RADIUS = 1000;
+
+const sphereRayCast = (function() {
+  // avoid create temp objects;
+  const inverseMatrix = new THREE.Matrix4();
+  const ray = new THREE.Ray();
+  const sphere = new THREE.Sphere(new THREE.Vector3(0, 0, 0), SPHERE_RADIUS);
+  const intersectionPoint = new THREE.Vector3();
+  const intersectionPointWorld = new THREE.Vector3();
+  return function(raycaster, intersects) {
+    // transform the ray into the space of the sphere
+    inverseMatrix.getInverse(this.matrixWorld);
+    ray.copy(raycaster.ray).applyMatrix4(inverseMatrix);
+    const intersect = ray.intersectSphere(sphere, intersectionPoint);
+    if (intersect === null) {
+      return;
+    }
+
+    // determine hit location in world space
+    intersectionPointWorld.copy(intersectionPoint);
+    intersectionPointWorld.applyMatrix4(this.matrixWorld);
+
+    const distance = raycaster.ray.origin.distanceTo(intersectionPointWorld);
+    if (distance < raycaster.near || distance > raycaster.far) {
+      return;
+    }
+
+    intersects.push({
+      distance: distance,
+      point: intersectionPointWorld.clone(),
+      object: this,
+    });
+  };
+})();
+
+export default class RCTLiveEnvCamera extends RCTBaseView {
+  /**
+   * constructor: allocates the required resources and sets defaults
+   */
+  constructor(guiSys, rnctx) {
+    super();
+
+    navigator.getUserMedia =
+      navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
+
+    const constraints = {
+      video: {facingMode: {exact: 'environment'}},
+    };
+    const video = document.createElement('video');
+    const videoTexture = new THREE.Texture(video);
+    videoTexture.minFilter = THREE.LinearFilter;
+    this._video = video;
+    this._videoTexture = videoTexture;
+    navigator.getUserMedia(
+      constraints,
+      stream => {
+        video.src = window.URL.createObjectURL(stream);
+      },
+      error => {
+        console.log('navigator.getUserMedia error: ', error);
+      }
+    );
+
+    this._sphereGeometry = new THREE.SphereGeometry(SPHERE_RADIUS, 5, 5);
+    this._material = new THREE.ShaderMaterial({
+      uniforms: {
+        map: {
+          value: videoTexture,
+          type: 't',
+        },
+      },
+      vertexShader: basic_vert,
+      fragmentShader: basic_frag,
+      side: THREE.DoubleSide,
+    });
+
+    this._onUpdate = this._onUpdate.bind(this);
+
+    this._globe = new THREE.Mesh(this._sphereGeometry, this._material);
+    this._globe.raycast = sphereRayCast.bind(this._globe);
+    this._globe.rotation.y = -Math.PI / 2;
+    this._globe.onUpdate = this._onUpdate;
+
+    this.view = new OVRUI.UIView(guiSys);
+    this.view.add(this._globe);
+  }
+
+  _onUpdate(scene, camera) {
+    if (this._video.readyState === this._video.HAVE_ENOUGH_DATA) {
+      this._videoTexture.needsUpdate = true;
+    }
+  }
+
+  presentLayout() {
+    super.presentLayout();
+    this._globe.visible = this.YGNode.getDisplay() !== Yoga.DISPLAY_NONE;
+  }
+
+  /**
+   * Dispose of any associated resources
+   */
+  dispose() {
+    if (this._localResource) {
+      this._localResource.dispose();
+    }
+    super.dispose();
+  }
+
+  /**
+   * Describes the properties representable by this view type and merges
+   * with super type
+   */
+  static describe() {
+    return merge(super.describe(), {
+      // declare the native props sent from react to runtime
+      NativeProps: {},
+    });
+  }
+}

website/server/docsList.js

@@ -21,6 +21,7 @@ path.join(rn, 'Libraries/Text/Text.js'),
 const components = [
 '../Libraries/Lights/AmbientLight.js',
 '../Libraries/Mesh/Box.js',
+'../Libraries/Camera/LiveEnvCamera.js',
 '../Libraries/Mesh/Cylinder.js',
 '../Libraries/VRLayers/CylindricalPanel.js',
 '../Libraries/Lights/DirectionalLight.js',