//================================================================================================================================
//
// Copyright (c) 2015-2021 VisionStar Information Technology (Shanghai) Co., Ltd. All Rights Reserved.
// EasyAR is the registered trademark or trademark of VisionStar Information Technology (Shanghai) Co., Ltd in China
// and other countries for the augmented reality technology developed by VisionStar Information Technology (Shanghai) Co., Ltd.
//
//================================================================================================================================
using System;
using System.Collections.Generic;
using UnityEngine;
namespace easyar
{
///
/// which controls AR session in the scene. One session contains a set of components assembled as and controls data flow in the whole life cycle. This class is the entrance of AR, it is possible to create a new session class and replace this one in the scene to implement fully different AR workflow.
/// 在场景中控制AR会话的。一个会话包含一组组装成的组件，并控制整个生命周期的数据流。这个类是AR的入口，如果要实现完全不同的AR工作流可以创建一个新的会话类并在场景中替换这个类。
///
public class ARSession : MonoBehaviour
{
///
/// AR center mode. Modify at any time and takes effect immediately.
/// AR中心模式。可随时修改，立即生效。
///
public ARCenterMode CenterMode;
///
/// AR center when == or == . Modify at any time and takes effect immediately.
/// == 或 == 时的AR中心。可随时修改，立即生效。
///
public TargetController CenterTarget;
///
/// AR center when == . Modify at any time and takes effect immediately.
/// == 时的AR中心。可随时修改，立即生效。
///
public WorldRootController WorldRootController;
///
/// Horizontal flip rendering mode for normal camera. Modify at any time and takes effect immediately.
/// 正常相机的水平镜像渲染模式。可随时修改，立即生效。
///
public ARHorizontalFlipMode HorizontalFlipNormal;
///
/// Horizontal flip rendering mode for front camera. Modify at any time and takes effect immediately.
/// 前置相机的水平镜像渲染模式。可随时修改，立即生效。
///
public ARHorizontalFlipMode HorizontalFlipFront = ARHorizontalFlipMode.World;
///
/// Assemble mode used in .
/// 在中使用的组装模式。
///
public ARAssembly.AssembleMode AssembleMode;
///
/// Assembly of AR components.
/// AR组件的组装体。
///
[HideInInspector, SerializeField]
public ARAssembly Assembly = new ARAssembly();
private WorldRootController previousWorldRootController;
private int frameIndex = -1;
private KeyValuePair frameStatus = new KeyValuePair();
///
/// Output frame change event delegate.
/// 输出帧发生改变的委托。
///
public delegate void FrameChangeAction(OutputFrame outputFrame, Matrix4x4 displayCompensation);
///
/// Output frame change event. It is triggered when the data itself changes, the frequency is affected by data change (like FPS).
/// 输出帧发生改变的事件。该事件会在数据本身产生变化的时候发生，频率受数据变化（比如帧率）影响。
///
public event FrameChangeAction FrameChange;
///
/// Output frame update event. It has the same frequency as MonoBehaviour Update.
/// 输出帧更新事件，该更新频率和MonoBehaviour Update频率相同。
///
public event Action FrameUpdate;
///
/// change event
/// 改变的事件。
///
public event Action WorldRootChanged;
///
/// AR center mode.
/// AR中心模式。
///
public enum ARCenterMode
{
///
/// The session will use the first tracked as center.
/// You can move or rotate the and the will follow. You cannot manually change the 's transform in this mode. The relative transform from to is controlled by code according to data every frame.
/// 当前session是以第一个跟踪到的为中心的。
/// 你可以移动或旋转，会跟着动。在这个模式下你将无法手动控制的transform。和的相对位置关系由代码根据每帧数据控制。
///
FirstTarget,
///
/// The session is centered.
/// You can move or rotate the and the will follow. You cannot manually change the 's transform in this mode. The relative transform from to is controlled by code according to data every frame.
/// 当前session是以为中心的。
/// 你可以移动或旋转，会跟着动。在这个模式下你将无法手动控制的transform。和的相对位置关系由代码根据每帧数据控制。
///
Camera,
///
/// The session will use the specified by as center.
/// If the specified is not found, will fall back to center mode. The relative transform from to is controlled by code according to data every frame.
/// 当前session是以所指定的为中心的。
/// 如果这个指定的没有被跟踪，将会回退到中心模式。和的相对位置关系由代码根据每帧数据控制。
///
SpecificTarget,
///
/// The session will use as center.
/// You can move or rotate the and the will follow. You cannot manually change the 's transform in this mode. The relative transform from to is controlled by code according to data every frame.
/// 当前session是以为中心的。
/// 你可以移动或旋转，会跟着动。在这个模式下你将无法手动控制的transform。和的相对位置关系由代码根据每帧数据控制。
///
WorldRoot,
///
/// The session will behave like mode, it is a mode designed for docking another AR system, like an AR eyewear SDK which usually has its own control of the and other objects from the system.
/// Everything from will be treated as camera centered, but the itself may be controlled by another system or code, while the whole system linked together may have another center.
/// You can move or rotate the and the will follow. You cannot manually change the 's transform in this mode. The relative transform from to is controlled by code according to data every frame.
/// 当前session与模式行为相同。它被用来连接另一个AR系统，比如AR眼镜SDK（通常它有自己对和其它物体的控制策略）。
/// 内所有物体都将被按照camera 中心来调整，但自身可能会由另一个系统或另一段代码来控制，而连接在一起的整个系统可能会有另一个中心。
/// 你可以移动或旋转，会跟着动。在这个模式下你将无法手动控制的transform。和的相对位置关系由代码根据每帧数据控制。
///
ExternalControl,
}
///
/// Horizontal flip rendering mode.
/// In a flip rendering mode, the camera image will be mirrored. And to display to tracked objects in the right way, it will affect the 3D object rendering as well, so there are two different ways of doing horizontal flip. Horizontal flip can only work in object sensing like image or object tracking algorithms.
/// 水平镜像渲染模式。
/// 在水平翻转状态下，相机图像将镜像显示，为确保物体跟踪正常，它同时会影响3D物体的渲染，因此提供两种不同的方式。水平翻转只能在物体感知（比如图像跟踪或物体跟踪）算法下工作。
///
public enum ARHorizontalFlipMode
{
///
/// No flip.
/// 不翻转。
///
None,
///
/// Render with horizontal flip, the camera image will be flipped in rendering, the camera projection matrix will be changed to do flip rendering. Target scale will not change.
/// 水平镜像渲染，camera图像会镜像显示，camera投影矩阵会变化进行镜像渲染，target scale不会改变。
///
World,
///
/// Render with horizontal flip, the camera image will be flipped in rendering, the target scale will be changed to do flip rendering. Camera projection matrix will not change.
/// 水平镜像渲染，camera图像会镜像显示，target scale会改变进行镜像渲染，camera投影矩阵不会改变。
///
Target,
}
///
/// from current frame.
/// 当前帧的。
///
public Optional FrameCameraParameters { get; private set; }
///
/// MonoBehaviour Start
///
private void Start()
{
if (!EasyARController.Initialized)
{
return;
}
Assembly.Assemble(this);
if (!WorldRootController) { WorldRootController = FindObjectOfType(); }
}
///
/// MonoBehaviour Update
///
private void Update()
{
if (!Assembly.Ready)
{
OnEmptyFrame();
return;
}
if (WorldRootController != previousWorldRootController)
{
if (WorldRootChanged != null)
{
WorldRootChanged(WorldRootController);
}
previousWorldRootController = WorldRootController;
}
var oFrame = Assembly.OutputFrame;
if (oFrame.OnNone)
{
OnEmptyFrame();
return;
}
using (var outputFrame = oFrame.Value)
using (var iFrame = outputFrame.inputFrame())
{
if (FrameCameraParameters.OnSome)
{
FrameCameraParameters.Value.Dispose();
}
FrameCameraParameters = iFrame.cameraParameters();
var displayCompensation = GetCompensation(FrameCameraParameters.Value);
var index = iFrame.index();
if (frameIndex != index && FrameChange != null)
{
FrameChange(outputFrame, displayCompensation);
}
frameIndex = index;
// update self first, some flags will pass down to other components
OnFrameUpdate(outputFrame, iFrame, displayCompensation);
if (FrameUpdate != null)
{
FrameUpdate(outputFrame);
}
}
}
///
/// MonoBehaviour OnDestroy
///
private void OnDestroy()
{
Assembly.Dispose();
if (FrameCameraParameters.OnSome)
{
FrameCameraParameters.Value.Dispose();
}
}
///
/// Transforms points from screen coordinate system ([0, 1]^2) to image coordinate system ([0, 1]^2). should be normalized to [0, 1]^2.
/// 从屏幕坐标系（[0, 1]^2）变换到图像坐标系（[0, 1]^2）。 需要被归一化到[0, 1]^2。
///
public Optional ImageCoordinatesFromScreenCoordinates(Vector2 pointInView)
{
if (FrameCameraParameters.OnNone || !Assembly.Camera)
{
return Optional.CreateNone();
}
return FrameCameraParameters.Value.imageCoordinatesFromScreenCoordinates(
Assembly.Camera.aspect, Assembly.Display.Rotation, true, false, new Vec2F(pointInView.x, 1 - pointInView.y)).ToUnityVector();
}
private void OnFrameUpdate(OutputFrame outputFrame, InputFrame inputFrame, Matrix4x4 displayCompensation)
{
// world root
if (Assembly.RequireWorldCenter && !WorldRootController)
{
Debug.Log("WorldRoot not found, create from " + typeof(ARSession));
var gameObject = new GameObject("WorldRoot");
WorldRootController = gameObject.AddComponent();
if (WorldRootChanged != null)
{
WorldRootChanged(WorldRootController);
}
previousWorldRootController = WorldRootController;
}
if (!Assembly.RequireWorldCenter && CenterMode == ARCenterMode.WorldRoot)
{
Debug.LogWarning("ARCenterMode.WorldRoot not available for target only tracking");
CenterMode = ARCenterMode.FirstTarget;
}
// horizontal flip
var hflip = HorizontalFlipNormal;
using (var cameraParameters = inputFrame.cameraParameters())
{
if (cameraParameters.cameraDeviceType() == CameraDeviceType.Front)
{
hflip = HorizontalFlipFront;
}
}
var worldHFlip = false;
var targetHFlip = false;
switch (hflip)
{
case ARHorizontalFlipMode.World:
worldHFlip = true;
targetHFlip = false;
break;
case ARHorizontalFlipMode.Target:
worldHFlip = false;
targetHFlip = true;
break;
default:
break;
}
foreach (var renderCamera in Assembly.RenderCameras)
{
renderCamera.SetProjectHFlip(worldHFlip);
renderCamera.SetRenderImageHFilp(worldHFlip || targetHFlip);
}
foreach (var filter in Assembly.FrameFilters)
{
filter.SetHFlip(targetHFlip);
}
// dispatch results
var results = outputFrame.results();
var motionTrackingStatus = Optional.CreateNone();
if (inputFrame.hasSpatialInformation())
{
motionTrackingStatus = inputFrame.trackingStatus();
}
var resultControllers = DispatchResults(results, motionTrackingStatus);
// get camera pose if available
var cameraPose = Optional.Empty;
if (Assembly.RequireWorldCenter)
{
if (motionTrackingStatus.OnSome)
{
if (motionTrackingStatus.Value != MotionTrackingStatus.NotTracking)
{
cameraPose = inputFrame.cameraTransform();
}
}
else
{
foreach (var result in resultControllers)
{
if (result.Key.OnNone)
{
cameraPose = result.Value;
break;
}
}
}
}
// get center target pose if available
var centerTargetPose = Optional.Empty;
if (CenterMode == ARCenterMode.FirstTarget || CenterMode == ARCenterMode.SpecificTarget)
{
foreach (var result in resultControllers)
{
if (!CenterTarget)
break;
if (result.Key.OnNone)
continue;
if (result.Key == CenterTarget)
{
centerTargetPose = result.Value;
break;
}
}
if (CenterMode == ARCenterMode.FirstTarget && centerTargetPose.OnNone)
{
foreach (var result in resultControllers)
{
if (result.Key.OnNone)
continue;
CenterTarget = result.Key.Value;
centerTargetPose = result.Value;
break;
}
}
}
else
{
CenterTarget = null;
}
// set camera transform first
if (CenterMode == ARCenterMode.FirstTarget || CenterMode == ARCenterMode.SpecificTarget)
{
if (CenterTarget && centerTargetPose.OnSome)
{
TransformUtil.SetTargetPoseOnCamera(Assembly.CameraRoot, CenterTarget, centerTargetPose.Value, displayCompensation, targetHFlip);
}
}
else if (CenterMode == ARCenterMode.WorldRoot)
{
if (WorldRootController && cameraPose.OnSome)
{
TransformUtil.SetCameraPoseOnCamera(Assembly.CameraRoot, WorldRootController, cameraPose.Value, displayCompensation, targetHFlip);
}
}
// set target and world root transform
if (CenterMode == ARCenterMode.Camera)
{
foreach (var result in resultControllers)
{
if (result.Key.OnSome)
{
TransformUtil.SetTargetPoseOnTarget(Assembly.CameraRoot, result.Key.Value, result.Value, displayCompensation, targetHFlip);
}
}
if (WorldRootController && cameraPose.OnSome)
{
TransformUtil.SetCameraPoseOnWorldRoot(Assembly.CameraRoot, WorldRootController, cameraPose.Value, displayCompensation, targetHFlip);
}
}
else if (CenterMode == ARCenterMode.WorldRoot)
{
foreach (var result in resultControllers)
{
if (result.Key.OnSome)
{
TransformUtil.SetTargetPoseOnTarget(Assembly.CameraRoot, result.Key.Value, result.Value, displayCompensation, targetHFlip);
}
}
}
else if (CenterMode == ARCenterMode.FirstTarget || CenterMode == ARCenterMode.SpecificTarget)
{
foreach (var result in resultControllers)
{
if (result.Key.OnSome && result.Key.Value != CenterTarget)
{
TransformUtil.SetTargetPoseOnTarget(Assembly.CameraRoot, result.Key.Value, result.Value, displayCompensation, targetHFlip);
}
}
if (WorldRootController && cameraPose.OnSome)
{
TransformUtil.SetCameraPoseOnWorldRoot(Assembly.CameraRoot, WorldRootController, cameraPose.Value, displayCompensation, targetHFlip);
}
}
else if (CenterMode == ARCenterMode.ExternalControl)
{
foreach (var result in resultControllers)
{
if (result.Key.OnSome)
{
TransformUtil.SetTargetPoseOnTarget(Assembly.CameraRoot, result.Key.Value, result.Value, displayCompensation, targetHFlip);
}
}
}
// dispose results
foreach (var result in results)
{
if (result.OnSome)
{
result.Value.Dispose();
}
}
}
private void OnEmptyFrame()
{
if (frameStatus.Key)
{
if (FrameChange != null)
{
FrameChange(null, Matrix4x4.identity);
}
DispatchResults(null, frameStatus.Value ? MotionTrackingStatus.NotTracking : Optional.CreateNone());
}
if (FrameCameraParameters.OnSome)
{
FrameCameraParameters.Value.Dispose();
FrameCameraParameters = Optional.CreateNone();
}
}
private List, Matrix44F>> DispatchResults(Optional>> results, Optional motionTrackingStatus)
{
var resultControllers = new List, Matrix44F>>();
var joinIndex = 0;
foreach (var filter in Assembly.FrameFilters)
{
if (!filter)
{
Assembly.Break();
}
if (filter is FrameFilter.IOutputFrameSource)
{
var outputFrameSource = filter as FrameFilter.IOutputFrameSource;
var list = outputFrameSource.OnResult(results.OnSome ? results.Value[joinIndex] : null);
if (list != null)
{
resultControllers.AddRange(list);
}
joinIndex++;
}
if (motionTrackingStatus.OnSome && filter is FrameFilter.ISpatialInformationSink)
{
(filter as FrameFilter.ISpatialInformationSink).OnTracking(motionTrackingStatus.Value);
}
}
if (Assembly.RequireWorldCenter)
{
if (motionTrackingStatus.OnSome)
{
WorldRootController.OnTracking(motionTrackingStatus.Value);
}
else
{
var trackingStatus = MotionTrackingStatus.NotTracking;
foreach (var result in resultControllers)
{
if (result.Key.OnNone)
{
trackingStatus = MotionTrackingStatus.Tracking;
break;
}
}
WorldRootController.OnTracking(trackingStatus);
}
}
frameStatus = new KeyValuePair(results.OnSome, motionTrackingStatus.OnSome);
return resultControllers;
}
private Matrix4x4 GetCompensation(CameraParameters camParams)
{
var imageRotation = camParams.imageOrientation(Assembly.Display.Rotation) / 180f * Mathf.PI;
Matrix4x4 rotationMatrix = Matrix4x4.identity;
rotationMatrix.m00 = Mathf.Cos(-imageRotation);
rotationMatrix.m01 = -Mathf.Sin(-imageRotation);
rotationMatrix.m10 = Mathf.Sin(-imageRotation);
rotationMatrix.m11 = Mathf.Cos(-imageRotation);
return rotationMatrix;
}
}
}