{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Set CUDA:2 as current device.\n" ] } ], "source": [ "import sys\n", "import os\n", "import torch\n", "import matplotlib.pyplot as plt\n", "\n", "sys.path.append(os.path.abspath(sys.path[0] + '/../'))\n", "torch.cuda.set_device(2)\n", "print(\"Set CUDA:%d as current device.\" % torch.cuda.current_device())\n", "\n", "from ..data.spherical_view_syn import *\n", "from ..configs.spherical_view_syn import SphericalViewSynConfig\n", "from utils import netio\n", "from utils import img\n", "from utils import device\n", "from utils import view\n", "from components.gen_final import GenFinal\n", "from utils.perf import Perf\n", "\n", "\n", "def load_net(path):\n", " config = SphericalViewSynConfig()\n", " config.from_id(path[:-4])\n", " config.SAMPLE_PARAMS['perturb_sample'] = False\n", " config.print()\n", " net = config.create_net().to(device.default())\n", " netio.load(path, net)\n", " return net\n", "\n", "\n", "def find_file(prefix):\n", " for path in os.listdir():\n", " if path.startswith(prefix):\n", " return path\n", " return None\n", "\n", "\n", "def load_views(data_desc_file) -> view.Trans:\n", " with open(data_desc_file, 'r', encoding='utf-8') as file:\n", " data_desc = json.loads(file.read())\n", " view_centers = torch.tensor(\n", " data_desc['view_centers'], device=device.default()).view(-1, 3)\n", " view_rots = torch.tensor(\n", " data_desc['view_rots'], device=device.default()).view(-1, 3, 3)\n", " return view.Trans(view_centers, view_rots)\n", "\n", "\n", "def plot_figures(images, center):\n", " plt.figure(figsize=(8, 4))\n", " plt.subplot(121)\n", " img.plot(images['fovea_raw'])\n", " plt.subplot(122)\n", " img.plot(images['fovea'])\n", "\n", " plt.figure(figsize=(8, 4))\n", " plt.subplot(121)\n", " img.plot(images['mid_raw'])\n", " plt.subplot(122)\n", " img.plot(images['mid'])\n", "\n", " plt.figure(figsize=(8, 4))\n", " plt.subplot(121)\n", " img.plot(images['periph_raw'])\n", " plt.subplot(122)\n", " img.plot(images['periph'])\n", "\n", " # Plot Blended\n", " plt.figure(figsize=(12, 6))\n", " plt.subplot(121)\n", " img.plot(images['blended_raw'])\n", " plt.subplot(122)\n", " img.plot(images['blended'])\n", " plt.plot([(res_full[1] - 1) / 2 + center[0] - 5, (res_full[1] - 1) / 2 + center[0] + 5],\n", " [(res_full[0] - 1) / 2 + center[1],\n", " (res_full[0] - 1) / 2 + center[1]],\n", " color=[0, 1, 0])\n", " plt.plot([(res_full[1] - 1) / 2 + center[0], (res_full[1] - 1) / 2 + center[0]],\n", " [(res_full[0] - 1) / 2 + center[1] - 5,\n", " (res_full[0] - 1) / 2 + center[1] + 5],\n", " color=[0, 1, 0])" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Change working directory to /home/dengnc/deep_view_syn/data/__0_user_study/us_gas_all_in_one\n", "==== Config fovea ====\n", "Net type: nmsl\n", "Encode dim: 10\n", "Optimizer decay: 0\n", "Normalize: False\n", "Direction as input: False\n", "Full-connected network parameters: {'nf': 128, 'n_layers': 4, 'skips': []}\n", "Sample parameters {'spherical': True, 'depth_range': (1.0, 50.0), 'n_samples': 32, 'perturb_sample': False, 'lindisp': True, 'inverse_r': True}\n", "==========================\n", "Load net from fovea@nmsl-rgb_e10_fc128x4_d1-50_s32.pth ...\n", "==== Config periph ====\n", "Net type: nnmsl\n", "Encode dim: 10\n", "Optimizer decay: 0\n", "Normalize: False\n", "Direction as input: False\n", "Full-connected network parameters: {'nf': 64, 'n_layers': 4, 'skips': []}\n", "Sample parameters {'spherical': True, 'depth_range': (1.0, 50.0), 'n_samples': 16, 'perturb_sample': False, 'lindisp': True, 'inverse_r': True}\n", "==========================\n", "Load net from periph@nnmsl-rgb_e10_fc64x4_d1-50_s16.pth ...\n", "Dataset loaded.\n", "views: [110]\n" ] } ], "source": [ "os.chdir(sys.path[0] + '/../data/__0_user_study/us_gas_all_in_one')\n", "#os.chdir(sys.path[0] + '/../data/__0_user_study/us_mc_all_in_one')\n", "#os.chdir(sys.path[0] + '/../data/lobby_all_in_one')\n", "print('Change working directory to ', os.getcwd())\n", "torch.autograd.set_grad_enabled(False)\n", "\n", "fovea_net = load_net(find_file('fovea'))\n", "periph_net = load_net(find_file('periph'))\n", "\n", "# Load Dataset\n", "views = load_views('nerf_views.json')\n", "print('Dataset loaded.')\n", "\n", "print('views:', views.size())\n", "#print('ref views:', ref_dataset.samples)\n", "\n", "fov_list = [20, 45, 110]\n", "res_list = [(128, 128), (256, 256), (256, 230)] # (192,256)]\n", "res_full = (1600, 1440)\n", "gen = GenFinal(fov_list, res_list, res_full, fovea_net, periph_net,\n", " device=device.default())\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GetRays: 580.4ms\n", "Sample: 2.1ms\n", "Encode: 2.2ms\n", "Rays: tensor([[-0.1711, 0.1711, 0.9702],\n", " [-0.1685, 0.1711, 0.9707],\n", " [-0.1659, 0.1713, 0.9712],\n", " ...,\n", " [ 0.1633, -0.1687, 0.9722],\n", " [ 0.1660, -0.1687, 0.9717],\n", " [ 0.1686, -0.1686, 0.9712]], device='cuda:2')\n", "Spherical coords: tensor([[[1.0000, 1.7454, 1.3988],\n", " [0.9684, 1.7454, 1.3988],\n", " [0.9368, 1.7454, 1.3988],\n", " ...,\n", " [0.0832, 1.7454, 1.3988],\n", " [0.0516, 1.7454, 1.3988],\n", " [0.0200, 1.7454, 1.3988]],\n", "\n", " [[1.0000, 1.7426, 1.3988],\n", " [0.9684, 1.7426, 1.3988],\n", " [0.9368, 1.7426, 1.3988],\n", " ...,\n", " [0.0832, 1.7426, 1.3988],\n", " [0.0516, 1.7426, 1.3988],\n", " [0.0200, 1.7426, 1.3988]],\n", "\n", " [[1.0000, 1.7400, 1.3987],\n", " [0.9684, 1.7400, 1.3987],\n", " [0.9368, 1.7400, 1.3987],\n", " ...,\n", " [0.0832, 1.7400, 1.3987],\n", " [0.0516, 1.7400, 1.3987],\n", " [0.0200, 1.7400, 1.3987]],\n", "\n", " ...,\n", "\n", " [[1.0000, 1.4043, 1.7403],\n", " [0.9684, 1.4043, 1.7403],\n", " [0.9368, 1.4043, 1.7403],\n", " ...,\n", " [0.0832, 1.4043, 1.7403],\n", " [0.0516, 1.4043, 1.7403],\n", " [0.0200, 1.4043, 1.7403]],\n", "\n", " [[1.0000, 1.4016, 1.7403],\n", " [0.9684, 1.4016, 1.7403],\n", " [0.9368, 1.4016, 1.7403],\n", " ...,\n", " [0.0832, 1.4016, 1.7403],\n", " [0.0516, 1.4016, 1.7403],\n", " [0.0200, 1.4016, 1.7403]],\n", "\n", " [[1.0000, 1.3989, 1.7402],\n", " [0.9684, 1.3989, 1.7402],\n", " [0.9368, 1.3989, 1.7402],\n", " ...,\n", " [0.0832, 1.3989, 1.7402],\n", " [0.0516, 1.3989, 1.7402],\n", " [0.0200, 1.3989, 1.7402]]], device='cuda:2')\n", "Depths: tensor([[ 1.0001, 1.0327, 1.0675, ..., 12.0161, 19.3760, 50.0026],\n", " [ 1.0000, 1.0327, 1.0675, ..., 12.0159, 19.3757, 50.0017],\n", " [ 1.0000, 1.0326, 1.0675, ..., 12.0151, 19.3744, 49.9984],\n", " ...,\n", " [ 0.9999, 1.0325, 1.0674, ..., 12.0140, 19.3726, 49.9938],\n", " [ 0.9999, 1.0326, 1.0674, ..., 12.0144, 19.3732, 49.9954],\n", " [ 1.0000, 1.0326, 1.0675, ..., 12.0152, 19.3745, 49.9987]],\n", " device='cuda:2')\n", "Encoded: tensor([[[ 1.0000, 1.7454, 1.3988, ..., -0.9968, 0.1395, 0.9952],\n", " [ 0.9684, 1.7454, 1.3988, ..., 0.8486, 0.1395, 0.9952],\n", " [ 0.9368, 1.7454, 1.3988, ..., -0.5103, 0.1395, 0.9952],\n", " ...,\n", " [ 0.0832, 1.7454, 1.3988, ..., 0.1988, 0.1395, 0.9952],\n", " [ 0.0516, 1.7454, 1.3988, ..., 0.2742, 0.1395, 0.9952],\n", " [ 0.0200, 1.7454, 1.3988, ..., -0.6857, 0.1395, 0.9952]],\n", "\n", " [[ 1.0000, 1.7426, 1.3988, ..., -0.9968, 0.9999, 0.9953],\n", " [ 0.9684, 1.7426, 1.3988, ..., 0.8486, 0.9999, 0.9953],\n", " [ 0.9368, 1.7426, 1.3988, ..., -0.5103, 0.9999, 0.9953],\n", " ...,\n", " [ 0.0832, 1.7426, 1.3988, ..., 0.1988, 0.9999, 0.9953],\n", " [ 0.0516, 1.7426, 1.3988, ..., 0.2742, 0.9999, 0.9953],\n", " [ 0.0200, 1.7426, 1.3988, ..., -0.6857, 0.9999, 0.9953]],\n", "\n", " [[ 1.0000, 1.7400, 1.3987, ..., -0.9968, 0.2253, 0.9881],\n", " [ 0.9684, 1.7400, 1.3987, ..., 0.8486, 0.2253, 0.9881],\n", " [ 0.9368, 1.7400, 1.3987, ..., -0.5103, 0.2253, 0.9881],\n", " ...,\n", " [ 0.0832, 1.7400, 1.3987, ..., 0.1988, 0.2253, 0.9881],\n", " [ 0.0516, 1.7400, 1.3987, ..., 0.2742, 0.2253, 0.9881],\n", " [ 0.0200, 1.7400, 1.3987, ..., -0.6857, 0.2253, 0.9881]],\n", "\n", " ...,\n", "\n", " [[ 1.0000, 1.4043, 1.7403, ..., -0.9968, -0.9210, 0.3760],\n", " [ 0.9684, 1.4043, 1.7403, ..., 0.8486, -0.9210, 0.3760],\n", " [ 0.9368, 1.4043, 1.7403, ..., -0.5103, -0.9210, 0.3760],\n", " ...,\n", " [ 0.0832, 1.4043, 1.7403, ..., 0.1988, -0.9210, 0.3760],\n", " [ 0.0516, 1.4043, 1.7403, ..., 0.2742, -0.9210, 0.3760],\n", " [ 0.0200, 1.4043, 1.7403, ..., -0.6857, -0.9210, 0.3760]],\n", "\n", " [[ 1.0000, 1.4016, 1.7403, ..., -0.9968, 0.2445, 0.3786],\n", " [ 0.9684, 1.4016, 1.7403, ..., 0.8486, 0.2445, 0.3786],\n", " [ 0.9368, 1.4016, 1.7403, ..., -0.5103, 0.2445, 0.3786],\n", " ...,\n", " [ 0.0832, 1.4016, 1.7403, ..., 0.1988, 0.2445, 0.3786],\n", " [ 0.0516, 1.4016, 1.7403, ..., 0.2742, 0.2445, 0.3786],\n", " [ 0.0200, 1.4016, 1.7403, ..., -0.6857, 0.2445, 0.3786]],\n", "\n", " [[ 1.0000, 1.3989, 1.7402, ..., -0.9968, 0.9995, 0.3247],\n", " [ 0.9684, 1.3989, 1.7402, ..., 0.8486, 0.9995, 0.3247],\n", " [ 0.9368, 1.3989, 1.7402, ..., -0.5103, 0.9995, 0.3247],\n", " ...,\n", " [ 0.0832, 1.3989, 1.7402, ..., 0.1988, 0.9995, 0.3247],\n", " [ 0.0516, 1.3989, 1.7402, ..., 0.2742, 0.9995, 0.3247],\n", " [ 0.0200, 1.3989, 1.7402, ..., -0.6857, 0.9995, 0.3247]]],\n", " device='cuda:2')\n" ] } ], "source": [ "test_view = view.Trans(\n", " torch.tensor([[0.0, 0.0, 0.0]], device=device.default()),\n", " torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device.default())\n", ")\n", "perf = Perf(True, True)\n", "rays_o, rays_d = gen.layer_cams[0].get_global_rays(test_view, True)\n", "perf.checkpoint(\"GetRays\")\n", "rays_o = rays_o.view(-1, 3)\n", "rays_d = rays_d.view(-1, 3)\n", "coords, pts, depths = fovea_net.sampler(rays_o, rays_d)\n", "perf.checkpoint(\"Sample\")\n", "encoded = fovea_net.input_encoder(coords)\n", "perf.checkpoint(\"Encode\")\n", "print(\"Rays:\", rays_d)\n", "print(\"Spherical coords:\", coords)\n", "print(\"Depths:\", depths)\n", "print(\"Encoded:\", encoded)\n", "#plot_figures(images, center)\n", "\n", "#misc.create_dir('output/teasers')\n", "#for key in images:\n", "# img.save(\n", "# images[key], 'output/teasers/view%04d_%s.png' % (view_idx, key))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3.7.9 64-bit ('pytorch': conda)", "name": "python379jvsc74a57bd0660ca2a75467d3af74a68fcc6f40bc78ab96b99ff17d2f100b5ca821fbb183f2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.9" }, "orig_nbformat": 2 }, "nbformat": 4, "nbformat_minor": 2 }