from .__common__ import * import torch.nn.functional as F __all__ = ["density2energy", "density2alpha", "VolumnRenderer"] def density2energy(densities: torch.Tensor, dists: torch.Tensor, raw_noise_std: float = 0) -> torch.Tensor: """ Calculate energies from densities inferred by model. :param densities `Tensor(N...)`: model's output densities :param dists `Tensor(N...)`: integration times :param raw_noise_std `float`: the noise std used to egularize network during training (prevents floater artifacts), defaults to 0, means no noise is added :return `Tensor(N...)`: energies which block light rays """ if raw_noise_std > 0: # Add noise to model's predictions for density. Can be used to # regularize network during training (prevents floater artifacts). densities = densities + torch.normal(0.0, raw_noise_std, densities.shape, device=densities.device) return F.relu(densities) * dists def energy2alpha(energies: torch.Tensor) -> torch.Tensor: """ Convert energies to alphas. :param energies `Tensor(N...)`: energies (calculated from densities) :return `Tensor(N...)`: alphas """ return 1.0 - torch.exp(-energies) def density2alpha(densities: torch.Tensor, dists: torch.Tensor, raw_noise_std: float = 0) -> torch.Tensor: """ Calculate alphas from densities inferred by model. :param densities `Tensor(N...)`: model's output densities :param dists `Tensor(N...)`: integration times :param raw_noise_std `float`: the noise std used to regularize network during training (prevents floater artifacts), defaults to 0, means no noise is added :return `Tensor(N...)`: alphas """ return energy2alpha(density2energy(densities, dists, raw_noise_std)) class VolumnRenderer(nn.Module): def __init__(self): super().__init__() # stub method def __call__(self, samples: Samples, densities: torch.Tensor, colors: torch.Tensor, *outputs: str, white_bg: bool, raw_noise_std: float) -> ReturnData: """ Perform volumn rendering. :param samples `Samples(B, P)`: samples :param rgbd `Tensor(B, P, C+1)`: colors and densities :param outputs `str...`: items should be contained in the result dict. Optional values include 'color', 'depth', 'layers', 'states' and attribute names in class `States` (e.g. 'weights'). Defaults to [] :return `ReturnData`: render result { 'color'[, 'depth', 'layers', 'states', ...] } """ ... @profile def forward(self, samples: Samples, rgbd: torch.Tensor, *outputs: str, white_bg: bool, raw_noise_std: float) -> ReturnData: energies = density2energy(rgbd[..., -1], samples.dists, raw_noise_std) # (B, P) alphas = energy2alpha(energies) # (B, P) weights = (alphas * torch.cumprod(union(1, 1. - alphas + 1e-10), -1)[..., :-1])[..., None] output_fn = { "color": lambda: torch.sum(weights * rgbd[..., :-1], -2) + (1. - torch.sum(weights, -2) if white_bg else 0.), "depth": lambda: torch.sum(weights * samples.depths[..., None], -2), "colors": lambda: rgbd[..., :-1], "densities": lambda: rgbd[..., -1:], "alphas": lambda: alphas[..., None], "energies": lambda: energies[..., None], "weights": lambda: weights } return ReturnData({key: output_fn[key]() for key in outputs if key in output_fn})