src/utils/render.py

# Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 
#
# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
# property and proprietary rights in and to this material, related
# documentation and any modifications thereto. Any use, reproduction, 
# disclosure or distribution of this material and related documentation 
# without an express license agreement from NVIDIA CORPORATION or 
# its affiliates is strictly prohibited.

import torch
import nvdiffrast.torch as dr

from . import render_utils
from src.models.geometry.render import renderutils as ru
import numpy as np
from PIL import Image
import torchvision

# ==============================================================================================
#  Helper functions
# ==============================================================================================
def interpolate(attr, rast, attr_idx, rast_db=None):
    return dr.interpolate(attr.contiguous(), rast, attr_idx, rast_db=rast_db, diff_attrs=None if rast_db is None else 'all')


def get_mip(roughness):
    return torch.where(roughness < 1.0
                    , (torch.clamp(roughness, 0.04, 1.0) - 0.04) / (1.0 - 0.04) * (6 - 2)
                    , (torch.clamp(roughness, 1.0, 1.0) - 1.0) / (1.0 - 1.0) + 6 - 2)

def shade_with_env(gb_pos, gb_normal, kd, metallic, roughness, view_pos, run_n_view, env, metallic_gt, roughness_gt, use_material_gt=True, gt_render=False):

    #mask = mask[..., 0]
    view_pos = view_pos.expand(-1,  gb_pos.shape[1],  gb_pos.shape[2], -1)  #.reshape(1, 512, 10240, 3)
 
    wo = render_utils.safe_normalize(view_pos - gb_pos)

    spec_col  = (1.0 - metallic_gt)*0.04 + kd * metallic_gt
    diff_col  = kd * (1.0 - metallic_gt)

    
    nrmvec = gb_normal
    reflvec = render_utils.safe_normalize(render_utils.reflect(wo, nrmvec))
    
    prb_rendered_list = []
    pbr_specular_light_list = []
    pbr_diffuse_light_list = []
    for i in range(run_n_view):
        specular_light, diffuse_light = env[i]
        diffuse_light = diffuse_light.cuda()
        specular_light_new = []
        for split_specular_light in specular_light:
            specular_light_new.append(split_specular_light.cuda())
        specular_light = specular_light_new

        shaded_col = torch.ones((gb_pos.shape[1], gb_pos.shape[2], 3)).cuda()

        diffuse = dr.texture(diffuse_light[None, ...], nrmvec[i,:,:,:][None, ...].contiguous(), filter_mode='linear', boundary_mode='cube')
        diffuse_comp = diffuse * diff_col[i,:,:,:][None, ...]
        
        # Lookup FG term from lookup texture
        NdotV = torch.clamp(render_utils.dot(wo[i,:,:,:], nrmvec[i,:,:,:]), min=1e-4)
        fg_uv = torch.cat((NdotV, roughness_gt[i,:,:,:]), dim=-1)
        _FG_LUT = torch.as_tensor(np.fromfile('src/data/bsdf_256_256.bin', dtype=np.float32).reshape(1, 256, 256, 2), dtype=torch.float32, device='cuda')
        fg_lookup = dr.texture(_FG_LUT, fg_uv[None, ...], filter_mode='linear', boundary_mode='clamp')
        
        miplevel = get_mip(roughness_gt[i,:,:,:])
        miplevel = miplevel[None, ...]
        spec = dr.texture(specular_light[0][None, ...], reflvec[i,:,:,:][None, ...].contiguous(), mip=list(m[None, ...] for m in specular_light[1:]), mip_level_bias=miplevel[..., 0], filter_mode='linear-mipmap-linear', boundary_mode='cube')

        # Compute aggregate lighting
        reflectance = spec_col[i,:,:,:][None, ...] * fg_lookup[...,0:1] + fg_lookup[...,1:2]
        specular_comp = spec * reflectance
        shaded_col = (specular_comp[0] + diffuse_comp[0])
        
        prb_rendered_list.append(shaded_col)
        pbr_specular_light_list.append(spec[0])
        pbr_diffuse_light_list.append(diffuse[0])
      
    shaded_col_all = torch.stack(prb_rendered_list, dim=0)
    pbr_specular_light =  torch.stack(pbr_specular_light_list, dim=0)
    pbr_diffuse_light =  torch.stack(pbr_diffuse_light_list, dim=0)

    shaded_col_all = render_utils.rgb_to_srgb(shaded_col_all).clamp(0.,1.)
    pbr_specular_light = render_utils.rgb_to_srgb(pbr_specular_light).clamp(0.,1.)
    pbr_diffuse_light = render_utils.rgb_to_srgb(pbr_diffuse_light).clamp(0.,1.)
    
    return shaded_col_all, pbr_specular_light, pbr_diffuse_light

# ==============================================================================================
#  pixel shader
# ==============================================================================================
def shade(
        gb_pos,
        gb_geometric_normal,
        gb_normal,
        gb_tangent,
        gb_texc,
        gb_texc_deriv,
        view_pos,
        env,
        planes,
        kd_fn,
        materials,
        material,
        mask,
        gt_render,
        gt_albedo_map=None,
    ):

    ################################################################################
    # Texture lookups
    ################################################################################
    perturbed_nrm = None
    resolution = gb_pos.shape[1]
    N_views = view_pos.shape[0]

    if planes is None:
        kd = material['kd'].sample(gb_texc, gb_texc_deriv)
        matellic_gt, roughness_gt =  (materials[0] * torch.ones(*kd.shape[:-1])).unsqueeze(-1).cuda(), (materials[1] * torch.ones(*kd.shape[:-1])).unsqueeze(-1).cuda()
        matellic, roughness = None, None
    else:
        # predict kd with MLP and interpolated feature
        gb_pos_interp, mask = [gb_pos], [mask]
        gb_pos_interp = [torch.cat([pos[i_view:i_view + 1] for i_view in range(N_views)], dim=2) for pos in gb_pos_interp]
        mask = [torch.cat([ma[i_view:i_view + 1] for i_view in range(N_views)], dim=2) for ma in mask]
        # gt_albedo_map
        if gt_albedo_map is not None:
            kd = gt_albedo_map[0].permute(0,2,3,1)
            matellic, roughness = None, None
        else:
            kd, matellic, roughness = kd_fn( planes[None,...], gb_pos_interp, mask[0])
            kd = torch.cat( [torch.cat([kd[i:i + 1, :, resolution * i_view: resolution * (i_view + 1)]for i_view in range(N_views)], dim=0) for i in range(len(kd))], dim=0)
            
        matellic_gt = torch.full((N_views, resolution, resolution, 1), fill_value=0, dtype=torch.float32)
        roughness_gt = torch.full((N_views, resolution, resolution, 1), fill_value=0, dtype=torch.float32)
       
        matellic_val = [x[0] for x in materials]
        roughness_val = [y[1] for y in materials]

        for i in range(len(matellic_gt)):
            matellic_gt[i, :, :, 0].fill_(matellic_val[i])
            roughness_gt[i, :, :, 0].fill_(roughness_val[i])

        matellic_gt = matellic_gt.cuda()
        roughness_gt = roughness_gt.cuda()

    # Separate kd into alpha and color, default alpha = 1
    alpha = kd[..., 3:4] if kd.shape[-1] == 4 else torch.ones_like(kd[..., 0:1]) 
    kd = kd[..., 0:3].clamp(0., 1.)
    ################################################################################
    # Normal perturbation & normal bend
    ################################################################################
    #if 'no_perturbed_nrm' in material and material['no_perturbed_nrm']:
    perturbed_nrm = None

    gb_normal_ = ru.prepare_shading_normal(gb_pos, view_pos, perturbed_nrm, gb_normal, gb_tangent, gb_geometric_normal, two_sided_shading=True, opengl=True)

    ################################################################################
    # Evaluate BSDF
    ################################################################################
    
    shaded_col, spec_light, diff_light = shade_with_env(gb_pos, gb_normal_, kd, matellic, roughness, view_pos, N_views, env, matellic_gt, roughness_gt, use_material_gt=True, gt_render=gt_render)
    
    buffers = {
        'shaded'    : torch.cat((shaded_col, alpha), dim=-1),
        'spec_light': torch.cat((spec_light, alpha), dim=-1),
        'diff_light': torch.cat((diff_light, alpha), dim=-1),
        'gb_normal' : torch.cat((gb_normal_, alpha), dim=-1),
        'normal'    : torch.cat((gb_normal, alpha), dim=-1),
        'albedo'    :  torch.cat((kd, alpha), dim=-1),
    }
    return buffers

# ==============================================================================================
#  Render a depth slice of the mesh (scene), some limitations:
#  - Single mesh
#  - Single light
#  - Single material
# ==============================================================================================
def render_layer(
        rast,
        rast_deriv,
        mesh,
        view_pos,
        env,
        planes,
        kd_fn,
        materials,
        v_pos_clip,
        resolution,
        spp,
        msaa,
        gt_render,
        gt_albedo_map=None,
    ):

    full_res = [resolution[0]*spp, resolution[1]*spp]

    ################################################################################
    # Rasterize
    ################################################################################

    # Scale down to shading resolution when MSAA is enabled, otherwise shade at full resolution
    if spp > 1 and msaa:
        rast_out_s = render_utils.scale_img_nhwc(rast, resolution, mag='nearest', min='nearest')
        rast_out_deriv_s = render_utils.scale_img_nhwc(rast_deriv, resolution, mag='nearest', min='nearest') * spp
    else:
        rast_out_s = rast
        rast_out_deriv_s = rast_deriv

    ################################################################################
    # Interpolate attributes
    ################################################################################

    # Interpolate world space position
    gb_pos, _ = interpolate(mesh.v_pos[None, ...], rast_out_s, mesh.t_pos_idx.int())

    # Compute geometric normals. We need those because of bent normals trick (for bump mapping)
    v0 = mesh.v_pos[mesh.t_pos_idx[:, 0], :]
    v1 = mesh.v_pos[mesh.t_pos_idx[:, 1], :]
    v2 = mesh.v_pos[mesh.t_pos_idx[:, 2], :]
    face_normals = render_utils.safe_normalize(torch.cross(v1 - v0, v2 - v0))
    face_normal_indices = (torch.arange(0, face_normals.shape[0], dtype=torch.int64, device='cuda')[:, None]).repeat(1, 3)
    gb_geometric_normal, _ = interpolate(face_normals[None, ...], rast_out_s, face_normal_indices.int())

    # Compute tangent space
    assert mesh.v_nrm is not None and mesh.v_tng is not None
    gb_normal, _ = interpolate(mesh.v_nrm[None, ...], rast_out_s, mesh.t_nrm_idx.int())
    gb_tangent, _ = interpolate(mesh.v_tng[None, ...], rast_out_s, mesh.t_tng_idx.int()) # Interpolate tangents

    # Texture coordinate
    assert mesh.v_tex is not None
    gb_texc, gb_texc_deriv = interpolate(mesh.v_tex[None, ...], rast_out_s, mesh.t_tex_idx.int(), rast_db=rast_out_deriv_s)
    
    # render depth
    depth = torch.linalg.norm(view_pos.expand_as(gb_pos) - gb_pos, dim=-1)
    
    mask = torch.clamp(rast[..., -1:], 0, 1)
    antialias_mask = dr.antialias(mask.clone().contiguous(), rast, v_pos_clip,mesh.t_pos_idx.int())

    ################################################################################
    # Shade
    ################################################################################
    buffers = shade(gb_pos, gb_geometric_normal, gb_normal, gb_tangent, gb_texc, gb_texc_deriv, view_pos, env, planes, kd_fn, materials, mesh.material, mask, gt_render, gt_albedo_map=gt_albedo_map)
    buffers['depth'] = torch.cat((depth.unsqueeze(-1).repeat(1,1,1,3), torch.ones_like(gb_pos[..., 0:1])), dim=-1 )
    buffers['mask'] = torch.cat((antialias_mask.repeat(1,1,1,3), torch.ones_like(gb_pos[..., 0:1])), dim=-1 )
    ################################################################################
    # Prepare output
    ################################################################################

    # Scale back up to visibility resolution if using MSAA
    if spp > 1 and msaa:
        for key in buffers.keys():
            buffers[key] = render_utils.scale_img_nhwc(buffers[key], full_res, mag='nearest', min='nearest')

    # Return buffers
    return buffers

# ==============================================================================================
#  Render a depth peeled mesh (scene), some limitations:
#  - Single mesh
#  - Single light
#  - Single material
# ==============================================================================================
def render_mesh(
        ctx,
        mesh,
        mtx_in,
        view_pos,
        env,
        planes,
        kd_fn,
        materials,
        resolution,
        spp         = 1,
        num_layers  = 1,
        msaa        = False,
        background  = None, 
        gt_render   = False,
        gt_albedo_map = None,
    ):
    def prepare_input_vector(x):
        x = torch.tensor(x, dtype=torch.float32, device='cuda') if not torch.is_tensor(x) else x
        return x[:, None, None, :] if len(x.shape) == 2 else x
    
    def composite_buffer(key, layers, background, antialias):
        accum = background
        for buffers, rast in reversed(layers):
            alpha = (rast[..., -1:] > 0).float() * buffers[key][..., -1:]
            accum = torch.lerp(accum, torch.cat((buffers[key][..., :-1], torch.ones_like(buffers[key][..., -1:])), dim=-1), alpha)
            if antialias:
                accum = dr.antialias(accum.contiguous(), rast, v_pos_clip, mesh.t_pos_idx.int())
        return accum

    assert mesh.t_pos_idx.shape[0] > 0, "Got empty training triangle mesh (unrecoverable discontinuity)"
    assert background is None or (background.shape[1] == resolution[0] and background.shape[2] == resolution[1])

    full_res = [resolution[0]*spp, resolution[1]*spp]

    # Convert numpy arrays to torch tensors
    mtx_in      = torch.tensor(mtx_in, dtype=torch.float32, device='cuda') if not torch.is_tensor(mtx_in) else mtx_in
    view_pos    = prepare_input_vector(view_pos)

    # clip space transform
    v_pos_clip = ru.xfm_points(mesh.v_pos[None, ...], mtx_in)

    # Render all layers front-to-back
    layers = []
    with dr.DepthPeeler(ctx, v_pos_clip, mesh.t_pos_idx.int(), full_res) as peeler:
        for _ in range(num_layers):
            rast, db = peeler.rasterize_next_layer()
            layers += [(render_layer(rast, db, mesh, view_pos, env, planes, kd_fn, materials, v_pos_clip, resolution, spp, msaa, gt_render, gt_albedo_map), rast)]

    # Setup background
    if background is not None:
        if spp > 1:
            background = render_utils.scale_img_nhwc(background, full_res, mag='nearest', min='nearest')
        background = torch.cat((background, torch.zeros_like(background[..., 0:1])), dim=-1)
    else:
        background = torch.ones(1, full_res[0], full_res[1], 4, dtype=torch.float32, device='cuda')
        background_black = torch.zeros(1, full_res[0], full_res[1], 4, dtype=torch.float32, device='cuda')
        
    # Composite layers front-to-back
    out_buffers = {}

    for key in layers[0][0].keys():
        if key == 'mask':
            accum = composite_buffer(key, layers, background_black, True)
        else:
            accum = composite_buffer(key, layers, background, True)

        # Downscale to framebuffer resolution. Use avg pooling 
        out_buffers[key] = render_utils.avg_pool_nhwc(accum, spp) if spp > 1 else accum
    
    return out_buffers

# ==============================================================================================
#  Render UVs
# ==============================================================================================
def render_uv(ctx, mesh, resolution, mlp_texture):

    # clip space transform 
    uv_clip = mesh.v_tex[None, ...]*2.0 - 1.0

    # pad to four component coordinate
    uv_clip4 = torch.cat((uv_clip, torch.zeros_like(uv_clip[...,0:1]), torch.ones_like(uv_clip[...,0:1])), dim = -1)

    # rasterize
    rast, _ = dr.rasterize(ctx, uv_clip4, mesh.t_tex_idx.int(), resolution)

    # Interpolate world space position
    gb_pos, _ = interpolate(mesh.v_pos[None, ...], rast, mesh.t_pos_idx.int())

    # Sample out textures from MLP
    all_tex = mlp_texture.sample(gb_pos)
    assert all_tex.shape[-1] == 9 or all_tex.shape[-1] == 10, "Combined kd_ks_normal must be 9 or 10 channels"
    perturbed_nrm = all_tex[..., -3:]
    return (rast[..., -1:] > 0).float(), all_tex[..., :-6], all_tex[..., -6:-3], render_utils.safe_normalize(perturbed_nrm)