livetalking/ernerf/data_utils/face_tracking/render_land.py

import torch
import torch.nn as nn
import render_util
import geo_transform
import numpy as np


def compute_tri_normal(geometry, tris):
    geometry = geometry.permute(0, 2, 1)
    tri_1 = tris[:, 0]
    tri_2 = tris[:, 1]
    tri_3 = tris[:, 2]

    vert_1 = torch.index_select(geometry, 2, tri_1)
    vert_2 = torch.index_select(geometry, 2, tri_2)
    vert_3 = torch.index_select(geometry, 2, tri_3)

    nnorm = torch.cross(vert_2 - vert_1, vert_3 - vert_1, 1)
    normal = nn.functional.normalize(nnorm).permute(0, 2, 1)
    return normal


class Compute_normal_base(torch.autograd.Function):
    @staticmethod
    def forward(ctx, normal):
        (normal_b,) = render_util.normal_base_forward(normal)
        ctx.save_for_backward(normal)
        return normal_b

    @staticmethod
    def backward(ctx, grad_normal_b):
        (normal,) = ctx.saved_tensors
        (grad_normal,) = render_util.normal_base_backward(grad_normal_b, normal)
        return grad_normal


class Normal_Base(torch.nn.Module):
    def __init__(self):
        super(Normal_Base, self).__init__()

    def forward(self, normal):
        return Compute_normal_base.apply(normal)


def preprocess_render(geometry, euler, trans, cam, tris, vert_tris, ori_img):
    point_num = geometry.shape[1]
    rott_geo = geo_transform.euler_trans_geo(geometry, euler, trans)
    proj_geo = geo_transform.proj_geo(rott_geo, cam)
    rot_tri_normal = compute_tri_normal(rott_geo, tris)
    rot_vert_normal = torch.index_select(rot_tri_normal, 1, vert_tris)
    is_visible = -torch.bmm(
        rot_vert_normal.reshape(-1, 1, 3),
        nn.functional.normalize(rott_geo.reshape(-1, 3, 1)),
    ).reshape(-1, point_num)
    is_visible[is_visible < 0.01] = -1
    pixel_valid = torch.zeros(
        (ori_img.shape[0], ori_img.shape[1] * ori_img.shape[2]),
        dtype=torch.float32,
        device=ori_img.device,
    )
    return rott_geo, proj_geo, rot_tri_normal, is_visible, pixel_valid


class Render_Face(torch.autograd.Function):
    @staticmethod
    def forward(
        ctx, proj_geo, texture, nbl, ori_img, is_visible, tri_inds, pixel_valid
    ):
        batch_size, h, w, _ = ori_img.shape
        ori_img = ori_img.view(batch_size, -1, 3)
        ori_size = torch.cat(
            (
                torch.ones((batch_size, 1), dtype=torch.int32, device=ori_img.device)
                * h,
                torch.ones((batch_size, 1), dtype=torch.int32, device=ori_img.device)
                * w,
            ),
            dim=1,
        ).view(-1)
        tri_index, tri_coord, render, real = render_util.render_face_forward(
            proj_geo, ori_img, ori_size, texture, nbl, is_visible, tri_inds, pixel_valid
        )
        ctx.save_for_backward(
            ori_img, ori_size, proj_geo, texture, nbl, tri_inds, tri_index, tri_coord
        )
        return render, real

    @staticmethod
    def backward(ctx, grad_render, grad_real):
        (
            ori_img,
            ori_size,
            proj_geo,
            texture,
            nbl,
            tri_inds,
            tri_index,
            tri_coord,
        ) = ctx.saved_tensors
        grad_proj_geo, grad_texture, grad_nbl = render_util.render_face_backward(
            grad_render,
            grad_real,
            ori_img,
            ori_size,
            proj_geo,
            texture,
            nbl,
            tri_inds,
            tri_index,
            tri_coord,
        )
        return grad_proj_geo, grad_texture, grad_nbl, None, None, None, None


class Render_RGB(nn.Module):
    def __init__(self):
        super(Render_RGB, self).__init__()

    def forward(
        self, proj_geo, texture, nbl, ori_img, is_visible, tri_inds, pixel_valid
    ):
        return Render_Face.apply(
            proj_geo, texture, nbl, ori_img, is_visible, tri_inds, pixel_valid
        )


def cal_land(proj_geo, is_visible, lands_info, land_num):
    (land_index,) = render_util.update_contour(lands_info, is_visible, land_num)
    proj_land = torch.index_select(proj_geo.reshape(-1, 3), 0, land_index)[
        :, :2
    ].reshape(-1, land_num, 2)
    return proj_land


class Render_Land(nn.Module):
    def __init__(self):
        super(Render_Land, self).__init__()
        lands_info = np.loadtxt("../data/3DMM/lands_info.txt", dtype=np.int32)
        self.lands_info = torch.as_tensor(lands_info).cuda()
        tris = np.loadtxt("../data/3DMM/tris.txt", dtype=np.int64)
        self.tris = torch.as_tensor(tris).cuda() - 1
        vert_tris = np.loadtxt("../data/3DMM/vert_tris.txt", dtype=np.int64)
        self.vert_tris = torch.as_tensor(vert_tris).cuda()
        self.normal_baser = Normal_Base().cuda()
        self.renderer = Render_RGB().cuda()

    def render_mesh(self, geometry, euler, trans, cam, ori_img, light):
        batch_size, h, w, _ = ori_img.shape
        ori_img = ori_img.view(batch_size, -1, 3)
        ori_size = torch.cat(
            (
                torch.ones((batch_size, 1), dtype=torch.int32, device=ori_img.device)
                * h,
                torch.ones((batch_size, 1), dtype=torch.int32, device=ori_img.device)
                * w,
            ),
            dim=1,
        ).view(-1)
        rott_geo, proj_geo, rot_tri_normal, _, _ = preprocess_render(
            geometry, euler, trans, cam, self.tris, self.vert_tris, ori_img
        )
        tri_nb = self.normal_baser(rot_tri_normal.contiguous())
        nbl = torch.bmm(
            tri_nb, (light.reshape(-1, 9, 3))[:, :, 0].unsqueeze(-1).repeat(1, 1, 3)
        )
        texture = torch.ones_like(geometry) * 200
        (render,) = render_util.render_mesh(
            proj_geo, ori_img, ori_size, texture, nbl, self.tris
        )
        return render.view(batch_size, h, w, 3).byte()

    def cal_loss_rgb(self, geometry, euler, trans, cam, ori_img, light, texture, lands):
        rott_geo, proj_geo, rot_tri_normal, is_visible, pixel_valid = preprocess_render(
            geometry, euler, trans, cam, self.tris, self.vert_tris, ori_img
        )
        tri_nb = self.normal_baser(rot_tri_normal.contiguous())
        nbl = torch.bmm(tri_nb, light.reshape(-1, 9, 3))
        render, real = self.renderer(
            proj_geo, texture, nbl, ori_img, is_visible, self.tris, pixel_valid
        )
        proj_land = cal_land(proj_geo, is_visible, self.lands_info, lands.shape[1])
        col_minus = torch.norm((render - real).reshape(-1, 3), dim=1).reshape(
            ori_img.shape[0], -1
        )
        col_dis = torch.mean(col_minus * pixel_valid) / (
            torch.mean(pixel_valid) + 0.00001
        )
        land_dists = torch.norm((proj_land - lands).reshape(-1, 2), dim=1).reshape(
            ori_img.shape[0], -1
        )
        lan_dis = torch.mean(land_dists)
        return col_dis, lan_dis