Convert triangle mesh model to tmesh map (#6758)

- ToString() for Material and string representation in Python.
- MaterialRecord to Material conversion - allows converting triangle mesh models to a map of tmeshes, including materials. Limitation: Only one material per tmesh (same as TriangleMeshModel)
- Support for emissive color reading in FileASSIMP (model) and writing for tmesh (tio FIleASSIMP)

Author: ssheorey

cpp/open3d/io/file_format/FileASSIMP.cpp

@@ -422,6 +422,9 @@ bool ReadModelUsingAssimp(const std::string& filename,
         mat->Get(AI_MATKEY_CLEARCOAT_ROUGHNESS_FACTOR,
                  o3d_mat.base_clearcoat_roughness);
         mat->Get(AI_MATKEY_ANISOTROPY, o3d_mat.base_anisotropy);
+        mat->Get(AI_MATKEY_COLOR_EMISSIVE, color);
+        o3d_mat.emissive_color =
+                Eigen::Vector4f(color.r, color.g, color.b, 1.f);
         aiString alpha_mode;
         mat->Get(AI_MATKEY_GLTF_ALPHAMODE, alpha_mode);
         std::string alpha_mode_str(alpha_mode.C_Str());

cpp/open3d/t/geometry/TriangleMesh.cpp

@@ -377,6 +377,7 @@ geometry::TriangleMesh TriangleMesh::FromLegacy(
         tmat.SetAnisotropy(mat.baseAnisotropy);
         tmat.SetBaseClearcoat(mat.baseClearCoat);
         tmat.SetBaseClearcoatRoughness(mat.baseClearCoatRoughness);
+        // no emissive_color in legacy mesh material
         if (mat.albedo) tmat.SetAlbedoMap(Image::FromLegacy(*mat.albedo));
         if (mat.normalMap) tmat.SetNormalMap(Image::FromLegacy(*mat.normalMap));
         if (mat.roughness)
@@ -453,10 +454,6 @@ open3d::geometry::TriangleMesh TriangleMesh::ToLegacy() const {
             legacy_mat.baseColor.f4[1] = tmat.GetBaseColor().y();
             legacy_mat.baseColor.f4[2] = tmat.GetBaseColor().z();
             legacy_mat.baseColor.f4[3] = tmat.GetBaseColor().w();
-            utility::LogWarning("{},{},{},{}", legacy_mat.baseColor.f4[0],
-                                legacy_mat.baseColor.f4[1],
-                                legacy_mat.baseColor.f4[2],
-                                legacy_mat.baseColor.f4[3]);
         }
         if (tmat.HasBaseRoughness()) {
             legacy_mat.baseRoughness = tmat.GetBaseRoughness();
@@ -523,6 +520,26 @@ open3d::geometry::TriangleMesh TriangleMesh::ToLegacy() const {
     return mesh_legacy;
 }
 
+std::unordered_map<std::string, geometry::TriangleMesh>
+TriangleMesh::FromTriangleMeshModel(
+        const open3d::visualization::rendering::TriangleMeshModel &model,
+        core::Dtype float_dtype,
+        core::Dtype int_dtype,
+        const core::Device &device) {
+    std::unordered_map<std::string, TriangleMesh> tmeshes;
+    for (const auto &mobj : model.meshes_) {
+        auto tmesh = TriangleMesh::FromLegacy(*mobj.mesh, float_dtype,
+                                              int_dtype, device);
+        // material textures will be on the CPU. GPU resident texture images is
+        // not yet supported. See comment in Material.cpp
+        tmesh.SetMaterial(
+                visualization::rendering::Material::FromMaterialRecord(
+                        model.materials_[mobj.material_idx]));
+        tmeshes.emplace(mobj.mesh_name, tmesh);
+    }
+    return tmeshes;
+}
+
 TriangleMesh TriangleMesh::To(const core::Device &device, bool copy) const {
     if (!copy && GetDevice() == device) {
         return *this;

cpp/open3d/t/geometry/TriangleMesh.h

@@ -8,6 +8,7 @@
 #pragma once
 
 #include <list>
+#include <unordered_map>
 
 #include "open3d/core/Tensor.h"
 #include "open3d/core/TensorCheck.h"
@@ -16,6 +17,7 @@
 #include "open3d/t/geometry/DrawableGeometry.h"
 #include "open3d/t/geometry/Geometry.h"
 #include "open3d/t/geometry/TensorMap.h"
+#include "open3d/visualization/rendering/Model.h"
 
 namespace open3d {
 namespace t {
@@ -701,7 +703,8 @@ class TriangleMesh : public Geometry, public DrawableGeometry {
     /// values, e.g. vertices, normals, colors.
     /// \param int_dtype Int32 or Int64, used to store index values, e.g.
     /// triangles.
-    /// \param device The device where the resulting TriangleMesh resides in.
+    /// \param device The device where the resulting TriangleMesh resides in
+    /// (default CPU:0).
     static geometry::TriangleMesh FromLegacy(
             const open3d::geometry::TriangleMesh &mesh_legacy,
             core::Dtype float_dtype = core::Float32,
@@ -711,6 +714,28 @@ class TriangleMesh : public Geometry, public DrawableGeometry {
     /// Convert to a legacy Open3D TriangleMesh.
     open3d::geometry::TriangleMesh ToLegacy() const;
 
+    /// Convert a TriangleMeshModel (e.g. as read from a file with
+    /// open3d::io::ReadTriangleMeshModel) to an unordered map of mesh names to
+    /// TriangleMeshes. Only one material is supported per mesh. Materials
+    /// common to multiple meshes will be dupicated. Textures (as
+    /// t::geometry::Image) will use shared storage.
+    /// \param model TriangleMeshModel to convert.
+    /// \param float_dtype Float32 or Float64, used to store floating point
+    /// values, e.g. vertices, normals, colors.
+    /// \param int_dtype Int32 or Int64, used to store index values, e.g.
+    /// triangles.
+    /// \param device The device where the resulting TriangleMesh resides in
+    /// (default CPU:0). Material textures use CPU storage - GPU resident
+    /// texture images are not yet supported.
+    /// \return unordered map of constituent mesh names to TriangleMeshes, with
+    /// materials.
+    static std::unordered_map<std::string, geometry::TriangleMesh>
+    FromTriangleMeshModel(
+            const open3d::visualization::rendering::TriangleMeshModel &model,
+            core::Dtype float_dtype = core::Float32,
+            core::Dtype int_dtype = core::Int64,
+            const core::Device &device = core::Device("CPU:0"));
+
     /// Compute the convex hull of the triangle mesh using qhull.
     ///
     /// This runs on the CPU.

cpp/open3d/t/io/file_format/FileASSIMP.cpp

@@ -6,6 +6,7 @@
 // ----------------------------------------------------------------------------
 
 #include <assimp/GltfMaterial.h>
+#include <assimp/material.h>
 #include <assimp/postprocess.h>
 #include <assimp/scene.h>
 
@@ -357,12 +358,17 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename,
             auto r = mesh.GetMaterial().GetBaseClearcoatRoughness();
             ai_mat->AddProperty(&r, 1, AI_MATKEY_CLEARCOAT_ROUGHNESS_FACTOR);
         }
+        if (mesh.GetMaterial().HasEmissiveColor()) {
+            auto c = mesh.GetMaterial().GetEmissiveColor();
+            auto ac = aiColor4D(c.x(), c.y(), c.z(), c.w());
+            ai_mat->AddProperty(&ac, 1, AI_MATKEY_COLOR_EMISSIVE);
+        }
 
         // Count texture maps...
         // NOTE: GLTF2 expects a single combined roughness/metal map. If the
         // model has one we just export it, otherwise if both roughness and
-        // metal maps are avaialbe we combine them, otherwise if only one or the
-        // other is available we just export the one map.
+        // metal maps are available we combine them, otherwise if only one or
+        // the other is available we just export the one map.
         int n_textures = 0;
         if (mesh.GetMaterial().HasAlbedoMap()) ++n_textures;
         if (mesh.GetMaterial().HasNormalMap()) ++n_textures;

cpp/open3d/visualization/rendering/Material.cpp

@@ -26,6 +26,7 @@ void Material::SetDefaultProperties() {
     SetTransmission(1.f);
     SetAbsorptionColor(Eigen::Vector4f(1.f, 1.f, 1.f, 1.f));
     SetAbsorptionDistance(1.f);
+    SetEmissiveColor(Eigen::Vector4f(1.f, 1.f, 1.f, 1.f));
     SetPointSize(3.f);
     SetLineWidth(1.f);
 }
@@ -39,6 +40,24 @@ void Material::SetTextureMap(const std::string &key,
     texture_maps_[key] = image.To(core::Device("CPU:0"), true);
 }
 
+std::string Material::ToString() const {
+    if (!IsValid()) {
+        return "Invalid Material\n";
+    }
+    std::ostringstream os;
+    os << "Material " << material_name_ << '\n';
+    for (const auto &kv : scalar_properties_) {
+        os << '\t' << kv.first << ": " << kv.second << '\n';
+    }
+    for (const auto &kv : vector_properties_) {
+        os << '\t' << kv.first << ": " << kv.second.transpose() << '\n';
+    }
+    for (const auto &kv : texture_maps_) {
+        os << '\t' << kv.first << ": " << kv.second.ToString() << '\n';
+    }
+    return os.str();
+}
+
 void Material::ToMaterialRecord(MaterialRecord &record) const {
     record.shader = GetMaterialName();
     // Convert base material properties
@@ -63,6 +82,9 @@ void Material::ToMaterialRecord(MaterialRecord &record) const {
     if (HasAnisotropy()) {
         record.base_anisotropy = GetAnisotropy();
     }
+    if (HasEmissiveColor()) {
+        record.emissive_color = GetEmissiveColor();
+    }
     if (HasThickness()) {
         record.thickness = GetThickness();
     }
@@ -124,6 +146,62 @@ void Material::ToMaterialRecord(MaterialRecord &record) const {
     }
 }
 
+Material Material::FromMaterialRecord(const MaterialRecord &record) {
+    using t::geometry::Image;
+    Material tmat(record.shader);
+    // scalar and vector properties
+    tmat.SetBaseColor(record.base_color);
+    tmat.SetBaseMetallic(record.base_metallic);
+    tmat.SetBaseRoughness(record.base_roughness);
+    tmat.SetBaseReflectance(record.base_reflectance);
+    tmat.SetBaseClearcoat(record.base_clearcoat);
+    tmat.SetBaseClearcoatRoughness(record.base_clearcoat_roughness);
+    tmat.SetAnisotropy(record.base_anisotropy);
+    tmat.SetEmissiveColor(record.emissive_color);
+    // refractive materials
+    tmat.SetThickness(record.thickness);
+    tmat.SetTransmission(record.transmission);
+    tmat.SetAbsorptionDistance(record.absorption_distance);
+    // points and lines
+    tmat.SetPointSize(record.point_size);
+    tmat.SetLineWidth(record.line_width);
+    // maps
+    if (record.albedo_img) {
+        tmat.SetAlbedoMap(Image::FromLegacy(*record.albedo_img));
+    }
+    if (record.normal_img) {
+        tmat.SetNormalMap(Image::FromLegacy(*record.normal_img));
+    }
+    if (record.ao_img) {
+        tmat.SetAOMap(Image::FromLegacy(*record.ao_img));
+    }
+    if (record.metallic_img) {
+        tmat.SetMetallicMap(Image::FromLegacy(*record.metallic_img));
+    }
+    if (record.roughness_img) {
+        tmat.SetRoughnessMap(Image::FromLegacy(*record.roughness_img));
+    }
+    if (record.reflectance_img) {
+        tmat.SetReflectanceMap(Image::FromLegacy(*record.reflectance_img));
+    }
+    if (record.clearcoat_img) {
+        tmat.SetClearcoatMap(Image::FromLegacy(*record.clearcoat_img));
+    }
+    if (record.clearcoat_roughness_img) {
+        tmat.SetClearcoatRoughnessMap(
+                Image::FromLegacy(*record.clearcoat_roughness_img));
+    }
+    if (record.anisotropy_img) {
+        tmat.SetAnisotropyMap(Image::FromLegacy(*record.anisotropy_img));
+    }
+    if (record.ao_rough_metal_img) {
+        tmat.SetAORoughnessMetalMap(
+                Image::FromLegacy(*record.ao_rough_metal_img));
+    }
+
+    return tmat;
+}
+
 }  // namespace rendering
 }  // namespace visualization
 }  // namespace open3d

cpp/open3d/visualization/rendering/Material.h

@@ -7,6 +7,7 @@
 
 #pragma once
 
+#include <sstream>
 #include <string>
 
 #include "open3d/t/geometry/Image.h"
@@ -34,6 +35,9 @@ class Material {
 
     Material(const Material &mat) = default;
 
+    /// Convert from MaterialRecord
+    static Material FromMaterialRecord(const MaterialRecord &mat);
+
     Material &operator=(const Material &other) = default;
 
     /// Create an empty but valid material for the specified material name
@@ -51,6 +55,9 @@ class Material {
     /// Get the name of the material.
     const std::string &GetMaterialName() const { return material_name_; }
 
+    /// String reprentation for printing.
+    std::string ToString() const;
+
     /// Returns the texture map map
     const TextureMaps &GetTextureMaps() const { return texture_maps_; }
 
@@ -249,6 +256,9 @@ class Material {
     float GetAbsorptionDistance() const {
         return GetScalarProperty("absorption_distance");
     }
+    Eigen::Vector4f GetEmissiveColor() const {
+        return GetVectorProperty("emissive_color");
+    }
 
     bool HasBaseColor() const { return HasVectorProperty("base_color"); }
     bool HasBaseMetallic() const { return HasScalarProperty("metallic"); }
@@ -267,6 +277,9 @@ class Material {
     bool HasAbsorptionDistance() const {
         return HasScalarProperty("absorption_distance");
     }
+    bool HasEmissiveColor() const {
+        return HasVectorProperty("emissive_color");
+    }
 
     void SetBaseColor(const Eigen::Vector4f &value) {
         SetVectorProperty("base_color", value);
@@ -295,6 +308,9 @@ class Material {
     void SetAbsorptionDistance(float value) {
         SetScalarProperty("absorption_distance", value);
     }
+    void SetEmissiveColor(const Eigen::Vector4f &value) {
+        SetVectorProperty("emissive_color", value);
+    }
 
     ////////////////////////////////////////////////////////////////////////////
     ///

cpp/pybind/t/geometry/trianglemesh.cpp

@@ -238,6 +238,28 @@ The attributes of the triangle mesh have different levels::
             "vertex_dtype"_a = core::Float32, "triangle_dtype"_a = core::Int64,
             "device"_a = core::Device("CPU:0"),
             "Create a TriangleMesh from a legacy Open3D TriangleMesh.");
+    triangle_mesh.def_static(
+            "from_triangle_mesh_model", &TriangleMesh::FromTriangleMeshModel,
+            "model"_a, "vertex_dtype"_a = core::Float32,
+            "triangle_dtype"_a = core::Int64,
+            "device"_a = core::Device("CPU:0"),
+            R"(Convert a TriangleMeshModel (e.g. as read from a file with
+`open3d.io.read_triangle_mesh_model()`) to a dictionary of mesh names to
+triangle meshes with the specified vertex and triangle dtypes and moved to the
+specified device. Only a single material per mesh is supported. Materials common
+to multiple meshes will be duplicated. Textures (as t.geometry.Image) will use
+shared storage on the CPU (GPU resident images for textures is not yet supported).
+
+Returns:
+    Dictionary of names to triangle meshes.
+
+Example:
+    flight_helmet = o3d.data.FlightHelmetModel()
+    model = o3d.io.read_triangle_model(flight_helmet.path)
+    mesh_dict = o3d.t.geometry.TriangleMesh.from_triangle_mesh_model(model)
+    o3d.visualization.draw(list({"name": name, "geometry": tmesh} for
+        (name, tmesh) in mesh_dict.items()))
+            )");
     // conversion
     triangle_mesh.def("to_legacy", &TriangleMesh::ToLegacy,
                       "Convert to a legacy Open3D TriangleMesh.");
@@ -706,7 +728,7 @@ This function always uses the CPU device.
 
 Returns:
     This function creates a face attribute "texture_uvs" and returns a tuple
-    with (max stretch, num_charts, num_partitions) storing the 
+    with (max stretch, num_charts, num_partitions) storing the
     actual amount of stretch, the number of created charts, and the number of
     parallel partitions created.
 
@@ -883,7 +905,7 @@ This function always uses the CPU device.
                       "max_faces"_a,
                       R"(Partition the mesh by recursively doing PCA.
 
-This function creates a new face attribute with the name "partition_ids" storing 
+This function creates a new face attribute with the name "partition_ids" storing
 the partition id for each face.
 
 Args:
@@ -892,7 +914,7 @@ the partition id for each face.
 
 Example:
 
-    This code partitions a mesh such that each partition contains at most 20k 
+    This code partitions a mesh such that each partition contains at most 20k
     faces::
 
         import open3d as o3d
@@ -911,15 +933,15 @@ the partition id for each face.
             R"(Returns a new mesh with the faces selected by a boolean mask.
 
 Args:
-    mask (open3d.core.Tensor): A boolean mask with the shape (N) with N as the 
+    mask (open3d.core.Tensor): A boolean mask with the shape (N) with N as the
         number of faces in the mesh.
-    
+
 Returns:
     A new mesh with the selected faces. If the original mesh is empty, return an empty mesh.
 
 Example:
 
-    This code partitions the mesh using PCA and then visualized the individual 
+    This code partitions the mesh using PCA and then visualized the individual
     parts::
 
         import open3d as o3d

cpp/pybind/visualization/rendering/material.cpp

@@ -12,6 +12,7 @@
 
 #include "open3d/visualization/rendering/Material.h"
 
+#include "open3d/visualization/rendering/MaterialRecord.h"
 #include "pybind/open3d_pybind.h"
 
 PYBIND11_MAKE_OPAQUE(
@@ -41,6 +42,9 @@ void pybind_material(py::module& m) {
     mat.def(py::init<>())
             .def(py::init<Material>(), "", "mat"_a)
             .def(py::init<const std::string&>(), "", "material_name"_a)
+            .def(py::init(&Material::FromMaterialRecord), "material_record"_a,
+                 "Convert from MaterialRecord.")
+            .def("__repr__", &Material::ToString)
             .def("set_default_properties", &Material::SetDefaultProperties,
                  "Fills material with defaults for common PBR material "
                  "properties used by Open3D")