diff --git a/CMakePresets.json b/CMakePresets.json index 06f1be41..72954721 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -31,7 +31,6 @@ "ASSIMP_BUILD_ASSIMP_TOOLS": "OFF", "ASSIMP_BUILD_ALL_IMPORTERS_BY_DEFAULT": "OFF", "ASSIMP_BUILD_OBJ_IMPORTER": "ON", - "ASSIMP_BUILD_FBX_IMPORTER": "ON", "ASSIMP_BUILD_ALL_EXPORTERS_BY_DEFAULT": "OFF", "ASSIMP_BUILD_OBJ_EXPORTER": "ON", diff --git a/Tetragrama/Components/AssetImporterUIComponent.cpp b/Tetragrama/Components/AssetImporterUIComponent.cpp index 08a25e41..aa8e00be 100644 --- a/Tetragrama/Components/AssetImporterUIComponent.cpp +++ b/Tetragrama/Components/AssetImporterUIComponent.cpp @@ -32,11 +32,13 @@ namespace Tetragrama::Components // import memory — engine importers and editor importers — is budget-tracked. auto* import_arena = &ZEngine::Engine::GetContext()->ImportPipelineArena; import_arena->CreateSubArena(ZMega(64), &GltfImporterArena); - import_arena->CreateSubArena(ZMega(350), &AssimpImporterArena); + import_arena->CreateSubArena(ZMega(128), &AssimpImporterArena); m_gltf_importer = ZPushStructCtor(import_arena, ZEngine::Importers::GltfImporter); + m_fbx_importer = ZPushStructCtor(import_arena, ZEngine::Importers::FbxImporter); m_assimp_importer = ZPushStructCtor(import_arena, ZEngine::Importers::AssimpImporter); m_gltf_importer->Initialize(&GltfImporterArena); + m_fbx_importer->Initialize(import_arena); m_assimp_importer->Initialize(&AssimpImporterArena); m_path_buf.init(&LocalStringArena, 1024); @@ -138,7 +140,7 @@ namespace Tetragrama::Components void AssetImporterUIComponent::StartImport() { - if (!m_gltf_importer || !m_assimp_importer) + if (!m_gltf_importer || !m_fbx_importer || !m_assimp_importer) return; auto* app = reinterpret_cast(ParentLayer->CurrentApp); @@ -198,6 +200,10 @@ namespace Tetragrama::Components { ZEngine::Helpers::ThreadPoolHelper::Submit([this, src = m_path_buf, cfg_copy, arena = &LocalArena, app]() mutable { m_gltf_importer->ImportFile(src.c_str(), cfg_copy, arena, this, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); }); } + else if (secure_strcmp(ext.Data, ".fbx") == 0) + { + ZEngine::Helpers::ThreadPoolHelper::Submit([this, src = m_path_buf, cfg_copy, arena = &LocalArena, app]() mutable { m_fbx_importer->ImportFile(src.c_str(), cfg_copy, arena, this, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); }); + } else { ZEngine::Helpers::ThreadPoolHelper::Submit([this, src = m_path_buf, cfg_copy, arena = &LocalArena, app]() mutable { m_assimp_importer->ImportFile(src.c_str(), cfg_copy, arena, this, OnImportFileComplete, OnImportProgress, OnImportError, OnImportLog); }); diff --git a/Tetragrama/Components/AssetImporterUIComponent.h b/Tetragrama/Components/AssetImporterUIComponent.h index 76fa8da8..f6f0bd0f 100644 --- a/Tetragrama/Components/AssetImporterUIComponent.h +++ b/Tetragrama/Components/AssetImporterUIComponent.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -92,6 +93,7 @@ namespace Tetragrama::Components // Importers — allocated from parent arena in Initialize() ZEngine::Importers::GltfImporter* m_gltf_importer = nullptr; + ZEngine::Importers::FbxImporter* m_fbx_importer = nullptr; ZEngine::Importers::AssimpImporter* m_assimp_importer = nullptr; void PushLog(cstring text, const float color[4]); diff --git a/ZEngine/ZEngine/Core/Memory/MemoryManager.h b/ZEngine/ZEngine/Core/Memory/MemoryManager.h index e9a49008..7714f881 100644 --- a/ZEngine/ZEngine/Core/Memory/MemoryManager.h +++ b/ZEngine/ZEngine/Core/Memory/MemoryManager.h @@ -57,12 +57,12 @@ namespace ZEngine::Core::Memory MemoryBudgetConfig cfg = {}; cfg.AudioEngine = {"AudioEngine", ZMega(128ULL)}; cfg.AnimationManager = {"AnimationManager", ZMega(256ULL)}; - cfg.AssetManager = {"AssetManager", ZMega(512ULL)}; + cfg.AssetManager = {"AssetManager", ZGiga(1ULL)}; cfg.ECSScene = {"ECSScene", ZMega(512ULL)}; cfg.Logging = {"Logging", ZMega(8ULL)}; cfg.VirtualFS = {"VirtualFS", ZMega(64ULL)}; cfg.VulkanDevice = {"VulkanDevice", ZGiga(1ULL)}; - cfg.ImportPipeline = {"ImportPipeline", ZGiga(1ULL)}; // glTF 64 + Assimp 128 + envmap 32 + editor ~414 MB; each carves directly + cfg.ImportPipeline = {"ImportPipeline", ZMega(3584ULL)}; // 3.5 GB — each importer gets generous headroom for large scenes cfg.UIContext = {"UIContext", ZMega(64ULL)}; cfg.Swapchain = {"Swapchain", ZMega(8ULL)}; cfg.ShaderCache = {"ShaderCache", ZMega(64ULL)}; diff --git a/ZEngine/ZEngine/Engine.cpp b/ZEngine/ZEngine/Engine.cpp index c6524d22..1a92485c 100644 --- a/ZEngine/ZEngine/Engine.cpp +++ b/ZEngine/ZEngine/Engine.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -95,12 +96,15 @@ namespace ZEngine g_engine_ctx->ImportCoordinator->Initialize(&g_engine_ctx->AssetArena, g_engine_ctx->VFS, Managers::AssetManager::Instance()->Registry); static Importers::GltfImporter s_gltf_importer; + static Importers::FbxImporter s_fbx_importer; static Importers::AssimpImporter s_assimp_importer; static Importers::EnvironmentMapImporter s_env_map_importer; s_gltf_importer.Initialize(&g_engine_ctx->ImportPipelineArena); + s_fbx_importer.Initialize(&g_engine_ctx->ImportPipelineArena); s_assimp_importer.Initialize(&g_engine_ctx->ImportPipelineArena); s_env_map_importer.Initialize(&g_engine_ctx->ImportPipelineArena); g_engine_ctx->ImportCoordinator->RegisterImporter(&s_gltf_importer); + g_engine_ctx->ImportCoordinator->RegisterImporter(&s_fbx_importer); g_engine_ctx->ImportCoordinator->RegisterImporter(&s_assimp_importer); g_engine_ctx->ImportCoordinator->RegisterImporter(&s_env_map_importer); diff --git a/ZEngine/ZEngine/Importers/AssimpImporter.cpp b/ZEngine/ZEngine/Importers/AssimpImporter.cpp index 3dc0bf8c..18267416 100644 --- a/ZEngine/ZEngine/Importers/AssimpImporter.cpp +++ b/ZEngine/ZEngine/Importers/AssimpImporter.cpp @@ -41,7 +41,7 @@ namespace ZEngine::Importers { if (!extension) return false; - return secure_strcmp(extension, "fbx") == 0 || secure_strcmp(extension, "obj") == 0; + return secure_strcmp(extension, "obj") == 0; } Core::VFS::VFSResult AssimpImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) diff --git a/ZEngine/ZEngine/Importers/FbxImporter.cpp b/ZEngine/ZEngine/Importers/FbxImporter.cpp new file mode 100644 index 00000000..5537c3dd --- /dev/null +++ b/ZEngine/ZEngine/Importers/FbxImporter.cpp @@ -0,0 +1,474 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using ZEngine::Core::VFS::VFSPath; +using namespace ZEngine::Core::Containers; +using namespace ZEngine::Core::Maths; +using namespace ZEngine::Importers; +using namespace uuids; + +namespace fs = std::filesystem; + +namespace ZEngine::Importers +{ + // Vertex deduplication key — (position, normal, uv) index triple. + struct VtxKey + { + uint32_t pos, nrm, uv; + bool operator==(const VtxKey& o) const + { + return pos == o.pos && nrm == o.nrm && uv == o.uv; + } + }; + struct VtxKeyHash + { + size_t operator()(const VtxKey& k) const + { + size_t h = (size_t) k.pos * 2654435761u; + h ^= (size_t) k.nrm * 2246822519u; + h ^= (size_t) k.uv * 3266489917u; + return h; + } + }; + + void FbxImporter::Initialize(Core::Memory::ArenaAllocator* arena) + { + arena->CreateSubArena(ZMega(512), &Arena); + } + + bool FbxImporter::CanImport(const char* extension) const + { + return extension && Helpers::secure_strcmp(extension, "fbx") == 0; + } + + Core::VFS::VFSResult FbxImporter::Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) + { + char native[MAX_FILE_PATH_COUNT] = {}; + const char* working_space = Managers::AssetManager::Instance() ? Managers::AssetManager::Instance()->CurrentWorkingSpacePath : ""; + if (working_space && working_space[0] != '\0') + path.ResolveNative(working_space, native, sizeof(native)); + else + path.ToNative(native, sizeof(native)); + + AssetCodec::ImportConfiguration config = {}; + config.VFS = &ctx; + + AssetMesh mesh = {}; + AssetNodeHierarchy hier = {}; + Array materials = {}; + Array textures = {}; + + auto scratch = ZGetScratch(&Arena); + + std::random_device rd; + std::mt19937 gen_mt(rd()); + uuid_random_generator gen(&gen_mt); + + ufbx_load_opts opts = {}; + opts.target_axes = ufbx_axes_right_handed_y_up; + opts.target_unit_meters = 1.0f; + opts.generate_missing_normals = true; + + ufbx_error error; + ufbx_scene* scene = ufbx_load_file(native, &opts, &error); + if (!scene) + { + ZENGINE_CORE_ERROR("[FbxImporter] Failed to load '{}': {}", native, error.description.data) + return Core::VFS::VFSResult::Fail(Core::VFS::VFSError::IOError); + } + + mesh.MeshUUID = meta.AssetUUID; + mesh.Vertices.init(scratch.Arena, 1024); + mesh.Indices.init(scratch.Arena, 1024); + mesh.SubMeshes.init(scratch.Arena, (uint32_t) scene->meshes.count); + materials.init(scratch.Arena, 64); + textures.init(scratch.Arena, 256); + + for (size_t mi = 0; mi < scene->meshes.count; ++mi) + { + ufbx_mesh* fbx_mesh = scene->meshes.data[mi]; + std::unordered_map vtx_map; + vtx_map.reserve(fbx_mesh->num_triangles * 3); + + uint32_t sub_vtx_start = static_cast(mesh.Vertices.size() / 8); + uint32_t sub_idx_start = static_cast(mesh.Indices.size()); + + uint32_t tri_buf[512]; + for (size_t fi = 0; fi < fbx_mesh->faces.count; ++fi) + { + ufbx_face face = fbx_mesh->faces.data[fi]; + uint32_t num_tris = ufbx_triangulate_face(tri_buf, 512, fbx_mesh, face); + + for (uint32_t ti = 0; ti < num_tris; ++ti) + { + for (uint32_t vi = 0; vi < 3; ++vi) + { + uint32_t fv = tri_buf[ti * 3 + vi]; + + uint32_t pi = (uint32_t) fbx_mesh->vertex_position.indices.data[fv]; + uint32_t ni = fbx_mesh->vertex_normal.exists ? (uint32_t) fbx_mesh->vertex_normal.indices.data[fv] : 0; + uint32_t ui = fbx_mesh->vertex_uv.exists ? (uint32_t) fbx_mesh->vertex_uv.indices.data[fv] : 0; + + VtxKey key{pi, ni, ui}; + auto it = vtx_map.find(key); + + uint32_t flat_ix; + if (it == vtx_map.end()) + { + flat_ix = sub_vtx_start + static_cast(vtx_map.size()); + vtx_map.emplace(key, flat_ix); + + ufbx_vec3 pos = fbx_mesh->vertex_position.values.data[pi]; + ufbx_vec3 nrm = fbx_mesh->vertex_normal.exists ? fbx_mesh->vertex_normal.values.data[ni] : ufbx_vec3{0, 1, 0}; + ufbx_vec2 uv = fbx_mesh->vertex_uv.exists ? fbx_mesh->vertex_uv.values.data[ui] : ufbx_vec2{0, 0}; + + mesh.Vertices.push(static_cast(pos.x)); + mesh.Vertices.push(static_cast(pos.y)); + mesh.Vertices.push(static_cast(pos.z)); + mesh.Vertices.push(static_cast(nrm.x)); + mesh.Vertices.push(static_cast(nrm.y)); + mesh.Vertices.push(static_cast(nrm.z)); + mesh.Vertices.push(static_cast(uv.x)); + mesh.Vertices.push(static_cast(1.0 - uv.y)); // flip V — FBX convention + } + else + { + flat_ix = it->second; + } + mesh.Indices.push(flat_ix); + } + } + } + + AssetSubMesh sub = {}; + sub.VertexOffset = sub_vtx_start; + sub.VertexCount = static_cast(vtx_map.size()); + sub.IndexOffset = sub_idx_start; + sub.IndexCount = static_cast(mesh.Indices.size()) - sub_idx_start; + sub.VertexUnitStreamSize = 8 * sizeof(float); + sub.IndexUnitStreamSize = sizeof(uint32_t); + sub.TotalByteSize = sub.VertexCount * sub.VertexUnitStreamSize; + + if (fbx_mesh->materials.count > 0 && fbx_mesh->materials.data[0]) + { + ufbx_material* mat = fbx_mesh->materials.data[0]; + AssetMaterial a_mat = {}; + a_mat.MaterialUUID = gen(); + a_mat.Name.init(scratch.Arena, mat->name.data); + sub.MaterialUUID = a_mat.MaterialUUID; + + auto tex_uuid = [&](ufbx_material_map& map) -> uuids::uuid { + if (map.texture_enabled && map.texture && map.texture->filename.length > 0) + { + uuids::uuid id = gen(); + AssetTexture t = {}; + t.TextureUUID = id; + t.Path.init(scratch.Arena, map.texture->filename.data); + textures.push(t); + return id; + } + return {}; + }; + + a_mat.AlbedoTexUUID = tex_uuid(mat->pbr.base_color); + a_mat.NormalTexUUID = tex_uuid(mat->pbr.normal_map); + a_mat.SpecularTexUUID = tex_uuid(mat->pbr.metalness); + + auto col = mat->pbr.base_color.value_vec4; + a_mat.AlbedoColor[0] = static_cast(col.x); + a_mat.AlbedoColor[1] = static_cast(col.y); + a_mat.AlbedoColor[2] = static_cast(col.z); + a_mat.AlbedoColor[3] = static_cast(col.w); + a_mat.Factors[1] = static_cast(mat->pbr.metalness.value_real); + a_mat.RoughnessColor[0] = static_cast(mat->pbr.roughness.value_real); + + materials.push(a_mat); + } + + mesh.SubMeshes.push(sub); + } + + ufbx_free_scene(scene); + + if (Managers::AssetManager::Instance()) + { + Managers::AssetManager::IngestTextures(std::move(textures)); + for (size_t i = 0; i < materials.size(); ++i) + Managers::AssetManager::IngestMaterial(std::move(materials[i])); + Managers::AssetManager::IngestMesh(std::move(mesh), std::move(hier)); + } + + ZReleaseScratch(scratch); + return Core::VFS::VFSResult::Ok(); + } + + void FbxImporter::ImportFile(const char* filename, const AssetCodec::ImportConfiguration& cfg, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log) + { + AssetCodec::ImportConfiguration config = {}; + config.OutputWorkingSpacePath.init(arena, cfg.OutputWorkingSpacePath.c_str()); + config.OutputTextureFilesPath.init(arena, cfg.OutputTextureFilesPath.c_str()); + config.OutputAssetsPath.init(arena, cfg.OutputAssetsPath.c_str()); + config.OutputMaterialPath.init(arena, cfg.OutputMaterialPath.c_str()); + config.AssetName.init(arena, cfg.AssetName.c_str()); + config.OutputAssetFile.init(arena, cfg.OutputAssetFile.c_str()); + config.InputBaseAssetFilePath.init(arena, cfg.InputBaseAssetFilePath.c_str()); + config.VFS = cfg.VFS; + config.Options = cfg.Options; + + ufbx_load_opts opts = {}; + opts.target_axes = ufbx_axes_right_handed_y_up; + opts.target_unit_meters = 1.0f; + opts.generate_missing_normals = (config.Options.NormalsMode > 0); + + ufbx_error error; + ufbx_scene* scene = ufbx_load_file(filename, &opts, &error); + + if (!scene) + { + if (on_error) + on_error(context, error.description.data); + return; + } + + if (on_progress) + on_progress(context, 0.2f); + + std::random_device rd; + std::mt19937 gen_mt(rd()); + uuid_random_generator gen(&gen_mt); + + auto scratch = ZGetScratch(&Arena); + + AssetMesh mesh = {}; + AssetNodeHierarchy hier = {}; + Array materials = {}; + Array textures = {}; + + // Pre-size vertex/index arrays from ufbx's triangle counts to avoid grow() dead blocks. + // Worst-case capacity: all face-vertices unique (no dedup) — actual used will be less. + uint32_t total_tris = 0; + for (size_t mi = 0; mi < scene->meshes.count; ++mi) + total_tris += static_cast(scene->meshes.data[mi]->num_triangles); + + mesh.MeshUUID = gen(); + mesh.Vertices.init(scratch.Arena, (size_t) total_tris * 3 * 8); + mesh.Indices.init(scratch.Arena, (size_t) total_tris * 3); + mesh.SubMeshes.init(scratch.Arena, (uint32_t) scene->meshes.count); + materials.init(scratch.Arena, 64); + textures.init(scratch.Arena, 256); + + const float scale = config.Options.UniformScale; + + for (size_t mi = 0; mi < scene->meshes.count; ++mi) + { + ufbx_mesh* fbx_mesh = scene->meshes.data[mi]; + std::unordered_map vtx_map; + vtx_map.reserve(fbx_mesh->num_triangles * 2); + + uint32_t sub_vtx_start = static_cast(mesh.Vertices.size() / 8); + uint32_t sub_idx_start = static_cast(mesh.Indices.size()); + + uint32_t tri_buf[512]; + for (size_t fi = 0; fi < fbx_mesh->faces.count; ++fi) + { + ufbx_face face = fbx_mesh->faces.data[fi]; + uint32_t num_tris = ufbx_triangulate_face(tri_buf, 512, fbx_mesh, face); + + for (uint32_t ti = 0; ti < num_tris; ++ti) + { + for (uint32_t vi = 0; vi < 3; ++vi) + { + uint32_t fv = tri_buf[ti * 3 + vi]; + uint32_t pi = (uint32_t) fbx_mesh->vertex_position.indices.data[fv]; + uint32_t ni = fbx_mesh->vertex_normal.exists ? (uint32_t) fbx_mesh->vertex_normal.indices.data[fv] : 0; + uint32_t ui = fbx_mesh->vertex_uv.exists ? (uint32_t) fbx_mesh->vertex_uv.indices.data[fv] : 0; + + VtxKey key{pi, ni, ui}; + auto it = vtx_map.find(key); + uint32_t flat_ix; + + if (it == vtx_map.end()) + { + flat_ix = sub_vtx_start + static_cast(vtx_map.size()); + vtx_map.emplace(key, flat_ix); + + ufbx_vec3 pos = fbx_mesh->vertex_position.values.data[pi]; + ufbx_vec3 nrm = fbx_mesh->vertex_normal.exists ? fbx_mesh->vertex_normal.values.data[ni] : ufbx_vec3{0, 1, 0}; + ufbx_vec2 uv = fbx_mesh->vertex_uv.exists ? fbx_mesh->vertex_uv.values.data[ui] : ufbx_vec2{0, 0}; + + float flip_v = config.Options.FlipUVs ? (float) uv.y : 1.0f - (float) uv.y; + + mesh.Vertices.push(static_cast(pos.x) * scale); + mesh.Vertices.push(static_cast(pos.y) * scale); + mesh.Vertices.push(static_cast(pos.z) * scale); + mesh.Vertices.push(static_cast(nrm.x)); + mesh.Vertices.push(static_cast(nrm.y)); + mesh.Vertices.push(static_cast(nrm.z)); + mesh.Vertices.push(static_cast(uv.x)); + mesh.Vertices.push(flip_v); + } + else + { + flat_ix = it->second; + } + mesh.Indices.push(flat_ix); + } + } + } + + { + // Optimize: convert absolute indices → relative, run 3-pass optimization, restore. + const uint32_t sub_vc = static_cast(vtx_map.size()); + const uint32_t sub_ic = static_cast(mesh.Indices.size()) - sub_idx_start; + uint32_t* sub_idx = mesh.Indices.data() + sub_idx_start; + for (uint32_t j = 0; j < sub_ic; ++j) + sub_idx[j] -= sub_vtx_start; + OptimizeMeshSubmesh(mesh.Vertices.data() + sub_vtx_start * 8, sub_vc, sub_idx, sub_ic); + for (uint32_t j = 0; j < sub_ic; ++j) + sub_idx[j] += sub_vtx_start; + } + + AssetSubMesh sub = {}; + sub.VertexOffset = sub_vtx_start; + sub.VertexCount = static_cast(vtx_map.size()); + sub.IndexOffset = sub_idx_start; + sub.IndexCount = static_cast(mesh.Indices.size()) - sub_idx_start; + sub.VertexUnitStreamSize = 8 * sizeof(float); + sub.IndexUnitStreamSize = sizeof(uint32_t); + sub.TotalByteSize = sub.VertexCount * sub.VertexUnitStreamSize; + + if (config.Options.ImportMaterials && fbx_mesh->materials.count > 0 && fbx_mesh->materials.data[0]) + { + ufbx_material* mat = fbx_mesh->materials.data[0]; + AssetMaterial a_mat = {}; + a_mat.MaterialUUID = gen(); + a_mat.Name.init(scratch.Arena, mat->name.data); + sub.MaterialUUID = a_mat.MaterialUUID; + + auto push_tex = [&](ufbx_material_map& map) -> uuids::uuid { + if (!config.Options.ImportTextures) + return {}; + if (map.texture_enabled && map.texture && map.texture->filename.length > 0) + { + uuids::uuid id = gen(); + AssetTexture t = {}; + t.TextureUUID = id; + t.Path.init(scratch.Arena, map.texture->filename.data); + textures.push(t); + return id; + } + return {}; + }; + + a_mat.AlbedoTexUUID = push_tex(mat->pbr.base_color); + a_mat.NormalTexUUID = push_tex(mat->pbr.normal_map); + a_mat.SpecularTexUUID = push_tex(mat->pbr.metalness); + + auto col = mat->pbr.base_color.value_vec4; + a_mat.AlbedoColor[0] = static_cast(col.x); + a_mat.AlbedoColor[1] = static_cast(col.y); + a_mat.AlbedoColor[2] = static_cast(col.z); + a_mat.AlbedoColor[3] = static_cast(col.w); + a_mat.Factors[1] = static_cast(mat->pbr.metalness.value_real); + a_mat.RoughnessColor[0] = static_cast(mat->pbr.roughness.value_real); + + materials.push(a_mat); + } + + mesh.SubMeshes.push(sub); + } + + if (on_progress) + on_progress(context, 0.5f); + + if (config.Options.ImportMaterials && config.Options.ImportTextures) + CopyTextureFiles(arena, textures, config); + + if (on_progress) + on_progress(context, 0.8f); + + ufbx_free_scene(scene); + + hier.MeshUUID = mesh.MeshUUID; + hier.NodeHierarchyUUID = mesh.MeshUUID; + hier.LocalTransforms.init(arena, 1); + hier.GlobalTransforms.init(arena, 1); + hier.Hierarchies.init(arena, 1); + hier.Names.init(arena, 1); + hier.MaterialNames.init(arena, 1); + hier.NodeNames.init(scratch.Arena, 64); + hier.NodeMeshes.init(scratch.Arena, 1); + hier.NodeMaterials.init(scratch.Arena, 1); + hier.LocalTransforms.push(Identity()); + hier.GlobalTransforms.push(Identity()); + + Array outputs = {}; + outputs.init(scratch.Arena, 16); + outputs.push(AssetCodec::SerializeMeshAssetFile(scratch.Arena, mesh, hier, config)); + if (config.Options.ImportMaterials) + for (size_t i = 0; i < materials.size(); ++i) + outputs.push(AssetCodec::SerializeMaterialAssetFile(scratch.Arena, materials[i], config)); + + auto* mgr = Managers::AssetManager::Instance(); + if (mgr) + { + if (config.Options.ImportTextures && config.Options.ImportMaterials) + Managers::AssetManager::IngestTextures(std::move(textures)); + if (config.Options.ImportMaterials) + for (size_t i = 0; i < materials.size(); ++i) + Managers::AssetManager::IngestMaterial(std::move(materials[i])); + Managers::AssetManager::IngestMesh(std::move(mesh), std::move(hier)); + } + + if (on_progress) + on_progress(context, 1.0f); + if (on_complete) + on_complete(context, ArrayView{outputs}); + + ZReleaseScratch(scratch); + } + + void FbxImporter::CopyTextureFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config) + { + if (textures.empty()) + return; + + char dst_dir_buf[MAX_FILE_PATH_COUNT] = {}; + (VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()).ResolveNative(config.OutputWorkingSpacePath.c_str(), dst_dir_buf, sizeof(dst_dir_buf)); + if (config.VFS) + config.VFS->CreateDir(VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str()); + + for (auto& tex : textures) + { + if (tex.Path.empty()) + continue; + + fs::path tex_path(tex.Path.c_str()); + fs::path src = tex_path.is_absolute() ? tex_path : fs::path(config.InputBaseAssetFilePath.c_str()) / tex_path; + fs::path dst = fs::path(dst_dir_buf) / tex_path.filename(); + + std::ifstream in(src, std::ios::binary); + if (!in.is_open()) + { + ZENGINE_CORE_WARN("[FbxImporter] Texture not found: {}", src.string()) + continue; + } + std::ofstream out(dst, std::ios::binary); + out << in.rdbuf(); + + auto new_path = std::string((VFSPath::Parse(config.OutputTextureFilesPath.c_str()).Value() / config.AssetName.c_str() / tex_path.filename().string().c_str()).CStr()); + tex.Path.clear(); + tex.Path.append(new_path.c_str()); + } + } +} // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/FbxImporter.h b/ZEngine/ZEngine/Importers/FbxImporter.h new file mode 100644 index 00000000..e548ccdd --- /dev/null +++ b/ZEngine/ZEngine/Importers/FbxImporter.h @@ -0,0 +1,23 @@ +#pragma once +#include +#include +#include +#include + +namespace ZEngine::Importers +{ + class FbxImporter : public IAssetImporter + { + public: + void Initialize(Core::Memory::ArenaAllocator* arena); + + Core::Memory::ArenaAllocator Arena = {}; + + bool CanImport(const char* extension) const override; + Core::VFS::VFSResult Import(Core::VFS::IVFSContext& ctx, const Core::VFS::VFSPath& path, const Core::VFS::MetaFileData& meta) override; + void ImportFile(const char* filename, const AssetCodec::ImportConfiguration& config, Core::Memory::ArenaAllocator* arena, void* context, ImportCompleteCallback on_complete, ImportProgressCallback on_progress, ImportErrorCallback on_error, ImportLogCallback on_log); + + private: + void CopyTextureFiles(Core::Memory::ArenaAllocator* arena, Core::Containers::Array& textures, const AssetCodec::ImportConfiguration& config); + }; +} // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Importers/GltfImporter.cpp b/ZEngine/ZEngine/Importers/GltfImporter.cpp index 99b7b2de..efe28417 100644 --- a/ZEngine/ZEngine/Importers/GltfImporter.cpp +++ b/ZEngine/ZEngine/Importers/GltfImporter.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -550,6 +551,18 @@ namespace ZEngine::Importers } } + // Optimize each submesh: vertex cache, overdraw, vertex fetch. + for (uint32_t si = 0; si < mesh.SubMeshes.size(); ++si) + { + auto& sub = mesh.SubMeshes[si]; + uint32_t* sub_idx = mesh.Indices.data() + sub.IndexOffset; + for (uint32_t j = 0; j < sub.IndexCount; ++j) + sub_idx[j] -= sub.VertexOffset; + Importers::OptimizeMeshSubmesh(mesh.Vertices.data() + sub.VertexOffset * 8, sub.VertexCount, sub_idx, sub.IndexCount); + for (uint32_t j = 0; j < sub.IndexCount; ++j) + sub_idx[j] += sub.VertexOffset; + } + if (config.Options.ImportMaterials) { ExtractMaterials(&scratch, asset, gen, materials); diff --git a/ZEngine/ZEngine/Importers/MeshOptimizer.h b/ZEngine/ZEngine/Importers/MeshOptimizer.h new file mode 100644 index 00000000..12b671db --- /dev/null +++ b/ZEngine/ZEngine/Importers/MeshOptimizer.h @@ -0,0 +1,41 @@ +#pragma once +#include +#include +#include + +namespace ZEngine::Importers +{ + // Three-pass mesh optimization for one submesh. + // Operates on RELATIVE (0-based) indices and a pointer to the submesh's vertex slice. + // Vertex format: 8 floats (pos.xyz nrm.xyz uv.xy) — 32 bytes per vertex. + // + // Pass 1: vertex cache — reorders indices for GPU post-transform cache + // Pass 2: overdraw — clusters triangles to reduce pixel shader overdraw + // Pass 3: vertex fetch — reorders vertices to match optimized index order + // + // The temp buffer for Pass 3 uses the system heap (malloc/free) so it is + // freed immediately per submesh — arena accumulation crashes are avoided. + // This is acceptable since optimization is an import-time, one-shot operation. + inline void OptimizeMeshSubmesh(float* vertices, uint32_t vertex_count, uint32_t* indices, uint32_t index_count) + { + if (!vertices || !indices || vertex_count == 0 || index_count == 0) + return; + + constexpr size_t stride = 8 * sizeof(float); // 32 bytes per vertex + + // Pass 1 — vertex cache + meshopt_optimizeVertexCache(indices, indices, index_count, vertex_count); + + // Pass 2 — overdraw (position at offset 0, stride 32) + meshopt_optimizeOverdraw(indices, indices, index_count, vertices, vertex_count, stride, 1.05f); + + // Pass 3 — vertex fetch: system heap for the temp buffer, freed immediately. + float* opt = static_cast(std::malloc(vertex_count * stride)); + if (!opt) + return; + + meshopt_optimizeVertexFetch(opt, indices, index_count, vertices, vertex_count, stride); + Helpers::secure_memcpy(vertices, vertex_count * stride, opt, vertex_count * stride); + std::free(opt); + } +} // namespace ZEngine::Importers diff --git a/ZEngine/ZEngine/Managers/AssetManager.cpp b/ZEngine/ZEngine/Managers/AssetManager.cpp index 308d0a14..6ed8d971 100644 --- a/ZEngine/ZEngine/Managers/AssetManager.cpp +++ b/ZEngine/ZEngine/Managers/AssetManager.cpp @@ -336,6 +336,8 @@ namespace ZEngine::Managers auto* rec = s_Instance->Registry->Access(result.Handles[i]); if (!rec || rec->UUID.is_nil()) continue; + if (rec->State == Core::VFS::AssetState::Loaded) + continue; if (s_Instance->MeshToHierarchySlot.find(rec->UUID) != nullptr) continue; @@ -344,7 +346,11 @@ namespace ZEngine::Managers AssetMesh mesh = {}; AssetNodeHierarchy hier = {}; - Importers::AssetCodec::DeserializeMeshAssetFile(scratch, native, mesh, hier); + // Use the AssetManager's own arena — meshes can be hundreds of MB, + // far exceeding the caller's scratch. After IngestMesh copies the data + // permanently, the deserialization buffers become dead weight but are + // acceptable as a one-time startup cost. + Importers::AssetCodec::DeserializeMeshAssetFile(s_Instance->Arena, native, mesh, hier); if (!mesh.MeshUUID.is_nil()) { ZENGINE_LOG_ASSET_INFO("Reloading mesh from disk: {}", native) diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.h b/ZEngine/ZEngine/Rendering/RenderResourceManager.h index 0c23deb2..8e4609fb 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.h +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.h @@ -391,8 +391,8 @@ namespace ZEngine::Rendering Core::VFS::AssetRegistry* m_registry = nullptr; // Global geometry buffers — all mesh vertices/indices packed together. - static constexpr VkDeviceSize GLOBAL_VTX_CAPACITY = 256 * 1024 * 1024; // 256 MB → ~8M DrawVertex - static constexpr VkDeviceSize GLOBAL_IDX_CAPACITY = 256 * 1024 * 1024; // 256 MB → ~64M uint32 + static constexpr VkDeviceSize GLOBAL_VTX_CAPACITY = 512 * 1024 * 1024; // 512 MB → ~16M DrawVertex + static constexpr VkDeviceSize GLOBAL_IDX_CAPACITY = 512 * 1024 * 1024; // 512 MB → ~128M uint32 Core::Memory::BufferView m_global_vertex_buf = {}; Core::Memory::BufferView m_global_index_buf = {}; VkDeviceSize m_vtx_cursor = 0; // byte offset of next write diff --git a/ZEngine/ZEngine/Rendering/Scenes/RenderScene.h b/ZEngine/ZEngine/Rendering/Scenes/RenderScene.h index d4e5ec80..f0a5f7ba 100644 --- a/ZEngine/ZEngine/Rendering/Scenes/RenderScene.h +++ b/ZEngine/ZEngine/Rendering/Scenes/RenderScene.h @@ -87,7 +87,7 @@ namespace ZEngine::Rendering::Scenes // The heap is reset every frame so we cache the commands here and re-push every frame. uint32_t IndirectHeapOffset = 0; uint32_t IndirectCommandCount = 0; - static constexpr uint32_t MAX_DRAW_COMMANDS = 512; + static constexpr uint32_t MAX_DRAW_COMMANDS = 8192; VkDrawIndirectCommand CachedDrawCmds[MAX_DRAW_COMMANDS] = {}; // RMM-owned HOST_VISIBLE buffers — written via RRM::UpdateBuffer every frame. diff --git a/dependencies.cmake b/dependencies.cmake index 14b0251d..993268b4 100644 --- a/dependencies.cmake +++ b/dependencies.cmake @@ -162,6 +162,33 @@ FetchContent_Declare(miniz GIT_SHALLOW TRUE ) +FetchContent_Declare(ufbx + GIT_REPOSITORY https://github.com/ufbx/ufbx.git + GIT_SHALLOW TRUE + GIT_TAG v0.14.3 + SOURCE_DIR ${FETCHCONTENT_BASE_DIR}/ufbx +) +FetchContent_GetProperties(ufbx) +if(NOT ufbx_POPULATED) + FetchContent_Populate(ufbx) +endif() +add_library(ufbx STATIC ${FETCHCONTENT_BASE_DIR}/ufbx/ufbx.c) +target_include_directories(ufbx PUBLIC ${FETCHCONTENT_BASE_DIR}/ufbx) + +FetchContent_Declare(meshoptimizer + GIT_REPOSITORY https://github.com/zeux/meshoptimizer.git + GIT_SHALLOW TRUE + GIT_TAG v0.22 + SOURCE_DIR ${FETCHCONTENT_BASE_DIR}/meshoptimizer +) +FetchContent_GetProperties(meshoptimizer) +if(NOT meshoptimizer_POPULATED) + FetchContent_Populate(meshoptimizer) +endif() +file(GLOB MESHOPT_SOURCES ${FETCHCONTENT_BASE_DIR}/meshoptimizer/src/*.cpp) +add_library(meshoptimizer STATIC ${MESHOPT_SOURCES}) +target_include_directories(meshoptimizer PUBLIC ${FETCHCONTENT_BASE_DIR}/meshoptimizer/src) + FetchContent_Declare(simdjson GIT_REPOSITORY https://github.com/simdjson/simdjson.git GIT_SHALLOW TRUE @@ -274,6 +301,8 @@ target_include_directories(External_libs ${FETCHCONTENT_BASE_DIR}/stb ${FETCHCONTENT_BASE_DIR}/CLI11 ${FETCHCONTENT_BASE_DIR}/tlsf + ${FETCHCONTENT_BASE_DIR}/ufbx + ${FETCHCONTENT_BASE_DIR}/meshoptimizer/src ) @@ -304,6 +333,8 @@ target_link_libraries(External_libs nlohmann_json::nlohmann_json miniz fastgltf::fastgltf + ufbx + meshoptimizer ) if(ZENGINE_TRACY)