From f8163caee159bee675f2fc4c775374c23ec60ac3 Mon Sep 17 00:00:00 2001 From: copygirl Date: Mon, 9 Dec 2024 06:49:36 +0100 Subject: [PATCH] Use lookup table instead of strings Whenever a model is changed, we prepare a bi-directional name <-> id (byte) lookup table using which we can further cut down the number of bytes used. I also decided to avoid sending any bones which are in their rest position. We're down to ~200 bytes, plus an additional ~200 for each hand tracked. This commit breaks blendshape synchronization. I would like to grab the data directly from the model / animation system, rather than relying on "blend_shape_last_values". --- copyMultiplayer.gd | 49 ++++++++++++++++++--- sync_controller.gd | 105 +++++++++++++++++++++++++++++---------------- 2 files changed, 111 insertions(+), 43 deletions(-) diff --git a/copyMultiplayer.gd b/copyMultiplayer.gd index dcc539c..be5f19d 100644 --- a/copyMultiplayer.gd +++ b/copyMultiplayer.gd @@ -11,6 +11,12 @@ var main_controller: ModelController ## Hardcoded list of bone names that will get syncronized. var tracked_bones: Array[String] +var version: int = -1 +var bone_lookup: Array[String] = [] +var bone_to_lookup: Dictionary = {} +var blendshape_lookup: Array[String] = [] +var blendshape_to_lookup: Dictionary = {} + # Temporary positioning system. # TODO: Add a setting that allows syncing model positions, as an alternative # to letting the local player choose where each model is going to appear. @@ -43,6 +49,7 @@ func _ready() -> void: # FIXME: Hardcoded way to get the main model controller. main_controller = $"/root/SnekStudio_Main/ModelController" main_controller.child_entered_tree.connect(on_model_controller_child_added) + on_model_controller_child_added(main_controller.get_node_or_null("Model")) setup_setting_widget("cache" , "Cache/LineEdit", true ) setup_setting_widget("nickname", "Name/LineEdit" , false) @@ -137,7 +144,7 @@ func on_peer_connected(id: int) -> void: # (Technically this doesn't need to be relayed through the server, but oh well.) if nickname: change_nickname.rpc_id(id, nickname) var filename = main_controller._last_loaded_vrm.get_file() - if filename.is_valid_filename(): change_model.rpc_id(id, filename) + if filename.is_valid_filename(): change_model.rpc_id(id, version, bone_lookup, blendshape_lookup, filename) update_status() print_log("%s connected" % sync_controller.get_display_name()) @@ -216,11 +223,30 @@ func _process(_delta: float) -> void: SyncController.send_model_animation(self) func on_model_controller_child_added(child: Node) -> void: - if child.name != "Model": return + if (child == null) or (child.name != "Model"): return + + var skeleton: Skeleton3D = main_controller._get_model_skeleton() + if not skeleton: return # Do nothing, I guess? Unsure if potential bug. + + version = (version + 1) % 256 + + bone_lookup.clear() + bone_to_lookup.clear() + for bone_name in tracked_bones: + bone_lookup.append(bone_name) + var bone_idx := skeleton.find_bone(bone_name) + if bone_idx == -1: continue # Need bone in bone_lookup, but not in bone_to_lookup! + bone_to_lookup[bone_name] = bone_lookup.size() - 1 + + blendshape_lookup.clear() + blendshape_to_lookup.clear() + # TODO: Redo the blendshapes. Consider grabbing data from animations directly? + # Wait for one frame, then "_last_loaded_vrm" is updated. await get_tree().process_frame var filename = main_controller._last_loaded_vrm.get_file() - if filename.is_valid_filename(): change_model.rpc(filename) + + change_model.rpc(version, bone_lookup, blendshape_lookup, filename) ## Gets the SyncController for the player with the specified peer id. @@ -236,13 +262,22 @@ func change_nickname(new_nickname: String) -> void: if controller: controller.change_nickname(new_nickname) @rpc("any_peer", "reliable") -func change_model(filename: String) -> void: +func change_model( + new_version: int, + new_bone_lookup: Array[String], + new_blendshape_lookup: Array[String], + filename: String, +) -> void: var peer_id := multiplayer.get_remote_sender_id() var controller := get_sync_controller(peer_id) - if controller: controller.change_model(filename) + if controller: controller.change_model(new_version, new_bone_lookup, new_blendshape_lookup, filename) @rpc("any_peer", "unreliable_ordered") -func sync_model_animation(uncompressed_length: int, buffer: PackedByteArray) -> void: +func sync_model_animation( + current_version: int, + uncompressed_length: int, + buffer: PackedByteArray, +) -> void: var peer_id := multiplayer.get_remote_sender_id() var controller := get_sync_controller(peer_id) - if controller: controller.sync_model_animation(uncompressed_length, buffer) + if controller: controller.sync_model_animation(current_version, uncompressed_length, buffer) diff --git a/sync_controller.gd b/sync_controller.gd index 90de462..84081fc 100644 --- a/sync_controller.gd +++ b/sync_controller.gd @@ -6,6 +6,11 @@ var model_controller: ModelController var peer_id: int var nickname: String + +var version: int = -1 +var bone_lookup: Array[String] +var blendshape_lookup: Array[String] + var model_name: String var model: Node var skeleton: Skeleton3D @@ -32,7 +37,18 @@ func change_nickname(new_nickname: String) -> void: nickname = new_nickname ## Attempts to change the model of this player. -func change_model(filename: String) -> void: +func change_model( + new_version: int, + new_bone_lookup: Array[String], + new_blendshape_lookup: Array[String], + filename: String, +) -> void: + # These should be safe to update even if the model doesn't load. + # We just need to stay in sync with the lookup tables. + version = new_version + bone_lookup = new_bone_lookup + blendshape_lookup = new_blendshape_lookup + if not filename.is_valid_filename(): module.print_log("ERROR: '%s' is not a valid file name!" % filename) return @@ -43,34 +59,51 @@ func change_model(filename: String) -> void: if not model_controller.load_vrm(full_path): module.print_log("ERROR: Model '%s' could not be loaded!" % filename) return - module.print_log("%s switched to '%s'" % [ get_display_name(), filename ]) + model_name = filename model = model_controller.get_node_or_null("Model") skeleton = model_controller._get_model_skeleton() -func sync_model_animation(uncompressed_length: int, buffer: PackedByteArray) -> void: + module.print_log("%s switched to '%s'" % [ get_display_name(), filename ]) + +func sync_model_animation( + current_version: int, + uncompressed_length: int, + buffer: PackedByteArray, +) -> void: + if version != current_version: return if (not model) or (not skeleton): return var uncompressed_buffer := buffer.decompress(uncompressed_length, FileAccess.COMPRESSION_ZSTD); var stream := StreamBuffer.from_buffer(uncompressed_buffer) model.transform = stream.read_transform32() + # We skipped some bones, so reset the skipped ones to the rest pose. + var all_bones := {} + for bone_name in module.bone_to_lookup: + var bone_idx := skeleton.find_bone(bone_name) + var bone_rest := skeleton.get_bone_rest(bone_idx) + all_bones[bone_name] = { idx = bone_idx, pose = bone_rest } + + # Override rest poses with ones from the packet. + var num_bones := stream.read_uint8() + for i in num_bones: + var bone_name := bone_lookup[stream.read_uint8()] + all_bones[bone_name].pose = stream.read_bone_pose() + + # Apply bone poses to skeleton. + for bone_name in all_bones: + var bone: Dictionary = all_bones[bone_name] + skeleton.set_bone_pose(bone.idx, bone.pose) + var shape_dict := {} - # 256 blendshapes (and bones) should be enough, right? - var num_shapes := stream.read_uint8() + var num_shapes := stream.read_uint8() # 256 blendshapes (and bones) should be enough, right? for i in num_shapes: - var shape_name := stream.read_string() - var shape_alpha := stream.read_float16() + var shape_name := blendshape_lookup[stream.read_uint8()] + var shape_alpha := stream.read_float16() shape_dict[shape_name] = shape_alpha BlendShapes.apply_animations(model, shape_dict) - var num_bones := stream.read_uint8() - for i in num_bones: - var bone_name := stream.read_string() - var bone_pose := stream.read_bone_pose() - var bone_idx := skeleton.find_bone(bone_name) - if bone_idx != -1: skeleton.set_bone_pose(bone_idx, bone_pose) - @warning_ignore("shadowed_variable") static func send_model_animation(module: copyMultiplayer) -> void: # Check if there's other players we're connected to. @@ -83,32 +116,32 @@ static func send_model_animation(module: copyMultiplayer) -> void: write_stream.write_transform32(model.transform) - # TODO: Do not write full strings. Use a lookup table! - # TODO: Only write non-default blendshapes. Anything missing = default. + # Pre-filter any bones that are in rest pose. + var restless_bones := {} + for bone_name in module.bone_to_lookup: + var bone_idx := skeleton.find_bone(bone_name) + var bone_pose := skeleton.get_bone_pose(bone_idx) + var bone_rest := skeleton.get_bone_rest(bone_idx) + if not bone_pose.is_equal_approx(bone_rest): + restless_bones[bone_name] = bone_pose + + write_stream.write_uint8(restless_bones.size()) + for bone_name in restless_bones: + var bone_pose: Transform3D = restless_bones[bone_name] + write_stream.write_uint8(module.bone_to_lookup[bone_name]) + write_stream.write_bone_pose(bone_pose) + # TODO: Only write non-default blendshapes. Anything missing = default. var shape_dict: Dictionary = media_pipe.blend_shape_last_values - write_stream.write_uint8(shape_dict.size()) - for shape_name in shape_dict: + write_stream.write_uint8(module.blendshape_to_lookup.size()) + for shape_name in module.blendshape_to_lookup: var shape_alpha: float = shape_dict[shape_name] - write_stream.write_string(shape_name) + write_stream.write_uint8(module.blendshape_to_lookup[shape_name]) write_stream.write_float16(shape_alpha) - var bone_poses = {} - for bone_name in module.tracked_bones: - var bone_idx = skeleton.find_bone(bone_name) - if bone_idx == -1: continue - var bone_pose = skeleton.get_bone_pose(bone_idx) - bone_poses[bone_name] = bone_pose - - write_stream.write_uint8(bone_poses.size()) - for bone_name in bone_poses: - var bone_pose: Transform3D = bone_poses[bone_name] - write_stream.write_string(bone_name) - write_stream.write_bone_pose(bone_pose) - - # TODO: Ideally, compression won't be needed once we remove strings. + # The compression still helps, so we'll keep it for now. var compressed_buffer := write_stream.slice().compress(FileAccess.COMPRESSION_ZSTD); - # DEBUG: Uncomment this to see packet size (ideally < 1024). - # module.set_status("Packet size: %d (%d uncompressed)" % [compressed_buffer.size(), write_stream.size]) - module.sync_model_animation.rpc(write_stream.size, compressed_buffer) + # Uncomment this to see packet size. Can we hit < 256 bytes? + # module.set_status("Packet size: %d bytes (%d uncompressed)" % [ compressed_buffer.size(), write_stream.size ]) + module.sync_model_animation.rpc(module.version, write_stream.size, compressed_buffer) write_stream.clear()