diff --git a/copyMultiplayer.gd b/copyMultiplayer.gd index be5f19d..318930b 100644 --- a/copyMultiplayer.gd +++ b/copyMultiplayer.gd @@ -13,9 +13,7 @@ var tracked_bones: Array[String] var version: int = -1 var bone_lookup: Array[String] = [] -var bone_to_lookup: Dictionary = {} -var blendshape_lookup: Array[String] = [] -var blendshape_to_lookup: Dictionary = {} +var blendshape_lookup: Array[NodePath] = [] # Temporary positioning system. # TODO: Add a setting that allows syncing model positions, as an alternative @@ -48,8 +46,8 @@ func _ready() -> void: # FIXME: Hardcoded way to get the main model controller. main_controller = $"/root/SnekStudio_Main/ModelController" - main_controller.child_entered_tree.connect(on_model_controller_child_added) - on_model_controller_child_added(main_controller.get_node_or_null("Model")) + main_controller.child_entered_tree.connect(on_model_changed) + on_model_changed(main_controller.get_node_or_null("Model")) setup_setting_widget("cache" , "Cache/LineEdit", true ) setup_setting_widget("nickname", "Name/LineEdit" , false) @@ -222,25 +220,33 @@ func clear_player_models() -> void: func _process(_delta: float) -> void: SyncController.send_model_animation(self) -func on_model_controller_child_added(child: Node) -> void: - if (child == null) or (child.name != "Model"): return +# Will be called once when the module is initialized, and also when a child is +# added to the main ModelController, which happens when the model is changed. +func on_model_changed(model: Node) -> void: + if (model == null) or (model.name != "Model"): return var skeleton: Skeleton3D = main_controller._get_model_skeleton() if not skeleton: return # Do nothing, I guess? Unsure if potential bug. + # This "version" is used to ensure that sync updates + # are not applied to the incorrect lookup tables. version = (version + 1) % 256 bone_lookup.clear() - bone_to_lookup.clear() for bone_name in tracked_bones: - bone_lookup.append(bone_name) var bone_idx := skeleton.find_bone(bone_name) - if bone_idx == -1: continue # Need bone in bone_lookup, but not in bone_to_lookup! - bone_to_lookup[bone_name] = bone_lookup.size() - 1 + if bone_idx == -1: continue + bone_lookup.append(bone_name) blendshape_lookup.clear() - blendshape_to_lookup.clear() - # TODO: Redo the blendshapes. Consider grabbing data from animations directly? + var anim_player: AnimationPlayer = model.find_child("AnimationPlayer", false, false) + for anim_name in anim_player.get_animation_list(): + if anim_name == "RESET": continue # Skip RESET animation? + var anim := anim_player.get_animation(anim_name) + for track_index in anim.get_track_count(): + if anim.track_get_type(track_index) == Animation.TYPE_BLEND_SHAPE: + blendshape_lookup.append(anim.track_get_path(track_index)) + continue # Wait for one frame, then "_last_loaded_vrm" is updated. await get_tree().process_frame @@ -265,7 +271,7 @@ func change_nickname(new_nickname: String) -> void: func change_model( new_version: int, new_bone_lookup: Array[String], - new_blendshape_lookup: Array[String], + new_blendshape_lookup: Array[NodePath], filename: String, ) -> void: var peer_id := multiplayer.get_remote_sender_id() diff --git a/sync_controller.gd b/sync_controller.gd index 51394d0..68f361f 100644 --- a/sync_controller.gd +++ b/sync_controller.gd @@ -9,18 +9,17 @@ var nickname: String var version: int = -1 var bone_lookup: Array[String] -var blendshape_lookup: Array[String] +var blendshape_lookup: Array[NodePath] -var model_name: String -var model: Node -var skeleton: Skeleton3D +var model_name : String +var model : Node +var skeleton : Skeleton3D +var anim_player : AnimationPlayer +var anim_root : Node # Reusable buffer to write data for synchronizing models. static var write_stream: StreamBuffer = StreamBuffer.with_capacity(2048) -# Allows us to use the "apply_animations" function to apply blendshapes to a model. -static var BlendShapes: Script = load("res://Mods/MediaPipe/MediaPipeController_BlendShapes.gd") - func _ready() -> void: module = get_parent().get_parent() model_controller = get_parent() @@ -40,7 +39,7 @@ func change_nickname(new_nickname: String) -> void: func change_model( new_version: int, new_bone_lookup: Array[String], - new_blendshape_lookup: Array[String], + new_blendshape_lookup: Array[NodePath], filename: String, ) -> void: # These should be safe to update even if the model doesn't load. @@ -60,9 +59,11 @@ func change_model( module.print_log("ERROR: Model '%s' could not be loaded!" % filename) return - model_name = filename - model = model_controller.get_node_or_null("Model") - skeleton = model_controller._get_model_skeleton() + model_name = filename + model = model_controller.get_node_or_null("Model") + skeleton = model_controller._get_model_skeleton() + anim_player = model.find_child("AnimationPlayer", false, false) + anim_root = anim_player.get_node(anim_player.root_node) module.print_log("%s switched to '%s'" % [ get_display_name(), filename ]) @@ -72,71 +73,84 @@ func sync_model_animation( buffer: PackedByteArray, ) -> void: if version != current_version: return - if (not model) or (not skeleton): return + if (not model) or (not skeleton) or (not anim_root): return var uncompressed_buffer := buffer.decompress(uncompressed_length, FileAccess.COMPRESSION_ZSTD); var stream := StreamBuffer.from_buffer(uncompressed_buffer) model.transform = stream.read_transform16() - # We skipped some bones, so reset the skipped ones to the rest pose. - var all_bones := {} - for bone_name in module.bone_to_lookup: + var all_bones: Array[Dictionary] = [] + var all_blendshapes: Array[Dictionary] = [] + + # We filter out bones at rest and blendshapes at zero, so this + # is initializing all_bones/blendshapes to the default values. + for bone_name in module.bone_lookup: var bone_idx := skeleton.find_bone(bone_name) var bone_rest := skeleton.get_bone_rest(bone_idx) - all_bones[bone_name] = { idx = bone_idx, pose = bone_rest } - - # Override rest poses with ones from the packet. - var num_bones := stream.read_uint8() - for i in num_bones: - var bone_name := bone_lookup[stream.read_uint8()] - all_bones[bone_name].pose = stream.read_bone_pose() - - # Apply bone poses to skeleton. - for bone_name in all_bones: - var bone: Dictionary = all_bones[bone_name] + all_bones.append({ name = bone_name, idx = bone_idx, pose = bone_rest }) + for anim_path in module.blendshape_lookup: + all_blendshapes.append({ path = anim_path, value = 0.0 }) + + # 256 bones (and blendshapes) should be enough, right? + for i in stream.read_uint8(): + var lookup := stream.read_uint8() + all_bones[lookup].pose = stream.read_bone_pose() + for i in stream.read_uint8(): + var lookup := stream.read_uint8() + all_blendshapes[lookup].value = stream.read_range16() + + # Apply all the values to bones / blendshapes. + for bone in all_bones: + if bone.idx == -1: continue # Different model might not have this bone. skeleton.set_bone_pose(bone.idx, bone.pose) - - var shape_dict := {} - var num_shapes := stream.read_uint8() # 256 blendshapes (and bones) should be enough, right? - for i in num_shapes: - var shape_name := blendshape_lookup[stream.read_uint8()] - var shape_alpha := stream.read_float16() - shape_dict[shape_name] = shape_alpha - BlendShapes.apply_animations(model, shape_dict) + for blendshape in all_blendshapes: + var anim_node := anim_root.get_node_or_null(blendshape.path) + if not anim_node: continue # Different model might not have this node. + anim_node.set("blend_shapes/" + blendshape.path.get_subname(0), blendshape.value) @warning_ignore("shadowed_variable") static func send_model_animation(module: copyMultiplayer) -> void: # Check if there's other players we're connected to. if module.multiplayer.get_peers().size() == 0: return - var model := module.get_model() - var skeleton := module.get_skeleton() - if (not model) or (not skeleton): return + var model := module.get_model() + var skeleton := module.get_skeleton() + var anim_player := model.find_child("AnimationPlayer", false, false) + var anim_root := anim_player.get_node_or_null(anim_player.root_node) + if (not model) or (not skeleton) or (not anim_root): return write_stream.write_transform16(model.transform) - # Pre-filter any bones that are in rest pose. - var restless_bones := {} - for bone_name in module.bone_to_lookup: + # Pre-filter any bones that are at rest / blendshapes that are at zero. + # Unless most bones / blendshapes are active, this should reduce packet size. + var restless_bones := [] + var active_blendshapes := [] + + for i in module.bone_lookup.size(): + var bone_name := module.bone_lookup[i] var bone_idx := skeleton.find_bone(bone_name) var bone_pose := skeleton.get_bone_pose(bone_idx) var bone_rest := skeleton.get_bone_rest(bone_idx) if not bone_pose.is_equal_approx(bone_rest): - restless_bones[bone_name] = bone_pose + restless_bones.append({ lookup = i, pose = bone_pose }) + + for i in module.blendshape_lookup.size(): + var anim_path := module.blendshape_lookup[i] + var anim_node := anim_root.get_node_or_null(anim_path) + var value: float = anim_node.get("blend_shapes/" + anim_path.get_subname(0)) + if not is_zero_approx(value): + active_blendshapes.append({ lookup = i, value = value }) write_stream.write_uint8(restless_bones.size()) - for bone_name in restless_bones: - write_stream.write_uint8(module.bone_to_lookup[bone_name]) - write_stream.write_bone_pose(restless_bones[bone_name]) - - # TODO: Only write non-default blendshapes. Anything missing = default. - var shape_dict: Dictionary = {} # TODO: Redo the blendshapes. - write_stream.write_uint8(module.blendshape_to_lookup.size()) - for shape_name in module.blendshape_to_lookup: - var shape_alpha: float = shape_dict[shape_name] - write_stream.write_uint8(module.blendshape_to_lookup[shape_name]) - write_stream.write_float16(shape_alpha) + for bone in restless_bones: + write_stream.write_uint8(bone.lookup) + write_stream.write_bone_pose(bone.pose) + + write_stream.write_uint8(active_blendshapes.size()) + for blendshape in active_blendshapes: + write_stream.write_uint8(blendshape.lookup) + write_stream.write_range16(blendshape.value) # The compression still helps, so we'll keep it for now. var compressed_buffer := write_stream.slice().compress(FileAccess.COMPRESSION_ZSTD);