Sync blendshapes once more

copyMultiplayer.gd

@@ -13,9 +13,7 @@ var tracked_bones: Array[String]
 
 var version: int = -1
 var bone_lookup: Array[String] = []
-var bone_to_lookup: Dictionary = {}
+var blendshape_lookup: Array[NodePath] = []
-var blendshape_lookup: Array[String] = []
-var blendshape_to_lookup: Dictionary = {}
 
 # Temporary positioning system.
 # TODO: Add a setting that allows syncing model positions, as an alternative
@@ -48,8 +46,8 @@ func _ready() -> void:
 
 	# FIXME: Hardcoded way to get the main model controller.
 	main_controller = $"/root/SnekStudio_Main/ModelController"
-	main_controller.child_entered_tree.connect(on_model_controller_child_added)
+	main_controller.child_entered_tree.connect(on_model_changed)
-	on_model_controller_child_added(main_controller.get_node_or_null("Model"))
+	on_model_changed(main_controller.get_node_or_null("Model"))
 
 	setup_setting_widget("cache"   , "Cache/LineEdit", true )
 	setup_setting_widget("nickname", "Name/LineEdit" , false)
@@ -222,25 +220,33 @@ func clear_player_models() -> void:
 func _process(_delta: float) -> void:
 	SyncController.send_model_animation(self)
 
-func on_model_controller_child_added(child: Node) -> void:
+# Will be called once when the module is initialized, and also when a child is
-	if (child == null) or (child.name != "Model"): return
+# added to the main ModelController, which happens when the model is changed.
+func on_model_changed(model: Node) -> void:
+	if (model == null) or (model.name != "Model"): return
 
 	var skeleton: Skeleton3D = main_controller._get_model_skeleton()
 	if not skeleton: return # Do nothing, I guess? Unsure if potential bug.
 
+	# This "version" is used to ensure that sync updates
+	# are not applied to the incorrect lookup tables.
 	version = (version + 1) % 256
 
 	bone_lookup.clear()
-	bone_to_lookup.clear()
 	for bone_name in tracked_bones:
-		bone_lookup.append(bone_name)
 		var bone_idx := skeleton.find_bone(bone_name)
-		if bone_idx == -1: continue # Need bone in bone_lookup, but not in bone_to_lookup!
+		if bone_idx == -1: continue
-		bone_to_lookup[bone_name] = bone_lookup.size() - 1
+		bone_lookup.append(bone_name)
 
 	blendshape_lookup.clear()
-	blendshape_to_lookup.clear()
+	var anim_player: AnimationPlayer = model.find_child("AnimationPlayer", false, false)
-	# TODO: Redo the blendshapes. Consider grabbing data from animations directly?
+	for anim_name in anim_player.get_animation_list():
+		if anim_name == "RESET": continue # Skip RESET animation?
+		var anim := anim_player.get_animation(anim_name)
+		for track_index in anim.get_track_count():
+			if anim.track_get_type(track_index) == Animation.TYPE_BLEND_SHAPE:
+				blendshape_lookup.append(anim.track_get_path(track_index))
+				continue
 
 	# Wait for one frame, then "_last_loaded_vrm" is updated.
 	await get_tree().process_frame
@@ -265,7 +271,7 @@ func change_nickname(new_nickname: String) -> void:
 func change_model(
 	new_version: int,
 	new_bone_lookup: Array[String],
-	new_blendshape_lookup: Array[String],
+	new_blendshape_lookup: Array[NodePath],
 	filename: String,
 ) -> void:
 	var peer_id := multiplayer.get_remote_sender_id()

sync_controller.gd

@@ -9,18 +9,17 @@ var nickname: String
 
 var version: int = -1
 var bone_lookup: Array[String]
-var blendshape_lookup: Array[String]
+var blendshape_lookup: Array[NodePath]
 
 var model_name  : String
 var model       : Node
 var skeleton    : Skeleton3D
+var anim_player : AnimationPlayer
+var anim_root   : Node
 
 # Reusable buffer to write data for synchronizing models.
 static var write_stream: StreamBuffer = StreamBuffer.with_capacity(2048)
 
-# Allows us to use the "apply_animations" function to apply blendshapes to a model.
-static var BlendShapes: Script = load("res://Mods/MediaPipe/MediaPipeController_BlendShapes.gd")
-
 func _ready() -> void:
 	module = get_parent().get_parent()
 	model_controller = get_parent()
@@ -40,7 +39,7 @@ func change_nickname(new_nickname: String) -> void:
 func change_model(
 	new_version: int,
 	new_bone_lookup: Array[String],
-	new_blendshape_lookup: Array[String],
+	new_blendshape_lookup: Array[NodePath],
 	filename: String,
 ) -> void:
 	# These should be safe to update even if the model doesn't load.
@@ -63,6 +62,8 @@ func change_model(
 	model_name  = filename
 	model       = model_controller.get_node_or_null("Model")
 	skeleton    = model_controller._get_model_skeleton()
+	anim_player = model.find_child("AnimationPlayer", false, false)
+	anim_root   = anim_player.get_node(anim_player.root_node)
 
 	module.print_log("%s switched to '%s'" % [ get_display_name(), filename ])
 
@@ -72,38 +73,41 @@ func sync_model_animation(
 	buffer: PackedByteArray,
 ) -> void:
 	if version != current_version: return
-	if (not model) or (not skeleton): return
+	if (not model) or (not skeleton) or (not anim_root): return
 
 	var uncompressed_buffer := buffer.decompress(uncompressed_length, FileAccess.COMPRESSION_ZSTD);
 	var stream := StreamBuffer.from_buffer(uncompressed_buffer)
 
 	model.transform = stream.read_transform16()
 
-	# We skipped some bones, so reset the skipped ones to the rest pose.
+	var all_bones: Array[Dictionary] = []
-	var all_bones := {}
+	var all_blendshapes: Array[Dictionary] = []
-	for bone_name in module.bone_to_lookup:
+
+	# We filter out bones at rest and blendshapes at zero, so this
+	# is initializing all_bones/blendshapes to the default values.
+	for bone_name in bone_lookup:
 		var bone_idx  := skeleton.find_bone(bone_name)
 		var bone_rest := skeleton.get_bone_rest(bone_idx)
-		all_bones[bone_name] = { idx = bone_idx, pose = bone_rest }
+		all_bones.append({ name = bone_name, idx = bone_idx, pose = bone_rest })
-
+	for anim_path in blendshape_lookup:
-	# Override rest poses with ones from the packet.
+		all_blendshapes.append({ path = anim_path, value = 0.0 })
-	var num_bones := stream.read_uint8()
+
-	for i in num_bones:
+	# 256 bones (and blendshapes) should be enough, right?
-		var bone_name := bone_lookup[stream.read_uint8()]
+	for i in stream.read_uint8():
-		all_bones[bone_name].pose = stream.read_bone_pose()
+		var lookup := stream.read_uint8()
-
+		all_bones[lookup].pose = stream.read_bone_pose()
-	# Apply bone poses to skeleton.
+	for i in stream.read_uint8():
-	for bone_name in all_bones:
+		var lookup := stream.read_uint8()
-		var bone: Dictionary = all_bones[bone_name]
+		all_blendshapes[lookup].value = stream.read_range16()
+
+	# Apply all the values to bones / blendshapes.
+	for bone in all_bones:
+		if bone.idx == -1: continue # Different model might not have this bone.
 		skeleton.set_bone_pose(bone.idx, bone.pose)
-
+	for blendshape in all_blendshapes:
-	var shape_dict := {}
+		var anim_node := anim_root.get_node_or_null(blendshape.path)
-	var num_shapes := stream.read_uint8() # 256 blendshapes (and bones) should be enough, right?
+		if not anim_node: continue # Different model might not have this node.
-	for i in num_shapes:
+		anim_node.set("blend_shapes/" + blendshape.path.get_subname(0), blendshape.value)
-		var shape_name   := blendshape_lookup[stream.read_uint8()]
-		var shape_alpha  := stream.read_float16()
-		shape_dict[shape_name] = shape_alpha
-	BlendShapes.apply_animations(model, shape_dict)
 
 @warning_ignore("shadowed_variable")
 static func send_model_animation(module: copyMultiplayer) -> void:
@@ -112,31 +116,41 @@ static func send_model_animation(module: copyMultiplayer) -> void:
 
 	var model       := module.get_model()
 	var skeleton    := module.get_skeleton()
-	if (not model) or (not skeleton): return
+	var anim_player := model.find_child("AnimationPlayer", false, false)
+	var anim_root   := anim_player.get_node_or_null(anim_player.root_node)
+	if (not model) or (not skeleton) or (not anim_root): return
 
 	write_stream.write_transform16(model.transform)
 
-	# Pre-filter any bones that are in rest pose.
+	# Pre-filter any bones that are at rest / blendshapes that are at zero.
-	var restless_bones := {}
+	# Unless most bones / blendshapes are active, this should reduce packet size.
-	for bone_name in module.bone_to_lookup:
+	var restless_bones     := []
+	var active_blendshapes := []
+
+	for i in module.bone_lookup.size():
+		var bone_name := module.bone_lookup[i]
 		var bone_idx  := skeleton.find_bone(bone_name)
 		var bone_pose := skeleton.get_bone_pose(bone_idx)
 		var bone_rest := skeleton.get_bone_rest(bone_idx)
 		if not bone_pose.is_equal_approx(bone_rest):
-			restless_bones[bone_name] = bone_pose
+			restless_bones.append({ lookup = i, pose = bone_pose })
+
+	for i in module.blendshape_lookup.size():
+		var anim_path := module.blendshape_lookup[i]
+		var anim_node := anim_root.get_node_or_null(anim_path)
+		var value: float = anim_node.get("blend_shapes/" + anim_path.get_subname(0))
+		if not is_zero_approx(value):
+			active_blendshapes.append({ lookup = i, value = value })
 
 	write_stream.write_uint8(restless_bones.size())
-	for bone_name in restless_bones:
+	for bone in restless_bones:
-		write_stream.write_uint8(module.bone_to_lookup[bone_name])
+		write_stream.write_uint8(bone.lookup)
-		write_stream.write_bone_pose(restless_bones[bone_name])
+		write_stream.write_bone_pose(bone.pose)
-
+
-	# TODO: Only write non-default blendshapes. Anything missing = default.
+	write_stream.write_uint8(active_blendshapes.size())
-	var shape_dict: Dictionary = {} # TODO: Redo the blendshapes.
+	for blendshape in active_blendshapes:
-	write_stream.write_uint8(module.blendshape_to_lookup.size())
+		write_stream.write_uint8(blendshape.lookup)
-	for shape_name in module.blendshape_to_lookup:
+		write_stream.write_range16(blendshape.value)
-		var shape_alpha: float = shape_dict[shape_name]
-		write_stream.write_uint8(module.blendshape_to_lookup[shape_name])
-		write_stream.write_float16(shape_alpha)
 
 	# The compression still helps, so we'll keep it for now.
 	var compressed_buffer := write_stream.slice().compress(FileAccess.COMPRESSION_ZSTD);