copygirl
/
copyMultiplayer
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Sync blendshapes once more

Browse Source
copygirl 2 weeks ago
parent bd180ec061
commit 7060809d7f
2 changed files with 86 additions and 66 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 34
      copyMultiplayer.gd
  2. 118
      sync_controller.gd

34
copyMultiplayer.gd
Unescape View File

@ -13,9 +13,7 @@ var tracked_bones: Array[String]
var version: int = -1
var bone_lookup: Array[String] = []
var bone_to_lookup: Dictionary = {}
var blendshape_lookup: Array[String] = []
var blendshape_to_lookup: Dictionary = {}
var blendshape_lookup: Array[NodePath] = []
# Temporary positioning system.
# TODO: Add a setting that allows syncing model positions, as an alternative
@ -48,8 +46,8 @@ func _ready() -> void:
# FIXME: Hardcoded way to get the main model controller.
main_controller = $"/root/SnekStudio_Main/ModelController"
main_controller.child_entered_tree.connect(on_model_controller_child_added)
on_model_controller_child_added(main_controller.get_node_or_null("Model"))
main_controller.child_entered_tree.connect(on_model_changed)
on_model_changed(main_controller.get_node_or_null("Model"))
setup_setting_widget("cache" , "Cache/LineEdit", true )
setup_setting_widget("nickname", "Name/LineEdit" , false)
@ -222,25 +220,33 @@ func clear_player_models() -> void:
func _process(_delta: float) -> void:
SyncController.send_model_animation(self)
func on_model_controller_child_added(child: Node) -> void:
if (child == null) or (child.name != "Model"): return
# Will be called once when the module is initialized, and also when a child is
# added to the main ModelController, which happens when the model is changed.
func on_model_changed(model: Node) -> void:
if (model == null) or (model.name != "Model"): return
var skeleton: Skeleton3D = main_controller._get_model_skeleton()
if not skeleton: return # Do nothing, I guess? Unsure if potential bug.
# This "version" is used to ensure that sync updates
# are not applied to the incorrect lookup tables.
version = (version + 1) % 256
bone_lookup.clear()
bone_to_lookup.clear()
for bone_name in tracked_bones:
bone_lookup.append(bone_name)
var bone_idx := skeleton.find_bone(bone_name)
if bone_idx == -1: continue # Need bone in bone_lookup, but not in bone_to_lookup!
bone_to_lookup[bone_name] = bone_lookup.size() - 1
if bone_idx == -1: continue
bone_lookup.append(bone_name)
blendshape_lookup.clear()
blendshape_to_lookup.clear()
# TODO: Redo the blendshapes. Consider grabbing data from animations directly?
var anim_player: AnimationPlayer = model.find_child("AnimationPlayer", false, false)
for anim_name in anim_player.get_animation_list():
if anim_name == "RESET": continue # Skip RESET animation?
var anim := anim_player.get_animation(anim_name)
for track_index in anim.get_track_count():
if anim.track_get_type(track_index) == Animation.TYPE_BLEND_SHAPE:
blendshape_lookup.append(anim.track_get_path(track_index))
continue
# Wait for one frame, then "_last_loaded_vrm" is updated.
await get_tree().process_frame
@ -265,7 +271,7 @@ func change_nickname(new_nickname: String) -> void:
func change_model(
new_version: int,
new_bone_lookup: Array[String],
new_blendshape_lookup: Array[String],
new_blendshape_lookup: Array[NodePath],
filename: String,
) -> void:
var peer_id := multiplayer.get_remote_sender_id()

118
sync_controller.gd
Unescape View File

@ -9,18 +9,17 @@ var nickname: String
var version: int = -1
var bone_lookup: Array[String]
var blendshape_lookup: Array[String]
var blendshape_lookup: Array[NodePath]
var model_name: String
var model: Node
var skeleton: Skeleton3D
var model_name : String
var model : Node
var skeleton : Skeleton3D
var anim_player : AnimationPlayer
var anim_root : Node
# Reusable buffer to write data for synchronizing models.
static var write_stream: StreamBuffer = StreamBuffer.with_capacity(2048)
# Allows us to use the "apply_animations" function to apply blendshapes to a model.
static var BlendShapes: Script = load("res://Mods/MediaPipe/MediaPipeController_BlendShapes.gd")
func _ready() -> void:
module = get_parent().get_parent()
model_controller = get_parent()
@ -40,7 +39,7 @@ func change_nickname(new_nickname: String) -> void:
func change_model(
new_version: int,
new_bone_lookup: Array[String],
new_blendshape_lookup: Array[String],
new_blendshape_lookup: Array[NodePath],
filename: String,
) -> void:
# These should be safe to update even if the model doesn't load.
@ -60,9 +59,11 @@ func change_model(
module.print_log("ERROR: Model '%s' could not be loaded!" % filename)
return
model_name = filename
model = model_controller.get_node_or_null("Model")
skeleton = model_controller._get_model_skeleton()
model_name = filename
model = model_controller.get_node_or_null("Model")
skeleton = model_controller._get_model_skeleton()
anim_player = model.find_child("AnimationPlayer", false, false)
anim_root = anim_player.get_node(anim_player.root_node)
module.print_log("%s switched to '%s'" % [ get_display_name(), filename ])
@ -72,71 +73,84 @@ func sync_model_animation(
buffer: PackedByteArray,
) -> void:
if version != current_version: return
if (not model) or (not skeleton): return
if (not model) or (not skeleton) or (not anim_root): return
var uncompressed_buffer := buffer.decompress(uncompressed_length, FileAccess.COMPRESSION_ZSTD);
var stream := StreamBuffer.from_buffer(uncompressed_buffer)
model.transform = stream.read_transform16()
# We skipped some bones, so reset the skipped ones to the rest pose.
var all_bones := {}
for bone_name in module.bone_to_lookup:
var all_bones: Array[Dictionary] = []
var all_blendshapes: Array[Dictionary] = []
# We filter out bones at rest and blendshapes at zero, so this
# is initializing all_bones/blendshapes to the default values.
for bone_name in module.bone_lookup:
var bone_idx := skeleton.find_bone(bone_name)
var bone_rest := skeleton.get_bone_rest(bone_idx)
all_bones[bone_name] = { idx = bone_idx, pose = bone_rest }
# Override rest poses with ones from the packet.
var num_bones := stream.read_uint8()
for i in num_bones:
var bone_name := bone_lookup[stream.read_uint8()]
all_bones[bone_name].pose = stream.read_bone_pose()
# Apply bone poses to skeleton.
for bone_name in all_bones:
var bone: Dictionary = all_bones[bone_name]
all_bones.append({ name = bone_name, idx = bone_idx, pose = bone_rest })
for anim_path in module.blendshape_lookup:
all_blendshapes.append({ path = anim_path, value = 0.0 })
# 256 bones (and blendshapes) should be enough, right?
for i in stream.read_uint8():
var lookup := stream.read_uint8()
all_bones[lookup].pose = stream.read_bone_pose()
for i in stream.read_uint8():
var lookup := stream.read_uint8()
all_blendshapes[lookup].value = stream.read_range16()
# Apply all the values to bones / blendshapes.
for bone in all_bones:
if bone.idx == -1: continue # Different model might not have this bone.
skeleton.set_bone_pose(bone.idx, bone.pose)
var shape_dict := {}
var num_shapes := stream.read_uint8() # 256 blendshapes (and bones) should be enough, right?
for i in num_shapes:
var shape_name := blendshape_lookup[stream.read_uint8()]
var shape_alpha := stream.read_float16()
shape_dict[shape_name] = shape_alpha
BlendShapes.apply_animations(model, shape_dict)
for blendshape in all_blendshapes:
var anim_node := anim_root.get_node_or_null(blendshape.path)
if not anim_node: continue # Different model might not have this node.
anim_node.set("blend_shapes/" + blendshape.path.get_subname(0), blendshape.value)
@warning_ignore("shadowed_variable")
static func send_model_animation(module: copyMultiplayer) -> void:
# Check if there's other players we're connected to.
if module.multiplayer.get_peers().size() == 0: return
var model := module.get_model()
var skeleton := module.get_skeleton()
if (not model) or (not skeleton): return
var model := module.get_model()
var skeleton := module.get_skeleton()
var anim_player := model.find_child("AnimationPlayer", false, false)
var anim_root := anim_player.get_node_or_null(anim_player.root_node)
if (not model) or (not skeleton) or (not anim_root): return
write_stream.write_transform16(model.transform)
# Pre-filter any bones that are in rest pose.
var restless_bones := {}
for bone_name in module.bone_to_lookup:
# Pre-filter any bones that are at rest / blendshapes that are at zero.
# Unless most bones / blendshapes are active, this should reduce packet size.
var restless_bones := []
var active_blendshapes := []
for i in module.bone_lookup.size():
var bone_name := module.bone_lookup[i]
var bone_idx := skeleton.find_bone(bone_name)
var bone_pose := skeleton.get_bone_pose(bone_idx)
var bone_rest := skeleton.get_bone_rest(bone_idx)
if not bone_pose.is_equal_approx(bone_rest):
restless_bones[bone_name] = bone_pose
restless_bones.append({ lookup = i, pose = bone_pose })
for i in module.blendshape_lookup.size():
var anim_path := module.blendshape_lookup[i]
var anim_node := anim_root.get_node_or_null(anim_path)
var value: float = anim_node.get("blend_shapes/" + anim_path.get_subname(0))
if not is_zero_approx(value):
active_blendshapes.append({ lookup = i, value = value })
write_stream.write_uint8(restless_bones.size())
for bone_name in restless_bones:
write_stream.write_uint8(module.bone_to_lookup[bone_name])
write_stream.write_bone_pose(restless_bones[bone_name])
# TODO: Only write non-default blendshapes. Anything missing = default.
var shape_dict: Dictionary = {} # TODO: Redo the blendshapes.
write_stream.write_uint8(module.blendshape_to_lookup.size())
for shape_name in module.blendshape_to_lookup:
var shape_alpha: float = shape_dict[shape_name]
write_stream.write_uint8(module.blendshape_to_lookup[shape_name])
write_stream.write_float16(shape_alpha)
for bone in restless_bones:
write_stream.write_uint8(bone.lookup)
write_stream.write_bone_pose(bone.pose)
write_stream.write_uint8(active_blendshapes.size())
for blendshape in active_blendshapes:
write_stream.write_uint8(blendshape.lookup)
write_stream.write_range16(blendshape.value)
# The compression still helps, so we'll keep it for now.
var compressed_buffer := write_stream.slice().compress(FileAccess.COMPRESSION_ZSTD);

Write Preview
Loading…
Cancel
Save