298977887/ComfyUI
1
0
Fork 0
mirror of https://github.com/comfyanonymous/ComfyUI.git synced 2025-01-10 18:05:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Properly calculate inner dim for t5 model.

Browse Source 
This is required to support some different types of t5 models.
This commit is contained in:
comfyanonymous 2025-01-07 17:32:29 -05:00
parent c515bdf371
commit d0f3752e33
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
comfy/text_encoders/t5.py
View File

@ -227,8 +227,9 @@ class T5(torch.nn.Module):
super().__init__()
self.num_layers = config_dict["num_layers"]
model_dim = config_dict["d_model"]
inner_dim = config_dict["d_kv"] * config_dict["num_heads"]
self.encoder = T5Stack(self.num_layers, model_dim, model_dim, config_dict["d_ff"], config_dict["dense_act_fn"], config_dict["is_gated_act"], config_dict["num_heads"], config_dict["model_type"] != "umt5", dtype, device, operations)
self.encoder = T5Stack(self.num_layers, model_dim, inner_dim, config_dict["d_ff"], config_dict["dense_act_fn"], config_dict["is_gated_act"], config_dict["num_heads"], config_dict["model_type"] != "umt5", dtype, device, operations)
self.dtype = dtype
self.shared = operations.Embedding(config_dict["vocab_size"], model_dim, device=device, dtype=dtype)