mirror of
https://github.com/comfyanonymous/ComfyUI.git
synced 2025-01-11 02:15:17 +00:00
31 lines
1.1 KiB
Python
31 lines
1.1 KiB
Python
import torch
|
|
import comfy.ops
|
|
|
|
def pad_to_patch_size(img, patch_size=(2, 2), padding_mode="circular"):
|
|
if padding_mode == "circular" and (torch.jit.is_tracing() or torch.jit.is_scripting()):
|
|
padding_mode = "reflect"
|
|
|
|
pad = ()
|
|
for i in range(img.ndim - 2):
|
|
pad = (0, (patch_size[i] - img.shape[i + 2] % patch_size[i]) % patch_size[i]) + pad
|
|
|
|
return torch.nn.functional.pad(img, pad, mode=padding_mode)
|
|
|
|
try:
|
|
rms_norm_torch = torch.nn.functional.rms_norm
|
|
except:
|
|
rms_norm_torch = None
|
|
|
|
def rms_norm(x, weight=None, eps=1e-6):
|
|
if rms_norm_torch is not None and not (torch.jit.is_tracing() or torch.jit.is_scripting()):
|
|
if weight is None:
|
|
return rms_norm_torch(x, (x.shape[-1],), eps=eps)
|
|
else:
|
|
return rms_norm_torch(x, weight.shape, weight=comfy.ops.cast_to(weight, dtype=x.dtype, device=x.device), eps=eps)
|
|
else:
|
|
r = x * torch.rsqrt(torch.mean(x**2, dim=-1, keepdim=True) + eps)
|
|
if weight is None:
|
|
return r
|
|
else:
|
|
return r * comfy.ops.cast_to(weight, dtype=x.dtype, device=x.device)
|