From f86c724ef29e7d13e9bcc7c894288f5ac03679da Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Mon, 3 Mar 2025 06:50:31 -0500 Subject: [PATCH] Temporal area composition. New ConditioningSetAreaPercentageVideo node. --- comfy/samplers.py | 54 +++++++++++++++++++++---------- comfy_extras/nodes_video_model.py | 27 ++++++++++++++++ 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/comfy/samplers.py b/comfy/samplers.py index 076c98791..7578ac1ef 100644 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -19,6 +19,12 @@ import comfy.hooks import scipy.stats import numpy + +def add_area_dims(area, num_dims): + while (len(area) // 2) < num_dims: + area = [2147483648] + area[:len(area) // 2] + [0] + area[len(area) // 2:] + return area + def get_area_and_mult(conds, x_in, timestep_in): dims = tuple(x_in.shape[2:]) area = None @@ -34,8 +40,9 @@ def get_area_and_mult(conds, x_in, timestep_in): return None if 'area' in conds: area = list(conds['area']) - while (len(area) // 2) < len(dims): - area = [2147483648] + area[:len(area) // 2] + [0] + area[len(area) // 2:] + area = add_area_dims(area, len(dims)) + if (len(area) // 2) > len(dims): + area = area[:len(dims)] + area[len(area) // 2:(len(area) // 2) + len(dims)] if 'strength' in conds: strength = conds['strength'] @@ -53,7 +60,7 @@ def get_area_and_mult(conds, x_in, timestep_in): if "mask_strength" in conds: mask_strength = conds["mask_strength"] mask = conds['mask'] - assert(mask.shape[1:] == x_in.shape[2:]) + assert (mask.shape[1:] == x_in.shape[2:]) mask = mask[:input_x.shape[0]] if area is not None: @@ -67,16 +74,17 @@ def get_area_and_mult(conds, x_in, timestep_in): mult = mask * strength if 'mask' not in conds and area is not None: - rr = 8 + fuzz = 8 for i in range(len(dims)): + rr = min(fuzz, mult.shape[2 + i] // 4) if area[len(dims) + i] != 0: for t in range(rr): m = mult.narrow(i + 2, t, 1) - m *= ((1.0/rr) * (t + 1)) + m *= ((1.0 / rr) * (t + 1)) if (area[i] + area[len(dims) + i]) < x_in.shape[i + 2]: for t in range(rr): m = mult.narrow(i + 2, area[i] - 1 - t, 1) - m *= ((1.0/rr) * (t + 1)) + m *= ((1.0 / rr) * (t + 1)) conditioning = {} model_conds = conds["model_conds"] @@ -551,25 +559,37 @@ def resolve_areas_and_cond_masks(conditions, h, w, device): logging.warning("WARNING: The comfy.samplers.resolve_areas_and_cond_masks function is deprecated please use the resolve_areas_and_cond_masks_multidim one instead.") return resolve_areas_and_cond_masks_multidim(conditions, [h, w], device) -def create_cond_with_same_area_if_none(conds, c): #TODO: handle dim != 2 +def create_cond_with_same_area_if_none(conds, c): if 'area' not in c: return + def area_inside(a, area_cmp): + a = add_area_dims(a, len(area_cmp) // 2) + area_cmp = add_area_dims(area_cmp, len(a) // 2) + + a_l = len(a) // 2 + area_cmp_l = len(area_cmp) // 2 + for i in range(min(a_l, area_cmp_l)): + if a[a_l + i] < area_cmp[area_cmp_l + i]: + return False + for i in range(min(a_l, area_cmp_l)): + if (a[i] + a[a_l + i]) > (area_cmp[i] + area_cmp[area_cmp_l + i]): + return False + return True + c_area = c['area'] smallest = None for x in conds: if 'area' in x: a = x['area'] - if c_area[2] >= a[2] and c_area[3] >= a[3]: - if a[0] + a[2] >= c_area[0] + c_area[2]: - if a[1] + a[3] >= c_area[1] + c_area[3]: - if smallest is None: - smallest = x - elif 'area' not in smallest: - smallest = x - else: - if smallest['area'][0] * smallest['area'][1] > a[0] * a[1]: - smallest = x + if area_inside(c_area, a): + if smallest is None: + smallest = x + elif 'area' not in smallest: + smallest = x + else: + if math.prod(smallest['area'][:len(smallest['area']) // 2]) > math.prod(a[:len(a) // 2]): + smallest = x else: if smallest is None: smallest = x diff --git a/comfy_extras/nodes_video_model.py b/comfy_extras/nodes_video_model.py index e7a7ec181..0f760aa26 100644 --- a/comfy_extras/nodes_video_model.py +++ b/comfy_extras/nodes_video_model.py @@ -4,6 +4,7 @@ import comfy.utils import comfy.sd import folder_paths import comfy_extras.nodes_model_merging +import node_helpers class ImageOnlyCheckpointLoader: @@ -121,12 +122,38 @@ class ImageOnlyCheckpointSave(comfy_extras.nodes_model_merging.CheckpointSave): comfy_extras.nodes_model_merging.save_checkpoint(model, clip_vision=clip_vision, vae=vae, filename_prefix=filename_prefix, output_dir=self.output_dir, prompt=prompt, extra_pnginfo=extra_pnginfo) return {} + +class ConditioningSetAreaPercentageVideo: + @classmethod + def INPUT_TYPES(s): + return {"required": {"conditioning": ("CONDITIONING", ), + "width": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}), + "height": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}), + "temporal": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}), + "x": ("FLOAT", {"default": 0, "min": 0, "max": 1.0, "step": 0.01}), + "y": ("FLOAT", {"default": 0, "min": 0, "max": 1.0, "step": 0.01}), + "z": ("FLOAT", {"default": 0, "min": 0, "max": 1.0, "step": 0.01}), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + }} + RETURN_TYPES = ("CONDITIONING",) + FUNCTION = "append" + + CATEGORY = "conditioning" + + def append(self, conditioning, width, height, temporal, x, y, z, strength): + c = node_helpers.conditioning_set_values(conditioning, {"area": ("percentage", temporal, height, width, z, y, x), + "strength": strength, + "set_area_to_bounds": False}) + return (c, ) + + NODE_CLASS_MAPPINGS = { "ImageOnlyCheckpointLoader": ImageOnlyCheckpointLoader, "SVD_img2vid_Conditioning": SVD_img2vid_Conditioning, "VideoLinearCFGGuidance": VideoLinearCFGGuidance, "VideoTriangleCFGGuidance": VideoTriangleCFGGuidance, "ImageOnlyCheckpointSave": ImageOnlyCheckpointSave, + "ConditioningSetAreaPercentageVideo": ConditioningSetAreaPercentageVideo, } NODE_DISPLAY_NAME_MAPPINGS = {