From 588a6e6d3053cb8001b97b1162050aec4acbdd56 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Sep 2024 12:08:33 -0400 Subject: [PATCH 1/4] fix flux controlnet pipelines --- src/diffusers/pipelines/auto_pipeline.py | 11 ++++++++++- .../pipelines/flux/pipeline_flux_controlnet.py | 15 +++++++++------ .../pipeline_flux_controlnet_image_to_image.py | 4 ++-- .../flux/pipeline_flux_controlnet_inpainting.py | 4 ++-- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 39ceadb5acef..a9a41a54999c 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -29,7 +29,14 @@ StableDiffusionXLControlNetPipeline, ) from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline -from .flux import FluxControlNetPipeline, FluxImg2ImgPipeline, FluxInpaintPipeline, FluxPipeline +from .flux import ( + FluxControlNetPipeline, + FluxControlNetImg2ImgPipeline, + FluxControlNetInpaintPipeline, + FluxImg2ImgPipeline, + FluxInpaintPipeline, + FluxPipeline +) from .hunyuandit import HunyuanDiTPipeline from .kandinsky import ( KandinskyCombinedPipeline, @@ -128,6 +135,7 @@ ("stable-diffusion-xl-controlnet-pag", StableDiffusionXLControlNetPAGImg2ImgPipeline), ("lcm", LatentConsistencyModelImg2ImgPipeline), ("flux", FluxImg2ImgPipeline), + ("flux-controlnet", FluxControlNetImg2ImgPipeline), ] ) @@ -143,6 +151,7 @@ ("stable-diffusion-xl-controlnet", StableDiffusionXLControlNetInpaintPipeline), ("stable-diffusion-xl-pag", StableDiffusionXLPAGInpaintPipeline), ("flux", FluxInpaintPipeline), + ("flux-controlnet", FluxControlNetInpaintPipeline), ] ) diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py index 481994903d3f..2809a47bf63a 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py @@ -840,12 +840,10 @@ def __call__( # broadcast to batch dimension in a way that's compatible with ONNX/Core ML timestep = t.expand(latents.shape[0]).to(latents.dtype) - # handle guidance - if self.transformer.config.guidance_embeds: - guidance = torch.tensor([guidance_scale], device=device) - guidance = guidance.expand(latents.shape[0]) - else: - guidance = None + guidance = ( + torch.tensor([guidance_scale], device=device) if self.controlnet.config.guidance_embeds else None + ) + guidance = guidance.expand(latents.shape[0]) if guidance is not None else None # controlnet controlnet_block_samples, controlnet_single_block_samples = self.controlnet( @@ -863,6 +861,11 @@ def __call__( return_dict=False, ) + guidance = ( + torch.tensor([guidance_scale], device=device) if self.transformer.config.guidance_embeds else None + ) + guidance = guidance.expand(latents.shape[0]) if guidance is not None else None + noise_pred = self.transformer( hidden_states=latents, timestep=timestep / 1000, diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py index 72803b180c34..deeb9e3f546a 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py @@ -767,7 +767,7 @@ def __call__( batch_size=batch_size * num_images_per_prompt, num_images_per_prompt=num_images_per_prompt, device=device, - dtype=dtype, + dtype=self.vae.dtype, ) height, width = control_image.shape[-2:] @@ -798,7 +798,7 @@ def __call__( batch_size=batch_size * num_images_per_prompt, num_images_per_prompt=num_images_per_prompt, device=device, - dtype=dtype, + dtype=self.vae.dtype, ) height, width = control_image_.shape[-2:] diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py index d43acdf38ea5..e763200155f6 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py @@ -899,7 +899,7 @@ def __call__( batch_size=batch_size * num_images_per_prompt, num_images_per_prompt=num_images_per_prompt, device=device, - dtype=dtype, + dtype=self.vae.dtype, ) height, width = control_image.shape[-2:] @@ -933,7 +933,7 @@ def __call__( batch_size=batch_size * num_images_per_prompt, num_images_per_prompt=num_images_per_prompt, device=device, - dtype=dtype, + dtype=self.vae.dtype, ) height, width = control_image_.shape[-2:] From 786e247c9fb845190cfdc39424cdfa8269d3d0b5 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Sep 2024 12:14:51 -0400 Subject: [PATCH 2/4] add missing vae dtype --- src/diffusers/pipelines/flux/pipeline_flux_controlnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py index 2809a47bf63a..11b71b1cbece 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py @@ -729,7 +729,7 @@ def __call__( batch_size=batch_size * num_images_per_prompt, num_images_per_prompt=num_images_per_prompt, device=device, - dtype=dtype, + dtype=self.vae.dtype, ) height, width = control_image.shape[-2:] @@ -763,7 +763,7 @@ def __call__( batch_size=batch_size * num_images_per_prompt, num_images_per_prompt=num_images_per_prompt, device=device, - dtype=dtype, + dtype=self.vae.dtype, ) height, width = control_image_.shape[-2:] From 733d9d5e8e92a3b2ea1ea093b830f9277ac620df Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Sep 2024 19:34:18 -0400 Subject: [PATCH 3/4] sort pipelines --- src/diffusers/pipelines/auto_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index a9a41a54999c..807bcb6cecd8 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -30,9 +30,9 @@ ) from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline from .flux import ( - FluxControlNetPipeline, FluxControlNetImg2ImgPipeline, FluxControlNetInpaintPipeline, + FluxControlNetPipeline, FluxImg2ImgPipeline, FluxInpaintPipeline, FluxPipeline From aee786e13f178a7e2b3dd491b24b975caaa706b6 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 20 Sep 2024 03:21:57 +0200 Subject: [PATCH 4/4] style --- src/diffusers/pipelines/auto_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 807bcb6cecd8..f6186da260ad 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -35,7 +35,7 @@ FluxControlNetPipeline, FluxImg2ImgPipeline, FluxInpaintPipeline, - FluxPipeline + FluxPipeline, ) from .hunyuandit import HunyuanDiTPipeline from .kandinsky import (