fix(lora): only drop adapter from _merged_adapters when unfused from all components

Signed-off-by: Aloys Jehwin <aloysjehwin@gmail.com>

Author: AloysJehwin

src/diffusers/loaders/lora_base.py

@@ -669,11 +669,21 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs):
                 if issubclass(model.__class__, (ModelMixin, PreTrainedModel)):
                     for module in model.modules():
                         if isinstance(module, BaseTunerLayer):
-                            for adapter in set(module.merged_adapters):
-                                if adapter and adapter in self._merged_adapters:
-                                    self._merged_adapters = self._merged_adapters - {adapter}
                             module.unmerge()
 
+        # Only remove an adapter from _merged_adapters once it is no longer
+        # physically merged in any remaining loadable component. Removing it
+        # on the first unfused component would desync the set when the adapter
+        # is still fused into other components.
+        remaining_merged: set[str] = set()
+        for component_name in self._lora_loadable_modules:
+            component_model = getattr(self, component_name, None)
+            if component_model is not None and issubclass(component_model.__class__, (ModelMixin, PreTrainedModel)):
+                for module in component_model.modules():
+                    if isinstance(module, BaseTunerLayer):
+                        remaining_merged.update(module.merged_adapters)
+        self._merged_adapters = self._merged_adapters & remaining_merged
+
     def set_adapters(
         self,
         adapter_names: list[str] | str,