LoKr

mindnlp.peft.tuners.lokr.config

lokr.

mindnlp.peft.tuners.lokr.config.LoKrConfig dataclass

Bases: PeftConfig

This is the configuration class to store the configuration of a [LoraModel].

PARAMETER	DESCRIPTION
r	lokr attention dimension. TYPE: `int` DEFAULT: 8
target_modules	The names of the modules to apply Lora to. TYPE: `Union[List[str],str]` DEFAULT: None
lora_alpha	The alpha parameter for Lokr scaling. TYPE: `float` DEFAULT: 8
rank_dropout	The dropout probability for rank dimension during training. TYPE: `float` DEFAULT: 0.0
module_dropout	The dropout probability for LoKR layers. TYPE: `float` DEFAULT: 0.0
use_effective_conv2d	Use parameter effective decomposition for Conv2d with ksize > 1 ("Proposition 3" from FedPara paper). TYPE: `bool` DEFAULT: False
decompose_both	Perform rank decomposition of left kronecker product matrix. TYPE: `bool` DEFAULT: False
decompose_factor	Kronecker product decomposition factor. TYPE: `int` DEFAULT: -1
bias	Bias type for Lora. Can be 'none', 'all' or 'lora_only' TYPE: `str` DEFAULT: 'none'
modules_to_save	List of modules apart from LoRA layers to be set as trainable and saved in the final checkpoint. TYPE: `List[str]` DEFAULT: None
init_weights	Whether to perform initialization of adapter weights. This defaults to True, passing False is discouraged. TYPE: `bool` DEFAULT: True
layers_to_transform	The layer indexes to transform, if this argument is specified, it will apply the LoRA transformations on the layer indexes that are specified in this list. If a single integer is passed, it will apply the LoRA transformations on the layer at this index. TYPE: `Union[List[int],int]` DEFAULT: None
layers_pattern	The layer pattern name, used only if layers_to_transform is different from None and if the layer pattern is not in the common layers pattern. TYPE: `str` DEFAULT: None
rank_pattern	The mapping from layer names or regexp expression to ranks which are different from the default rank specified by r. TYPE: `dict` DEFAULT: dict()
alpha_pattern	The mapping from layer names or regexp expression to alphas which are different from the default alpha specified by alpha. TYPE: `dict` DEFAULT: dict()

Source code in mindnlp/peft/tuners/lokr/config.py

@dataclass
class LoKrConfig(PeftConfig):
    """
    This is the configuration class to store the configuration of a [`LoraModel`].

    Args:
        r (`int`): lokr attention dimension.
        target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to.
        lora_alpha (`float`): The alpha parameter for Lokr scaling.
        rank_dropout (`float`):The dropout probability for rank dimension during training.
        module_dropout (`float`): The dropout probability for LoKR layers.
        use_effective_conv2d (`bool`):
            Use parameter effective decomposition for
            Conv2d with ksize > 1 ("Proposition 3" from FedPara paper).
        decompose_both (`bool`):Perform rank decomposition of left kronecker product matrix.
        decompose_factor (`int`):Kronecker product decomposition factor.

        bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only'
        modules_to_save (`List[str]`):
            List of modules apart from LoRA layers to be set as trainable
            and saved in the final checkpoint.
        init_weights (`bool`):
            Whether to perform initialization of adapter weights. This defaults to `True`, 
            passing `False` is discouraged.
        layers_to_transform (`Union[List[int],int]`):
            The layer indexes to transform, if this argument is specified, it will apply the LoRA transformations on
            the layer indexes that are specified in this list. If a single integer is passed, it will apply the LoRA
            transformations on the layer at this index.
        layers_pattern (`str`):
            The layer pattern name, used only if `layers_to_transform` is different from `None` and if the layer
            pattern is not in the common layers pattern.
        rank_pattern (`dict`):
            The mapping from layer names or regexp expression to ranks which are different from the default rank
            specified by `r`.
        alpha_pattern (`dict`):
            The mapping from layer names or regexp expression to alphas which are different from the default alpha
            specified by `alpha`.
    """

    r: int = field(default=8, metadata={"help": "lokr attention dimension"})
    target_modules: Optional[Union[List[str], str]] = field(
        default=None,
        metadata={
            "help": "List of module names or regex expression of the module names to replace with Lora."
            "For example, ['q', 'v'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$' "
        },
    )
    lora_alpha: int = field(default=8, metadata={"help": "lokr alpha"})
    rank_dropout: float = field(
        default=0.0,
        metadata={"help": "The dropout probability for rank dimension during training"},
    )
    module_dropout: float = field(default=0.0, metadata={"help": "lokr dropout"})
    use_effective_conv2d: bool = field(
        default=False,
        metadata={
            "help": 'Use parameter effective decomposition for Conv2d 3x3 with ksize > 1 ("Proposition 3" from FedPara paper)'
        },
    )
    decompose_both: bool = field(
        default=False,
        metadata={
            "help": "Perform rank decomposition of left kronecker product matrix."
        },
    )
    decompose_factor: int = field(
        default=-1, metadata={"help": "Kronecker product decomposition factor."}
    )

    bias: str = field(
        default="none",
        metadata={"help": "Bias type for Lora. Can be 'none', 'all' or 'lora_only'"},
    )
    modules_to_save: Optional[List[str]] = field(
        default=None,
        metadata={
            "help": "List of modules apart from LoRA layers to be set as trainable and saved in the final checkpoint. "
            "For example, in Sequence Classification or Token Classification tasks, "
            "the final layer `classifier/score` are randomly initialized and as such need to be trainable and saved."
        },
    )
    init_weights: bool = field(
        default=True,
        metadata={"help": "Whether to initialize the weights of the Lora layers."},
    )
    layers_to_transform: Optional[Union[List, int]] = field(
        default=None,
        metadata={
            "help": "The layer indexes to transform, is this argument is specified, \
                PEFT will transform only the layers indexes that are specified inside this list. \
                If a single integer is passed, PEFT will transform only the layer at this index."
        },
    )
    layers_pattern: Optional[str] = field(
        default=None,
        metadata={
            "help": "The layer pattern name, used only if `layers_to_transform` is different to None and \
                  if the layer pattern is not in the common layers pattern."
        },
    )
    rank_pattern: Optional[dict] = field(
        default_factory=dict,
        metadata={
            "help": (
                "The mapping from layer names or regexp expression to ranks which are different from the default rank specified by `r`. "
                "For example, `{model.decoder.layers.0.encoder_attn.k_proj: 8`}"
            )
        },
    )
    alpha_pattern: Optional[dict] = field(
        default_factory=dict,
        metadata={
            "help": (
                "The mapping from layer names or regexp expression to alphas which are different from the default alpha specified by `alpha`. "
                "For example, `{model.decoder.layers.0.encoder_attn.k_proj: 32`}"
            )
        },
    )

    def __post_init__(self):
        self.peft_type = PeftType.LOKR

    @property
    def is_prompt_learning(self):
        r"""
        Utility method to check if the configuration is for prompt learning.
        """
        return False

mindnlp.peft.tuners.lokr.config.LoKrConfig.is_prompt_learning property

Utility method to check if the configuration is for prompt learning.

mindnlp.peft.tuners.lokr.model

Lokr.

mindnlp.peft.tuners.lokr.model.LoKrModel

Bases: BaseTuner

Creates Low-Rank Kronecker Product model from a pretrained model. The original method is partially described in https://arxiv.org/abs/2108.06098 and in https://arxiv.org/abs/2309.14859 Current implementation heavily borrows from https://github.com/KohakuBlueleaf/LyCORIS/blob/eb460098187f752a5d66406d3affade6f0a07ece/lycoris/modules/lokr.py

PARAMETER	DESCRIPTION
model	The model to which the adapter tuner layers will be attached. TYPE: `mindspore.nn.Cell`
peft_config	The configuration of the LoKr model. TYPE: [`LoKrConfig`]
adapter_name	The name of the adapter, defaults to "default". TYPE: `str`

RETURNS	DESCRIPTION
LoKrModel	The LoKr model. TYPE: [`mindspore.nn.Cell`]

Example

>>> from diffusers import StableDiffusionPipeline
>>> from peft import LoKrModel, LoKrConfig

>>> config_te = LoKrConfig(
...     r=8,
...     lora_alpha=32,
...     target_modules=["k_proj", "q_proj", "v_proj", "out_proj", "fc1", "fc2"],
...     rank_dropout=0.0,
...     module_dropout=0.0,
...     init_weights=True,
... )
>>> config_unet = LoKrConfig(
...     r=8,
...     lora_alpha=32,
...     target_modules=[
...         "proj_in",
...         "proj_out",
...         "to_k",
...         "to_q",
...         "to_v",
...         "to_out.0",
...         "ff.net.0.proj",
...         "ff.net.2",
...     ],
...     rank_dropout=0.0,
...     module_dropout=0.0,
...     init_weights=True,
...     use_effective_conv2d=True,
... )

>>> model = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
>>> model.text_encoder = LoKrModel(model.text_encoder, config_te, "default")
>>> model.unet = LoKrModel(model.unet, config_unet, "default")

Attributes:

• model ([~nn.Cell])— The model to be adapted.

• peft_config ([LoKrConfig]): The configuration of the LoKr model.

Source code in mindnlp/peft/tuners/lokr/model.py

class LoKrModel(BaseTuner):
    """
    Creates Low-Rank Kronecker Product model from a pretrained model. The original method is partially described in
    https://arxiv.org/abs/2108.06098 and in https://arxiv.org/abs/2309.14859 Current implementation heavily borrows
    from
    https://github.com/KohakuBlueleaf/LyCORIS/blob/eb460098187f752a5d66406d3affade6f0a07ece/lycoris/modules/lokr.py

    Args:
        model (`mindspore.nn.Cell`): The model to which the adapter tuner layers will be attached.
        peft_config ([`LoKrConfig`]): The configuration of the LoKr model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.

    Returns:
        LoKrModel ([`mindspore.nn.Cell`]): The LoKr model.

    Example:
        ```py
        >>> from diffusers import StableDiffusionPipeline
        >>> from peft import LoKrModel, LoKrConfig

        >>> config_te = LoKrConfig(
        ...     r=8,
        ...     lora_alpha=32,
        ...     target_modules=["k_proj", "q_proj", "v_proj", "out_proj", "fc1", "fc2"],
        ...     rank_dropout=0.0,
        ...     module_dropout=0.0,
        ...     init_weights=True,
        ... )
        >>> config_unet = LoKrConfig(
        ...     r=8,
        ...     lora_alpha=32,
        ...     target_modules=[
        ...         "proj_in",
        ...         "proj_out",
        ...         "to_k",
        ...         "to_q",
        ...         "to_v",
        ...         "to_out.0",
        ...         "ff.net.0.proj",
        ...         "ff.net.2",
        ...     ],
        ...     rank_dropout=0.0,
        ...     module_dropout=0.0,
        ...     init_weights=True,
        ...     use_effective_conv2d=True,
        ... )

        >>> model = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
        >>> model.text_encoder = LoKrModel(model.text_encoder, config_te, "default")
        >>> model.unet = LoKrModel(model.unet, config_unet, "default")
        ```

    > **Attributes**:  

    >   - **model** ([`~nn.Cell`])— The model to be adapted. 

    >   - **peft_config** ([`LoKrConfig`]): The configuration of the LoKr  model. 

    """

    prefix: str = "lokr_"
    layers_mapping: Dict[Type[nn.Cell], Type[LoKrLayer]] = {
        nn.Conv2d: Conv2d,
        nn.Dense: Dense,
    }

    def _create_and_replace(
        self,
        config: LoKrConfig,
        adapter_name: str,
        target: Union[LoKrLayer, nn.Cell],
        target_name: str,
        parent: nn.Cell,
        current_key: str,
        loaded_in_8bit: Optional[bool] = False,
        loaded_in_4bit: Optional[bool] = False,
    ) -> None:
        """
        A private method to create and replace the target module with the adapter module.
        """

        # Regexp matching - Find key which matches current target_name in patterns provided
        pattern_keys = list(
            chain(config.rank_pattern.keys(), config.alpha_pattern.keys())
        )
        target_name_key = next(
            filter(lambda key: re.match(rf"(.*\.)?{key}$", current_key), pattern_keys),
            target_name,
        )

        kwargs = config.to_dict()
        kwargs["r"] = config.rank_pattern.get(target_name_key, config.r)
        kwargs["alpha"] = config.alpha_pattern.get(target_name_key, config.lora_alpha)

        if isinstance(target, LoKrLayer):
            target.update_layer(adapter_name, **kwargs)
        else:
            new_module = self._create_new_module(config, adapter_name, target, **kwargs)
            self._replace_module(parent, target_name, new_module, target)

    @classmethod
    def _create_new_module(
        cls, config: LoKrConfig, adapter_name: str, target: nn.Cell, **kwargs
    ) -> LoKrLayer:
        # Find corresponding subtype of provided target module
        new_module_cls = None
        for subtype, target_cls in cls.layers_mapping.items():
            if (
                hasattr(target, "base_layer")
                and isinstance(target.get_base_layer(), subtype)
                and isinstance(target, BaseTunerLayer)
            ):
                # nested tuner layers are allowed
                new_module_cls = target_cls
                break
            elif isinstance(target, subtype):
                new_module_cls = target_cls
                break

        # We didn't find corresponding type, so adapter for this layer is not supported
        if new_module_cls is None:
            supported_modules = ", ".join(
                layer.__name__ for layer in cls.layers_mapping.keys()
            )
            raise ValueError(
                f"Target module of type {type(target)} not supported, "
                f"currently only adapters for {supported_modules} are supported"
            )

        if isinstance(target, BaseTunerLayer):
            target_base_layer = target.get_base_layer()
        else:
            target_base_layer = target

        if isinstance(target_base_layer, nn.Cell):
            new_module = new_module_cls(target, adapter_name=adapter_name, **kwargs)
        elif isinstance(target_base_layer, nn.Cell):
            new_module = new_module_cls(target, adapter_name=adapter_name, **kwargs)
        else:
            supported_modules = ", ".join(
                layer.__name__ for layer in cls.layers_mapping.keys()
            )
            raise ValueError(
                f"Target module of type {type(target)} not supported, "
                f"currently only adapters for {supported_modules} are supported"
            )

        return new_module

    def __getattr__(self, name: str):
        """Forward missing attributes to the wrapped module."""
        try:
            return super().__getattr__(name)  # defer to nn.Module's logic
        except AttributeError:
            return getattr(self.model, name)

    def _replace_module(self, parent, child_name, new_module, child):
        setattr(parent, child_name, new_module)

        # child layer wraps the original module, unpack it
        if hasattr(child, "base_layer"):
            child = child.base_layer

        # layers with base_layer don't need the weight to be copied, as they have a reference already
        if not hasattr(new_module, "base_layer"):
            new_module.weight = child.weight
            if hasattr(child, "bias"):
                new_module.bias = child.bias

        if getattr(child, "state", None) is not None:
            if hasattr(new_module, "base_layer"):
                new_module.base_layer.state = child.state
            else:
                new_module.state = child.state

    def _mark_only_adapters_as_trainable(self, model: nn.Cell) -> None:
        for n, p in model.parameters_and_names():
            if self.prefix not in n:
                p.requires_grad = False

    def _set_adapter_layers(self, enabled=True):
        for module in self.model.modules():
            if isinstance(module, (BaseTunerLayer, ModulesToSaveWrapper)):
                module.enable_adapters(enabled)

    def _unload_and_optionally_merge(
        self,
        merge: bool = True,
        progressbar: bool = False,
        safe_merge: bool = False,
        adapter_names: Optional[list[str]] = None,
    ):
        if merge:
            if getattr(self.model, "quantization_method", None) == "gptq":
                raise ValueError(
                    "Cannot merge LOHA layers when the model is gptq quantized"
                )

        self._unloading_checks(adapter_names)
        key_list = [
            key for key, _ in self.model.named_modules() if self.prefix not in key
        ]
        desc = "Unloading " + ("and merging " if merge else "") + "model"
        for key in tqdm(key_list, disable=not progressbar, desc=desc):
            try:
                parent, target, target_name = _get_submodules(self.model, key)
            except AttributeError:
                continue

            if hasattr(target, "base_layer"):
                if merge:
                    target.merge(safe_merge=safe_merge, adapter_names=adapter_names)
                self._replace_module(
                    parent, target_name, target.get_base_layer(), target
                )
            elif isinstance(target, ModulesToSaveWrapper):
                # save any additional trainable modules part of `modules_to_save`
                new_module = target.modules_to_save[target.active_adapter]
                if hasattr(new_module, "base_layer"):
                    # check if the module is itself a tuner layer
                    if merge:
                        new_module.merge(
                            safe_merge=safe_merge, adapter_names=adapter_names
                        )
                    new_module = new_module.get_base_layer()
                setattr(parent, target_name, new_module)

        return self.model

    def _unloading_checks(self, adapter_names: Optional[list[str]]):
        adapters_to_consider = adapter_names or self.active_adapters
        is_modules_to_save_available = any(
            self.peft_config[adapter].modules_to_save
            for adapter in adapters_to_consider
        )
        if is_modules_to_save_available and len(adapters_to_consider) > 1:
            raise ValueError(
                "Cannot unload multiple adapters that specify `modules_to_save`."
            )

    @staticmethod
    def _prepare_adapter_config(peft_config, model_config):
        if peft_config.target_modules is None:
            raise ValueError("Please specify `target_modules` in `peft_config`")
        return peft_config

    @staticmethod
    def _check_target_module_exists(LoKR_config, key):
        return check_target_module_exists(LoKR_config, key)

mindnlp.peft.tuners.lokr.model.LoKrModel.__getattr__(name)

Forward missing attributes to the wrapped module.

Source code in mindnlp/peft/tuners/lokr/model.py

def __getattr__(self, name: str):
    """Forward missing attributes to the wrapped module."""
    try:
        return super().__getattr__(name)  # defer to nn.Module's logic
    except AttributeError:
        return getattr(self.model, name)