Skip to content

Adaption_Prompt

mindnlp.peft.tuners.adaption_prompt.config.AdaptionPromptConfig dataclass

Bases: PeftConfig

Stores the configuration of an [AdaptionPromptModel].

Source code in mindnlp/peft/tuners/adaption_prompt/config.py 
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
@dataclass
class AdaptionPromptConfig(PeftConfig):
    """Stores the configuration of an [`AdaptionPromptModel`]."""

    target_modules: str = field(
        default=None, metadata={"help": "Name of the attention submodules to insert adaption prompts into."}
    )
    adapter_len: int = field(default=None, metadata={"help": "Number of adapter tokens to insert"})
    adapter_layers: int = field(default=None, metadata={"help": "Number of adapter layers (from the top)"})

    def __post_init__(self):
        self.peft_type = PeftType.ADAPTION_PROMPT

    @property
    def is_adaption_prompt(self) -> bool:
        """Return True if this is an adaption prompt config."""
        return True

mindnlp.peft.tuners.adaption_prompt.config.AdaptionPromptConfig.is_adaption_prompt: bool property

Return True if this is an adaption prompt config.

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel

Bases: Cell

Implements adaption prompts as described in https://arxiv.org/pdf/2303.16199.pdf.

The top L attention modules are replaced with AdaptedAttention modules that wrap the original ones, but insert trainable prompts with gates (for zero init).

Notes on the multi-adapter pattern: - We store the states of different adapters by keeping a dictionary of AdaptedAttention modules indexed by adapter name. - Every time we switch adapters, we remove the modules of the currently active adapter from the model, store them in the dictionary, and replace them with the modules of the new adapter. - To avoid duplicated and potentially inconsistent state, the currently active adapter is always removed from the dictionary. - Disabling the adapter would also result in the modules being removed from the model.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
class AdaptionPromptModel(nn.Cell):
    """
    Implements adaption prompts as described in https://arxiv.org/pdf/2303.16199.pdf.

    The top L attention modules are replaced with AdaptedAttention modules that wrap the original ones, but insert
    trainable prompts with gates (for zero init).

    Notes on the multi-adapter pattern:
    - We store the states of different adapters by keeping a dictionary of AdaptedAttention modules indexed by adapter
      name.
    - Every time we switch adapters, we remove the modules of the currently active adapter from the model, store them
      in the dictionary, and replace them with the modules of the new adapter.
    - To avoid duplicated and potentially inconsistent state, the currently active adapter is always removed from the
      dictionary.
    - Disabling the adapter would also result in the modules being removed from the model.
    """

    def __init__(self, model, configs: Dict, adapter_name: str):
        super(AdaptionPromptModel, self).__init__()
        self.model = model
        self.peft_config = {}
        self._parents = {}
        self._cached_adapters = {}
        self._active_adapter = None
        self._enabled = True
        self.forward = self.model.construct
        self.add_adapter(adapter_name, configs[adapter_name])
        self._mark_only_adaption_prompts_as_trainable(self.model)

    def add_adapter(self, adapter_name: str, config: AdaptionPromptConfig) -> None:
        """Add an adapter with the given name and config."""
        config = prepare_config(config, self.model)
        if adapter_name in self.peft_config:
            raise ValueError(f"Adapter named '{adapter_name}' already exists.")

        parents = []
        # 获取模型的所有子模块及其名称
        for name, submodule in self.model.name_cells().items():
            if name.endswith(config.target_modules):
                # 对每个符合条件的子模块调用 _get_submodules 函数
                parent, target, target_name = _get_submodules(self.model, name)
                if target == submodule:
                    parents.append(parent)

        if len(parents) < config.adapter_layers:
            raise ValueError("Config specifies more adapter layers than available in the model.")

        parents = parents[-config.adapter_layers:]
        self._parents[adapter_name] = parents

        if self._active_adapter and self._enabled:
            self._remove_adapted_attentions(self._active_adapter)
        self._active_adapter = adapter_name
        self.peft_config[adapter_name] = config
        self._create_adapted_attentions(config, parents)
        if not self._enabled:
            self._remove_adapted_attentions(adapter_name)

        if config.inference_mode:
            _freeze_adapter(self.model, adapter_name)

    def set_adapter(self, adapter_name: str) -> None:
        """Set the model to use the adapter with the given name."""
        if self._active_adapter == adapter_name:
            return
        if adapter_name not in self.peft_config:
            raise ValueError(f"Adapter with name '{adapter_name}' does not exist.")

        if self._enabled:
            self._remove_adapted_attentions(self._active_adapter)
            self._set_adapted_attentions(adapter_name)

        self._active_adapter = adapter_name

    def enable_adapter_layers(self):
        """Enable adapter layers by swapping in cached AdaptedAttention modules."""
        self._enabled = True
        self._set_adapted_attentions(self._active_adapter)

    def disable_adapter_layers(self):
        """Disable adapter layers by swapping out AdaptedAttention modules."""
        self._enabled = False
        self._remove_adapted_attentions(self._active_adapter)

    def _create_adapted_attentions(self, config: AdaptionPromptConfig, parents: List[nn.Cell]) -> None:
        """Wrap LlamaAttention modules with newly created AdaptedAttention modules."""
        for par in parents:
            attn = AdaptedAttention(
                model_type=self.model.config.model_type,
                adapter_len=config.adapter_len,
                model=getattr(par, config.target_modules),
            )
            setattr(par, config.target_modules, attn)

    def _set_adapted_attentions(self, adapter_name: str) -> None:
        """Replace LlamaAttention modules with cached AdaptedAttention modules."""
        cached = self._cached_adapters[adapter_name]
        del self._cached_adapters[adapter_name]
        config = self.peft_config[adapter_name]
        for i, par in enumerate(self._parents[adapter_name]):
            setattr(par, config.target_modules, cached[i])

    def _remove_adapted_attentions(self, adapter_name: str) -> None:
        """Remove AdaptedAttention modules from the model and store them in the cache."""
        config = self.peft_config[adapter_name]
        adapted_attentions = []
        for par in self._parents[adapter_name]:
            attn = getattr(par, config.target_modules)
            adapted_attentions.append(attn)
            setattr(par, config.target_modules, attn.model)
        self._cached_adapters[adapter_name] = adapted_attentions

    def _mark_only_adaption_prompts_as_trainable(self, model: nn.Cell) -> None:
        for param in model.trainable_params():
            if not is_adaption_prompt_trainable(param.name):
                param.requires_grad = False
    def __getattr__(self, name: str):
        """Forward missing attributes to the wrapped module."""
        try:
            return super().__getattr__(name)  # defer to nn.Module's logic
        except AttributeError:
            # This is necessary as e.g. causal models have various methods that we
            # don't want to re-implement here.
            return getattr(self.model, name)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.__getattr__(name)

Forward missing attributes to the wrapped module.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
142
143
144
145
146
147
148
149
def __getattr__(self, name: str):
    """Forward missing attributes to the wrapped module."""
    try:
        return super().__getattr__(name)  # defer to nn.Module's logic
    except AttributeError:
        # This is necessary as e.g. causal models have various methods that we
        # don't want to re-implement here.
        return getattr(self.model, name)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.add_adapter(adapter_name, config)

Add an adapter with the given name and config.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def add_adapter(self, adapter_name: str, config: AdaptionPromptConfig) -> None:
    """Add an adapter with the given name and config."""
    config = prepare_config(config, self.model)
    if adapter_name in self.peft_config:
        raise ValueError(f"Adapter named '{adapter_name}' already exists.")

    parents = []
    # 获取模型的所有子模块及其名称
    for name, submodule in self.model.name_cells().items():
        if name.endswith(config.target_modules):
            # 对每个符合条件的子模块调用 _get_submodules 函数
            parent, target, target_name = _get_submodules(self.model, name)
            if target == submodule:
                parents.append(parent)

    if len(parents) < config.adapter_layers:
        raise ValueError("Config specifies more adapter layers than available in the model.")

    parents = parents[-config.adapter_layers:]
    self._parents[adapter_name] = parents

    if self._active_adapter and self._enabled:
        self._remove_adapted_attentions(self._active_adapter)
    self._active_adapter = adapter_name
    self.peft_config[adapter_name] = config
    self._create_adapted_attentions(config, parents)
    if not self._enabled:
        self._remove_adapted_attentions(adapter_name)

    if config.inference_mode:
        _freeze_adapter(self.model, adapter_name)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.disable_adapter_layers()

Disable adapter layers by swapping out AdaptedAttention modules.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
105
106
107
108
def disable_adapter_layers(self):
    """Disable adapter layers by swapping out AdaptedAttention modules."""
    self._enabled = False
    self._remove_adapted_attentions(self._active_adapter)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.enable_adapter_layers()

Enable adapter layers by swapping in cached AdaptedAttention modules.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
100
101
102
103
def enable_adapter_layers(self):
    """Enable adapter layers by swapping in cached AdaptedAttention modules."""
    self._enabled = True
    self._set_adapted_attentions(self._active_adapter)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.set_adapter(adapter_name)

Set the model to use the adapter with the given name.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
87
88
89
90
91
92
93
94
95
96
97
98
def set_adapter(self, adapter_name: str) -> None:
    """Set the model to use the adapter with the given name."""
    if self._active_adapter == adapter_name:
        return
    if adapter_name not in self.peft_config:
        raise ValueError(f"Adapter with name '{adapter_name}' does not exist.")

    if self._enabled:
        self._remove_adapted_attentions(self._active_adapter)
        self._set_adapted_attentions(adapter_name)

    self._active_adapter = adapter_name