bert

mindnlp.transformers.models.bert.configuration_bert.BertConfig

Bases: PretrainedConfig

Configuration for BERT-base

Source code in mindnlp/transformers/models/bert/configuration_bert.py

class BertConfig(PretrainedConfig):
    """
    Configuration for BERT-base
    """
    model_type = "bert"

    def __init__(
        self,
        vocab_size=30522,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=2,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        pad_token_id=0,
        position_embedding_type="absolute",
        use_cache=True,
        classifier_dropout=None,
        **kwargs,
    ):
        """
        Initialize a BertConfig object with the specified parameters.

        Args:
            self (object): The object instance.
            vocab_size (int): The size of the vocabulary. Defaults to 30522.
            hidden_size (int): The size of the hidden layers. Defaults to 768.
            num_hidden_layers (int): The number of hidden layers. Defaults to 12.
            num_attention_heads (int): The number of attention heads. Defaults to 12.
            intermediate_size (int): The size of the intermediate layer in the transformer encoder. Defaults to 3072.
            hidden_act (str): The activation function for the hidden layers. Defaults to 'gelu'.
            hidden_dropout_prob (float): The dropout probability for the hidden layers. Defaults to 0.1.
            attention_probs_dropout_prob (float): The dropout probability for the attention probabilities. Defaults to 0.1.
            max_position_embeddings (int): The maximum position index. Defaults to 512.
            type_vocab_size (int): The size of the type vocabulary. Defaults to 2.
            initializer_range (float): The range for weight initialization. Defaults to 0.02.
            layer_norm_eps (float): The epsilon value for layer normalization. Defaults to 1e-12.
            pad_token_id (int): The token ID for padding. Defaults to 0.
            position_embedding_type (str): The type of position embeddings. Defaults to 'absolute'.
            use_cache (bool): Whether to use cache during inference. Defaults to True.
            classifier_dropout (float): The dropout probability for the classifier layer. Defaults to None.

        Returns:
            None.

        Raises:
            ValueError: If any of the input parameters are invalid or out of range.
        """
        super().__init__(pad_token_id=pad_token_id, **kwargs)
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.hidden_act = hidden_act
        self.intermediate_size = intermediate_size
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.position_embedding_type = position_embedding_type
        self.use_cache = use_cache
        self.classifier_dropout = classifier_dropout

mindnlp.transformers.models.bert.configuration_bert.BertConfig.__init__(vocab_size=30522, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072, hidden_act='gelu', hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, max_position_embeddings=512, type_vocab_size=2, initializer_range=0.02, layer_norm_eps=1e-12, pad_token_id=0, position_embedding_type='absolute', use_cache=True, classifier_dropout=None, **kwargs)

Initialize a BertConfig object with the specified parameters.

PARAMETER	DESCRIPTION
self	The object instance. TYPE: object
vocab_size	The size of the vocabulary. Defaults to 30522. TYPE: int DEFAULT: 30522
hidden_size	The size of the hidden layers. Defaults to 768. TYPE: int DEFAULT: 768
num_hidden_layers	The number of hidden layers. Defaults to 12. TYPE: int DEFAULT: 12
num_attention_heads	The number of attention heads. Defaults to 12. TYPE: int DEFAULT: 12
intermediate_size	The size of the intermediate layer in the transformer encoder. Defaults to 3072. TYPE: int DEFAULT: 3072
hidden_act	The activation function for the hidden layers. Defaults to 'gelu'. TYPE: str DEFAULT: 'gelu'
hidden_dropout_prob	The dropout probability for the hidden layers. Defaults to 0.1. TYPE: float DEFAULT: 0.1
attention_probs_dropout_prob	The dropout probability for the attention probabilities. Defaults to 0.1. TYPE: float DEFAULT: 0.1
max_position_embeddings	The maximum position index. Defaults to 512. TYPE: int DEFAULT: 512
type_vocab_size	The size of the type vocabulary. Defaults to 2. TYPE: int DEFAULT: 2
initializer_range	The range for weight initialization. Defaults to 0.02. TYPE: float DEFAULT: 0.02
layer_norm_eps	The epsilon value for layer normalization. Defaults to 1e-12. TYPE: float DEFAULT: 1e-12
pad_token_id	The token ID for padding. Defaults to 0. TYPE: int DEFAULT: 0
position_embedding_type	The type of position embeddings. Defaults to 'absolute'. TYPE: str DEFAULT: 'absolute'
use_cache	Whether to use cache during inference. Defaults to True. TYPE: bool DEFAULT: True
classifier_dropout	The dropout probability for the classifier layer. Defaults to None. TYPE: float DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If any of the input parameters are invalid or out of range.

Source code in mindnlp/transformers/models/bert/configuration_bert.py

def __init__(
    self,
    vocab_size=30522,
    hidden_size=768,
    num_hidden_layers=12,
    num_attention_heads=12,
    intermediate_size=3072,
    hidden_act="gelu",
    hidden_dropout_prob=0.1,
    attention_probs_dropout_prob=0.1,
    max_position_embeddings=512,
    type_vocab_size=2,
    initializer_range=0.02,
    layer_norm_eps=1e-12,
    pad_token_id=0,
    position_embedding_type="absolute",
    use_cache=True,
    classifier_dropout=None,
    **kwargs,
):
    """
    Initialize a BertConfig object with the specified parameters.

    Args:
        self (object): The object instance.
        vocab_size (int): The size of the vocabulary. Defaults to 30522.
        hidden_size (int): The size of the hidden layers. Defaults to 768.
        num_hidden_layers (int): The number of hidden layers. Defaults to 12.
        num_attention_heads (int): The number of attention heads. Defaults to 12.
        intermediate_size (int): The size of the intermediate layer in the transformer encoder. Defaults to 3072.
        hidden_act (str): The activation function for the hidden layers. Defaults to 'gelu'.
        hidden_dropout_prob (float): The dropout probability for the hidden layers. Defaults to 0.1.
        attention_probs_dropout_prob (float): The dropout probability for the attention probabilities. Defaults to 0.1.
        max_position_embeddings (int): The maximum position index. Defaults to 512.
        type_vocab_size (int): The size of the type vocabulary. Defaults to 2.
        initializer_range (float): The range for weight initialization. Defaults to 0.02.
        layer_norm_eps (float): The epsilon value for layer normalization. Defaults to 1e-12.
        pad_token_id (int): The token ID for padding. Defaults to 0.
        position_embedding_type (str): The type of position embeddings. Defaults to 'absolute'.
        use_cache (bool): Whether to use cache during inference. Defaults to True.
        classifier_dropout (float): The dropout probability for the classifier layer. Defaults to None.

    Returns:
        None.

    Raises:
        ValueError: If any of the input parameters are invalid or out of range.
    """
    super().__init__(pad_token_id=pad_token_id, **kwargs)
    self.vocab_size = vocab_size
    self.hidden_size = hidden_size
    self.num_hidden_layers = num_hidden_layers
    self.num_attention_heads = num_attention_heads
    self.hidden_act = hidden_act
    self.intermediate_size = intermediate_size
    self.hidden_dropout_prob = hidden_dropout_prob
    self.attention_probs_dropout_prob = attention_probs_dropout_prob
    self.max_position_embeddings = max_position_embeddings
    self.type_vocab_size = type_vocab_size
    self.initializer_range = initializer_range
    self.layer_norm_eps = layer_norm_eps
    self.position_embedding_type = position_embedding_type
    self.use_cache = use_cache
    self.classifier_dropout = classifier_dropout

mindnlp.transformers.models.bert.modeling_bert

MindNLP bert model

mindnlp.transformers.models.bert.modeling_bert.BertAttention

Bases: Module

Bert Attention

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertAttention(nn.Module):
    r"""
    Bert Attention
    """
    def __init__(self, config, position_embedding_type=None):
        """
        Initializes a BertAttention object.

        Args:
            self: The instance of the class itself.
            config (object): The configuration object containing settings for the BertAttention.
            position_embedding_type (str, optional): The type of position embedding to be used. Defaults to None.

        Returns:
            None: This method initializes the BertAttention object and does not return any value.

        Raises:
            None
        """
        super().__init__()
        self.self = BertSelfAttention(config, position_embedding_type=position_embedding_type)
        self.output = BertSelfOutput(config)
        self.pruned_heads = set()

    def prune_heads(self, heads):
        """prune heads"""
        if len(heads) == 0:
            return
        heads, index = find_pruneable_heads_and_indices(
            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
        )

        # Prune linear layers
        self.self.query = prune_linear_layer(self.self.query, index)
        self.self.key = prune_linear_layer(self.self.key, index)
        self.self.value = prune_linear_layer(self.self.value, index)
        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

        # Update hyper params and store pruned heads
        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
        self.pruned_heads = self.pruned_heads.union(heads)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ):
        """
        This method forwards the BertAttention layer.

        Args:
            self (BertAttention): The instance of the BertAttention class.
            hidden_states (mindspore.Tensor): The input tensor containing the hidden states of the model.
                The shape should be [batch_size, sequence_length, hidden_size].
            attention_mask (Optional[mindspore.Tensor]): An optional tensor containing the attention mask for the input.
                If provided, the shape should be [batch_size, 1, sequence_length, sequence_length] and
                the values should be 0 or 1. Default is None.
            head_mask (Optional[mindspore.Tensor]): An optional tensor containing the head mask for the input.
                If provided, the shape should be [num_heads] and the values should be 0 or 1. Default is None.
            encoder_hidden_states (Optional[mindspore.Tensor]):
                An optional tensor containing the hidden states of the encoder.
                If provided, the shape should be [batch_size, sequence_length, hidden_size].
                Default is None.
            encoder_attention_mask (Optional[mindspore.Tensor]):
                An optional tensor containing the attention mask for the encoder input.
                If provided, the shape should be [batch_size, 1, sequence_length,
                sequence_length] and the values should be 0 or 1. Default is None.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                An optional tuple containing the past key and value tensors.
                If provided, the shape should be [(batch_size, num_heads, sequence_length,
                head_size), (batch_size, num_heads, sequence_length, head_size)]. Default is None.
            output_attentions (Optional[bool]): An optional boolean value indicating whether to output attentions.
                Default is False.

        Returns:
            outputs (Tuple[mindspore.Tensor]):
                A tuple of output tensors containing the attention_output and any additional outputs from the layer.

        Raises:
            ValueError: If the shapes or types of input tensors are invalid.
            RuntimeError: If there is a runtime error during the execution of the method.
        """
        self_outputs = self.self(
            hidden_states,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            past_key_value,
            output_attentions,
        )
        attention_output = self.output(self_outputs[0], hidden_states)
        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
        return outputs

mindnlp.transformers.models.bert.modeling_bert.BertAttention.__init__(config, position_embedding_type=None)

Initializes a BertAttention object.

PARAMETER	DESCRIPTION
self	The instance of the class itself.
config	The configuration object containing settings for the BertAttention. TYPE: object
position_embedding_type	The type of position embedding to be used. Defaults to None. TYPE: str DEFAULT: None

RETURNS	DESCRIPTION
None	This method initializes the BertAttention object and does not return any value.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config, position_embedding_type=None):
    """
    Initializes a BertAttention object.

    Args:
        self: The instance of the class itself.
        config (object): The configuration object containing settings for the BertAttention.
        position_embedding_type (str, optional): The type of position embedding to be used. Defaults to None.

    Returns:
        None: This method initializes the BertAttention object and does not return any value.

    Raises:
        None
    """
    super().__init__()
    self.self = BertSelfAttention(config, position_embedding_type=position_embedding_type)
    self.output = BertSelfOutput(config)
    self.pruned_heads = set()

mindnlp.transformers.models.bert.modeling_bert.BertAttention.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

This method forwards the BertAttention layer.

PARAMETER	DESCRIPTION
self	The instance of the BertAttention class. TYPE: BertAttention
hidden_states	The input tensor containing the hidden states of the model. The shape should be [batch_size, sequence_length, hidden_size]. TYPE: Tensor
attention_mask	An optional tensor containing the attention mask for the input. If provided, the shape should be [batch_size, 1, sequence_length, sequence_length] and the values should be 0 or 1. Default is None. TYPE: Optional[Tensor] DEFAULT: None
head_mask	An optional tensor containing the head mask for the input. If provided, the shape should be [num_heads] and the values should be 0 or 1. Default is None. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	An optional tensor containing the hidden states of the encoder. If provided, the shape should be [batch_size, sequence_length, hidden_size]. Default is None. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	An optional tensor containing the attention mask for the encoder input. If provided, the shape should be [batch_size, 1, sequence_length, sequence_length] and the values should be 0 or 1. Default is None. TYPE: Optional[Tensor] DEFAULT: None
past_key_value	An optional tuple containing the past key and value tensors. If provided, the shape should be [(batch_size, num_heads, sequence_length, head_size), (batch_size, num_heads, sequence_length, head_size)]. Default is None. TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
output_attentions	An optional boolean value indicating whether to output attentions. Default is False. TYPE: Optional[bool] DEFAULT: False

RETURNS	DESCRIPTION
outputs	A tuple of output tensors containing the attention_output and any additional outputs from the layer. TYPE: Tuple[Tensor]

RAISES	DESCRIPTION
ValueError	If the shapes or types of input tensors are invalid.
RuntimeError	If there is a runtime error during the execution of the method.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
):
    """
    This method forwards the BertAttention layer.

    Args:
        self (BertAttention): The instance of the BertAttention class.
        hidden_states (mindspore.Tensor): The input tensor containing the hidden states of the model.
            The shape should be [batch_size, sequence_length, hidden_size].
        attention_mask (Optional[mindspore.Tensor]): An optional tensor containing the attention mask for the input.
            If provided, the shape should be [batch_size, 1, sequence_length, sequence_length] and
            the values should be 0 or 1. Default is None.
        head_mask (Optional[mindspore.Tensor]): An optional tensor containing the head mask for the input.
            If provided, the shape should be [num_heads] and the values should be 0 or 1. Default is None.
        encoder_hidden_states (Optional[mindspore.Tensor]):
            An optional tensor containing the hidden states of the encoder.
            If provided, the shape should be [batch_size, sequence_length, hidden_size].
            Default is None.
        encoder_attention_mask (Optional[mindspore.Tensor]):
            An optional tensor containing the attention mask for the encoder input.
            If provided, the shape should be [batch_size, 1, sequence_length,
            sequence_length] and the values should be 0 or 1. Default is None.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            An optional tuple containing the past key and value tensors.
            If provided, the shape should be [(batch_size, num_heads, sequence_length,
            head_size), (batch_size, num_heads, sequence_length, head_size)]. Default is None.
        output_attentions (Optional[bool]): An optional boolean value indicating whether to output attentions.
            Default is False.

    Returns:
        outputs (Tuple[mindspore.Tensor]):
            A tuple of output tensors containing the attention_output and any additional outputs from the layer.

    Raises:
        ValueError: If the shapes or types of input tensors are invalid.
        RuntimeError: If there is a runtime error during the execution of the method.
    """
    self_outputs = self.self(
        hidden_states,
        attention_mask,
        head_mask,
        encoder_hidden_states,
        encoder_attention_mask,
        past_key_value,
        output_attentions,
    )
    attention_output = self.output(self_outputs[0], hidden_states)
    outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
    return outputs

mindnlp.transformers.models.bert.modeling_bert.BertAttention.prune_heads(heads)

prune heads

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def prune_heads(self, heads):
    """prune heads"""
    if len(heads) == 0:
        return
    heads, index = find_pruneable_heads_and_indices(
        heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
    )

    # Prune linear layers
    self.self.query = prune_linear_layer(self.self.query, index)
    self.self.key = prune_linear_layer(self.self.key, index)
    self.self.value = prune_linear_layer(self.self.value, index)
    self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

    # Update hyper params and store pruned heads
    self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
    self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
    self.pruned_heads = self.pruned_heads.union(heads)

mindnlp.transformers.models.bert.modeling_bert.BertDualAttention

Bases: Module

This class represents a BertDualAttention module that inherits from nn.Module. It contains methods for initializing the module, pruning attention heads, and forwarding the attention mechanism for BERT models.

ATTRIBUTE	DESCRIPTION
config	Configuration for the BertDualAttention module.
position_embedding_type	Type of position embedding to be used (optional).

METHOD	DESCRIPTION
__init__	Initializes the BertDualAttention module with the given configuration and position embedding type.
prune_heads	Prunes the specified attention heads from the self-attention mechanism.
encoder_attention_mask=None, past_key_value=None, output_attentions=False)	Constructs the attention mechanism for BERT models using the provided inputs and past key values.

RAISES	DESCRIPTION
ValueError	If the number of heads to be pruned is invalid.

RETURNS	DESCRIPTION
outputs	Tuple containing the attention output and optional additional outputs.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualAttention(nn.Module):

    """
    This class represents a BertDualAttention module that inherits from nn.Module.
    It contains methods for initializing the module, pruning attention heads,
    and forwarding the attention mechanism for BERT models.

    Attributes:
        config: Configuration for the BertDualAttention module.
        position_embedding_type: Type of position embedding to be used (optional).

    Methods:
        __init__(self, config, position_embedding_type=None):
            Initializes the BertDualAttention module with the given configuration and position embedding type.

        prune_heads(self, heads):
            Prunes the specified attention heads from the self-attention mechanism.

        forward(self, hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None,
        encoder_attention_mask=None, past_key_value=None, output_attentions=False):
            Constructs the attention mechanism for BERT models using the provided inputs and past key values.

    Raises:
        ValueError: If the number of heads to be pruned is invalid.

    Returns:
        outputs: Tuple containing the attention output and optional additional outputs.
    """
    def __init__(self, config, position_embedding_type=None):
        """
        Initializes the BertDualAttention class with the provided configuration and position embedding type.

        Args:
            self (object): The instance of the class.
            config (object): The configuration object containing settings for the dual attention mechanism.
            position_embedding_type (str, optional): The type of position embedding to be used. Default is None.

        Returns:
            None: This method does not return any value.

        Raises:
            None
        """
        super().__init__()
        self.self = BertDualSelfAttention(config, position_embedding_type=position_embedding_type)
        self.output = BertDualSelfOutput(config)
        self.pruned_heads = set()

    def prune_heads(self, heads):
        """prune heads"""
        if len(heads) == 0:
            return
        heads, index = find_pruneable_heads_and_indices(
            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
        )

        # Prune linear layers
        self.self.query = prune_linear_layer(self.self.query, index)
        self.self.key = prune_linear_layer(self.self.key, index)
        self.self.value = prune_linear_layer(self.self.value, index)
        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

        # Update hyper params and store pruned heads
        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
        self.pruned_heads = self.pruned_heads.union(heads)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ):
        """
        Constructs the attention mechanism for the BertDualAttention class.

        Args:
            self (BertDualAttention): The instance of the BertDualAttention class.
            hidden_states (mindspore.Tensor):
                The input hidden states tensor of shape (batch_size, seq_length, hidden_size).
            attention_mask (Optional[mindspore.Tensor]):
                The attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length).
                Defaults to None.
            head_mask (Optional[mindspore.Tensor]):
                The head mask tensor of shape (num_heads, seq_length, seq_length). Defaults to None.
            encoder_hidden_states (Optional[mindspore.Tensor]):
                The encoder hidden states tensor of shape (batch_size, seq_length, hidden_size). Defaults to None.
            encoder_attention_mask (Optional[mindspore.Tensor]):
                The encoder attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length).
                Defaults to None.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                The previous key-value pairs tensor. Defaults to None.
            output_attentions (Optional[bool]): Whether to output the attention weights. Defaults to False.

        Returns:
            outputs (tuple):
                A tuple containing the attention output tensor of shape (batch_size, seq_length, hidden_size)
                and any additional outputs.

        Raises:
            None
        """
        self_outputs = self.self(
            hidden_states,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            past_key_value,
            output_attentions,
        )
        attention_output = self.output(self_outputs[0], hidden_states)
        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
        return outputs

mindnlp.transformers.models.bert.modeling_bert.BertDualAttention.__init__(config, position_embedding_type=None)

Initializes the BertDualAttention class with the provided configuration and position embedding type.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
config	The configuration object containing settings for the dual attention mechanism. TYPE: object
position_embedding_type	The type of position embedding to be used. Default is None. TYPE: str DEFAULT: None

RETURNS	DESCRIPTION
None	This method does not return any value.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config, position_embedding_type=None):
    """
    Initializes the BertDualAttention class with the provided configuration and position embedding type.

    Args:
        self (object): The instance of the class.
        config (object): The configuration object containing settings for the dual attention mechanism.
        position_embedding_type (str, optional): The type of position embedding to be used. Default is None.

    Returns:
        None: This method does not return any value.

    Raises:
        None
    """
    super().__init__()
    self.self = BertDualSelfAttention(config, position_embedding_type=position_embedding_type)
    self.output = BertDualSelfOutput(config)
    self.pruned_heads = set()

mindnlp.transformers.models.bert.modeling_bert.BertDualAttention.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

Constructs the attention mechanism for the BertDualAttention class.

PARAMETER	DESCRIPTION
self	The instance of the BertDualAttention class. TYPE: BertDualAttention
hidden_states	The input hidden states tensor of shape (batch_size, seq_length, hidden_size). TYPE: Tensor
attention_mask	The attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length). Defaults to None. TYPE: Optional[Tensor] DEFAULT: None
head_mask	The head mask tensor of shape (num_heads, seq_length, seq_length). Defaults to None. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	The encoder hidden states tensor of shape (batch_size, seq_length, hidden_size). Defaults to None. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	The encoder attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length). Defaults to None. TYPE: Optional[Tensor] DEFAULT: None
past_key_value	The previous key-value pairs tensor. Defaults to None. TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
output_attentions	Whether to output the attention weights. Defaults to False. TYPE: Optional[bool] DEFAULT: False

RETURNS	DESCRIPTION
outputs	A tuple containing the attention output tensor of shape (batch_size, seq_length, hidden_size) and any additional outputs. TYPE: tuple

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
):
    """
    Constructs the attention mechanism for the BertDualAttention class.

    Args:
        self (BertDualAttention): The instance of the BertDualAttention class.
        hidden_states (mindspore.Tensor):
            The input hidden states tensor of shape (batch_size, seq_length, hidden_size).
        attention_mask (Optional[mindspore.Tensor]):
            The attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length).
            Defaults to None.
        head_mask (Optional[mindspore.Tensor]):
            The head mask tensor of shape (num_heads, seq_length, seq_length). Defaults to None.
        encoder_hidden_states (Optional[mindspore.Tensor]):
            The encoder hidden states tensor of shape (batch_size, seq_length, hidden_size). Defaults to None.
        encoder_attention_mask (Optional[mindspore.Tensor]):
            The encoder attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length).
            Defaults to None.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            The previous key-value pairs tensor. Defaults to None.
        output_attentions (Optional[bool]): Whether to output the attention weights. Defaults to False.

    Returns:
        outputs (tuple):
            A tuple containing the attention output tensor of shape (batch_size, seq_length, hidden_size)
            and any additional outputs.

    Raises:
        None
    """
    self_outputs = self.self(
        hidden_states,
        attention_mask,
        head_mask,
        encoder_hidden_states,
        encoder_attention_mask,
        past_key_value,
        output_attentions,
    )
    attention_output = self.output(self_outputs[0], hidden_states)
    outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
    return outputs

mindnlp.transformers.models.bert.modeling_bert.BertDualAttention.prune_heads(heads)

prune heads

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def prune_heads(self, heads):
    """prune heads"""
    if len(heads) == 0:
        return
    heads, index = find_pruneable_heads_and_indices(
        heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
    )

    # Prune linear layers
    self.self.query = prune_linear_layer(self.self.query, index)
    self.self.key = prune_linear_layer(self.self.key, index)
    self.self.value = prune_linear_layer(self.self.value, index)
    self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

    # Update hyper params and store pruned heads
    self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
    self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
    self.pruned_heads = self.pruned_heads.union(heads)

mindnlp.transformers.models.bert.modeling_bert.BertDualEncoder

Bases: Module

The BertDualEncoder class represents a dual encoder model based on the BERT architecture. This class inherits from the nn.Module class in MindSpore.

ATTRIBUTE	DESCRIPTION
config	The configuration parameters for the model.
layer	A list of BertDualLayer instances representing the stacked layers in the encoder.
gradient_checkpointing	A boolean indicating whether gradient checkpointing is enabled in the model.

METHOD	DESCRIPTION
__init__	Initializes the BertDualEncoder instance with the provided configuration.
forward	Constructs the dual encoder model with the given input tensors and parameters. Returns the final hidden states, past key values, hidden states at all layers, self-attentions at all layers, and cross-attentions at all layers.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualEncoder(nn.Module):

    """
    The BertDualEncoder class represents a dual encoder model based on the BERT architecture.
    This class inherits from the nn.Module class in MindSpore.

    Attributes:
        config: The configuration parameters for the model.
        layer: A list of BertDualLayer instances representing the stacked layers in the encoder.
        gradient_checkpointing: A boolean indicating whether gradient checkpointing is enabled in the model.

    Methods:
        __init__(self, config):
            Initializes the BertDualEncoder instance with the provided configuration.

        forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict):
            Constructs the dual encoder model with the given input tensors and parameters.
            Returns the final hidden states, past key values, hidden states at all layers,
            self-attentions at all layers, and cross-attentions at all layers.
    """
    def __init__(self, config):
        """Initialize the BertDualEncoder class.

        Args:
            self: The instance of the BertDualEncoder class.
            config:
                A dictionary containing the configuration parameters for the BertDualEncoder.

                - Type: dict
                - Purpose: Specifies the configuration settings for the BertDualEncoder.
                - Restrictions: Must be a valid dictionary object.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.config = config
        self.layer = nn.ModuleList([BertDualLayer(config) for _ in range(config.num_hidden_layers)])
        self.gradient_checkpointing = False

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = False,
        output_hidden_states: Optional[bool] = False,
        return_dict: Optional[bool] = True,
    ):
        """
        This method forwards the BertDualEncoder model.

        Args:
            self: The object instance.
            hidden_states (mindspore.Tensor): The input hidden states tensor.
            attention_mask (Optional[mindspore.Tensor]): Mask indicating which elements in the input should be attended to.
            head_mask (Optional[mindspore.Tensor]): Mask for attention heads.
            encoder_hidden_states (Optional[mindspore.Tensor]): Hidden states from the encoder.
            encoder_attention_mask (Optional[mindspore.Tensor]): Mask for encoder attention.
            past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): Past key values for caching.
            use_cache (Optional[bool]): Flag indicating whether to use caching.
            output_attentions (Optional[bool]): Flag indicating whether to output attentions.
            output_hidden_states (Optional[bool]): Flag indicating whether to output hidden states.
            return_dict (Optional[bool]): Flag indicating whether to return a dictionary.

        Returns:
            None.

        Raises:
            None
        """
        all_hidden_states = () if output_hidden_states else None
        all_self_attentions = () if output_attentions else None
        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

        next_decoder_cache = () if use_cache else None
        for i, layer_module in enumerate(self.layer):
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_head_mask = head_mask[i] if head_mask is not None else None
            past_key_value = past_key_values[i] if past_key_values is not None else None
            layer_outputs = layer_module(
                hidden_states,
                attention_mask,
                layer_head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                past_key_value,
                output_attentions,
            )
            hidden_states = layer_outputs[0]
            if use_cache:
                next_decoder_cache += (layer_outputs[-1],)
            if output_attentions:
                all_self_attentions = all_self_attentions + (layer_outputs[1],)
                if self.config.add_cross_attention:
                    all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        if not return_dict:
            return tuple(
                v
                for v in [
                    hidden_states,
                    next_decoder_cache,
                    all_hidden_states,
                    all_self_attentions,
                    all_cross_attentions,
                ]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_decoder_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attentions,
            cross_attentions=all_cross_attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertDualEncoder.__init__(config)

Initialize the BertDualEncoder class.

PARAMETER	DESCRIPTION
self	The instance of the BertDualEncoder class.
config	A dictionary containing the configuration parameters for the BertDualEncoder. Type: dict Purpose: Specifies the configuration settings for the BertDualEncoder. Restrictions: Must be a valid dictionary object.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """Initialize the BertDualEncoder class.

    Args:
        self: The instance of the BertDualEncoder class.
        config:
            A dictionary containing the configuration parameters for the BertDualEncoder.

            - Type: dict
            - Purpose: Specifies the configuration settings for the BertDualEncoder.
            - Restrictions: Must be a valid dictionary object.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.config = config
    self.layer = nn.ModuleList([BertDualLayer(config) for _ in range(config.num_hidden_layers)])
    self.gradient_checkpointing = False

mindnlp.transformers.models.bert.modeling_bert.BertDualEncoder.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, use_cache=None, output_attentions=False, output_hidden_states=False, return_dict=True)

This method forwards the BertDualEncoder model.

PARAMETER	DESCRIPTION
self	The object instance.
hidden_states	The input hidden states tensor. TYPE: Tensor
attention_mask	Mask indicating which elements in the input should be attended to. TYPE: Optional[Tensor] DEFAULT: None
head_mask	Mask for attention heads. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	Hidden states from the encoder. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	Mask for encoder attention. TYPE: Optional[Tensor] DEFAULT: None
past_key_values	Past key values for caching. TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
use_cache	Flag indicating whether to use caching. TYPE: Optional[bool] DEFAULT: None
output_attentions	Flag indicating whether to output attentions. TYPE: Optional[bool] DEFAULT: False
output_hidden_states	Flag indicating whether to output hidden states. TYPE: Optional[bool] DEFAULT: False
return_dict	Flag indicating whether to return a dictionary. TYPE: Optional[bool] DEFAULT: True

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = False,
    output_hidden_states: Optional[bool] = False,
    return_dict: Optional[bool] = True,
):
    """
    This method forwards the BertDualEncoder model.

    Args:
        self: The object instance.
        hidden_states (mindspore.Tensor): The input hidden states tensor.
        attention_mask (Optional[mindspore.Tensor]): Mask indicating which elements in the input should be attended to.
        head_mask (Optional[mindspore.Tensor]): Mask for attention heads.
        encoder_hidden_states (Optional[mindspore.Tensor]): Hidden states from the encoder.
        encoder_attention_mask (Optional[mindspore.Tensor]): Mask for encoder attention.
        past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): Past key values for caching.
        use_cache (Optional[bool]): Flag indicating whether to use caching.
        output_attentions (Optional[bool]): Flag indicating whether to output attentions.
        output_hidden_states (Optional[bool]): Flag indicating whether to output hidden states.
        return_dict (Optional[bool]): Flag indicating whether to return a dictionary.

    Returns:
        None.

    Raises:
        None
    """
    all_hidden_states = () if output_hidden_states else None
    all_self_attentions = () if output_attentions else None
    all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

    next_decoder_cache = () if use_cache else None
    for i, layer_module in enumerate(self.layer):
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        layer_head_mask = head_mask[i] if head_mask is not None else None
        past_key_value = past_key_values[i] if past_key_values is not None else None
        layer_outputs = layer_module(
            hidden_states,
            attention_mask,
            layer_head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            past_key_value,
            output_attentions,
        )
        hidden_states = layer_outputs[0]
        if use_cache:
            next_decoder_cache += (layer_outputs[-1],)
        if output_attentions:
            all_self_attentions = all_self_attentions + (layer_outputs[1],)
            if self.config.add_cross_attention:
                all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

    if output_hidden_states:
        all_hidden_states = all_hidden_states + (hidden_states,)

    if not return_dict:
        return tuple(
            v
            for v in [
                hidden_states,
                next_decoder_cache,
                all_hidden_states,
                all_self_attentions,
                all_cross_attentions,
            ]
            if v is not None
        )
    return BaseModelOutputWithPastAndCrossAttentions(
        last_hidden_state=hidden_states,
        past_key_values=next_decoder_cache,
        hidden_states=all_hidden_states,
        attentions=all_self_attentions,
        cross_attentions=all_cross_attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertDualForSequenceClassification

Bases: BertPreTrainedModel

The BertDualForSequenceClassification class represents a dual BERT model for sequence classification tasks. This class inherits from BertPreTrainedModel and provides methods for initializing the model and processing input data for sequence classification.

The init method initializes the BertDualForSequenceClassification instance by setting the number of labels, BERT model configuration, dropout, and classifier layers.

The forward method processes input data for sequence classification using the BERT model. It accepts input tensors such as input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, and additional parameters for controlling the output format. The method returns the classification logits and can also calculate the loss based on the problem type and labels provided.

Note

This docstring is based on the provided code and may need to be updated with additional information about the class attributes, methods, and usage.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualForSequenceClassification(BertPreTrainedModel):

    """
    The BertDualForSequenceClassification class represents a dual BERT model for sequence classification tasks.
    This class inherits from BertPreTrainedModel and provides methods for initializing the model and processing
    input data for sequence classification.

    The __init__ method initializes the BertDualForSequenceClassification instance by setting the number of labels,
    BERT model configuration, dropout, and classifier layers.

    The forward method processes input data for sequence classification using the BERT model.
    It accepts input tensors such as input_ids, attention_mask, token_type_ids, position_ids, head_mask,
    inputs_embeds, labels, and additional parameters for controlling the output format.
    The method returns the classification logits and can also calculate the loss based on the problem type
    and labels provided.

    Note:
        This docstring is based on the provided code and may need to be updated with additional information
        about the class attributes, methods, and usage.
    """
    def __init__(self, config):
        """
        Initializes a new instance of the BertDualForSequenceClassification class.

        Args:
            self: The instance of the class.
            config:
                An instance of the configuration class containing the model configuration parameters.

                - Type: config class
                - Purpose: Specifies the configuration parameters for the model.
                - Restrictions: Must be a valid instance of the configuration class.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not of the expected type.
            ValueError: If the config.num_labels is not provided or is invalid.
            AttributeError: If the required attributes are not found in the config object.
            RuntimeError: If an error occurs during model initialization or post-initialization.
        """
        super().__init__(config)
        self.num_labels = config.num_labels
        self.config = config

        self.bert = BertDualModel(config)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        """
        This method forwards a dual BERT model for sequence classification.

        Args:
            self (BertDualForSequenceClassification): The instance of the BertDualForSequenceClassification class.
            input_ids (Optional[mindspore.Tensor]): The input token IDs representing the sequences. Default is None.
            attention_mask (Optional[mindspore.Tensor]): The attention mask to avoid attending to padding tokens. Default is None.
            token_type_ids (Optional[mindspore.Tensor]): The token type IDs to distinguish different sequences in the input. Default is None.
            position_ids (Optional[mindspore.Tensor]): The position IDs to specify the position of each token in the input. Default is None.
            head_mask (Optional[mindspore.Tensor]): The head mask to nullify selected heads of the self-attention mechanism. Default is None.
            inputs_embeds (Optional[mindspore.Tensor]): The embedded representation of the input sequences. Default is None.
            labels (Optional[mindspore.Tensor]): The labels for the input sequences. Default is None.
            output_attentions (Optional[bool]): Whether to return the attentions of all layers. Default is None.
            output_hidden_states (Optional[bool]): Whether to return the hidden states of all layers. Default is None.
            return_dict (Optional[bool]): Whether to return outputs as a dictionary. Default is None.

        Returns:
            None.

        Raises:
            ValueError: If the provided problem type is not supported or recognized.
            RuntimeError: If the number of labels is not compatible with the problem type.
            NotImplementedError: If the problem type is not implemented.

        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = outputs[1]

        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)

        loss = None
        if labels is not None:
            if self.config.problem_type is None:
                if self.num_labels == 1:
                    self.config.problem_type = "regression"
                elif self.num_labels > 1 and labels.dtype in (mindspore.int32, mindspore.int64):
                    self.config.problem_type = "single_label_classification"
                else:
                    self.config.problem_type = "multi_label_classification"

            if self.config.problem_type == "regression":
                if self.num_labels == 1:
                    loss = ops.mse_loss(logits.squeeze(), labels.squeeze())
                else:
                    loss = ops.mse_loss(logits, labels)
            elif self.config.problem_type == "single_label_classification":
                loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))
            elif self.config.problem_type == "multi_label_classification":
                loss = ops.binary_cross_entropy_with_logits(logits, labels)
        if not return_dict:
            output = (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertDualForSequenceClassification.__init__(config)

Initializes a new instance of the BertDualForSequenceClassification class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An instance of the configuration class containing the model configuration parameters. Type: config class Purpose: Specifies the configuration parameters for the model. Restrictions: Must be a valid instance of the configuration class.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the config parameter is not of the expected type.
ValueError	If the config.num_labels is not provided or is invalid.
AttributeError	If the required attributes are not found in the config object.
RuntimeError	If an error occurs during model initialization or post-initialization.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes a new instance of the BertDualForSequenceClassification class.

    Args:
        self: The instance of the class.
        config:
            An instance of the configuration class containing the model configuration parameters.

            - Type: config class
            - Purpose: Specifies the configuration parameters for the model.
            - Restrictions: Must be a valid instance of the configuration class.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not of the expected type.
        ValueError: If the config.num_labels is not provided or is invalid.
        AttributeError: If the required attributes are not found in the config object.
        RuntimeError: If an error occurs during model initialization or post-initialization.
    """
    super().__init__(config)
    self.num_labels = config.num_labels
    self.config = config

    self.bert = BertDualModel(config)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertDualForSequenceClassification.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

This method forwards a dual BERT model for sequence classification.

PARAMETER	DESCRIPTION
self	The instance of the BertDualForSequenceClassification class. TYPE: BertDualForSequenceClassification
input_ids	The input token IDs representing the sequences. Default is None. TYPE: Optional[Tensor] DEFAULT: None
attention_mask	The attention mask to avoid attending to padding tokens. Default is None. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	The token type IDs to distinguish different sequences in the input. Default is None. TYPE: Optional[Tensor] DEFAULT: None
position_ids	The position IDs to specify the position of each token in the input. Default is None. TYPE: Optional[Tensor] DEFAULT: None
head_mask	The head mask to nullify selected heads of the self-attention mechanism. Default is None. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The embedded representation of the input sequences. Default is None. TYPE: Optional[Tensor] DEFAULT: None
labels	The labels for the input sequences. Default is None. TYPE: Optional[Tensor] DEFAULT: None
output_attentions	Whether to return the attentions of all layers. Default is None. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Whether to return the hidden states of all layers. Default is None. TYPE: Optional[bool] DEFAULT: None
return_dict	Whether to return outputs as a dictionary. Default is None. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the provided problem type is not supported or recognized.
RuntimeError	If the number of labels is not compatible with the problem type.
NotImplementedError	If the problem type is not implemented.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    """
    This method forwards a dual BERT model for sequence classification.

    Args:
        self (BertDualForSequenceClassification): The instance of the BertDualForSequenceClassification class.
        input_ids (Optional[mindspore.Tensor]): The input token IDs representing the sequences. Default is None.
        attention_mask (Optional[mindspore.Tensor]): The attention mask to avoid attending to padding tokens. Default is None.
        token_type_ids (Optional[mindspore.Tensor]): The token type IDs to distinguish different sequences in the input. Default is None.
        position_ids (Optional[mindspore.Tensor]): The position IDs to specify the position of each token in the input. Default is None.
        head_mask (Optional[mindspore.Tensor]): The head mask to nullify selected heads of the self-attention mechanism. Default is None.
        inputs_embeds (Optional[mindspore.Tensor]): The embedded representation of the input sequences. Default is None.
        labels (Optional[mindspore.Tensor]): The labels for the input sequences. Default is None.
        output_attentions (Optional[bool]): Whether to return the attentions of all layers. Default is None.
        output_hidden_states (Optional[bool]): Whether to return the hidden states of all layers. Default is None.
        return_dict (Optional[bool]): Whether to return outputs as a dictionary. Default is None.

    Returns:
        None.

    Raises:
        ValueError: If the provided problem type is not supported or recognized.
        RuntimeError: If the number of labels is not compatible with the problem type.
        NotImplementedError: If the problem type is not implemented.

    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    pooled_output = outputs[1]

    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    loss = None
    if labels is not None:
        if self.config.problem_type is None:
            if self.num_labels == 1:
                self.config.problem_type = "regression"
            elif self.num_labels > 1 and labels.dtype in (mindspore.int32, mindspore.int64):
                self.config.problem_type = "single_label_classification"
            else:
                self.config.problem_type = "multi_label_classification"

        if self.config.problem_type == "regression":
            if self.num_labels == 1:
                loss = ops.mse_loss(logits.squeeze(), labels.squeeze())
            else:
                loss = ops.mse_loss(logits, labels)
        elif self.config.problem_type == "single_label_classification":
            loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))
        elif self.config.problem_type == "multi_label_classification":
            loss = ops.binary_cross_entropy_with_logits(logits, labels)
    if not return_dict:
        output = (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

    return SequenceClassifierOutput(
        loss=loss,
        logits=logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertDualIntermediate

Bases: Module

This class represents a dual intermediate layer in a BERT model.

The BertDualIntermediate class is a subclass of nn.Module and is used to forward the dual intermediate layer in a BERT model. It takes in a configuration object as input, which specifies the hidden size and intermediate size. The class initializes the hidden size and intermediate size attributes based on the provided configuration.

ATTRIBUTE	DESCRIPTION
hidden_size	The size of the hidden state in the dual intermediate layer. TYPE: int
intermediate_size	The size of the intermediate state in the dual intermediate layer. TYPE: int
dense	A dense layer that transforms the input hidden states. TYPE: Dense
intermediate_act_fn	The activation function to be applied to the intermediate states. TYPE: function

METHOD

DESCRIPTION

forward

Constructs the dual intermediate layer using the given hidden states as input. The method first splits the input hidden states into two channels: hidden_states_r and hidden_states_d. Then, it combines the two channels into a single input using the to_2channel function. The combined input is passed through the dense layer. The resulting intermediate states are then split back into two channels: hidden_states_r and hidden_states_d. Finally, the two channels are concatenated and passed through the intermediate activation function. The method returns the resulting hidden states.

Note

• This class assumes that the given configuration object contains the necessary parameters for initialization.
• The intermediate activation function can be either a string representing a predefined activation function or a custom activation function.

Example

>>> # Create a configuration object
>>> config = {
>>>     'hidden_size': 768,
>>>     'intermediate_size': 3072,
>>>     'hidden_act': 'gelu'
>>> }
...
>>> # Create an instance of the BertDualIntermediate class
>>> dual_intermediate = BertDualIntermediate(config)
...
>>> # Use the dual_intermediate instance to forward the dual intermediate layer
>>> hidden_states = ... # input hidden states
>>> output = dual_intermediate.forward(hidden_states)

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualIntermediate(nn.Module):

    """
    This class represents a dual intermediate layer in a BERT model.

    The BertDualIntermediate class is a subclass of nn.Module and is used to forward the dual intermediate layer in a BERT model.
    It takes in a configuration object as input, which specifies the hidden size and intermediate size.
    The class initializes the hidden size and intermediate size attributes based on the provided configuration.

    Attributes:
        hidden_size (int): The size of the hidden state in the dual intermediate layer.
        intermediate_size (int): The size of the intermediate state in the dual intermediate layer.
        dense (Dense): A dense layer that transforms the input hidden states.
        intermediate_act_fn (function): The activation function to be applied to the intermediate states.

    Methods:
        forward(hidden_states):
            Constructs the dual intermediate layer using the given hidden states as input.
            The method first splits the input hidden states into two channels: hidden_states_r and hidden_states_d.
            Then, it combines the two channels into a single input using the to_2channel function.
            The combined input is passed through the dense layer.
            The resulting intermediate states are then split back into two channels: hidden_states_r and hidden_states_d.
            Finally, the two channels are concatenated and passed through the intermediate activation function.
            The method returns the resulting hidden states.

    Note:
        - This class assumes that the given configuration object contains the necessary parameters for initialization.
        - The intermediate activation function can be either a string representing a predefined activation function or a custom activation function.

    Example:
        ```python
        >>> # Create a configuration object
        >>> config = {
        >>>     'hidden_size': 768,
        >>>     'intermediate_size': 3072,
        >>>     'hidden_act': 'gelu'
        >>> }
        ...
        >>> # Create an instance of the BertDualIntermediate class
        >>> dual_intermediate = BertDualIntermediate(config)
        ...
        >>> # Use the dual_intermediate instance to forward the dual intermediate layer
        >>> hidden_states = ... # input hidden states
        >>> output = dual_intermediate.forward(hidden_states)
        ```
    """
    def __init__(self, config):
        """
        Initializes an instance of the BertDualIntermediate class.

        Args:
            self: The current object instance.
            config:
                An object of the configuration class that holds the configuration settings.

                - Type: object
                - Purpose: To provide the necessary configuration for initializing the BertDualIntermediate instance.
                - Restrictions: None

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.hidden_size = config.hidden_size
        self.intermediate_size = config.intermediate_size
        self.dense = Dense(config.hidden_size//2, config.intermediate_size//2)
        if isinstance(config.hidden_act, str):
            self.intermediate_act_fn = ACT2FN[config.hidden_act]
        else:
            self.intermediate_act_fn = config.hidden_act

    def forward(self, hidden_states):
        """
        The forward method in the BertDualIntermediate class processes the hidden_states tensor to produce an intermediate representation.

        Args:
            self (BertDualIntermediate): The instance of the BertDualIntermediate class.
            hidden_states (tensor): A tensor of shape (batch_size, sequence_length, hidden_size)
                representing the hidden states of the input sequence. The hidden_size is expected to be an even number.

        Returns:
            None: This method does not return any value. The input hidden_states tensor is modified in place.

        Raises:
            ValueError: If the hidden_states tensor does not have the expected shape or if the hidden_size is not an even number.
            RuntimeError: If any runtime error occurs during the processing of hidden_states.
        """
        hidden_states_r = hidden_states[:,:,:self.hidden_size//2]
        hidden_states_d = hidden_states[:,:,self.hidden_size//2:]
        hidden_states = to_2channel(hidden_states_r, hidden_states_d)
        hidden_states = self.dense(hidden_states)
        hidden_states_r, hidden_states_d = get_x_and_y(hidden_states)
        hidden_states = ops.cat([hidden_states_r, hidden_states_d], -1)
        hidden_states = self.intermediate_act_fn(hidden_states)
        return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertDualIntermediate.__init__(config)

Initializes an instance of the BertDualIntermediate class.

PARAMETER	DESCRIPTION
self	The current object instance.
config	An object of the configuration class that holds the configuration settings. Type: object Purpose: To provide the necessary configuration for initializing the BertDualIntermediate instance. Restrictions: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of the BertDualIntermediate class.

    Args:
        self: The current object instance.
        config:
            An object of the configuration class that holds the configuration settings.

            - Type: object
            - Purpose: To provide the necessary configuration for initializing the BertDualIntermediate instance.
            - Restrictions: None

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.hidden_size = config.hidden_size
    self.intermediate_size = config.intermediate_size
    self.dense = Dense(config.hidden_size//2, config.intermediate_size//2)
    if isinstance(config.hidden_act, str):
        self.intermediate_act_fn = ACT2FN[config.hidden_act]
    else:
        self.intermediate_act_fn = config.hidden_act

mindnlp.transformers.models.bert.modeling_bert.BertDualIntermediate.forward(hidden_states)

The forward method in the BertDualIntermediate class processes the hidden_states tensor to produce an intermediate representation.

PARAMETER	DESCRIPTION
self	The instance of the BertDualIntermediate class. TYPE: BertDualIntermediate
hidden_states	A tensor of shape (batch_size, sequence_length, hidden_size) representing the hidden states of the input sequence. The hidden_size is expected to be an even number. TYPE: tensor

RETURNS	DESCRIPTION
None	This method does not return any value. The input hidden_states tensor is modified in place.

RAISES	DESCRIPTION
ValueError	If the hidden_states tensor does not have the expected shape or if the hidden_size is not an even number.
RuntimeError	If any runtime error occurs during the processing of hidden_states.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states):
    """
    The forward method in the BertDualIntermediate class processes the hidden_states tensor to produce an intermediate representation.

    Args:
        self (BertDualIntermediate): The instance of the BertDualIntermediate class.
        hidden_states (tensor): A tensor of shape (batch_size, sequence_length, hidden_size)
            representing the hidden states of the input sequence. The hidden_size is expected to be an even number.

    Returns:
        None: This method does not return any value. The input hidden_states tensor is modified in place.

    Raises:
        ValueError: If the hidden_states tensor does not have the expected shape or if the hidden_size is not an even number.
        RuntimeError: If any runtime error occurs during the processing of hidden_states.
    """
    hidden_states_r = hidden_states[:,:,:self.hidden_size//2]
    hidden_states_d = hidden_states[:,:,self.hidden_size//2:]
    hidden_states = to_2channel(hidden_states_r, hidden_states_d)
    hidden_states = self.dense(hidden_states)
    hidden_states_r, hidden_states_d = get_x_and_y(hidden_states)
    hidden_states = ops.cat([hidden_states_r, hidden_states_d], -1)
    hidden_states = self.intermediate_act_fn(hidden_states)
    return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertDualLayer

Bases: Module

BertDualLayer

This class represents a layer in a dual-attention BERT model. It is a subclass of nn.Module and is responsible for performing attention and feed-forward operations.

ATTRIBUTE	DESCRIPTION
chunk_size_feed_forward	The size of chunks for feed-forward operation. TYPE: int
seq_len_dim	The dimension of the sequence length. TYPE: int
attention	The attention module used for self-attention. TYPE: BertDualAttention
is_decoder	Indicates whether the layer is used as a decoder model. TYPE: bool
add_cross_attention	Indicates whether cross-attention is added. TYPE: bool
crossattention	The attention module used for cross-attention (if add_cross_attention is True). TYPE: BertAttention
intermediate	The intermediate module used in the feed-forward operation. TYPE: BertDualIntermediate
output	The output module used in the feed-forward operation. TYPE: BertDualOutput

METHOD	DESCRIPTION
forward	Constructs the layer by performing attention and feed-forward operations.
feed_forward_chunk	Performs the feed-forward operation on a chunk of attention output.

Note

The class assumes that the imported modules (BertDualAttention, BertAttention, BertDualIntermediate, BertDualOutput) are available and properly implemented.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualLayer(nn.Module):

    """
    BertDualLayer

    This class represents a layer in a dual-attention BERT model.
    It is a subclass of nn.Module and is responsible for performing attention and feed-forward operations.

    Attributes:
        chunk_size_feed_forward (int): The size of chunks for feed-forward operation.
        seq_len_dim (int): The dimension of the sequence length.
        attention (BertDualAttention): The attention module used for self-attention.
        is_decoder (bool): Indicates whether the layer is used as a decoder model.
        add_cross_attention (bool): Indicates whether cross-attention is added.
        crossattention (BertAttention): The attention module used for cross-attention (if add_cross_attention is True).
        intermediate (BertDualIntermediate): The intermediate module used in the feed-forward operation.
        output (BertDualOutput): The output module used in the feed-forward operation.

    Methods:
        forward(hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions):
            Constructs the layer by performing attention and feed-forward operations.

        feed_forward_chunk(attention_output):
            Performs the feed-forward operation on a chunk of attention output.

    Note:
        The class assumes that the imported modules (BertDualAttention, BertAttention, BertDualIntermediate, BertDualOutput)
        are available and properly implemented.
    """
    def __init__(self, config):
        """
        Initializes a new instance of the BertDualLayer class.

        Args:
            self: The object instance.
            config (object):
                The configuration object that contains the settings for the BertDualLayer.

                - chunk_size_feed_forward (int): The chunk size for feed-forward attention.
                - is_decoder (bool): Indicates whether the model is a decoder.
                - add_cross_attention (bool): Indicates whether cross attention is added.
                    Raises a ValueError if cross attention is added and the model is not a decoder.
                - position_embedding_type (str): The type of position embedding for cross attention.

        Returns:
            None

        Raises:
            ValueError: If cross attention is added and the model is not a decoder.
        """
        super().__init__()
        self.chunk_size_feed_forward = config.chunk_size_feed_forward
        self.seq_len_dim = 1
        self.attention = BertDualAttention(config)
        self.is_decoder = config.is_decoder
        self.add_cross_attention = config.add_cross_attention
        if self.add_cross_attention:
            if not self.is_decoder:
                raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
            self.crossattention = BertAttention(config, position_embedding_type="absolute")
        self.intermediate = BertDualIntermediate(config)
        self.output = BertDualOutput(config)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ):
        """
        This method forwards a BertDualLayer by performing self-attention and potentially cross-attention operations.

        Args:
            self: The instance of the BertDualLayer class.
            hidden_states (mindspore.Tensor): The input hidden states to be processed.
            attention_mask (Optional[mindspore.Tensor]): An optional tensor specifying which elements should be attended to.
            head_mask (Optional[mindspore.Tensor]): An optional tensor providing a mask for the attention heads.
            encoder_hidden_states (Optional[mindspore.Tensor]): Optional hidden states from an encoder layer for cross-attention.
            encoder_attention_mask (Optional[mindspore.Tensor]): Optional attention mask for the encoder hidden states.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]): Optional tuple containing the past key and value tensors.
            output_attentions (Optional[bool]): Flag indicating whether to output attention weights.

        Returns:
            None.

        Raises:
            ValueError: Raised if `encoder_hidden_states` are provided but cross-attention layers are not instantiated in the BertDualLayer instance.
        """
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        self_attention_outputs = self.attention(
            hidden_states,
            attention_mask,
            head_mask,
            output_attentions=output_attentions,
            past_key_value=self_attn_past_key_value,
        )
        attention_output = self_attention_outputs[0]

        # if decoder, the last output is tuple of self-attn cache
        if self.is_decoder:
            outputs = self_attention_outputs[1:-1]
            present_key_value = self_attention_outputs[-1]
        else:
            outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights

        cross_attn_present_key_value = None
        if self.is_decoder and encoder_hidden_states is not None:
            if not hasattr(self, "crossattention"):
                raise ValueError(
                    f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
                    " by setting `config.add_cross_attention=True`"
                )

            # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            cross_attention_outputs = self.crossattention(
                attention_output,
                attention_mask,
                head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                cross_attn_past_key_value,
                output_attentions,
            )
            attention_output = cross_attention_outputs[0]
            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

            # add cross-attn cache to positions 3,4 of present_key_value tuple
            cross_attn_present_key_value = cross_attention_outputs[-1]
            present_key_value = present_key_value + cross_attn_present_key_value

        # layer_output = apply_chunking_to_forward(
        #     self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
        # )
        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)

        outputs = (layer_output,) + outputs

        # if decoder, return the attn key/values as the last output
        if self.is_decoder:
            outputs = outputs + (present_key_value,)

        return outputs

    def feed_forward_chunk(self, attention_output):
        """feed forward chunk"""
        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)
        return layer_output

mindnlp.transformers.models.bert.modeling_bert.BertDualLayer.__init__(config)

Initializes a new instance of the BertDualLayer class.

PARAMETER	DESCRIPTION
self	The object instance.
config	The configuration object that contains the settings for the BertDualLayer. chunk_size_feed_forward (int): The chunk size for feed-forward attention. is_decoder (bool): Indicates whether the model is a decoder. add_cross_attention (bool): Indicates whether cross attention is added. Raises a ValueError if cross attention is added and the model is not a decoder. position_embedding_type (str): The type of position embedding for cross attention. TYPE: object

RETURNS	DESCRIPTION
	None

RAISES	DESCRIPTION
ValueError	If cross attention is added and the model is not a decoder.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes a new instance of the BertDualLayer class.

    Args:
        self: The object instance.
        config (object):
            The configuration object that contains the settings for the BertDualLayer.

            - chunk_size_feed_forward (int): The chunk size for feed-forward attention.
            - is_decoder (bool): Indicates whether the model is a decoder.
            - add_cross_attention (bool): Indicates whether cross attention is added.
                Raises a ValueError if cross attention is added and the model is not a decoder.
            - position_embedding_type (str): The type of position embedding for cross attention.

    Returns:
        None

    Raises:
        ValueError: If cross attention is added and the model is not a decoder.
    """
    super().__init__()
    self.chunk_size_feed_forward = config.chunk_size_feed_forward
    self.seq_len_dim = 1
    self.attention = BertDualAttention(config)
    self.is_decoder = config.is_decoder
    self.add_cross_attention = config.add_cross_attention
    if self.add_cross_attention:
        if not self.is_decoder:
            raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
        self.crossattention = BertAttention(config, position_embedding_type="absolute")
    self.intermediate = BertDualIntermediate(config)
    self.output = BertDualOutput(config)

mindnlp.transformers.models.bert.modeling_bert.BertDualLayer.feed_forward_chunk(attention_output)

feed forward chunk

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def feed_forward_chunk(self, attention_output):
    """feed forward chunk"""
    intermediate_output = self.intermediate(attention_output)
    layer_output = self.output(intermediate_output, attention_output)
    return layer_output

mindnlp.transformers.models.bert.modeling_bert.BertDualLayer.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

This method forwards a BertDualLayer by performing self-attention and potentially cross-attention operations.

PARAMETER	DESCRIPTION
self	The instance of the BertDualLayer class.
hidden_states	The input hidden states to be processed. TYPE: Tensor
attention_mask	An optional tensor specifying which elements should be attended to. TYPE: Optional[Tensor] DEFAULT: None
head_mask	An optional tensor providing a mask for the attention heads. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	Optional hidden states from an encoder layer for cross-attention. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	Optional attention mask for the encoder hidden states. TYPE: Optional[Tensor] DEFAULT: None
past_key_value	Optional tuple containing the past key and value tensors. TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
output_attentions	Flag indicating whether to output attention weights. TYPE: Optional[bool] DEFAULT: False

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	Raised if encoder_hidden_states are provided but cross-attention layers are not instantiated in the BertDualLayer instance.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
):
    """
    This method forwards a BertDualLayer by performing self-attention and potentially cross-attention operations.

    Args:
        self: The instance of the BertDualLayer class.
        hidden_states (mindspore.Tensor): The input hidden states to be processed.
        attention_mask (Optional[mindspore.Tensor]): An optional tensor specifying which elements should be attended to.
        head_mask (Optional[mindspore.Tensor]): An optional tensor providing a mask for the attention heads.
        encoder_hidden_states (Optional[mindspore.Tensor]): Optional hidden states from an encoder layer for cross-attention.
        encoder_attention_mask (Optional[mindspore.Tensor]): Optional attention mask for the encoder hidden states.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]): Optional tuple containing the past key and value tensors.
        output_attentions (Optional[bool]): Flag indicating whether to output attention weights.

    Returns:
        None.

    Raises:
        ValueError: Raised if `encoder_hidden_states` are provided but cross-attention layers are not instantiated in the BertDualLayer instance.
    """
    # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
    self_attention_outputs = self.attention(
        hidden_states,
        attention_mask,
        head_mask,
        output_attentions=output_attentions,
        past_key_value=self_attn_past_key_value,
    )
    attention_output = self_attention_outputs[0]

    # if decoder, the last output is tuple of self-attn cache
    if self.is_decoder:
        outputs = self_attention_outputs[1:-1]
        present_key_value = self_attention_outputs[-1]
    else:
        outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights

    cross_attn_present_key_value = None
    if self.is_decoder and encoder_hidden_states is not None:
        if not hasattr(self, "crossattention"):
            raise ValueError(
                f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
                " by setting `config.add_cross_attention=True`"
            )

        # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
        cross_attention_outputs = self.crossattention(
            attention_output,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            cross_attn_past_key_value,
            output_attentions,
        )
        attention_output = cross_attention_outputs[0]
        outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

        # add cross-attn cache to positions 3,4 of present_key_value tuple
        cross_attn_present_key_value = cross_attention_outputs[-1]
        present_key_value = present_key_value + cross_attn_present_key_value

    # layer_output = apply_chunking_to_forward(
    #     self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
    # )
    intermediate_output = self.intermediate(attention_output)
    layer_output = self.output(intermediate_output, attention_output)

    outputs = (layer_output,) + outputs

    # if decoder, return the attn key/values as the last output
    if self.is_decoder:
        outputs = outputs + (present_key_value,)

    return outputs

mindnlp.transformers.models.bert.modeling_bert.BertDualModel

Bases: BertPreTrainedModel

The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in Attention is all you need by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

To behave as an decoder the model needs to be initialized with the is_decoder argument of the configuration set to True. To be used in a Seq2Seq model, the model needs to initialized with both is_decoder argument and add_cross_attention set to True; an encoder_hidden_states is then expected as an input to the forward pass.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualModel(BertPreTrainedModel):
    """

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    """
    def __init__(self, config, add_pooling_layer=True):
        """
        Initializes an instance of the BertDualModel class.

        Args:
            self: The instance of the class.
            config (object): The configuration object that contains the settings for the model.
            add_pooling_layer (bool): A flag indicating whether to add a pooling layer. Defaults to True.

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.config = config

        self.embeddings = BertEmbeddings(config)
        self.encoder = BertDualEncoder(config)

        self.pooler = BertPooler(config) if add_pooling_layer else None

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        This method retrieves the input embeddings from the BertDualModel instance.

        Args:
            self: The instance of the BertDualModel class.

        Returns:
            None.

        Raises:
            None.

        This method retrieves the input embeddings, represented by the 'word_embeddings' attribute of the BertDualModel instance.
        The embeddings are used to encode the input data into numerical representations suitable for processing by the model.

        Note that this method does not modify any attributes or perform any calculations. It simply returns the existing input embeddings.

        Example:
            ```python
            >>> model = BertDualModel()
            >>> embeddings = model.get_input_embeddings()
            ```
        """
        return self.embeddings.word_embeddings

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the BertDualModel.

        Args:
            self (BertDualModel): The instance of the BertDualModel class.
            value: The input embeddings to be set for the model. Should be of type WordEmbeddings.

        Returns:
            None: This method updates the input embeddings for the BertDualModel in-place.

        Raises:
            TypeError: If the provided 'value' is not of type WordEmbeddings.
        """
        self.embeddings.word_embeddings = value

    def _prune_heads(self, heads_to_prune):
        """
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        """
        for layer, heads in heads_to_prune.items():
            self.encoder.layer[layer].attention.prune_heads(heads)

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        r"""
        Args:
            encoder_hidden_states  (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
                the model is configured as a decoder.
            encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on the padding token indices of the encoder input.
                This mask is used in the cross-attention if the model is configured as a decoder.
                Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
            past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with
                each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
                Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
                don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
                `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
                `past_key_values`).
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if self.config.is_decoder:
            use_cache = use_cache if use_cache is not None else self.config.use_cache
        else:
            use_cache = False

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        if input_ids is not None:
            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
            input_shape = input_ids.shape
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        batch_size, seq_length = input_shape

        # past_key_values_length
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if attention_mask is None:
            attention_mask = ops.ones(batch_size, seq_length + past_key_values_length)

        if token_type_ids is None:
            if hasattr(self.embeddings, "token_type_ids"):
                buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
                buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((batch_size, seq_length))
                token_type_ids = buffered_token_type_ids_expanded
            else:
                token_type_ids = ops.zeros(*input_shape, dtype=mindspore.int64)
        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)

        # If a 2D or 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
            if encoder_attention_mask is None:
                encoder_attention_mask = ops.ones(*encoder_hidden_shape)
            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
        else:
            encoder_extended_attention_mask = None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

        embedding_output = self.embeddings(
            input_ids=input_ids,
            position_ids=position_ids,
            token_type_ids=token_type_ids,
            inputs_embeds=inputs_embeds,
            past_key_values_length=past_key_values_length,
        )

        encoder_outputs = self.encoder(
            embedding_output,
            attention_mask=extended_attention_mask,
            head_mask=head_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_extended_attention_mask,
            past_key_values=past_key_values,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = encoder_outputs[0]
        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

        if not return_dict:
            return (sequence_output, pooled_output) + encoder_outputs[1:]

        return BaseModelOutputWithPoolingAndCrossAttentions(
            last_hidden_state=sequence_output,
            pooler_output=pooled_output,
            past_key_values=encoder_outputs.past_key_values,
            hidden_states=encoder_outputs.hidden_states,
            attentions=encoder_outputs.attentions,
            cross_attentions=encoder_outputs.cross_attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertDualModel.__init__(config, add_pooling_layer=True)

Initializes an instance of the BertDualModel class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object that contains the settings for the model. TYPE: object
add_pooling_layer	A flag indicating whether to add a pooling layer. Defaults to True. TYPE: bool DEFAULT: True

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config, add_pooling_layer=True):
    """
    Initializes an instance of the BertDualModel class.

    Args:
        self: The instance of the class.
        config (object): The configuration object that contains the settings for the model.
        add_pooling_layer (bool): A flag indicating whether to add a pooling layer. Defaults to True.

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.config = config

    self.embeddings = BertEmbeddings(config)
    self.encoder = BertDualEncoder(config)

    self.pooler = BertPooler(config) if add_pooling_layer else None

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertDualModel.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
encoder_hidden_states	Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. TYPE: (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
encoder_attention_mask	Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. TYPE: `torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
use_cache	If set to True, past_key_values key value states are returned and can be used to speed up decoding (see past_key_values). TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    r"""
    Args:
        encoder_hidden_states  (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input.
            This mask is used in the cross-attention if the model is configured as a decoder.
            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with
            each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if self.config.is_decoder:
        use_cache = use_cache if use_cache is not None else self.config.use_cache
    else:
        use_cache = False

    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    if input_ids is not None:
        self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
        input_shape = input_ids.shape
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
    else:
        raise ValueError("You have to specify either input_ids or inputs_embeds")

    batch_size, seq_length = input_shape

    # past_key_values_length
    past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

    if attention_mask is None:
        attention_mask = ops.ones(batch_size, seq_length + past_key_values_length)

    if token_type_ids is None:
        if hasattr(self.embeddings, "token_type_ids"):
            buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
            buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((batch_size, seq_length))
            token_type_ids = buffered_token_type_ids_expanded
        else:
            token_type_ids = ops.zeros(*input_shape, dtype=mindspore.int64)
    # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
    # ourselves in which case we just need to make it broadcastable to all heads.
    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)

    # If a 2D or 3D attention mask is provided for the cross-attention
    # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
    if self.config.is_decoder and encoder_hidden_states is not None:
        encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
        encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
        if encoder_attention_mask is None:
            encoder_attention_mask = ops.ones(*encoder_hidden_shape)
        encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
    else:
        encoder_extended_attention_mask = None

    # Prepare head mask if needed
    # 1.0 in head_mask indicate we keep the head
    # attention_probs has shape bsz x n_heads x N x N
    # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
    # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
    head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

    embedding_output = self.embeddings(
        input_ids=input_ids,
        position_ids=position_ids,
        token_type_ids=token_type_ids,
        inputs_embeds=inputs_embeds,
        past_key_values_length=past_key_values_length,
    )

    encoder_outputs = self.encoder(
        embedding_output,
        attention_mask=extended_attention_mask,
        head_mask=head_mask,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_extended_attention_mask,
        past_key_values=past_key_values,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = encoder_outputs[0]
    pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

    if not return_dict:
        return (sequence_output, pooled_output) + encoder_outputs[1:]

    return BaseModelOutputWithPoolingAndCrossAttentions(
        last_hidden_state=sequence_output,
        pooler_output=pooled_output,
        past_key_values=encoder_outputs.past_key_values,
        hidden_states=encoder_outputs.hidden_states,
        attentions=encoder_outputs.attentions,
        cross_attentions=encoder_outputs.cross_attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertDualModel.get_input_embeddings()

This method retrieves the input embeddings from the BertDualModel instance.

PARAMETER	DESCRIPTION
self	The instance of the BertDualModel class.

RETURNS	DESCRIPTION
	None.

This method retrieves the input embeddings, represented by the 'word_embeddings' attribute of the BertDualModel instance. The embeddings are used to encode the input data into numerical representations suitable for processing by the model.

Note that this method does not modify any attributes or perform any calculations. It simply returns the existing input embeddings.

Example

>>> model = BertDualModel()
>>> embeddings = model.get_input_embeddings()

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def get_input_embeddings(self):
    """
    This method retrieves the input embeddings from the BertDualModel instance.

    Args:
        self: The instance of the BertDualModel class.

    Returns:
        None.

    Raises:
        None.

    This method retrieves the input embeddings, represented by the 'word_embeddings' attribute of the BertDualModel instance.
    The embeddings are used to encode the input data into numerical representations suitable for processing by the model.

    Note that this method does not modify any attributes or perform any calculations. It simply returns the existing input embeddings.

    Example:
        ```python
        >>> model = BertDualModel()
        >>> embeddings = model.get_input_embeddings()
        ```
    """
    return self.embeddings.word_embeddings

mindnlp.transformers.models.bert.modeling_bert.BertDualModel.set_input_embeddings(value)

Sets the input embeddings for the BertDualModel.

PARAMETER	DESCRIPTION
self	The instance of the BertDualModel class. TYPE: BertDualModel
value	The input embeddings to be set for the model. Should be of type WordEmbeddings.

RETURNS	DESCRIPTION
None	This method updates the input embeddings for the BertDualModel in-place.

RAISES	DESCRIPTION
TypeError	If the provided 'value' is not of type WordEmbeddings.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the BertDualModel.

    Args:
        self (BertDualModel): The instance of the BertDualModel class.
        value: The input embeddings to be set for the model. Should be of type WordEmbeddings.

    Returns:
        None: This method updates the input embeddings for the BertDualModel in-place.

    Raises:
        TypeError: If the provided 'value' is not of type WordEmbeddings.
    """
    self.embeddings.word_embeddings = value

mindnlp.transformers.models.bert.modeling_bert.BertDualOutput

Bases: Module

The 'BertDualOutput' class represents a custom neural network layer for processing dual outputs in a BERT model. This class inherits functionality from nn.Module and implements methods for initialization and processing of hidden states.

ATTRIBUTE	DESCRIPTION
intermediate_size	The size of the intermediate layer in the network. TYPE: int
dense	A dense layer for processing the intermediate hidden states. TYPE: Dense
LayerNorm	A layer normalization module for normalizing hidden states. TYPE: LayerNorm
dropout	A dropout layer for regularization during training. TYPE: Dropout

METHOD	DESCRIPTION
__init__	Initializes the BertDualOutput instance with the provided configuration.
forward	Processes the hidden states and input tensor to produce the final output.

The 'init' method initializes the instance by setting the intermediate_size, dense layer, LayerNorm module, and dropout layer based on the provided configuration. The 'forward' method processes the hidden states by splitting them, applying transformations, and combining the outputs to produce the final hidden states.

This class is designed to be used as a component in BERT models for handling dual outputs efficiently.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualOutput(nn.Module):

    """
    The 'BertDualOutput' class represents a custom neural network layer for processing dual outputs in a BERT model.
    This class inherits functionality from nn.Module and implements methods for initialization and processing of hidden states.

    Attributes:
        intermediate_size (int): The size of the intermediate layer in the network.
        dense (Dense): A dense layer for processing the intermediate hidden states.
        LayerNorm (nn.LayerNorm): A layer normalization module for normalizing hidden states.
        dropout (nn.Dropout): A dropout layer for regularization during training.

    Methods:
        __init__(self, config): Initializes the BertDualOutput instance with the provided configuration.
        forward(self, hidden_states, input_tensor): Processes the hidden states and input tensor to produce the final output.

    The '__init__' method initializes the instance by setting the intermediate_size, dense layer,
    LayerNorm module, and dropout layer based on the provided configuration.
    The 'forward' method processes the hidden states by splitting them, applying transformations,
    and combining the outputs to produce the final hidden states.

    This class is designed to be used as a component in BERT models for handling dual outputs efficiently.
    """
    def __init__(self, config):
        """
        Initializes an instance of the BertDualOutput class.

        Args:
            self: The instance of the BertDualOutput class.
            config:
                A configuration object containing the following attributes:

                - intermediate_size (int): The size of the intermediate layer.
                - hidden_size (int): The size of the hidden layer.
                - layer_norm_eps (float): The epsilon value for layer normalization.
                - hidden_dropout_prob (float): The dropout probability for the hidden layer.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not provided or is not of the expected type.
            ValueError: If the config parameter does not contain the required attributes
                or if their values are not within the expected range.
            AttributeError: If the config parameter does not have the necessary attributes.
        """
        super().__init__()
        self.intermediate_size = config.intermediate_size
        self.dense = Dense(config.intermediate_size//2, config.hidden_size//2)
        self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        """
        This method 'forward' is a member of the class 'BertDualOutput' and is used to process hidden states
        and input tensors in a specific manner.

        Args:
            self: The instance of the class.
            hidden_states (tensor): The hidden states to be processed.
                It is expected to be a tensor with shape (batch_size, sequence_length, hidden_size).
            input_tensor (tensor): The input tensor to be added to the processed hidden states.
                It is expected to be a tensor with the same shape as hidden_states.

        Returns:
            None: This method does not return any value explicitly,
                but it modifies the hidden_states and input_tensor in place.

        Raises:
            None: This method does not raise any exceptions explicitly.
        """
        hidden_states_r = hidden_states[:,:,:self.intermediate_size//2]
        hidden_states_d = hidden_states[:,:,self.intermediate_size//2:]
        hidden_states = to_2channel(hidden_states_r, hidden_states_d)
        hidden_states = self.dense(hidden_states)
        hidden_states_r, hidden_states_d = get_x_and_y(hidden_states)
        hidden_states = ops.cat([hidden_states_r, hidden_states_d], -1)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertDualOutput.__init__(config)

Initializes an instance of the BertDualOutput class.

PARAMETER	DESCRIPTION
self	The instance of the BertDualOutput class.
config	A configuration object containing the following attributes: intermediate_size (int): The size of the intermediate layer. hidden_size (int): The size of the hidden layer. layer_norm_eps (float): The epsilon value for layer normalization. hidden_dropout_prob (float): The dropout probability for the hidden layer.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the config parameter is not provided or is not of the expected type.
ValueError	If the config parameter does not contain the required attributes or if their values are not within the expected range.
AttributeError	If the config parameter does not have the necessary attributes.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of the BertDualOutput class.

    Args:
        self: The instance of the BertDualOutput class.
        config:
            A configuration object containing the following attributes:

            - intermediate_size (int): The size of the intermediate layer.
            - hidden_size (int): The size of the hidden layer.
            - layer_norm_eps (float): The epsilon value for layer normalization.
            - hidden_dropout_prob (float): The dropout probability for the hidden layer.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not provided or is not of the expected type.
        ValueError: If the config parameter does not contain the required attributes
            or if their values are not within the expected range.
        AttributeError: If the config parameter does not have the necessary attributes.
    """
    super().__init__()
    self.intermediate_size = config.intermediate_size
    self.dense = Dense(config.intermediate_size//2, config.hidden_size//2)
    self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(config.hidden_dropout_prob)

mindnlp.transformers.models.bert.modeling_bert.BertDualOutput.forward(hidden_states, input_tensor)

This method 'forward' is a member of the class 'BertDualOutput' and is used to process hidden states and input tensors in a specific manner.

PARAMETER	DESCRIPTION
self	The instance of the class.
hidden_states	The hidden states to be processed. It is expected to be a tensor with shape (batch_size, sequence_length, hidden_size). TYPE: tensor
input_tensor	The input tensor to be added to the processed hidden states. It is expected to be a tensor with the same shape as hidden_states. TYPE: tensor

RETURNS	DESCRIPTION
None	This method does not return any value explicitly, but it modifies the hidden_states and input_tensor in place.

RAISES	DESCRIPTION
None	This method does not raise any exceptions explicitly.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states, input_tensor):
    """
    This method 'forward' is a member of the class 'BertDualOutput' and is used to process hidden states
    and input tensors in a specific manner.

    Args:
        self: The instance of the class.
        hidden_states (tensor): The hidden states to be processed.
            It is expected to be a tensor with shape (batch_size, sequence_length, hidden_size).
        input_tensor (tensor): The input tensor to be added to the processed hidden states.
            It is expected to be a tensor with the same shape as hidden_states.

    Returns:
        None: This method does not return any value explicitly,
            but it modifies the hidden_states and input_tensor in place.

    Raises:
        None: This method does not raise any exceptions explicitly.
    """
    hidden_states_r = hidden_states[:,:,:self.intermediate_size//2]
    hidden_states_d = hidden_states[:,:,self.intermediate_size//2:]
    hidden_states = to_2channel(hidden_states_r, hidden_states_d)
    hidden_states = self.dense(hidden_states)
    hidden_states_r, hidden_states_d = get_x_and_y(hidden_states)
    hidden_states = ops.cat([hidden_states_r, hidden_states_d], -1)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertDualSelfAttention

Bases: Module

The BertDualSelfAttention class represents the dual self-attention mechanism used in the BERT model. This class implements the mechanism for both real and imaginary parts of the self-attention mechanism. It inherits from the nn.Module class and provides methods for attention score computation and context layer generation.

ATTRIBUTE	DESCRIPTION
config	A configuration object containing the model's hyperparameters.
output_attentions	A boolean indicating whether to output attention scores.
num_attention_heads	An integer representing the number of attention heads.
attention_head_size	An integer representing the size of each attention head.
all_head_size	An integer representing the total size of all attention heads combined.
query	A Dense layer for computing queries for the attention mechanism.
key	A Dense layer for computing keys for the attention mechanism.
value	A Dense layer for computing values for the attention mechanism.
dropout	A dropout layer for performing dropout on the attention scores.
position_embedding_type	A string representing the type of position embedding used.

METHOD	DESCRIPTION
transpose_for_scores	Transposes the input tensor for computing attention scores.
forward	Constructs the dual self-attention mechanism using the provided input tensors.

Note

The forward method raises a NotImplementedError for cross-attention and past_key_value arguments, as these functionalities are not implemented yet.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualSelfAttention(nn.Module):

    """
    The BertDualSelfAttention class represents the dual self-attention mechanism used in the BERT model.
    This class implements the mechanism for both real and imaginary parts of the self-attention mechanism.
    It inherits from the nn.Module class and provides methods for attention score computation and context layer generation.

    Attributes:
        config: A configuration object containing the model's hyperparameters.
        output_attentions: A boolean indicating whether to output attention scores.
        num_attention_heads: An integer representing the number of attention heads.
        attention_head_size: An integer representing the size of each attention head.
        all_head_size: An integer representing the total size of all attention heads combined.
        query: A Dense layer for computing queries for the attention mechanism.
        key: A Dense layer for computing keys for the attention mechanism.
        value: A Dense layer for computing values for the attention mechanism.
        dropout: A dropout layer for performing dropout on the attention scores.
        position_embedding_type: A string representing the type of position embedding used.

    Methods:
        transpose_for_scores(input_x): Transposes the input tensor for computing attention scores.
        forward(hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions):
            Constructs the dual self-attention mechanism using the provided input tensors.

    Note:
        The forward method raises a NotImplementedError for cross-attention and past_key_value arguments,
        as these functionalities are not implemented yet.
    """
    def __init__(self, config, position_embedding_type=None):
        """
        Initializes an instance of the BertDualSelfAttention class.

        Args:
            self: The instance of the class.
            config (object): An object of the configuration class containing the model's configuration parameters.
            position_embedding_type (str, optional): The type of position embedding. Defaults to None.

        Returns:
            None

        Raises:
            ValueError: If the hidden size is not a multiple of the number of attention heads.

        """
        super().__init__()
        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
            raise ValueError(
                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
                f"heads ({config.num_attention_heads})"
            )
        self.config = config
        self.output_attentions = config.output_attentions
        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.query = Dense(config.hidden_size//2, self.all_head_size//2)
        self.key = Dense(config.hidden_size//2, self.all_head_size//2)
        self.value = Dense(config.hidden_size//2, self.all_head_size//2)

        self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
        self.position_embedding_type = position_embedding_type or getattr(
            config, "position_embedding_type", "absolute"
        )

    def transpose_for_scores(self, input_x):
        r"""
        transpose for scores
        """
        new_x_shape = input_x.shape[:-1] + (self.num_attention_heads, self.attention_head_size //2)
        input_x = input_x.view(*new_x_shape)
        return input_x.transpose(0, 1, 3, 2, 4)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ):
        """
        This method 'forward' in the class 'BertDualSelfAttention' implements the dual self-attention mechanism
        for the BERT model.

        Args:
            self: The instance of the class.
            hidden_states (mindspore.Tensor):
                The input hidden states tensor with shape (batch_size, sequence_length, hidden_size).
            attention_mask (Optional[mindspore.Tensor]):
                An optional tensor with shape (batch_size, sequence_length) containing values of 0 or 1 to mask
                the attention scores for padded tokens.
            head_mask (Optional[mindspore.Tensor]): An optional tensor to mask the attention scores of specific heads.
            encoder_hidden_states (Optional[mindspore.Tensor]):
                An optional tensor containing the hidden states of the encoder if performing cross-attention.
            encoder_attention_mask (Optional[mindspore.Tensor]):
                An optional tensor to mask the attention scores for cross-attention.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                An optional tuple containing the past key and value tensors for incremental decoding.
            output_attentions (Optional[bool]):
                An optional boolean flag indicating whether to output the attention scores.

        Returns:
            Tuple[mindspore.Tensor, Optional[mindspore.Tensor]]:
                A tuple containing the context layer tensor with shape (batch_size, sequence_length, hidden_size)
                and optionally the attention scores tensor with shape
                (batch_size, num_attention_heads, sequence_length, sequence_length).

        Raises:
            NotImplementedError: If the functionality for cross-attention or incremental decoding is not implemented.
        """
        hidden_states_r = hidden_states[:,:,:self.config.hidden_size//2]
        hidden_states_d = hidden_states[:,:,self.config.hidden_size//2:]

        new_hidden_states = to_2channel(hidden_states_r, hidden_states_d)
        mixed_query_layer = self.query(new_hidden_states)

        is_cross_attention = encoder_hidden_states is not None

        if is_cross_attention or past_key_value is not None:
            raise NotImplementedError("This functionality is not implemented.")

        mixed_key_layer = self.key(new_hidden_states)

        key_layer = self.transpose_for_scores(mixed_key_layer)

        mixed_value_layer = self.value(new_hidden_states)

        value_layer = self.transpose_for_scores(mixed_value_layer)

        query_layer = self.transpose_for_scores(mixed_query_layer)

        # Take the dot product between "query" and "key" to get the raw attention scores.
        attention_scores = matmul(query_layer, key_layer.swapaxes(-1, -2))

        attention_scores = attention_scores / math.sqrt(self.attention_head_size)

        if attention_mask is not None:
            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
            attention_scores = attention_scores + attention_mask

        attention_scores_r, attention_scores_i = get_x_and_y(attention_scores)
        attention_scores_r = ops.softmax(attention_scores_r, dim=-1)
        attention_scores_i = ops.softmax(attention_scores_i, dim=-1)

        p_attn = to_2channel(attention_scores_r, attention_scores_i)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        if self.dropout is not None:
            p_attn = self.dropout(p_attn)

        context_layer = matmul(p_attn, value_layer)

        context_layer_r, context_layer_i = get_x_and_y(context_layer)
        context_layer = ops.cat([context_layer_r, context_layer_i], -1)

        context_layer = context_layer.transpose(0, 2, 1, 3)
        new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(new_context_layer_shape)
        outputs = (context_layer, p_attn) if output_attentions else (context_layer,)

        return outputs

mindnlp.transformers.models.bert.modeling_bert.BertDualSelfAttention.__init__(config, position_embedding_type=None)

Initializes an instance of the BertDualSelfAttention class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An object of the configuration class containing the model's configuration parameters. TYPE: object
position_embedding_type	The type of position embedding. Defaults to None. TYPE: str DEFAULT: None

RETURNS	DESCRIPTION
	None

RAISES	DESCRIPTION
ValueError	If the hidden size is not a multiple of the number of attention heads.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config, position_embedding_type=None):
    """
    Initializes an instance of the BertDualSelfAttention class.

    Args:
        self: The instance of the class.
        config (object): An object of the configuration class containing the model's configuration parameters.
        position_embedding_type (str, optional): The type of position embedding. Defaults to None.

    Returns:
        None

    Raises:
        ValueError: If the hidden size is not a multiple of the number of attention heads.

    """
    super().__init__()
    if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
        raise ValueError(
            f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
            f"heads ({config.num_attention_heads})"
        )
    self.config = config
    self.output_attentions = config.output_attentions
    self.num_attention_heads = config.num_attention_heads
    self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
    self.all_head_size = self.num_attention_heads * self.attention_head_size

    self.query = Dense(config.hidden_size//2, self.all_head_size//2)
    self.key = Dense(config.hidden_size//2, self.all_head_size//2)
    self.value = Dense(config.hidden_size//2, self.all_head_size//2)

    self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
    self.position_embedding_type = position_embedding_type or getattr(
        config, "position_embedding_type", "absolute"
    )

mindnlp.transformers.models.bert.modeling_bert.BertDualSelfAttention.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

This method 'forward' in the class 'BertDualSelfAttention' implements the dual self-attention mechanism for the BERT model.

PARAMETER	DESCRIPTION
self	The instance of the class.
hidden_states	The input hidden states tensor with shape (batch_size, sequence_length, hidden_size). TYPE: Tensor
attention_mask	An optional tensor with shape (batch_size, sequence_length) containing values of 0 or 1 to mask the attention scores for padded tokens. TYPE: Optional[Tensor] DEFAULT: None
head_mask	An optional tensor to mask the attention scores of specific heads. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	An optional tensor containing the hidden states of the encoder if performing cross-attention. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	An optional tensor to mask the attention scores for cross-attention. TYPE: Optional[Tensor] DEFAULT: None
past_key_value	An optional tuple containing the past key and value tensors for incremental decoding. TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
output_attentions	An optional boolean flag indicating whether to output the attention scores. TYPE: Optional[bool] DEFAULT: False

RETURNS	DESCRIPTION
	Tuple[mindspore.Tensor, Optional[mindspore.Tensor]]: A tuple containing the context layer tensor with shape (batch_size, sequence_length, hidden_size) and optionally the attention scores tensor with shape (batch_size, num_attention_heads, sequence_length, sequence_length).

RAISES	DESCRIPTION
NotImplementedError	If the functionality for cross-attention or incremental decoding is not implemented.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
):
    """
    This method 'forward' in the class 'BertDualSelfAttention' implements the dual self-attention mechanism
    for the BERT model.

    Args:
        self: The instance of the class.
        hidden_states (mindspore.Tensor):
            The input hidden states tensor with shape (batch_size, sequence_length, hidden_size).
        attention_mask (Optional[mindspore.Tensor]):
            An optional tensor with shape (batch_size, sequence_length) containing values of 0 or 1 to mask
            the attention scores for padded tokens.
        head_mask (Optional[mindspore.Tensor]): An optional tensor to mask the attention scores of specific heads.
        encoder_hidden_states (Optional[mindspore.Tensor]):
            An optional tensor containing the hidden states of the encoder if performing cross-attention.
        encoder_attention_mask (Optional[mindspore.Tensor]):
            An optional tensor to mask the attention scores for cross-attention.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            An optional tuple containing the past key and value tensors for incremental decoding.
        output_attentions (Optional[bool]):
            An optional boolean flag indicating whether to output the attention scores.

    Returns:
        Tuple[mindspore.Tensor, Optional[mindspore.Tensor]]:
            A tuple containing the context layer tensor with shape (batch_size, sequence_length, hidden_size)
            and optionally the attention scores tensor with shape
            (batch_size, num_attention_heads, sequence_length, sequence_length).

    Raises:
        NotImplementedError: If the functionality for cross-attention or incremental decoding is not implemented.
    """
    hidden_states_r = hidden_states[:,:,:self.config.hidden_size//2]
    hidden_states_d = hidden_states[:,:,self.config.hidden_size//2:]

    new_hidden_states = to_2channel(hidden_states_r, hidden_states_d)
    mixed_query_layer = self.query(new_hidden_states)

    is_cross_attention = encoder_hidden_states is not None

    if is_cross_attention or past_key_value is not None:
        raise NotImplementedError("This functionality is not implemented.")

    mixed_key_layer = self.key(new_hidden_states)

    key_layer = self.transpose_for_scores(mixed_key_layer)

    mixed_value_layer = self.value(new_hidden_states)

    value_layer = self.transpose_for_scores(mixed_value_layer)

    query_layer = self.transpose_for_scores(mixed_query_layer)

    # Take the dot product between "query" and "key" to get the raw attention scores.
    attention_scores = matmul(query_layer, key_layer.swapaxes(-1, -2))

    attention_scores = attention_scores / math.sqrt(self.attention_head_size)

    if attention_mask is not None:
        # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
        attention_scores = attention_scores + attention_mask

    attention_scores_r, attention_scores_i = get_x_and_y(attention_scores)
    attention_scores_r = ops.softmax(attention_scores_r, dim=-1)
    attention_scores_i = ops.softmax(attention_scores_i, dim=-1)

    p_attn = to_2channel(attention_scores_r, attention_scores_i)

    # This is actually dropping out entire tokens to attend to, which might
    # seem a bit unusual, but is taken from the original Transformer paper.
    if self.dropout is not None:
        p_attn = self.dropout(p_attn)

    context_layer = matmul(p_attn, value_layer)

    context_layer_r, context_layer_i = get_x_and_y(context_layer)
    context_layer = ops.cat([context_layer_r, context_layer_i], -1)

    context_layer = context_layer.transpose(0, 2, 1, 3)
    new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
    context_layer = context_layer.view(new_context_layer_shape)
    outputs = (context_layer, p_attn) if output_attentions else (context_layer,)

    return outputs

mindnlp.transformers.models.bert.modeling_bert.BertDualSelfAttention.transpose_for_scores(input_x)

transpose for scores

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def transpose_for_scores(self, input_x):
    r"""
    transpose for scores
    """
    new_x_shape = input_x.shape[:-1] + (self.num_attention_heads, self.attention_head_size //2)
    input_x = input_x.view(*new_x_shape)
    return input_x.transpose(0, 1, 3, 2, 4)

mindnlp.transformers.models.bert.modeling_bert.BertDualSelfOutput

Bases: Module

The 'BertDualSelfOutput' class represents a module that performs dual self-attention mechanism for BERT. It inherits from nn.Module and contains methods for initializing the module and forwarding the dual self-attention mechanism.

ATTRIBUTE	DESCRIPTION
hidden_size	The size of the hidden states. TYPE: int
dense	The dense layer for the dual self-attention mechanism. TYPE: Dense
LayerNorm	The layer normalization for the dual self-attention mechanism. TYPE: LayerNorm
dropout	The dropout layer for the dual self-attention mechanism. TYPE: Dropout

METHOD	DESCRIPTION
__init__	Initializes the 'BertDualSelfOutput' module with the provided configuration.
forward	Constructs the dual self-attention mechanism using the provided hidden states and input tensor.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertDualSelfOutput(nn.Module):

    """
    The 'BertDualSelfOutput' class represents a module that performs dual self-attention mechanism for BERT.
    It inherits from nn.Module and contains methods for initializing the module and forwarding
    the dual self-attention mechanism.

    Attributes:
        hidden_size (int): The size of the hidden states.
        dense (Dense): The dense layer for the dual self-attention mechanism.
        LayerNorm (LayerNorm): The layer normalization for the dual self-attention mechanism.
        dropout (Dropout): The dropout layer for the dual self-attention mechanism.

    Methods:
        __init__(config): Initializes the 'BertDualSelfOutput' module with the provided configuration.
        forward(hidden_states, input_tensor):
            Constructs the dual self-attention mechanism using the provided hidden states and input tensor.
    """
    def __init__(self, config):
        """
        Initializes an instance of the BertDualSelfOutput class.

        Args:
            self (BertDualSelfOutput): An instance of the BertDualSelfOutput class.
            config: A configuration object containing the parameters for the model.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.hidden_size = config.hidden_size
        self.dense = Dense(config.hidden_size//2, config.hidden_size//2)
        self.LayerNorm  = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        """
        Method 'forward' in the class 'BertDualSelfOutput'.

        This method forwards the hidden states by processing the input hidden states and input tensor.

        Args:
            self: Instance of the class BertDualSelfOutput. It represents the current instance of the class.
            hidden_states: Tensor of shape (batch_size, sequence_length, hidden_size).
                The input hidden states to be processed.
            input_tensor: Tensor of shape (batch_size, sequence_length, hidden_size).
                The input tensor to be added to the processed hidden states.

        Returns:
            None: This method does not return any value.

        Raises:
            None.
        """
        hidden_states_r = hidden_states[:,:,:self.hidden_size//2]
        hidden_states_d = hidden_states[:,:,self.hidden_size//2:]

        hidden_states = to_2channel(hidden_states_r, hidden_states_d)
        hidden_states = self.dense(hidden_states)
        hidden_states_r, hidden_states_d = get_x_and_y(hidden_states)
        hidden_states = ops.cat([hidden_states_r, hidden_states_d], -1)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertDualSelfOutput.__init__(config)

Initializes an instance of the BertDualSelfOutput class.

PARAMETER	DESCRIPTION
self	An instance of the BertDualSelfOutput class. TYPE: BertDualSelfOutput
config	A configuration object containing the parameters for the model.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of the BertDualSelfOutput class.

    Args:
        self (BertDualSelfOutput): An instance of the BertDualSelfOutput class.
        config: A configuration object containing the parameters for the model.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.hidden_size = config.hidden_size
    self.dense = Dense(config.hidden_size//2, config.hidden_size//2)
    self.LayerNorm  = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(config.hidden_dropout_prob)

mindnlp.transformers.models.bert.modeling_bert.BertDualSelfOutput.forward(hidden_states, input_tensor)

Method 'forward' in the class 'BertDualSelfOutput'.

This method forwards the hidden states by processing the input hidden states and input tensor.

PARAMETER	DESCRIPTION
self	Instance of the class BertDualSelfOutput. It represents the current instance of the class.
hidden_states	Tensor of shape (batch_size, sequence_length, hidden_size). The input hidden states to be processed.
input_tensor	Tensor of shape (batch_size, sequence_length, hidden_size). The input tensor to be added to the processed hidden states.

RETURNS	DESCRIPTION
None	This method does not return any value.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states, input_tensor):
    """
    Method 'forward' in the class 'BertDualSelfOutput'.

    This method forwards the hidden states by processing the input hidden states and input tensor.

    Args:
        self: Instance of the class BertDualSelfOutput. It represents the current instance of the class.
        hidden_states: Tensor of shape (batch_size, sequence_length, hidden_size).
            The input hidden states to be processed.
        input_tensor: Tensor of shape (batch_size, sequence_length, hidden_size).
            The input tensor to be added to the processed hidden states.

    Returns:
        None: This method does not return any value.

    Raises:
        None.
    """
    hidden_states_r = hidden_states[:,:,:self.hidden_size//2]
    hidden_states_d = hidden_states[:,:,self.hidden_size//2:]

    hidden_states = to_2channel(hidden_states_r, hidden_states_d)
    hidden_states = self.dense(hidden_states)
    hidden_states_r, hidden_states_d = get_x_and_y(hidden_states)
    hidden_states = ops.cat([hidden_states_r, hidden_states_d], -1)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertEmbeddings

Bases: Module

Embeddings for BERT, include word, position and token_type

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertEmbeddings(nn.Module):
    """
    Embeddings for BERT, include word, position and token_type
    """
    def __init__(self, config):
        """
        This method initializes an instance of the BertEmbeddings class.

        Args:
            self: The instance of the BertEmbeddings class.
            config: An object containing configuration parameters for the embeddings.
                It should have the following attributes:

                - vocab_size (int): The size of the vocabulary.
                - hidden_size (int): The size of the hidden layer.
                - pad_token_id (int): The index of the padding token.
                - max_position_embeddings (int): The maximum number of positional embeddings.
                - type_vocab_size (int): The size of the token type vocabulary.
                - layer_norm_eps (float): The epsilon value for layer normalization.
                - hidden_dropout_prob (float): The dropout probability for the hidden layer.
                - position_embedding_type (str, optional): The type of positional embedding, defaults to 'absolute'.

        Returns:
            None.

        Raises:
            AttributeError: If the config object does not have the required attributes.
            ValueError: If the config attributes have invalid values or types.
            TypeError: If the config parameters are of incorrect types.
            RuntimeError: If there is an error during the initialization process.
        """
        super().__init__()
        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
        self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)

        self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)
        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
        self.position_ids = ops.arange(config.max_position_embeddings).reshape((1, -1))
        self.token_type_ids = ops.zeros(*self.position_ids.shape, dtype=mindspore.int64)

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        past_key_values_length: int = 0,
    ):
        """
        This method forwards the embeddings for input tokens in the BERT model.

        Args:
            self (BertEmbeddings): The instance of the BertEmbeddings class.
            input_ids (Optional[mindspore.Tensor]): The input token IDs. Default is None.
            token_type_ids (Optional[mindspore.Tensor]): The token type IDs for the input tokens. Default is None.
            position_ids (Optional[mindspore.Tensor]): The position IDs for the input tokens. Default is None.
            inputs_embeds (Optional[mindspore.Tensor]): The pre-computed input embeddings. Default is None.
            past_key_values_length (int): The length of past key values. Default is 0.

        Returns:
            None.

        Raises:
            TypeError: If the input_ids, token_type_ids, position_ids, or inputs_embeds are not of type mindspore.Tensor.
            ValueError: If the input_shape is not valid or if there is an issue with the dimensions of the input tensors.
            RuntimeError: If there is a runtime issue during the forwardion of embeddings.
        """
        if input_ids is not None:
            input_shape = input_ids.shape
        else:
            input_shape = inputs_embeds.shape[:-1]

        seq_length = input_shape[1]

        if position_ids is None:
            position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]

        # Setting the token_type_ids to the registered buffer in forwardor where it is all zeros, which usually occurs
        # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
        # issue #5664
        if token_type_ids is None:
            if hasattr(self, "token_type_ids"):
                buffered_token_type_ids = self.token_type_ids[:, :seq_length]
                buffered_token_type_ids_expanded = buffered_token_type_ids.expand(input_shape[0], seq_length)
                token_type_ids = buffered_token_type_ids_expanded
            else:
                token_type_ids = ops.zeros(*input_shape, dtype=mindspore.int64)
        if inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)
        token_type_embeddings = self.token_type_embeddings(token_type_ids)

        embeddings = inputs_embeds + token_type_embeddings
        if self.position_embedding_type == "absolute":
            position_embeddings = self.position_embeddings(position_ids)
            embeddings += position_embeddings
        embeddings = self.LayerNorm(embeddings)
        embeddings = self.dropout(embeddings)
        return embeddings

mindnlp.transformers.models.bert.modeling_bert.BertEmbeddings.__init__(config)

This method initializes an instance of the BertEmbeddings class.

PARAMETER	DESCRIPTION
self	The instance of the BertEmbeddings class.
config	An object containing configuration parameters for the embeddings. It should have the following attributes: vocab_size (int): The size of the vocabulary. hidden_size (int): The size of the hidden layer. pad_token_id (int): The index of the padding token. max_position_embeddings (int): The maximum number of positional embeddings. type_vocab_size (int): The size of the token type vocabulary. layer_norm_eps (float): The epsilon value for layer normalization. hidden_dropout_prob (float): The dropout probability for the hidden layer. position_embedding_type (str, optional): The type of positional embedding, defaults to 'absolute'.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
AttributeError	If the config object does not have the required attributes.
ValueError	If the config attributes have invalid values or types.
TypeError	If the config parameters are of incorrect types.
RuntimeError	If there is an error during the initialization process.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    This method initializes an instance of the BertEmbeddings class.

    Args:
        self: The instance of the BertEmbeddings class.
        config: An object containing configuration parameters for the embeddings.
            It should have the following attributes:

            - vocab_size (int): The size of the vocabulary.
            - hidden_size (int): The size of the hidden layer.
            - pad_token_id (int): The index of the padding token.
            - max_position_embeddings (int): The maximum number of positional embeddings.
            - type_vocab_size (int): The size of the token type vocabulary.
            - layer_norm_eps (float): The epsilon value for layer normalization.
            - hidden_dropout_prob (float): The dropout probability for the hidden layer.
            - position_embedding_type (str, optional): The type of positional embedding, defaults to 'absolute'.

    Returns:
        None.

    Raises:
        AttributeError: If the config object does not have the required attributes.
        ValueError: If the config attributes have invalid values or types.
        TypeError: If the config parameters are of incorrect types.
        RuntimeError: If there is an error during the initialization process.
    """
    super().__init__()
    self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
    self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
    self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)

    self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)
    # position_ids (1, len position emb) is contiguous in memory and exported when serialized
    self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
    self.position_ids = ops.arange(config.max_position_embeddings).reshape((1, -1))
    self.token_type_ids = ops.zeros(*self.position_ids.shape, dtype=mindspore.int64)

mindnlp.transformers.models.bert.modeling_bert.BertEmbeddings.forward(input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0)

This method forwards the embeddings for input tokens in the BERT model.

PARAMETER	DESCRIPTION
self	The instance of the BertEmbeddings class. TYPE: BertEmbeddings
input_ids	The input token IDs. Default is None. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	The token type IDs for the input tokens. Default is None. TYPE: Optional[Tensor] DEFAULT: None
position_ids	The position IDs for the input tokens. Default is None. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The pre-computed input embeddings. Default is None. TYPE: Optional[Tensor] DEFAULT: None
past_key_values_length	The length of past key values. Default is 0. TYPE: int DEFAULT: 0

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the input_ids, token_type_ids, position_ids, or inputs_embeds are not of type mindspore.Tensor.
ValueError	If the input_shape is not valid or if there is an issue with the dimensions of the input tensors.
RuntimeError	If there is a runtime issue during the forwardion of embeddings.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    past_key_values_length: int = 0,
):
    """
    This method forwards the embeddings for input tokens in the BERT model.

    Args:
        self (BertEmbeddings): The instance of the BertEmbeddings class.
        input_ids (Optional[mindspore.Tensor]): The input token IDs. Default is None.
        token_type_ids (Optional[mindspore.Tensor]): The token type IDs for the input tokens. Default is None.
        position_ids (Optional[mindspore.Tensor]): The position IDs for the input tokens. Default is None.
        inputs_embeds (Optional[mindspore.Tensor]): The pre-computed input embeddings. Default is None.
        past_key_values_length (int): The length of past key values. Default is 0.

    Returns:
        None.

    Raises:
        TypeError: If the input_ids, token_type_ids, position_ids, or inputs_embeds are not of type mindspore.Tensor.
        ValueError: If the input_shape is not valid or if there is an issue with the dimensions of the input tensors.
        RuntimeError: If there is a runtime issue during the forwardion of embeddings.
    """
    if input_ids is not None:
        input_shape = input_ids.shape
    else:
        input_shape = inputs_embeds.shape[:-1]

    seq_length = input_shape[1]

    if position_ids is None:
        position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]

    # Setting the token_type_ids to the registered buffer in forwardor where it is all zeros, which usually occurs
    # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
    # issue #5664
    if token_type_ids is None:
        if hasattr(self, "token_type_ids"):
            buffered_token_type_ids = self.token_type_ids[:, :seq_length]
            buffered_token_type_ids_expanded = buffered_token_type_ids.expand(input_shape[0], seq_length)
            token_type_ids = buffered_token_type_ids_expanded
        else:
            token_type_ids = ops.zeros(*input_shape, dtype=mindspore.int64)
    if inputs_embeds is None:
        inputs_embeds = self.word_embeddings(input_ids)
    token_type_embeddings = self.token_type_embeddings(token_type_ids)

    embeddings = inputs_embeds + token_type_embeddings
    if self.position_embedding_type == "absolute":
        position_embeddings = self.position_embeddings(position_ids)
        embeddings += position_embeddings
    embeddings = self.LayerNorm(embeddings)
    embeddings = self.dropout(embeddings)
    return embeddings

mindnlp.transformers.models.bert.modeling_bert.BertEncoder

Bases: Module

Bert Encoder

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertEncoder(nn.Module):
    r"""
    Bert Encoder
    """
    def __init__(self, config):
        """
        BertEncoder.__init__

        Initializes a new BertEncoder object.

        Args:
            self (object): The instance of the BertEncoder class.
            config (object): The configuration object containing settings for the BertEncoder.
                This parameter is required to initialize the BertEncoder object.

                - It should be an instance of the configuration class containing the necessary settings.

                    - Example: config = BertConfig(num_hidden_layers=12, ...)

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.config = config
        self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = False,
        output_hidden_states: Optional[bool] = False,
        return_dict: Optional[bool] = True,
    ):
        """
        This method 'forward' is a part of the class 'BertEncoder' and is responsible for processing
        hidden states through the encoder layers.

        Args:
            self: The instance of the class.
            hidden_states (mindspore.Tensor): The input hidden states to be processed through the encoder layers.
            attention_mask (Optional[mindspore.Tensor]): Mask to avoid attention on padding tokens, defaults to None.
            head_mask (Optional[mindspore.Tensor]): Mask for attention heads in the encoder layers, defaults to None.
            encoder_hidden_states (Optional[mindspore.Tensor]): Hidden states of the encoder, defaults to None.
            encoder_attention_mask (Optional[mindspore.Tensor]): Mask to avoid attention on padding tokens in the encoder, defaults to None.
            past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): Past key values for caching, defaults to None.
            use_cache (Optional[bool]): Indicates whether to use cache for the next decoder step, defaults to None.
            output_attentions (Optional[bool]): Flag to output attention weights, defaults to False.
            output_hidden_states (Optional[bool]): Flag to output hidden states, defaults to False.
            return_dict (Optional[bool]): Flag to return the output as a dictionary, defaults to True.

        Returns:
            None: This method does not return any value directly.
                It processes the input hidden states through the encoder layers and updates the states internally.

        Raises:
            None: This method does not raise any exceptions explicitly.
        """
        all_hidden_states = () if output_hidden_states else None
        all_self_attentions = () if output_attentions else None
        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

        next_decoder_cache = () if use_cache else None
        for i, layer_module in enumerate(self.layer):
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_head_mask = head_mask[i] if head_mask is not None else None
            past_key_value = past_key_values[i] if past_key_values is not None else None
            layer_outputs = layer_module(
                hidden_states,
                attention_mask,
                layer_head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                past_key_value,
                output_attentions,
            )
            hidden_states = layer_outputs[0]
            if use_cache:
                next_decoder_cache += (layer_outputs[-1],)
            if output_attentions:
                all_self_attentions = all_self_attentions + (layer_outputs[1],)
                if self.config.add_cross_attention:
                    all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        if not return_dict:
            return tuple(
                v
                for v in [
                    hidden_states,
                    next_decoder_cache,
                    all_hidden_states,
                    all_self_attentions,
                    all_cross_attentions,
                ]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_decoder_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attentions,
            cross_attentions=all_cross_attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertEncoder.__init__(config)

BertEncoder.init

Initializes a new BertEncoder object.

PARAMETER	DESCRIPTION
self	The instance of the BertEncoder class. TYPE: object
config	The configuration object containing settings for the BertEncoder. This parameter is required to initialize the BertEncoder object. It should be an instance of the configuration class containing the necessary settings. Example: config = BertConfig(num_hidden_layers=12, ...) TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    BertEncoder.__init__

    Initializes a new BertEncoder object.

    Args:
        self (object): The instance of the BertEncoder class.
        config (object): The configuration object containing settings for the BertEncoder.
            This parameter is required to initialize the BertEncoder object.

            - It should be an instance of the configuration class containing the necessary settings.

                - Example: config = BertConfig(num_hidden_layers=12, ...)

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.config = config
    self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])

mindnlp.transformers.models.bert.modeling_bert.BertEncoder.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, use_cache=None, output_attentions=False, output_hidden_states=False, return_dict=True)

This method 'forward' is a part of the class 'BertEncoder' and is responsible for processing hidden states through the encoder layers.

PARAMETER	DESCRIPTION
self	The instance of the class.
hidden_states	The input hidden states to be processed through the encoder layers. TYPE: Tensor
attention_mask	Mask to avoid attention on padding tokens, defaults to None. TYPE: Optional[Tensor] DEFAULT: None
head_mask	Mask for attention heads in the encoder layers, defaults to None. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	Hidden states of the encoder, defaults to None. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	Mask to avoid attention on padding tokens in the encoder, defaults to None. TYPE: Optional[Tensor] DEFAULT: None
past_key_values	Past key values for caching, defaults to None. TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
use_cache	Indicates whether to use cache for the next decoder step, defaults to None. TYPE: Optional[bool] DEFAULT: None
output_attentions	Flag to output attention weights, defaults to False. TYPE: Optional[bool] DEFAULT: False
output_hidden_states	Flag to output hidden states, defaults to False. TYPE: Optional[bool] DEFAULT: False
return_dict	Flag to return the output as a dictionary, defaults to True. TYPE: Optional[bool] DEFAULT: True

RETURNS	DESCRIPTION
None	This method does not return any value directly. It processes the input hidden states through the encoder layers and updates the states internally.

RAISES	DESCRIPTION
None	This method does not raise any exceptions explicitly.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = False,
    output_hidden_states: Optional[bool] = False,
    return_dict: Optional[bool] = True,
):
    """
    This method 'forward' is a part of the class 'BertEncoder' and is responsible for processing
    hidden states through the encoder layers.

    Args:
        self: The instance of the class.
        hidden_states (mindspore.Tensor): The input hidden states to be processed through the encoder layers.
        attention_mask (Optional[mindspore.Tensor]): Mask to avoid attention on padding tokens, defaults to None.
        head_mask (Optional[mindspore.Tensor]): Mask for attention heads in the encoder layers, defaults to None.
        encoder_hidden_states (Optional[mindspore.Tensor]): Hidden states of the encoder, defaults to None.
        encoder_attention_mask (Optional[mindspore.Tensor]): Mask to avoid attention on padding tokens in the encoder, defaults to None.
        past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): Past key values for caching, defaults to None.
        use_cache (Optional[bool]): Indicates whether to use cache for the next decoder step, defaults to None.
        output_attentions (Optional[bool]): Flag to output attention weights, defaults to False.
        output_hidden_states (Optional[bool]): Flag to output hidden states, defaults to False.
        return_dict (Optional[bool]): Flag to return the output as a dictionary, defaults to True.

    Returns:
        None: This method does not return any value directly.
            It processes the input hidden states through the encoder layers and updates the states internally.

    Raises:
        None: This method does not raise any exceptions explicitly.
    """
    all_hidden_states = () if output_hidden_states else None
    all_self_attentions = () if output_attentions else None
    all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

    next_decoder_cache = () if use_cache else None
    for i, layer_module in enumerate(self.layer):
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        layer_head_mask = head_mask[i] if head_mask is not None else None
        past_key_value = past_key_values[i] if past_key_values is not None else None
        layer_outputs = layer_module(
            hidden_states,
            attention_mask,
            layer_head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            past_key_value,
            output_attentions,
        )
        hidden_states = layer_outputs[0]
        if use_cache:
            next_decoder_cache += (layer_outputs[-1],)
        if output_attentions:
            all_self_attentions = all_self_attentions + (layer_outputs[1],)
            if self.config.add_cross_attention:
                all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

    if output_hidden_states:
        all_hidden_states = all_hidden_states + (hidden_states,)

    if not return_dict:
        return tuple(
            v
            for v in [
                hidden_states,
                next_decoder_cache,
                all_hidden_states,
                all_self_attentions,
                all_cross_attentions,
            ]
            if v is not None
        )
    return BaseModelOutputWithPastAndCrossAttentions(
        last_hidden_state=hidden_states,
        past_key_values=next_decoder_cache,
        hidden_states=all_hidden_states,
        attentions=all_self_attentions,
        cross_attentions=all_cross_attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertForMaskedLM

Bases: BertPreTrainedModel

BertForMaskedLM

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertForMaskedLM(BertPreTrainedModel):
    """BertForMaskedLM"""
    _tied_weights_keys = ["predictions.decoder.bias", "cls.predictions.decoder.weight"]

    def __init__(self, config):
        """
        Initializes an instance of the BertForMaskedLM class.

        Args:
            self: The instance of the class.
            config (BertConfig): The configuration object for the BertForMaskedLM model.

        Returns:
            None

        Raises:
            None

        Description:
        This method is the forwardor for the BertForMaskedLM class. It initializes the instance by setting up
        the model architecture and loading the configuration.

        The 'config' parameter is an instance of the BertConfig class, which contains various settings
        and hyperparameters for the model.
        It is used to configure the model architecture and behavior.

        Note that if the 'is_decoder' attribute of the 'config' parameter is set to True, a warning message is logged,
        reminding the user to set 'is_decoder' to False when using the 'BertForMaskedLM' model
        with bi-directional self-attention.

        The method initializes two attributes of the instance:

        - 'bert': An instance of the 'BertModel' class, which represents the BERT model without the MLM head.
        The 'config' parameter is passed to the 'BertModel' forwardor to configure the model architecture.
        - 'cls': An instance of the 'BertOnlyMLMHead' class, which represents the MLM head of the BERT model.
        The 'config' parameter is passed to the 'BertOnlyMLMHead' forwardor to configure the MLM head.

        After the initialization, the 'post_init' method is called to execute any additional setup steps specific to the BertForMaskedLM class.
        """
        super().__init__(config)

        if config.is_decoder:
            logger.warning(
                "If you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for "
                "bi-directional self-attention."
            )

        self.bert = BertModel(config, add_pooling_layer=False)
        self.cls = BertOnlyMLMHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    def get_output_embeddings(self):
        """
        This method returns the output embeddings for the BertForMaskedLM model.

        Args:
            self (BertForMaskedLM): The instance of the BertForMaskedLM class.

        Returns:
            None.

        Raises:
            None
        """
        return self.cls.predictions.decoder

    def set_output_embeddings(self, new_embeddings):
        """
        Set the output embeddings for the BertForMaskedLM model.

        Args:
            self (BertForMaskedLM): The instance of the BertForMaskedLM class.
            new_embeddings (Any): The new embeddings to set for the output layer.

        Returns:
            None.

        Raises:
            None
        """
        self.cls.predictions.decoder = new_embeddings

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        r"""
        Args:
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
                config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
                loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]
        prediction_scores = self.cls(sequence_output)

        masked_lm_loss = None
        if labels is not None:
            masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (prediction_scores,) + outputs[2:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return MaskedLMOutput(
            loss=masked_lm_loss,
            logits=prediction_scores,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

    def prepare_inputs_for_generation(self, input_ids, attention_mask=None):
        """
        Method: prepare_inputs_for_generation

        Description:
            This method prepares inputs for generation by adding a dummy token at the end of the input_ids
            and updating the attention_mask accordingly.

        Args:
            self: The instance of the BertForMaskedLM class.
            input_ids (Tensor): The input token IDs for generation.
            attention_mask (Tensor, optional): The attention mask tensor. Defaults to None.

        Returns:
            dict: A dictionary containing the updated 'input_ids' and 'attention_mask'.

        Raises:
            ValueError: If the PAD token is not defined in the configuration.
        """
        input_shape = input_ids.shape
        effective_batch_size = input_shape[0]

        #  add a dummy token
        if self.config.pad_token_id is None:
            raise ValueError("The PAD token should be defined for generation")

        attention_mask = ops.cat([attention_mask, attention_mask.new_zeros((attention_mask.shape[0], 1))], dim=-1)
        dummy_token = ops.full(
            (effective_batch_size, 1), self.config.pad_token_id, dtype=mindspore.int64)
        input_ids = ops.cat([input_ids, dummy_token], dim=1)

        return {"input_ids": input_ids, "attention_mask": attention_mask}

mindnlp.transformers.models.bert.modeling_bert.BertForMaskedLM.__init__(config)

Initializes an instance of the BertForMaskedLM class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object for the BertForMaskedLM model. TYPE: BertConfig

RETURNS	DESCRIPTION
	None

This method is the forwardor for the BertForMaskedLM class. It initializes the instance by setting up the model architecture and loading the configuration.

The 'config' parameter is an instance of the BertConfig class, which contains various settings and hyperparameters for the model. It is used to configure the model architecture and behavior.

Note that if the 'is_decoder' attribute of the 'config' parameter is set to True, a warning message is logged, reminding the user to set 'is_decoder' to False when using the 'BertForMaskedLM' model with bi-directional self-attention.

The method initializes two attributes of the instance:

• 'bert': An instance of the 'BertModel' class, which represents the BERT model without the MLM head. The 'config' parameter is passed to the 'BertModel' forwardor to configure the model architecture.
• 'cls': An instance of the 'BertOnlyMLMHead' class, which represents the MLM head of the BERT model. The 'config' parameter is passed to the 'BertOnlyMLMHead' forwardor to configure the MLM head.

After the initialization, the 'post_init' method is called to execute any additional setup steps specific to the BertForMaskedLM class.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of the BertForMaskedLM class.

    Args:
        self: The instance of the class.
        config (BertConfig): The configuration object for the BertForMaskedLM model.

    Returns:
        None

    Raises:
        None

    Description:
    This method is the forwardor for the BertForMaskedLM class. It initializes the instance by setting up
    the model architecture and loading the configuration.

    The 'config' parameter is an instance of the BertConfig class, which contains various settings
    and hyperparameters for the model.
    It is used to configure the model architecture and behavior.

    Note that if the 'is_decoder' attribute of the 'config' parameter is set to True, a warning message is logged,
    reminding the user to set 'is_decoder' to False when using the 'BertForMaskedLM' model
    with bi-directional self-attention.

    The method initializes two attributes of the instance:

    - 'bert': An instance of the 'BertModel' class, which represents the BERT model without the MLM head.
    The 'config' parameter is passed to the 'BertModel' forwardor to configure the model architecture.
    - 'cls': An instance of the 'BertOnlyMLMHead' class, which represents the MLM head of the BERT model.
    The 'config' parameter is passed to the 'BertOnlyMLMHead' forwardor to configure the MLM head.

    After the initialization, the 'post_init' method is called to execute any additional setup steps specific to the BertForMaskedLM class.
    """
    super().__init__(config)

    if config.is_decoder:
        logger.warning(
            "If you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for "
            "bi-directional self-attention."
        )

    self.bert = BertModel(config, add_pooling_layer=False)
    self.cls = BertOnlyMLMHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertForMaskedLM.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the masked language modeling loss. Indices should be in [-100, 0, ..., config.vocab_size] (see input_ids docstring) Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size] TYPE: `torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    r"""
    Args:
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_attention_mask,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = outputs[0]
    prediction_scores = self.cls(sequence_output)

    masked_lm_loss = None
    if labels is not None:
        masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

    if not return_dict:
        output = (prediction_scores,) + outputs[2:]
        return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

    return MaskedLMOutput(
        loss=masked_lm_loss,
        logits=prediction_scores,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertForMaskedLM.get_output_embeddings()

This method returns the output embeddings for the BertForMaskedLM model.

PARAMETER	DESCRIPTION
self	The instance of the BertForMaskedLM class. TYPE: BertForMaskedLM

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def get_output_embeddings(self):
    """
    This method returns the output embeddings for the BertForMaskedLM model.

    Args:
        self (BertForMaskedLM): The instance of the BertForMaskedLM class.

    Returns:
        None.

    Raises:
        None
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.bert.modeling_bert.BertForMaskedLM.prepare_inputs_for_generation(input_ids, attention_mask=None)

Description

This method prepares inputs for generation by adding a dummy token at the end of the input_ids and updating the attention_mask accordingly.

PARAMETER	DESCRIPTION
self	The instance of the BertForMaskedLM class.
input_ids	The input token IDs for generation. TYPE: Tensor
attention_mask	The attention mask tensor. Defaults to None. TYPE: Tensor DEFAULT: None

RETURNS	DESCRIPTION
dict	A dictionary containing the updated 'input_ids' and 'attention_mask'.

RAISES	DESCRIPTION
ValueError	If the PAD token is not defined in the configuration.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def prepare_inputs_for_generation(self, input_ids, attention_mask=None):
    """
    Method: prepare_inputs_for_generation

    Description:
        This method prepares inputs for generation by adding a dummy token at the end of the input_ids
        and updating the attention_mask accordingly.

    Args:
        self: The instance of the BertForMaskedLM class.
        input_ids (Tensor): The input token IDs for generation.
        attention_mask (Tensor, optional): The attention mask tensor. Defaults to None.

    Returns:
        dict: A dictionary containing the updated 'input_ids' and 'attention_mask'.

    Raises:
        ValueError: If the PAD token is not defined in the configuration.
    """
    input_shape = input_ids.shape
    effective_batch_size = input_shape[0]

    #  add a dummy token
    if self.config.pad_token_id is None:
        raise ValueError("The PAD token should be defined for generation")

    attention_mask = ops.cat([attention_mask, attention_mask.new_zeros((attention_mask.shape[0], 1))], dim=-1)
    dummy_token = ops.full(
        (effective_batch_size, 1), self.config.pad_token_id, dtype=mindspore.int64)
    input_ids = ops.cat([input_ids, dummy_token], dim=1)

    return {"input_ids": input_ids, "attention_mask": attention_mask}

mindnlp.transformers.models.bert.modeling_bert.BertForMaskedLM.set_output_embeddings(new_embeddings)

Set the output embeddings for the BertForMaskedLM model.

PARAMETER	DESCRIPTION
self	The instance of the BertForMaskedLM class. TYPE: BertForMaskedLM
new_embeddings	The new embeddings to set for the output layer. TYPE: Any

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def set_output_embeddings(self, new_embeddings):
    """
    Set the output embeddings for the BertForMaskedLM model.

    Args:
        self (BertForMaskedLM): The instance of the BertForMaskedLM class.
        new_embeddings (Any): The new embeddings to set for the output layer.

    Returns:
        None.

    Raises:
        None
    """
    self.cls.predictions.decoder = new_embeddings

mindnlp.transformers.models.bert.modeling_bert.BertForMultipleChoice

Bases: BertPreTrainedModel

BertForMultipleChoice

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertForMultipleChoice(BertPreTrainedModel):
    """BertForMultipleChoice"""
    def __init__(self, config):
        """
        Initializes a BertForMultipleChoice instance.

        Args:
            self (BertForMultipleChoice): The current instance of the BertForMultipleChoice class.
            config: An instance of the configuration class that holds various hyperparameters and settings for the model.

        Returns:
            None.

        Raises:
            TypeError: If the provided config is not of the expected type.
            ValueError: If the provided config does not contain necessary attributes.
            RuntimeError: If there are issues during the initialization process.
        """
        super().__init__(config)

        self.bert = BertModel(config)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, 1)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        r"""
        Args:
            labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
                num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
                `input_ids` above)
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]

        input_ids = input_ids.view(-1, input_ids.shape[-1]) if input_ids is not None else None
        attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) if attention_mask is not None else None
        token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) if token_type_ids is not None else None
        position_ids = position_ids.view(-1, position_ids.shape[-1]) if position_ids is not None else None
        inputs_embeds = (
            inputs_embeds.view(-1, inputs_embeds.shape[-2], inputs_embeds.shape[-1])
            if inputs_embeds is not None
            else None
        )

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = outputs[1]

        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)
        reshaped_logits = logits.view(-1, num_choices)

        loss = None
        if labels is not None:
            loss = F.cross_entropy(reshaped_logits, labels)

        if not return_dict:
            output = (reshaped_logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return MultipleChoiceModelOutput(
            loss=loss,
            logits=reshaped_logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertForMultipleChoice.__init__(config)

Initializes a BertForMultipleChoice instance.

PARAMETER	DESCRIPTION
self	The current instance of the BertForMultipleChoice class. TYPE: BertForMultipleChoice
config	An instance of the configuration class that holds various hyperparameters and settings for the model.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the provided config is not of the expected type.
ValueError	If the provided config does not contain necessary attributes.
RuntimeError	If there are issues during the initialization process.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes a BertForMultipleChoice instance.

    Args:
        self (BertForMultipleChoice): The current instance of the BertForMultipleChoice class.
        config: An instance of the configuration class that holds various hyperparameters and settings for the model.

    Returns:
        None.

    Raises:
        TypeError: If the provided config is not of the expected type.
        ValueError: If the provided config does not contain necessary attributes.
        RuntimeError: If there are issues during the initialization process.
    """
    super().__init__(config)

    self.bert = BertModel(config)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, 1)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertForMultipleChoice.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the multiple choice classification loss. Indices should be in [0, ..., num_choices-1] where num_choices is the size of the second dimension of the input tensors. (See input_ids above) TYPE: `torch.LongTensor` of shape `(batch_size,)`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    r"""
    Args:
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]

    input_ids = input_ids.view(-1, input_ids.shape[-1]) if input_ids is not None else None
    attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) if attention_mask is not None else None
    token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) if token_type_ids is not None else None
    position_ids = position_ids.view(-1, position_ids.shape[-1]) if position_ids is not None else None
    inputs_embeds = (
        inputs_embeds.view(-1, inputs_embeds.shape[-2], inputs_embeds.shape[-1])
        if inputs_embeds is not None
        else None
    )

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    pooled_output = outputs[1]

    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)
    reshaped_logits = logits.view(-1, num_choices)

    loss = None
    if labels is not None:
        loss = F.cross_entropy(reshaped_logits, labels)

    if not return_dict:
        output = (reshaped_logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

    return MultipleChoiceModelOutput(
        loss=loss,
        logits=reshaped_logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertForNextSentencePrediction

Bases: BertPreTrainedModel

BertForNextSentencePrediction

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertForNextSentencePrediction(BertPreTrainedModel):
    """BertForNextSentencePrediction"""
    def __init__(self, config):
        """
        Initializes an instance of BertForNextSentencePrediction class.

        Args:
            self (BertForNextSentencePrediction): The instance of the BertForNextSentencePrediction class.
            config: The configuration object containing settings for the BERT model.

        Returns:
            None: This method initializes the BertForNextSentencePrediction instance with the specified config settings.

        Raises:
            None.
        """
        super().__init__(config)

        self.bert = BertModel(config)
        self.cls = BertOnlyNSPHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        """Constructs the BertForNextSentencePrediction model.

        Args:
            self (BertForNextSentencePrediction): An instance of the BertForNextSentencePrediction class.
            input_ids (Optional[mindspore.Tensor]): The input tensor containing the indices of input sequence tokens.
            attention_mask (Optional[mindspore.Tensor]):
                The attention mask tensor indicating which tokens should be attended to (1) and which should not (0).
            token_type_ids (Optional[mindspore.Tensor]):
                The token type tensor indicating the type of each token in the input sequence.
            position_ids (Optional[mindspore.Tensor]): The tensor containing the position indices of each input token.
            head_mask (Optional[mindspore.Tensor]):
                The tensor indicating which heads should be masked in the attention layers.
            inputs_embeds (Optional[mindspore.Tensor]):
                The tensor containing the embedded representation of the input tokens.
            labels (Optional[mindspore.Tensor]): The tensor containing the labels for the next sentence prediction task.
            output_attentions (Optional[bool]): Whether to include the attention probabilities in the output.
            output_hidden_states (Optional[bool]): Whether to include the hidden states in the output.
            return_dict (Optional[bool]): Whether to return a dictionary instead of a tuple as the output.

        Returns:
            None.

        Raises:
            None.
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = outputs[1]

        seq_relationship_scores = self.cls(pooled_output)

        next_sentence_loss = None
        if labels is not None:
            next_sentence_loss = F.cross_entropy(seq_relationship_scores.view(-1, 2), labels.view(-1))

        if not return_dict:
            output = (seq_relationship_scores,) + outputs[2:]
            return ((next_sentence_loss,) + output) if next_sentence_loss is not None else output

        return NextSentencePredictorOutput(
            loss=next_sentence_loss,
            logits=seq_relationship_scores,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertForNextSentencePrediction.__init__(config)

Initializes an instance of BertForNextSentencePrediction class.

PARAMETER	DESCRIPTION
self	The instance of the BertForNextSentencePrediction class. TYPE: BertForNextSentencePrediction
config	The configuration object containing settings for the BERT model.

RETURNS	DESCRIPTION
None	This method initializes the BertForNextSentencePrediction instance with the specified config settings.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of BertForNextSentencePrediction class.

    Args:
        self (BertForNextSentencePrediction): The instance of the BertForNextSentencePrediction class.
        config: The configuration object containing settings for the BERT model.

    Returns:
        None: This method initializes the BertForNextSentencePrediction instance with the specified config settings.

    Raises:
        None.
    """
    super().__init__(config)

    self.bert = BertModel(config)
    self.cls = BertOnlyNSPHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertForNextSentencePrediction.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Constructs the BertForNextSentencePrediction model.

PARAMETER	DESCRIPTION
self	An instance of the BertForNextSentencePrediction class. TYPE: BertForNextSentencePrediction
input_ids	The input tensor containing the indices of input sequence tokens. TYPE: Optional[Tensor] DEFAULT: None
attention_mask	The attention mask tensor indicating which tokens should be attended to (1) and which should not (0). TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	The token type tensor indicating the type of each token in the input sequence. TYPE: Optional[Tensor] DEFAULT: None
position_ids	The tensor containing the position indices of each input token. TYPE: Optional[Tensor] DEFAULT: None
head_mask	The tensor indicating which heads should be masked in the attention layers. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The tensor containing the embedded representation of the input tokens. TYPE: Optional[Tensor] DEFAULT: None
labels	The tensor containing the labels for the next sentence prediction task. TYPE: Optional[Tensor] DEFAULT: None
output_attentions	Whether to include the attention probabilities in the output. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Whether to include the hidden states in the output. TYPE: Optional[bool] DEFAULT: None
return_dict	Whether to return a dictionary instead of a tuple as the output. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    """Constructs the BertForNextSentencePrediction model.

    Args:
        self (BertForNextSentencePrediction): An instance of the BertForNextSentencePrediction class.
        input_ids (Optional[mindspore.Tensor]): The input tensor containing the indices of input sequence tokens.
        attention_mask (Optional[mindspore.Tensor]):
            The attention mask tensor indicating which tokens should be attended to (1) and which should not (0).
        token_type_ids (Optional[mindspore.Tensor]):
            The token type tensor indicating the type of each token in the input sequence.
        position_ids (Optional[mindspore.Tensor]): The tensor containing the position indices of each input token.
        head_mask (Optional[mindspore.Tensor]):
            The tensor indicating which heads should be masked in the attention layers.
        inputs_embeds (Optional[mindspore.Tensor]):
            The tensor containing the embedded representation of the input tokens.
        labels (Optional[mindspore.Tensor]): The tensor containing the labels for the next sentence prediction task.
        output_attentions (Optional[bool]): Whether to include the attention probabilities in the output.
        output_hidden_states (Optional[bool]): Whether to include the hidden states in the output.
        return_dict (Optional[bool]): Whether to return a dictionary instead of a tuple as the output.

    Returns:
        None.

    Raises:
        None.
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    pooled_output = outputs[1]

    seq_relationship_scores = self.cls(pooled_output)

    next_sentence_loss = None
    if labels is not None:
        next_sentence_loss = F.cross_entropy(seq_relationship_scores.view(-1, 2), labels.view(-1))

    if not return_dict:
        output = (seq_relationship_scores,) + outputs[2:]
        return ((next_sentence_loss,) + output) if next_sentence_loss is not None else output

    return NextSentencePredictorOutput(
        loss=next_sentence_loss,
        logits=seq_relationship_scores,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertForPreTraining

Bases: BertPreTrainedModel

BertForPreTraining

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertForPreTraining(BertPreTrainedModel):
    """BertForPreTraining"""
    _tied_weights_keys = ["predictions.decoder.bias", "cls.predictions.decoder.weight"]

    def __init__(self, config):
        """
        Initializes a new instance of the BertForPreTraining class.

        Args:
            self: The object itself.
            config: A configuration object that specifies the model hyperparameters and other settings.
                It should be an instance of the BertConfig class.

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)

        self.bert = BertModel(config)
        self.cls = BertPreTrainingHeads(config)

        # Initialize weights and apply final processing
        self.post_init()

    def get_output_embeddings(self):
        """
        Method to retrieve the output embeddings from a BERT model for pre-training.

        Args:
            self: Instance of the BertForPreTraining class.
                This parameter refers to the current instance of the BertForPreTraining class.

        Returns:
            None:
                This method returns None, as it retrieves the output embeddings for further processing.

        Raises:
            None.
        """
        return self.cls.predictions.decoder

    def set_output_embeddings(self, new_embeddings):
        """
        Sets the output embeddings of the model with the provided new embeddings.

        Args:
            self (BertForPreTraining): An instance of the BertForPreTraining class.
            new_embeddings (Any): The new embeddings to be set for the model's output.

        Returns:
            None: This method modifies the model's output embeddings in-place.

        Raises:
            None.

        Note:
            The 'new_embeddings' parameter should be of the same shape and type as the original output embeddings.
            Modifying the output embeddings may affect the model's performance and downstream tasks.
        """
        self.cls.predictions.decoder = new_embeddings

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        next_sentence_label: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], BertForPreTrainingOutput]:
        """
        Constructs the pre-training model for BERT.

        Args:
            self: The instance of the BertForPreTraining class.
            input_ids (Optional[mindspore.Tensor]): The input token IDs. Default is None.
            attention_mask (Optional[mindspore.Tensor]): The attention mask for the input. Default is None.
            token_type_ids (Optional[mindspore.Tensor]): The token type IDs. Default is None.
            position_ids (Optional[mindspore.Tensor]): The position IDs of tokens. Default is None.
            head_mask (Optional[mindspore.Tensor]): The mask for heads in the self-attention mechanism. Default is None.
            inputs_embeds (Optional[mindspore.Tensor]): The embedded input tokens. Default is None.
            labels (Optional[mindspore.Tensor]): The labels for masked language modeling task. Default is None.
            next_sentence_label (Optional[mindspore.Tensor]): The label for next sentence prediction task. Default is None.
            output_attentions (Optional[bool]): Whether to output attentions. Default is None.
            output_hidden_states (Optional[bool]): Whether to output hidden states. Default is None.
            return_dict (Optional[bool]): Whether to return the output as a dictionary. Default is None.

        Returns:
            Union[Tuple[mindspore.Tensor], BertForPreTrainingOutput]:
                A tuple containing the prediction scores for masked language modeling
                and next sentence prediction tasks, and additional outputs if specified.

        Raises:
            None
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output, pooled_output = outputs[:2]
        prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)

        total_loss = None
        if labels is not None and next_sentence_label is not None:
            masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
            next_sentence_loss = F.cross_entropy(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1))
            total_loss = masked_lm_loss + next_sentence_loss

        if not return_dict:
            output = (prediction_scores, seq_relationship_score) + outputs[2:]
            return ((total_loss,) + output) if total_loss is not None else output

        return BertForPreTrainingOutput(
            loss=total_loss,
            prediction_logits=prediction_scores,
            seq_relationship_logits=seq_relationship_score,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertForPreTraining.__init__(config)

Initializes a new instance of the BertForPreTraining class.

PARAMETER	DESCRIPTION
self	The object itself.
config	A configuration object that specifies the model hyperparameters and other settings. It should be an instance of the BertConfig class.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes a new instance of the BertForPreTraining class.

    Args:
        self: The object itself.
        config: A configuration object that specifies the model hyperparameters and other settings.
            It should be an instance of the BertConfig class.

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)

    self.bert = BertModel(config)
    self.cls = BertPreTrainingHeads(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertForPreTraining.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, next_sentence_label=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Constructs the pre-training model for BERT.

PARAMETER	DESCRIPTION
self	The instance of the BertForPreTraining class.
input_ids	The input token IDs. Default is None. TYPE: Optional[Tensor] DEFAULT: None
attention_mask	The attention mask for the input. Default is None. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	The token type IDs. Default is None. TYPE: Optional[Tensor] DEFAULT: None
position_ids	The position IDs of tokens. Default is None. TYPE: Optional[Tensor] DEFAULT: None
head_mask	The mask for heads in the self-attention mechanism. Default is None. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The embedded input tokens. Default is None. TYPE: Optional[Tensor] DEFAULT: None
labels	The labels for masked language modeling task. Default is None. TYPE: Optional[Tensor] DEFAULT: None
next_sentence_label	The label for next sentence prediction task. Default is None. TYPE: Optional[Tensor] DEFAULT: None
output_attentions	Whether to output attentions. Default is None. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Whether to output hidden states. Default is None. TYPE: Optional[bool] DEFAULT: None
return_dict	Whether to return the output as a dictionary. Default is None. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple[Tensor], BertForPreTrainingOutput]	Union[Tuple[mindspore.Tensor], BertForPreTrainingOutput]: A tuple containing the prediction scores for masked language modeling and next sentence prediction tasks, and additional outputs if specified.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    next_sentence_label: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], BertForPreTrainingOutput]:
    """
    Constructs the pre-training model for BERT.

    Args:
        self: The instance of the BertForPreTraining class.
        input_ids (Optional[mindspore.Tensor]): The input token IDs. Default is None.
        attention_mask (Optional[mindspore.Tensor]): The attention mask for the input. Default is None.
        token_type_ids (Optional[mindspore.Tensor]): The token type IDs. Default is None.
        position_ids (Optional[mindspore.Tensor]): The position IDs of tokens. Default is None.
        head_mask (Optional[mindspore.Tensor]): The mask for heads in the self-attention mechanism. Default is None.
        inputs_embeds (Optional[mindspore.Tensor]): The embedded input tokens. Default is None.
        labels (Optional[mindspore.Tensor]): The labels for masked language modeling task. Default is None.
        next_sentence_label (Optional[mindspore.Tensor]): The label for next sentence prediction task. Default is None.
        output_attentions (Optional[bool]): Whether to output attentions. Default is None.
        output_hidden_states (Optional[bool]): Whether to output hidden states. Default is None.
        return_dict (Optional[bool]): Whether to return the output as a dictionary. Default is None.

    Returns:
        Union[Tuple[mindspore.Tensor], BertForPreTrainingOutput]:
            A tuple containing the prediction scores for masked language modeling
            and next sentence prediction tasks, and additional outputs if specified.

    Raises:
        None
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output, pooled_output = outputs[:2]
    prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)

    total_loss = None
    if labels is not None and next_sentence_label is not None:
        masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
        next_sentence_loss = F.cross_entropy(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1))
        total_loss = masked_lm_loss + next_sentence_loss

    if not return_dict:
        output = (prediction_scores, seq_relationship_score) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

    return BertForPreTrainingOutput(
        loss=total_loss,
        prediction_logits=prediction_scores,
        seq_relationship_logits=seq_relationship_score,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertForPreTraining.get_output_embeddings()

Method to retrieve the output embeddings from a BERT model for pre-training.

PARAMETER	DESCRIPTION
self	Instance of the BertForPreTraining class. This parameter refers to the current instance of the BertForPreTraining class.

RETURNS	DESCRIPTION
None	This method returns None, as it retrieves the output embeddings for further processing.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def get_output_embeddings(self):
    """
    Method to retrieve the output embeddings from a BERT model for pre-training.

    Args:
        self: Instance of the BertForPreTraining class.
            This parameter refers to the current instance of the BertForPreTraining class.

    Returns:
        None:
            This method returns None, as it retrieves the output embeddings for further processing.

    Raises:
        None.
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.bert.modeling_bert.BertForPreTraining.set_output_embeddings(new_embeddings)

Sets the output embeddings of the model with the provided new embeddings.

PARAMETER	DESCRIPTION
self	An instance of the BertForPreTraining class. TYPE: BertForPreTraining
new_embeddings	The new embeddings to be set for the model's output. TYPE: Any

RETURNS	DESCRIPTION
None	This method modifies the model's output embeddings in-place.

Note

The 'new_embeddings' parameter should be of the same shape and type as the original output embeddings. Modifying the output embeddings may affect the model's performance and downstream tasks.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def set_output_embeddings(self, new_embeddings):
    """
    Sets the output embeddings of the model with the provided new embeddings.

    Args:
        self (BertForPreTraining): An instance of the BertForPreTraining class.
        new_embeddings (Any): The new embeddings to be set for the model's output.

    Returns:
        None: This method modifies the model's output embeddings in-place.

    Raises:
        None.

    Note:
        The 'new_embeddings' parameter should be of the same shape and type as the original output embeddings.
        Modifying the output embeddings may affect the model's performance and downstream tasks.
    """
    self.cls.predictions.decoder = new_embeddings

mindnlp.transformers.models.bert.modeling_bert.BertForPreTrainingOutput dataclass

Bases: ModelOutput

Output type of [BertForPreTraining].

PARAMETER	DESCRIPTION
loss	Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. TYPE: *optional*, returned when `labels` is provided, `mindspore.Tensor` of shape `(1,)` DEFAULT: None
prediction_logits	Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)` DEFAULT: None
seq_relationship_logits	Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). TYPE: `mindspore.Tensor` of shape `(batch_size, 2)` DEFAULT: None
hidden_states	Tuple of mindspore.Tensor (one for the output of the embeddings + one for the output of each layer) of shape (batch_size, sequence_length, hidden_size). Hidden-states of the model at the output of each layer plus the initial embedding outputs. TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True` DEFAULT: None
attentions	Tuple of mindspore.Tensor (one for each layer) of shape (batch_size, num_heads, sequence_length, sequence_length). Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. TYPE: `tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True` DEFAULT: None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

@dataclass
class BertForPreTrainingOutput(ModelOutput):
    """
    Output type of [`BertForPreTraining`].

    Args:
        loss (*optional*, returned when `labels` is provided, `mindspore.Tensor` of shape `(1,)`):
            Total loss as the sum of the masked language modeling loss and the next sequence prediction
            (classification) loss.
        prediction_logits (`mindspore.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        seq_relationship_logits (`mindspore.Tensor` of shape `(batch_size, 2)`):
            Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
            before SoftMax).
        hidden_states (`tuple(mindspore.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `mindspore.Tensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(mindspore.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `mindspore.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    """
    loss: Optional[mindspore.Tensor] = None
    prediction_logits: mindspore.Tensor = None
    seq_relationship_logits: mindspore.Tensor = None
    hidden_states: Optional[Tuple[mindspore.Tensor]] = None
    attentions: Optional[Tuple[mindspore.Tensor]] = None

mindnlp.transformers.models.bert.modeling_bert.BertForPretraining

Bases: BertPreTrainedModel

Bert For Pretraining

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertForPretraining(BertPreTrainedModel):
    r"""
    Bert For Pretraining
    """
    _tied_weights_keys = ["predictions.decoder.bias", "cls.predictions.decoder.weight"]

    def __init__(self, config):
        """
        Initializes a new instance of BertForPretraining.

        Args:
            self: The instance of the class.
            config: A dictionary containing the configuration settings for the model.

        Returns:
            None.

        Raises:
            TypeError: If the provided config parameter is not a dictionary.
            ValueError: If the configuration settings are invalid or missing required fields.
        """
        super().__init__(config)

        self.bert = BertModel(config)
        self.cls = BertPreTrainingHeads(config)

        # Initialize weights and apply final processing
        self.post_init()

    def get_output_embeddings(self):
        """
        This method retrieves the output embeddings for the BertForPretraining model.

        Args:
            self: The instance of the class BertForPretraining.
                It is the implicit parameter representing the instance of the class itself.

        Returns:
            None: This method returns the output embeddings through the self.cls.predictions.decoder attribute.

        Raises:
            None.
        """
        return self.cls.predictions.decoder

    def set_output_embeddings(self, new_embeddings):
        """
        Sets the output embeddings for the BertForPretraining model.

        Args:
            self (BertForPretraining): An instance of the BertForPretraining class.
            new_embeddings: The new embeddings that will be set as the output embeddings.
                Should be compatible with the model's architecture.

        Returns:
            None: This method modifies the model in-place.

        Raises:
            None.
        """
        self.cls.predictions.decoder = new_embeddings

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        next_sentence_label: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        """
        Construct method in the BertForPretraining class.

        Args:
            self: The instance of the class.
            input_ids (Optional[mindspore.Tensor]):
                The input tensor containing the indices of input sequence tokens in the vocabulary. Default is None.
            attention_mask (Optional[mindspore.Tensor]):
                The input tensor containing the mask to avoid performing attention on padding token indices.
                Default is None.
            token_type_ids (Optional[mindspore.Tensor]):
                The input tensor containing the token type ids to differentiate two sequences in the input.
                Default is None.
            position_ids (Optional[mindspore.Tensor]):
                The input tensor containing the position indices of each input token in the sequence. Default is None.
            head_mask (Optional[mindspore.Tensor]):
                The input tensor containing the mask to nullify selected heads of the self-attention modules.
                Default is None.
            inputs_embeds (Optional[mindspore.Tensor]):
                The input tensor containing the embedded input sequence tokens. Default is None.
            labels (Optional[mindspore.Tensor]):
                The input tensor containing the labels for the masked language model. Default is None.
            next_sentence_label (Optional[mindspore.Tensor]):
                The input tensor containing the labels for the next sentence prediction. Default is None.
            output_attentions (Optional[bool]): Whether to return the attentions array. Default is None.
            output_hidden_states (Optional[bool]): Whether to return the hidden states. Default is None.
            return_dict (Optional[bool]): Whether to return outputs as a dict. Default is None.

        Returns:
            None.

        Raises:
            None

        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output, pooled_output = outputs[:2]
        prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)

        total_loss = None
        if labels is not None and next_sentence_label is not None:
            masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
            next_sentence_loss = F.cross_entropy(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1))
            total_loss = masked_lm_loss + next_sentence_loss

        if not return_dict:
            output = (prediction_scores, seq_relationship_score) + outputs[2:]
            return ((total_loss,) + output) if total_loss is not None else output

        return BertForPreTrainingOutput(
            loss=total_loss,
            prediction_logits=prediction_scores,
            seq_relationship_logits=seq_relationship_score,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertForPretraining.__init__(config)

Initializes a new instance of BertForPretraining.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	A dictionary containing the configuration settings for the model.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the provided config parameter is not a dictionary.
ValueError	If the configuration settings are invalid or missing required fields.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes a new instance of BertForPretraining.

    Args:
        self: The instance of the class.
        config: A dictionary containing the configuration settings for the model.

    Returns:
        None.

    Raises:
        TypeError: If the provided config parameter is not a dictionary.
        ValueError: If the configuration settings are invalid or missing required fields.
    """
    super().__init__(config)

    self.bert = BertModel(config)
    self.cls = BertPreTrainingHeads(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertForPretraining.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, next_sentence_label=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Construct method in the BertForPretraining class.

PARAMETER	DESCRIPTION
self	The instance of the class.
input_ids	The input tensor containing the indices of input sequence tokens in the vocabulary. Default is None. TYPE: Optional[Tensor] DEFAULT: None
attention_mask	The input tensor containing the mask to avoid performing attention on padding token indices. Default is None. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	The input tensor containing the token type ids to differentiate two sequences in the input. Default is None. TYPE: Optional[Tensor] DEFAULT: None
position_ids	The input tensor containing the position indices of each input token in the sequence. Default is None. TYPE: Optional[Tensor] DEFAULT: None
head_mask	The input tensor containing the mask to nullify selected heads of the self-attention modules. Default is None. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The input tensor containing the embedded input sequence tokens. Default is None. TYPE: Optional[Tensor] DEFAULT: None
labels	The input tensor containing the labels for the masked language model. Default is None. TYPE: Optional[Tensor] DEFAULT: None
next_sentence_label	The input tensor containing the labels for the next sentence prediction. Default is None. TYPE: Optional[Tensor] DEFAULT: None
output_attentions	Whether to return the attentions array. Default is None. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Whether to return the hidden states. Default is None. TYPE: Optional[bool] DEFAULT: None
return_dict	Whether to return outputs as a dict. Default is None. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    next_sentence_label: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    """
    Construct method in the BertForPretraining class.

    Args:
        self: The instance of the class.
        input_ids (Optional[mindspore.Tensor]):
            The input tensor containing the indices of input sequence tokens in the vocabulary. Default is None.
        attention_mask (Optional[mindspore.Tensor]):
            The input tensor containing the mask to avoid performing attention on padding token indices.
            Default is None.
        token_type_ids (Optional[mindspore.Tensor]):
            The input tensor containing the token type ids to differentiate two sequences in the input.
            Default is None.
        position_ids (Optional[mindspore.Tensor]):
            The input tensor containing the position indices of each input token in the sequence. Default is None.
        head_mask (Optional[mindspore.Tensor]):
            The input tensor containing the mask to nullify selected heads of the self-attention modules.
            Default is None.
        inputs_embeds (Optional[mindspore.Tensor]):
            The input tensor containing the embedded input sequence tokens. Default is None.
        labels (Optional[mindspore.Tensor]):
            The input tensor containing the labels for the masked language model. Default is None.
        next_sentence_label (Optional[mindspore.Tensor]):
            The input tensor containing the labels for the next sentence prediction. Default is None.
        output_attentions (Optional[bool]): Whether to return the attentions array. Default is None.
        output_hidden_states (Optional[bool]): Whether to return the hidden states. Default is None.
        return_dict (Optional[bool]): Whether to return outputs as a dict. Default is None.

    Returns:
        None.

    Raises:
        None

    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output, pooled_output = outputs[:2]
    prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)

    total_loss = None
    if labels is not None and next_sentence_label is not None:
        masked_lm_loss = F.cross_entropy(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
        next_sentence_loss = F.cross_entropy(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1))
        total_loss = masked_lm_loss + next_sentence_loss

    if not return_dict:
        output = (prediction_scores, seq_relationship_score) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

    return BertForPreTrainingOutput(
        loss=total_loss,
        prediction_logits=prediction_scores,
        seq_relationship_logits=seq_relationship_score,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertForPretraining.get_output_embeddings()

This method retrieves the output embeddings for the BertForPretraining model.

PARAMETER	DESCRIPTION
self	The instance of the class BertForPretraining. It is the implicit parameter representing the instance of the class itself.

RETURNS	DESCRIPTION
None	This method returns the output embeddings through the self.cls.predictions.decoder attribute.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def get_output_embeddings(self):
    """
    This method retrieves the output embeddings for the BertForPretraining model.

    Args:
        self: The instance of the class BertForPretraining.
            It is the implicit parameter representing the instance of the class itself.

    Returns:
        None: This method returns the output embeddings through the self.cls.predictions.decoder attribute.

    Raises:
        None.
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.bert.modeling_bert.BertForPretraining.set_output_embeddings(new_embeddings)

Sets the output embeddings for the BertForPretraining model.

PARAMETER	DESCRIPTION
self	An instance of the BertForPretraining class. TYPE: BertForPretraining
new_embeddings	The new embeddings that will be set as the output embeddings. Should be compatible with the model's architecture.

RETURNS	DESCRIPTION
None	This method modifies the model in-place.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def set_output_embeddings(self, new_embeddings):
    """
    Sets the output embeddings for the BertForPretraining model.

    Args:
        self (BertForPretraining): An instance of the BertForPretraining class.
        new_embeddings: The new embeddings that will be set as the output embeddings.
            Should be compatible with the model's architecture.

    Returns:
        None: This method modifies the model in-place.

    Raises:
        None.
    """
    self.cls.predictions.decoder = new_embeddings

mindnlp.transformers.models.bert.modeling_bert.BertForQuestionAnswering

Bases: BertPreTrainedModel

BertForQuestionAnswering

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertForQuestionAnswering(BertPreTrainedModel):
    """BertForQuestionAnswering"""
    def __init__(self, config):
        """
        Initializes a new instance of the BertForQuestionAnswering class.

        Args:
            self: The object itself.
            config (BertConfig):
                The configuration for the Bert model. It contains various hyperparameters and settings.

                - Type: BertConfig
                - Purpose: Specifies the configuration for the Bert model.
                - Restrictions: None

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.num_labels = config.num_labels

        self.bert = BertModel(config, add_pooling_layer=False)
        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        start_positions: Optional[mindspore.Tensor] = None,
        end_positions: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        """
        Constructs the BertForQuestionAnswering model.

        Args:
            self (BertForQuestionAnswering): The instance of the BertForQuestionAnswering class.
            input_ids (Optional[mindspore.Tensor]): The input tensor containing the indices of the input sequence tokens.
            attention_mask (Optional[mindspore.Tensor]):
                The input tensor containing the attention mask to avoid performing attention on padding tokens.
            token_type_ids (Optional[mindspore.Tensor]):
                The input tensor containing the segment token indices to indicate which tokens belong to the question
                and which belong to the context.
            position_ids (Optional[mindspore.Tensor]):
                The input tensor containing the position indices to indicate the position of each token in the
                input sequence.
            head_mask (Optional[mindspore.Tensor]):
                The input tensor containing the mask to nullify selected heads of the self-attention modules.
            inputs_embeds (Optional[mindspore.Tensor]):
                The input tensor containing the embedded representation of the inputs.
            start_positions (Optional[mindspore.Tensor]):
                The input tensor containing the indices of the start positions for the answer span.
            end_positions (Optional[mindspore.Tensor]):
                The input tensor containing the indices of the end positions for the answer span.
            output_attentions (Optional[bool]): Whether to return the attentions weights of each layer in the outputs.
            output_hidden_states (Optional[bool]): Whether to return the hidden states of all layers in the outputs.
            return_dict (Optional[bool]): Whether to return a dictionary as the output instead of a tuple.

        Returns:
            None.

        Raises:
            None.
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]

        logits = self.qa_outputs(sequence_output)
        start_logits, end_logits = logits.split(1, axis=-1)
        start_logits = start_logits.squeeze(-1)
        end_logits = end_logits.squeeze(-1)

        total_loss = None
        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, split add a dimension
            if len(start_positions.shape) > 1:
                start_positions = start_positions.squeeze(-1)
            if len(end_positions.shape) > 1:
                end_positions = end_positions.squeeze(-1)
            # sometimes the start/end positions are outside our model inputs, we ignore these terms
            ignored_index = start_logits.shape[1]
            start_positions = start_positions.clamp(0, ignored_index)
            end_positions = end_positions.clamp(0, ignored_index)

            start_loss = F.cross_entropy(start_logits, start_positions, ignore_index=ignored_index)
            end_loss = F.cross_entropy(end_logits, end_positions)
            total_loss = (start_loss + end_loss) / 2

        if not return_dict:
            output = (start_logits, end_logits) + outputs[2:]
            return ((total_loss,) + output) if total_loss is not None else output

        return QuestionAnsweringModelOutput(
            loss=total_loss,
            start_logits=start_logits,
            end_logits=end_logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertForQuestionAnswering.__init__(config)

Initializes a new instance of the BertForQuestionAnswering class.

PARAMETER	DESCRIPTION
self	The object itself.
config	The configuration for the Bert model. It contains various hyperparameters and settings. Type: BertConfig Purpose: Specifies the configuration for the Bert model. Restrictions: None TYPE: BertConfig

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes a new instance of the BertForQuestionAnswering class.

    Args:
        self: The object itself.
        config (BertConfig):
            The configuration for the Bert model. It contains various hyperparameters and settings.

            - Type: BertConfig
            - Purpose: Specifies the configuration for the Bert model.
            - Restrictions: None

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.num_labels = config.num_labels

    self.bert = BertModel(config, add_pooling_layer=False)
    self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertForQuestionAnswering.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, start_positions=None, end_positions=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Constructs the BertForQuestionAnswering model.

PARAMETER	DESCRIPTION
self	The instance of the BertForQuestionAnswering class. TYPE: BertForQuestionAnswering
input_ids	The input tensor containing the indices of the input sequence tokens. TYPE: Optional[Tensor] DEFAULT: None
attention_mask	The input tensor containing the attention mask to avoid performing attention on padding tokens. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	The input tensor containing the segment token indices to indicate which tokens belong to the question and which belong to the context. TYPE: Optional[Tensor] DEFAULT: None
position_ids	The input tensor containing the position indices to indicate the position of each token in the input sequence. TYPE: Optional[Tensor] DEFAULT: None
head_mask	The input tensor containing the mask to nullify selected heads of the self-attention modules. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The input tensor containing the embedded representation of the inputs. TYPE: Optional[Tensor] DEFAULT: None
start_positions	The input tensor containing the indices of the start positions for the answer span. TYPE: Optional[Tensor] DEFAULT: None
end_positions	The input tensor containing the indices of the end positions for the answer span. TYPE: Optional[Tensor] DEFAULT: None
output_attentions	Whether to return the attentions weights of each layer in the outputs. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Whether to return the hidden states of all layers in the outputs. TYPE: Optional[bool] DEFAULT: None
return_dict	Whether to return a dictionary as the output instead of a tuple. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    start_positions: Optional[mindspore.Tensor] = None,
    end_positions: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    """
    Constructs the BertForQuestionAnswering model.

    Args:
        self (BertForQuestionAnswering): The instance of the BertForQuestionAnswering class.
        input_ids (Optional[mindspore.Tensor]): The input tensor containing the indices of the input sequence tokens.
        attention_mask (Optional[mindspore.Tensor]):
            The input tensor containing the attention mask to avoid performing attention on padding tokens.
        token_type_ids (Optional[mindspore.Tensor]):
            The input tensor containing the segment token indices to indicate which tokens belong to the question
            and which belong to the context.
        position_ids (Optional[mindspore.Tensor]):
            The input tensor containing the position indices to indicate the position of each token in the
            input sequence.
        head_mask (Optional[mindspore.Tensor]):
            The input tensor containing the mask to nullify selected heads of the self-attention modules.
        inputs_embeds (Optional[mindspore.Tensor]):
            The input tensor containing the embedded representation of the inputs.
        start_positions (Optional[mindspore.Tensor]):
            The input tensor containing the indices of the start positions for the answer span.
        end_positions (Optional[mindspore.Tensor]):
            The input tensor containing the indices of the end positions for the answer span.
        output_attentions (Optional[bool]): Whether to return the attentions weights of each layer in the outputs.
        output_hidden_states (Optional[bool]): Whether to return the hidden states of all layers in the outputs.
        return_dict (Optional[bool]): Whether to return a dictionary as the output instead of a tuple.

    Returns:
        None.

    Raises:
        None.
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = outputs[0]

    logits = self.qa_outputs(sequence_output)
    start_logits, end_logits = logits.split(1, axis=-1)
    start_logits = start_logits.squeeze(-1)
    end_logits = end_logits.squeeze(-1)

    total_loss = None
    if start_positions is not None and end_positions is not None:
        # If we are on multi-GPU, split add a dimension
        if len(start_positions.shape) > 1:
            start_positions = start_positions.squeeze(-1)
        if len(end_positions.shape) > 1:
            end_positions = end_positions.squeeze(-1)
        # sometimes the start/end positions are outside our model inputs, we ignore these terms
        ignored_index = start_logits.shape[1]
        start_positions = start_positions.clamp(0, ignored_index)
        end_positions = end_positions.clamp(0, ignored_index)

        start_loss = F.cross_entropy(start_logits, start_positions, ignore_index=ignored_index)
        end_loss = F.cross_entropy(end_logits, end_positions)
        total_loss = (start_loss + end_loss) / 2

    if not return_dict:
        output = (start_logits, end_logits) + outputs[2:]
        return ((total_loss,) + output) if total_loss is not None else output

    return QuestionAnsweringModelOutput(
        loss=total_loss,
        start_logits=start_logits,
        end_logits=end_logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertForSequenceClassification

Bases: BertPreTrainedModel

Bert Model for classification tasks

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertForSequenceClassification(BertPreTrainedModel):
    """Bert Model for classification tasks"""
    def __init__(self, config):
        """
        Initializes the BertForSequenceClassification class.

        Args:
            self (BertForSequenceClassification): The current instance of the BertForSequenceClassification class.
            config (BertConfig): The configuration object for the BertModel.
                It specifies the model architecture and parameters.

        Returns:
            None.

        Raises:
            ValueError: If the provided configuration object is invalid or missing required parameters.
            TypeError: If the configuration object is not of type BertConfig.
        """
        super().__init__(config)
        self.num_labels = config.num_labels
        self.config = config

        self.bert = BertModel(config)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        '''
        This method forwards the Bert model for sequence classification.

        Args:
            self (BertForSequenceClassification): The instance of the BertForSequenceClassification class.
            input_ids (Optional[mindspore.Tensor]):
                The input tensor containing the indices of input sequence tokens in the vocabulary.
            attention_mask (Optional[mindspore.Tensor]):
                The input tensor containing the attention mask to avoid performing attention on padding token indices.
            token_type_ids (Optional[mindspore.Tensor]):
                The input tensor containing the token type ids to differentiate between two sequences in the input.
            position_ids (Optional[mindspore.Tensor]):
                The input tensor containing the position indices to position embeddings.
            head_mask (Optional[mindspore.Tensor]):
                The input tensor containing the mask for the heads which controls which head is executed.
            inputs_embeds (Optional[mindspore.Tensor]):
                The input tensor containing the embeddings of the input sequence tokens.
            labels (Optional[mindspore.Tensor]): The input tensor containing the labels for computing the loss.
            output_attentions (Optional[bool]): Whether to return attentions.
            output_hidden_states (Optional[bool]): Whether to return hidden states.
            return_dict (Optional[bool]): Whether to return a sequence classifier output as a dictionary.

        Returns:
            None

        Raises:
            TypeError: If the input tensors are not of type mindspore.Tensor.
            ValueError: If there is a mismatch in the dimensions or types of the input tensors.
        '''
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = outputs[1]

        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)

        loss = None
        if labels is not None:
            if self.config.problem_type is None:
                if self.num_labels == 1:
                    self.config.problem_type = "regression"
                elif self.num_labels > 1 and labels.dtype in (mindspore.int32, mindspore.int64):
                    self.config.problem_type = "single_label_classification"
                else:
                    self.config.problem_type = "multi_label_classification"

            if self.config.problem_type == "regression":
                if self.num_labels == 1:
                    loss = ops.mse_loss(logits.squeeze(), labels.squeeze())
                else:
                    loss = ops.mse_loss(logits, labels)
            elif self.config.problem_type == "single_label_classification":
                loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))
            elif self.config.problem_type == "multi_label_classification":
                loss = ops.binary_cross_entropy_with_logits(logits, labels)
        if not return_dict:
            output = (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertForSequenceClassification.__init__(config)

Initializes the BertForSequenceClassification class.

PARAMETER	DESCRIPTION
self	The current instance of the BertForSequenceClassification class. TYPE: BertForSequenceClassification
config	The configuration object for the BertModel. It specifies the model architecture and parameters. TYPE: BertConfig

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the provided configuration object is invalid or missing required parameters.
TypeError	If the configuration object is not of type BertConfig.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes the BertForSequenceClassification class.

    Args:
        self (BertForSequenceClassification): The current instance of the BertForSequenceClassification class.
        config (BertConfig): The configuration object for the BertModel.
            It specifies the model architecture and parameters.

    Returns:
        None.

    Raises:
        ValueError: If the provided configuration object is invalid or missing required parameters.
        TypeError: If the configuration object is not of type BertConfig.
    """
    super().__init__(config)
    self.num_labels = config.num_labels
    self.config = config

    self.bert = BertModel(config)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertForSequenceClassification.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

This method forwards the Bert model for sequence classification.

PARAMETER	DESCRIPTION
self	The instance of the BertForSequenceClassification class. TYPE: BertForSequenceClassification
input_ids	The input tensor containing the indices of input sequence tokens in the vocabulary. TYPE: Optional[Tensor] DEFAULT: None
attention_mask	The input tensor containing the attention mask to avoid performing attention on padding token indices. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	The input tensor containing the token type ids to differentiate between two sequences in the input. TYPE: Optional[Tensor] DEFAULT: None
position_ids	The input tensor containing the position indices to position embeddings. TYPE: Optional[Tensor] DEFAULT: None
head_mask	The input tensor containing the mask for the heads which controls which head is executed. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The input tensor containing the embeddings of the input sequence tokens. TYPE: Optional[Tensor] DEFAULT: None
labels	The input tensor containing the labels for computing the loss. TYPE: Optional[Tensor] DEFAULT: None
output_attentions	Whether to return attentions. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Whether to return hidden states. TYPE: Optional[bool] DEFAULT: None
return_dict	Whether to return a sequence classifier output as a dictionary. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
	None

RAISES	DESCRIPTION
TypeError	If the input tensors are not of type mindspore.Tensor.
ValueError	If there is a mismatch in the dimensions or types of the input tensors.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    '''
    This method forwards the Bert model for sequence classification.

    Args:
        self (BertForSequenceClassification): The instance of the BertForSequenceClassification class.
        input_ids (Optional[mindspore.Tensor]):
            The input tensor containing the indices of input sequence tokens in the vocabulary.
        attention_mask (Optional[mindspore.Tensor]):
            The input tensor containing the attention mask to avoid performing attention on padding token indices.
        token_type_ids (Optional[mindspore.Tensor]):
            The input tensor containing the token type ids to differentiate between two sequences in the input.
        position_ids (Optional[mindspore.Tensor]):
            The input tensor containing the position indices to position embeddings.
        head_mask (Optional[mindspore.Tensor]):
            The input tensor containing the mask for the heads which controls which head is executed.
        inputs_embeds (Optional[mindspore.Tensor]):
            The input tensor containing the embeddings of the input sequence tokens.
        labels (Optional[mindspore.Tensor]): The input tensor containing the labels for computing the loss.
        output_attentions (Optional[bool]): Whether to return attentions.
        output_hidden_states (Optional[bool]): Whether to return hidden states.
        return_dict (Optional[bool]): Whether to return a sequence classifier output as a dictionary.

    Returns:
        None

    Raises:
        TypeError: If the input tensors are not of type mindspore.Tensor.
        ValueError: If there is a mismatch in the dimensions or types of the input tensors.
    '''
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    pooled_output = outputs[1]

    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    loss = None
    if labels is not None:
        if self.config.problem_type is None:
            if self.num_labels == 1:
                self.config.problem_type = "regression"
            elif self.num_labels > 1 and labels.dtype in (mindspore.int32, mindspore.int64):
                self.config.problem_type = "single_label_classification"
            else:
                self.config.problem_type = "multi_label_classification"

        if self.config.problem_type == "regression":
            if self.num_labels == 1:
                loss = ops.mse_loss(logits.squeeze(), labels.squeeze())
            else:
                loss = ops.mse_loss(logits, labels)
        elif self.config.problem_type == "single_label_classification":
            loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))
        elif self.config.problem_type == "multi_label_classification":
            loss = ops.binary_cross_entropy_with_logits(logits, labels)
    if not return_dict:
        output = (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

    return SequenceClassifierOutput(
        loss=loss,
        logits=logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertForTokenClassification

Bases: BertPreTrainedModel

BertForTokenClassification

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertForTokenClassification(BertPreTrainedModel):
    """BertForTokenClassification"""
    def __init__(self, config):
        """
        Initialize the BertForTokenClassification model.

        Args:
            self (BertForTokenClassification): The instance of the BertForTokenClassification class.
            config:
                A configuration object containing settings for the model.

                - Type: object
                - Purpose: Specifies the configuration settings for the model.
                - Restrictions: Must be a valid configuration object.

        Returns:
            None.

        Raises:
            None
        """
        super().__init__(config)
        self.num_labels = config.num_labels

        self.bert = BertModel(config, add_pooling_layer=False)
        classifier_dropout = (
            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
        )
        self.dropout = nn.Dropout(p=classifier_dropout)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        r"""
        Args:
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]

        sequence_output = self.dropout(sequence_output)
        logits = self.classifier(sequence_output)

        loss = None
        if labels is not None:
            loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))

        if not return_dict:
            output = (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return TokenClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertForTokenClassification.__init__(config)

Initialize the BertForTokenClassification model.

PARAMETER	DESCRIPTION
self	The instance of the BertForTokenClassification class. TYPE: BertForTokenClassification
config	A configuration object containing settings for the model. Type: object Purpose: Specifies the configuration settings for the model. Restrictions: Must be a valid configuration object.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initialize the BertForTokenClassification model.

    Args:
        self (BertForTokenClassification): The instance of the BertForTokenClassification class.
        config:
            A configuration object containing settings for the model.

            - Type: object
            - Purpose: Specifies the configuration settings for the model.
            - Restrictions: Must be a valid configuration object.

    Returns:
        None.

    Raises:
        None
    """
    super().__init__(config)
    self.num_labels = config.num_labels

    self.bert = BertModel(config, add_pooling_layer=False)
    classifier_dropout = (
        config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
    )
    self.dropout = nn.Dropout(p=classifier_dropout)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertForTokenClassification.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the token classification loss. Indices should be in [0, ..., config.num_labels - 1]. TYPE: `torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    r"""
    Args:
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = outputs[0]

    sequence_output = self.dropout(sequence_output)
    logits = self.classifier(sequence_output)

    loss = None
    if labels is not None:
        loss = F.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))

    if not return_dict:
        output = (logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

    return TokenClassifierOutput(
        loss=loss,
        logits=logits,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertIntermediate

Bases: Module

Bert Intermediate

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertIntermediate(nn.Module):
    r"""
    Bert Intermediate
    """
    def __init__(self, config):
        """
        Initializes an instance of the BertIntermediate class.

        Args:
            self: The instance of the class.
            config: An object of type 'Config' containing the configuration settings.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
        if isinstance(config.hidden_act, str):
            self.intermediate_act_fn = ACT2FN[config.hidden_act]
        else:
            self.intermediate_act_fn = config.hidden_act

    def forward(self, hidden_states):
        """
        Constructs the intermediate layer of the BERT model.

        Args:
            self (BertIntermediate): An instance of the BertIntermediate class.
            hidden_states: The input hidden states to the intermediate layer.
                It should be a tensor of shape (batch_size, sequence_length, hidden_size).

        Returns:
            None

        Raises:
            None
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.intermediate_act_fn(hidden_states)
        return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertIntermediate.__init__(config)

Initializes an instance of the BertIntermediate class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An object of type 'Config' containing the configuration settings.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of the BertIntermediate class.

    Args:
        self: The instance of the class.
        config: An object of type 'Config' containing the configuration settings.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
    if isinstance(config.hidden_act, str):
        self.intermediate_act_fn = ACT2FN[config.hidden_act]
    else:
        self.intermediate_act_fn = config.hidden_act

mindnlp.transformers.models.bert.modeling_bert.BertIntermediate.forward(hidden_states)

Constructs the intermediate layer of the BERT model.

PARAMETER	DESCRIPTION
self	An instance of the BertIntermediate class. TYPE: BertIntermediate
hidden_states	The input hidden states to the intermediate layer. It should be a tensor of shape (batch_size, sequence_length, hidden_size).

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states):
    """
    Constructs the intermediate layer of the BERT model.

    Args:
        self (BertIntermediate): An instance of the BertIntermediate class.
        hidden_states: The input hidden states to the intermediate layer.
            It should be a tensor of shape (batch_size, sequence_length, hidden_size).

    Returns:
        None

    Raises:
        None
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.intermediate_act_fn(hidden_states)
    return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertLMHeadModel

Bases: BertPreTrainedModel

BertLMHeadModel

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertLMHeadModel(BertPreTrainedModel):
    """BertLMHeadModel"""
    _tied_weights_keys = ["predictions.decoder.bias", "cls.predictions.decoder.weight"]

    def __init__(self, config):
        """
        Initializes a new instance of BertLMHeadModel.

        Args:
            self: The instance of the class.
            config: A dictionary containing the configuration parameters for the model.
                It should include the following keys:

                - is_decoder (bool): Indicates whether the model is used as a decoder.

                    - If False, a warning will be logged.
                    - If True, the model will be initialized with is_decoder set to True.
                    - Default is False.

        Returns:
            None.

        Raises:
            None
        """
        super().__init__(config)

        if not config.is_decoder:
            logger.warning("If you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True.`")

        self.bert = BertModel(config, add_pooling_layer=False)
        self.cls = BertOnlyMLMHead(config)

        # Initialize weights and apply final processing
        self.post_init()

    def get_output_embeddings(self):
        """
        Returns the output embeddings of the BertLMHeadModel.

        Args:
            self (BertLMHeadModel): An instance of the BertLMHeadModel class.

        Returns:
            None.

        Raises:
            None.

        This method retrieves the output embeddings of the BertLMHeadModel.
        The output embeddings are obtained by predicting the decoder of the model's predictions.

        Note:
            The output embeddings represent the encoded representation of the input tokens in the model.
            They are useful for downstream tasks such as clustering or classification.
        """
        return self.cls.predictions.decoder

    def set_output_embeddings(self, new_embeddings):
        """
        Method to set new output embeddings for the language model head in a BERT model.

        Args:
            self (BertLMHeadModel): The instance of the BertLMHeadModel class.
            new_embeddings (Tensor): The new embeddings to be set for the output layer.
                Should be a tensor compatible with the existing model architecture.

        Returns:
            None.

        Raises:
            None.
        """
        self.cls.predictions.decoder = new_embeddings

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        '''
        This method forwards the BertLMHeadModel.

        Args:
            self: The instance of the class.
            input_ids (Optional[mindspore.Tensor]):
                Input tensor containing the indices of input tokens in the vocabulary.
            attention_mask (Optional[mindspore.Tensor]):
                Masking tensor used to avoid performing attention on padding token indices.
            token_type_ids (Optional[mindspore.Tensor]):
                Tensor containing the segment indices to differentiate between two sequences in the input.
            position_ids (Optional[mindspore.Tensor]):
                Tensor containing the position indices of each input token in the sequence.
            head_mask (Optional[mindspore.Tensor]):
                Masking tensor for attention heads.
            inputs_embeds (Optional[mindspore.Tensor]):
                Tensor containing the embedded input tokens.
            encoder_hidden_states (Optional[mindspore.Tensor]):
                Tensor containing the hidden states of the encoder.
            encoder_attention_mask (Optional[mindspore.Tensor]):
                Masking tensor for encoder attention.
            labels (Optional[mindspore.Tensor]):
                Tensor containing the labels for the prediction scores.
            past_key_values (Optional[List[mindspore.Tensor]]):
                List of tensors containing cached key and value states from previous attention mechanisms.
            use_cache (Optional[bool]):
                Flag indicating whether to use the cached key and value states for attention mechanisms.
            output_attentions (Optional[bool]):
                Flag indicating whether to output the attention weights of all layers.
            output_hidden_states (Optional[bool]):
                Flag indicating whether to output the hidden states of all layers.
            return_dict (Optional[bool]):
                Flag indicating whether to return the output as a dictionary.

        Returns:
            None.

        Raises:
            None
        '''
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        if labels is not None:
            use_cache = False

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            past_key_values=past_key_values,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]
        prediction_scores = self.cls(sequence_output)

        lm_loss = None
        if labels is not None:
            # we are doing next-token prediction; shift prediction scores and input ids by one
            shifted_prediction_scores = prediction_scores[:, :-1, :]
            labels = labels[:, 1:]
            lm_loss = F.cross_entropy(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (prediction_scores,) + outputs[2:]
            return ((lm_loss,) + output) if lm_loss is not None else output

        return CausalLMOutputWithCrossAttentions(
            loss=lm_loss,
            logits=prediction_scores,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            cross_attentions=outputs.cross_attentions,
        )

    def prepare_inputs_for_generation(
        self, input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs
    ):
        """
        Prepare inputs for generation.

        Args:
            self (BertLMHeadModel): The instance of the BertLMHeadModel class.
            input_ids (torch.Tensor): The input tensor of shape (batch_size, sequence_length) containing the input ids.
            past_key_values (tuple, optional): The tuple of past key values used for generation. Defaults to None.
            attention_mask (torch.Tensor, optional):
                The attention mask tensor of shape (batch_size, sequence_length) containing the attention masks.
                Defaults to None.
            use_cache (bool, optional): Whether to use cache for generation. Defaults to True.
            **model_kwargs: Additional keyword arguments for the model.

        Returns:
            dict:
                A dictionary containing the prepared inputs for generation with the following keys:

                - 'input_ids' (torch.Tensor):
                The input tensor of shape (batch_size, sequence_length) containing the updated input ids.
                - 'attention_mask' (torch.Tensor):
                The attention mask tensor of shape (batch_size, sequence_length) containing the updated attention masks.
                - 'past_key_values' (tuple):
                The tuple of past key values used for generation.
                - 'use_cache' (bool):
                Whether to use cache for generation.

        Raises:
            None.
        """
        input_shape = input_ids.shape
        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
        if attention_mask is None:
            attention_mask = input_ids.new_ones(input_shape)

        # cut decoder_input_ids if past_key_values is used
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]

        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "past_key_values": past_key_values,
            "use_cache": use_cache,
        }

    def _reorder_cache(self, past_key_values, beam_idx):
        """
        Reorders the cache based on the provided beam index for a BERT language model head model.

        Args:
            self (BertLMHeadModel): The instance of the BertLMHeadModel class.
            past_key_values (tuple): A tuple containing past key values for each layer of the model.
            beam_idx (torch.Tensor): A tensor representing the beam index to reorder the cache.

        Returns:
            None: This method modifies the cache in-place.

        Raises:
            None
        """
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),
            )
        return reordered_past

mindnlp.transformers.models.bert.modeling_bert.BertLMHeadModel.__init__(config)

Initializes a new instance of BertLMHeadModel.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	A dictionary containing the configuration parameters for the model. It should include the following keys: is_decoder (bool): Indicates whether the model is used as a decoder. If False, a warning will be logged. If True, the model will be initialized with is_decoder set to True. Default is False.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes a new instance of BertLMHeadModel.

    Args:
        self: The instance of the class.
        config: A dictionary containing the configuration parameters for the model.
            It should include the following keys:

            - is_decoder (bool): Indicates whether the model is used as a decoder.

                - If False, a warning will be logged.
                - If True, the model will be initialized with is_decoder set to True.
                - Default is False.

    Returns:
        None.

    Raises:
        None
    """
    super().__init__(config)

    if not config.is_decoder:
        logger.warning("If you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True.`")

    self.bert = BertModel(config, add_pooling_layer=False)
    self.cls = BertOnlyMLMHead(config)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertLMHeadModel.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, labels=None, past_key_values=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

This method forwards the BertLMHeadModel.

PARAMETER	DESCRIPTION
self	The instance of the class.
input_ids	Input tensor containing the indices of input tokens in the vocabulary. TYPE: Optional[Tensor] DEFAULT: None
attention_mask	Masking tensor used to avoid performing attention on padding token indices. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	Tensor containing the segment indices to differentiate between two sequences in the input. TYPE: Optional[Tensor] DEFAULT: None
position_ids	Tensor containing the position indices of each input token in the sequence. TYPE: Optional[Tensor] DEFAULT: None
head_mask	Masking tensor for attention heads. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	Tensor containing the embedded input tokens. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	Tensor containing the hidden states of the encoder. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	Masking tensor for encoder attention. TYPE: Optional[Tensor] DEFAULT: None
labels	Tensor containing the labels for the prediction scores. TYPE: Optional[Tensor] DEFAULT: None
past_key_values	List of tensors containing cached key and value states from previous attention mechanisms. TYPE: Optional[List[Tensor]] DEFAULT: None
use_cache	Flag indicating whether to use the cached key and value states for attention mechanisms. TYPE: Optional[bool] DEFAULT: None
output_attentions	Flag indicating whether to output the attention weights of all layers. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Flag indicating whether to output the hidden states of all layers. TYPE: Optional[bool] DEFAULT: None
return_dict	Flag indicating whether to return the output as a dictionary. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    '''
    This method forwards the BertLMHeadModel.

    Args:
        self: The instance of the class.
        input_ids (Optional[mindspore.Tensor]):
            Input tensor containing the indices of input tokens in the vocabulary.
        attention_mask (Optional[mindspore.Tensor]):
            Masking tensor used to avoid performing attention on padding token indices.
        token_type_ids (Optional[mindspore.Tensor]):
            Tensor containing the segment indices to differentiate between two sequences in the input.
        position_ids (Optional[mindspore.Tensor]):
            Tensor containing the position indices of each input token in the sequence.
        head_mask (Optional[mindspore.Tensor]):
            Masking tensor for attention heads.
        inputs_embeds (Optional[mindspore.Tensor]):
            Tensor containing the embedded input tokens.
        encoder_hidden_states (Optional[mindspore.Tensor]):
            Tensor containing the hidden states of the encoder.
        encoder_attention_mask (Optional[mindspore.Tensor]):
            Masking tensor for encoder attention.
        labels (Optional[mindspore.Tensor]):
            Tensor containing the labels for the prediction scores.
        past_key_values (Optional[List[mindspore.Tensor]]):
            List of tensors containing cached key and value states from previous attention mechanisms.
        use_cache (Optional[bool]):
            Flag indicating whether to use the cached key and value states for attention mechanisms.
        output_attentions (Optional[bool]):
            Flag indicating whether to output the attention weights of all layers.
        output_hidden_states (Optional[bool]):
            Flag indicating whether to output the hidden states of all layers.
        return_dict (Optional[bool]):
            Flag indicating whether to return the output as a dictionary.

    Returns:
        None.

    Raises:
        None
    '''
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    if labels is not None:
        use_cache = False

    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_attention_mask,
        past_key_values=past_key_values,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = outputs[0]
    prediction_scores = self.cls(sequence_output)

    lm_loss = None
    if labels is not None:
        # we are doing next-token prediction; shift prediction scores and input ids by one
        shifted_prediction_scores = prediction_scores[:, :-1, :]
        labels = labels[:, 1:]
        lm_loss = F.cross_entropy(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))

    if not return_dict:
        output = (prediction_scores,) + outputs[2:]
        return ((lm_loss,) + output) if lm_loss is not None else output

    return CausalLMOutputWithCrossAttentions(
        loss=lm_loss,
        logits=prediction_scores,
        past_key_values=outputs.past_key_values,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
        cross_attentions=outputs.cross_attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertLMHeadModel.get_output_embeddings()

Returns the output embeddings of the BertLMHeadModel.

PARAMETER	DESCRIPTION
self	An instance of the BertLMHeadModel class. TYPE: BertLMHeadModel

RETURNS	DESCRIPTION
	None.

This method retrieves the output embeddings of the BertLMHeadModel. The output embeddings are obtained by predicting the decoder of the model's predictions.

Note

The output embeddings represent the encoded representation of the input tokens in the model. They are useful for downstream tasks such as clustering or classification.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def get_output_embeddings(self):
    """
    Returns the output embeddings of the BertLMHeadModel.

    Args:
        self (BertLMHeadModel): An instance of the BertLMHeadModel class.

    Returns:
        None.

    Raises:
        None.

    This method retrieves the output embeddings of the BertLMHeadModel.
    The output embeddings are obtained by predicting the decoder of the model's predictions.

    Note:
        The output embeddings represent the encoded representation of the input tokens in the model.
        They are useful for downstream tasks such as clustering or classification.
    """
    return self.cls.predictions.decoder

mindnlp.transformers.models.bert.modeling_bert.BertLMHeadModel.prepare_inputs_for_generation(input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs)

Prepare inputs for generation.

PARAMETER	DESCRIPTION
self	The instance of the BertLMHeadModel class. TYPE: BertLMHeadModel
input_ids	The input tensor of shape (batch_size, sequence_length) containing the input ids. TYPE: Tensor
past_key_values	The tuple of past key values used for generation. Defaults to None. TYPE: tuple DEFAULT: None
attention_mask	The attention mask tensor of shape (batch_size, sequence_length) containing the attention masks. Defaults to None. TYPE: Tensor DEFAULT: None
use_cache	Whether to use cache for generation. Defaults to True. TYPE: bool DEFAULT: True
**model_kwargs	Additional keyword arguments for the model. DEFAULT: {}

RETURNS

DESCRIPTION

dict

A dictionary containing the prepared inputs for generation with the following keys: 'input_ids' (torch.Tensor): The input tensor of shape (batch_size, sequence_length) containing the updated input ids. 'attention_mask' (torch.Tensor): The attention mask tensor of shape (batch_size, sequence_length) containing the updated attention masks. 'past_key_values' (tuple): The tuple of past key values used for generation. 'use_cache' (bool): Whether to use cache for generation.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def prepare_inputs_for_generation(
    self, input_ids, past_key_values=None, attention_mask=None, use_cache=True, **model_kwargs
):
    """
    Prepare inputs for generation.

    Args:
        self (BertLMHeadModel): The instance of the BertLMHeadModel class.
        input_ids (torch.Tensor): The input tensor of shape (batch_size, sequence_length) containing the input ids.
        past_key_values (tuple, optional): The tuple of past key values used for generation. Defaults to None.
        attention_mask (torch.Tensor, optional):
            The attention mask tensor of shape (batch_size, sequence_length) containing the attention masks.
            Defaults to None.
        use_cache (bool, optional): Whether to use cache for generation. Defaults to True.
        **model_kwargs: Additional keyword arguments for the model.

    Returns:
        dict:
            A dictionary containing the prepared inputs for generation with the following keys:

            - 'input_ids' (torch.Tensor):
            The input tensor of shape (batch_size, sequence_length) containing the updated input ids.
            - 'attention_mask' (torch.Tensor):
            The attention mask tensor of shape (batch_size, sequence_length) containing the updated attention masks.
            - 'past_key_values' (tuple):
            The tuple of past key values used for generation.
            - 'use_cache' (bool):
            Whether to use cache for generation.

    Raises:
        None.
    """
    input_shape = input_ids.shape
    # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
    if attention_mask is None:
        attention_mask = input_ids.new_ones(input_shape)

    # cut decoder_input_ids if past_key_values is used
    if past_key_values is not None:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = input_ids.shape[1] - 1

        input_ids = input_ids[:, remove_prefix_length:]

    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "past_key_values": past_key_values,
        "use_cache": use_cache,
    }

mindnlp.transformers.models.bert.modeling_bert.BertLMHeadModel.set_output_embeddings(new_embeddings)

Method to set new output embeddings for the language model head in a BERT model.

PARAMETER	DESCRIPTION
self	The instance of the BertLMHeadModel class. TYPE: BertLMHeadModel
new_embeddings	The new embeddings to be set for the output layer. Should be a tensor compatible with the existing model architecture. TYPE: Tensor

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def set_output_embeddings(self, new_embeddings):
    """
    Method to set new output embeddings for the language model head in a BERT model.

    Args:
        self (BertLMHeadModel): The instance of the BertLMHeadModel class.
        new_embeddings (Tensor): The new embeddings to be set for the output layer.
            Should be a tensor compatible with the existing model architecture.

    Returns:
        None.

    Raises:
        None.
    """
    self.cls.predictions.decoder = new_embeddings

mindnlp.transformers.models.bert.modeling_bert.BertLMPredictionHead

Bases: Module

Bert LM Prediction Head

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertLMPredictionHead(nn.Module):
    r"""
    Bert LM Prediction Head
    """
    def __init__(self, config):
        """
        This method initializes the BertLMPredictionHead class.

        Args:
            self: The object instance of the BertLMPredictionHead class.
            config: A configuration object containing settings for the prediction head.
                It is of type dict or a custom configuration class.
                The config parameter is used to configure the prediction head's behavior and settings.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = Parameter(initializer('zeros', config.vocab_size), 'bias')

        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        """
        This method 'forward' is defined in the class 'BertLMPredictionHead' and is responsible for processing the hidden states.

        Args:
            self: The instance of the class.
            hidden_states (tensor): The input hidden states to be processed.
                It should be of type tensor and contain the information about the hidden states.

        Returns:
            hidden_states (tensor): The processed hidden states.
                It is of type tensor and contains the transformed and decoded information from the input hidden states.

        Raises:
            This method does not raise any exceptions.
        """
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states

    def _tie_weights(self):
        """
        Ties the weights of the bias in the BertLMPredictionHead decoder to the main decoder weights.

        Args:
            self (BertLMPredictionHead): The instance of the BertLMPredictionHead class.
                This parameter is a reference to the current object.

        Returns:
            None: This method does not return any value. It updates the bias weights in-place.

        Raises:
            None
        """
        self.bias = self.decoder.bias

mindnlp.transformers.models.bert.modeling_bert.BertLMPredictionHead.__init__(config)

This method initializes the BertLMPredictionHead class.

PARAMETER	DESCRIPTION
self	The object instance of the BertLMPredictionHead class.
config	A configuration object containing settings for the prediction head. It is of type dict or a custom configuration class. The config parameter is used to configure the prediction head's behavior and settings.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    This method initializes the BertLMPredictionHead class.

    Args:
        self: The object instance of the BertLMPredictionHead class.
        config: A configuration object containing settings for the prediction head.
            It is of type dict or a custom configuration class.
            The config parameter is used to configure the prediction head's behavior and settings.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.transform = BertPredictionHeadTransform(config)

    # The output weights are the same as the input embeddings, but there is
    # an output-only bias for each token.
    self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

    self.bias = Parameter(initializer('zeros', config.vocab_size), 'bias')

    self.decoder.bias = self.bias

mindnlp.transformers.models.bert.modeling_bert.BertLMPredictionHead.forward(hidden_states)

This method 'forward' is defined in the class 'BertLMPredictionHead' and is responsible for processing the hidden states.

PARAMETER	DESCRIPTION
self	The instance of the class.
hidden_states	The input hidden states to be processed. It should be of type tensor and contain the information about the hidden states. TYPE: tensor

RETURNS	DESCRIPTION
hidden_states	The processed hidden states. It is of type tensor and contains the transformed and decoded information from the input hidden states. TYPE: tensor

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states):
    """
    This method 'forward' is defined in the class 'BertLMPredictionHead' and is responsible for processing the hidden states.

    Args:
        self: The instance of the class.
        hidden_states (tensor): The input hidden states to be processed.
            It should be of type tensor and contain the information about the hidden states.

    Returns:
        hidden_states (tensor): The processed hidden states.
            It is of type tensor and contains the transformed and decoded information from the input hidden states.

    Raises:
        This method does not raise any exceptions.
    """
    hidden_states = self.transform(hidden_states)
    hidden_states = self.decoder(hidden_states)
    return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertLayer

Bases: Module

Bert Layer

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertLayer(nn.Module):
    r"""
    Bert Layer
    """
    def __init__(self, config):
        """
        Initialize a BertLayer object.

        Args:
            self (BertLayer): The instance of the BertLayer class.
            config (object):
                A configuration object containing various settings for the BertLayer.

                - chunk_size_feed_forward (int): The chunk size used for feed-forward operations.
                - is_decoder (bool): Indicates whether the model is designed as a decoder.
                - add_cross_attention (bool): Specifies if cross attention is to be added.
                - position_embedding_type (str): The type of position embedding to be used if cross attention is added.

        Returns:
            None.

        Raises:
            ValueError:
                If add_cross_attention is True and the model is not configured as a decoder, an exception is raised.
        """
        super().__init__()
        self.chunk_size_feed_forward = config.chunk_size_feed_forward
        self.seq_len_dim = 1
        self.attention = BertAttention(config)
        self.is_decoder = config.is_decoder
        self.add_cross_attention = config.add_cross_attention
        if self.add_cross_attention:
            if not self.is_decoder:
                raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
            self.crossattention = BertAttention(config, position_embedding_type="absolute")
        self.intermediate = BertIntermediate(config)
        self.output = BertOutput(config)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ):
        """
        This method forwards a BertLayer by processing the input hidden_states through self-attention
        and potentially cross-attention mechanisms.

        Args:
            self: The instance of the BertLayer class.
            hidden_states (mindspore.Tensor): The input tensor representing the hidden states.
            attention_mask (Optional[mindspore.Tensor]):
                An optional tensor for masking the attention scores. Defaults to None.
            head_mask (Optional[mindspore.Tensor]):
                An optional tensor to mask the heads of the attention mechanism. Defaults to None.
            encoder_hidden_states (Optional[mindspore.Tensor]):
                An optional tensor representing hidden states from the encoder. Defaults to None.
            encoder_attention_mask (Optional[mindspore.Tensor]):
                An optional tensor for masking the encoder attention scores. Defaults to None.
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                An optional tuple of past key and value tensors for efficient incremental decoding.
                Defaults to None.
            output_attentions (Optional[bool]): A flag indicating whether to output attention weights. Defaults to False.

        Returns:
            None:
                This method does not return any value explicitly,
                but it updates the internal state of the BertLayer instance.

        Raises:
            ValueError:
                If `encoder_hidden_states` are provided but cross-attention layers are not instantiated
                by setting `config.add_cross_attention=True`.
        """
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        self_attention_outputs = self.attention(
            hidden_states,
            attention_mask,
            head_mask,
            output_attentions=output_attentions,
            past_key_value=self_attn_past_key_value,
        )
        attention_output = self_attention_outputs[0]

        # if decoder, the last output is tuple of self-attn cache
        if self.is_decoder:
            outputs = self_attention_outputs[1:-1]
            present_key_value = self_attention_outputs[-1]
        else:
            outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights

        cross_attn_present_key_value = None
        if self.is_decoder and encoder_hidden_states is not None:
            if not hasattr(self, "crossattention"):
                raise ValueError(
                    f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
                    " by setting `config.add_cross_attention=True`"
                )

            # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            cross_attention_outputs = self.crossattention(
                attention_output,
                attention_mask,
                head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                cross_attn_past_key_value,
                output_attentions,
            )
            attention_output = cross_attention_outputs[0]
            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

            # add cross-attn cache to positions 3,4 of present_key_value tuple
            cross_attn_present_key_value = cross_attention_outputs[-1]
            present_key_value = present_key_value + cross_attn_present_key_value

        layer_output = apply_chunking_to_forward(
            self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
        )
        outputs = (layer_output,) + outputs

        # if decoder, return the attn key/values as the last output
        if self.is_decoder:
            outputs = outputs + (present_key_value,)

        return outputs

    def feed_forward_chunk(self, attention_output):
        """feed forward chunk"""
        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)
        return layer_output

mindnlp.transformers.models.bert.modeling_bert.BertLayer.__init__(config)

Initialize a BertLayer object.

PARAMETER	DESCRIPTION
self	The instance of the BertLayer class. TYPE: BertLayer
config	A configuration object containing various settings for the BertLayer. chunk_size_feed_forward (int): The chunk size used for feed-forward operations. is_decoder (bool): Indicates whether the model is designed as a decoder. add_cross_attention (bool): Specifies if cross attention is to be added. position_embedding_type (str): The type of position embedding to be used if cross attention is added. TYPE: object

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If add_cross_attention is True and the model is not configured as a decoder, an exception is raised.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initialize a BertLayer object.

    Args:
        self (BertLayer): The instance of the BertLayer class.
        config (object):
            A configuration object containing various settings for the BertLayer.

            - chunk_size_feed_forward (int): The chunk size used for feed-forward operations.
            - is_decoder (bool): Indicates whether the model is designed as a decoder.
            - add_cross_attention (bool): Specifies if cross attention is to be added.
            - position_embedding_type (str): The type of position embedding to be used if cross attention is added.

    Returns:
        None.

    Raises:
        ValueError:
            If add_cross_attention is True and the model is not configured as a decoder, an exception is raised.
    """
    super().__init__()
    self.chunk_size_feed_forward = config.chunk_size_feed_forward
    self.seq_len_dim = 1
    self.attention = BertAttention(config)
    self.is_decoder = config.is_decoder
    self.add_cross_attention = config.add_cross_attention
    if self.add_cross_attention:
        if not self.is_decoder:
            raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
        self.crossattention = BertAttention(config, position_embedding_type="absolute")
    self.intermediate = BertIntermediate(config)
    self.output = BertOutput(config)

mindnlp.transformers.models.bert.modeling_bert.BertLayer.feed_forward_chunk(attention_output)

feed forward chunk

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def feed_forward_chunk(self, attention_output):
    """feed forward chunk"""
    intermediate_output = self.intermediate(attention_output)
    layer_output = self.output(intermediate_output, attention_output)
    return layer_output

mindnlp.transformers.models.bert.modeling_bert.BertLayer.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

This method forwards a BertLayer by processing the input hidden_states through self-attention and potentially cross-attention mechanisms.

PARAMETER	DESCRIPTION
self	The instance of the BertLayer class.
hidden_states	The input tensor representing the hidden states. TYPE: Tensor
attention_mask	An optional tensor for masking the attention scores. Defaults to None. TYPE: Optional[Tensor] DEFAULT: None
head_mask	An optional tensor to mask the heads of the attention mechanism. Defaults to None. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	An optional tensor representing hidden states from the encoder. Defaults to None. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	An optional tensor for masking the encoder attention scores. Defaults to None. TYPE: Optional[Tensor] DEFAULT: None
past_key_value	An optional tuple of past key and value tensors for efficient incremental decoding. Defaults to None. TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
output_attentions	A flag indicating whether to output attention weights. Defaults to False. TYPE: Optional[bool] DEFAULT: False

RETURNS	DESCRIPTION
None	This method does not return any value explicitly, but it updates the internal state of the BertLayer instance.

RAISES	DESCRIPTION
ValueError	If encoder_hidden_states are provided but cross-attention layers are not instantiated by setting config.add_cross_attention=True.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
):
    """
    This method forwards a BertLayer by processing the input hidden_states through self-attention
    and potentially cross-attention mechanisms.

    Args:
        self: The instance of the BertLayer class.
        hidden_states (mindspore.Tensor): The input tensor representing the hidden states.
        attention_mask (Optional[mindspore.Tensor]):
            An optional tensor for masking the attention scores. Defaults to None.
        head_mask (Optional[mindspore.Tensor]):
            An optional tensor to mask the heads of the attention mechanism. Defaults to None.
        encoder_hidden_states (Optional[mindspore.Tensor]):
            An optional tensor representing hidden states from the encoder. Defaults to None.
        encoder_attention_mask (Optional[mindspore.Tensor]):
            An optional tensor for masking the encoder attention scores. Defaults to None.
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            An optional tuple of past key and value tensors for efficient incremental decoding.
            Defaults to None.
        output_attentions (Optional[bool]): A flag indicating whether to output attention weights. Defaults to False.

    Returns:
        None:
            This method does not return any value explicitly,
            but it updates the internal state of the BertLayer instance.

    Raises:
        ValueError:
            If `encoder_hidden_states` are provided but cross-attention layers are not instantiated
            by setting `config.add_cross_attention=True`.
    """
    # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
    self_attention_outputs = self.attention(
        hidden_states,
        attention_mask,
        head_mask,
        output_attentions=output_attentions,
        past_key_value=self_attn_past_key_value,
    )
    attention_output = self_attention_outputs[0]

    # if decoder, the last output is tuple of self-attn cache
    if self.is_decoder:
        outputs = self_attention_outputs[1:-1]
        present_key_value = self_attention_outputs[-1]
    else:
        outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights

    cross_attn_present_key_value = None
    if self.is_decoder and encoder_hidden_states is not None:
        if not hasattr(self, "crossattention"):
            raise ValueError(
                f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
                " by setting `config.add_cross_attention=True`"
            )

        # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
        cross_attention_outputs = self.crossattention(
            attention_output,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            cross_attn_past_key_value,
            output_attentions,
        )
        attention_output = cross_attention_outputs[0]
        outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

        # add cross-attn cache to positions 3,4 of present_key_value tuple
        cross_attn_present_key_value = cross_attention_outputs[-1]
        present_key_value = present_key_value + cross_attn_present_key_value

    layer_output = apply_chunking_to_forward(
        self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
    )
    outputs = (layer_output,) + outputs

    # if decoder, return the attn key/values as the last output
    if self.is_decoder:
        outputs = outputs + (present_key_value,)

    return outputs

mindnlp.transformers.models.bert.modeling_bert.BertModel

Bases: BertPreTrainedModel

Bert Model

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertModel(BertPreTrainedModel):
    r"""
    Bert Model
    """
    def __init__(self, config, add_pooling_layer=True):
        """
        Initializes a BertModel instance.

        Args:
            self: The instance of the BertModel class.
            config (object): The configuration object for the BertModel.
                It contains the required parameters for initializing the model.
            add_pooling_layer (bool): A flag indicating whether to add a pooling layer to the model.
                Defaults to True.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.config = config
        self.embeddings = BertEmbeddings(config)
        self.encoder = BertEncoder(config)

        self.pooler = BertPooler(config) if add_pooling_layer else None

        self.post_init()

    def get_input_embeddings(self):
        """
        Gets the input embeddings for the BertModel.

        Args:
            self (BertModel): The instance of the BertModel class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.embeddings.word_embeddings

    def set_input_embeddings(self, new_embeddings):
        """
        This method sets the input embeddings for the BertModel.

        Args:
            self (BertModel): The instance of the BertModel class.
            new_embeddings (object): The new input embeddings to be set for the BertModel.
                It can be of any valid object type.

        Returns:
            None.

        Raises:
            None.
        """
        self.embeddings.word_embeddings = new_embeddings

    def _prune_heads(self, heads_to_prune):
        """
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        """
        for layer, heads in heads_to_prune.items():
            self.encoder.layer[layer].attention.prune_heads(heads)

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        """
        This method forwards a BERT model with the specified input parameters.

        Args:
            self: The instance of the class.
            input_ids (Optional[mindspore.Tensor]): The input tensor containing token indices.
            attention_mask (Optional[mindspore.Tensor]): Mask tensor indicating which tokens should be attended to.
            token_type_ids (Optional[mindspore.Tensor]): Tensor indicating token types for different sequences in the input.
            position_ids (Optional[mindspore.Tensor]): Tensor containing position indices.
            head_mask (Optional[mindspore.Tensor]): Mask tensor specifying which heads to prune in the attention layers.
            inputs_embeds (Optional[mindspore.Tensor]): The embedded input tensor.
            encoder_hidden_states (Optional[mindspore.Tensor]): Tensor containing hidden states from the encoder.
            encoder_attention_mask (Optional[mindspore.Tensor]): Mask tensor for encoder attention.
            past_key_values (Optional[List[mindspore.Tensor]]): List of tensors containing past key values.
            use_cache (Optional[bool]): Flag indicating whether to use cache in the decoder.
            output_attentions (Optional[bool]): Flag indicating whether to output attentions.
            output_hidden_states (Optional[bool]): Flag indicating whether to output hidden states.
            return_dict (Optional[bool]): Flag indicating whether to return a dictionary.

        Returns:
            None.

        Raises:
            ValueError: If both input_ids and inputs_embeds are specified simultaneously.
            ValueError: If neither input_ids nor inputs_embeds are specified.
            ValueError: If padding is present without an attention mask.
            ValueError: If the function encounters any other invalid input configuration.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if self.config.is_decoder:
            use_cache = use_cache if use_cache is not None else self.config.use_cache
        else:
            use_cache = False

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        if input_ids is not None:
            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
            input_shape = input_ids.shape
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        batch_size, seq_length = input_shape

        # past_key_values_length
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if attention_mask is None:
            attention_mask = ops.ones(batch_size, seq_length + past_key_values_length)

        if token_type_ids is None:
            if hasattr(self.embeddings, "token_type_ids"):
                buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
                buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((batch_size, seq_length))
                token_type_ids = buffered_token_type_ids_expanded
            else:
                token_type_ids = ops.zeros(*input_shape, dtype=mindspore.int64)
        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)

        # If a 2D or 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
            if encoder_attention_mask is None:
                encoder_attention_mask = ops.ones(*encoder_hidden_shape)
            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
        else:
            encoder_extended_attention_mask = None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

        embedding_output = self.embeddings(
            input_ids=input_ids,
            position_ids=position_ids,
            token_type_ids=token_type_ids,
            inputs_embeds=inputs_embeds,
            past_key_values_length=past_key_values_length,
        )

        encoder_outputs = self.encoder(
            embedding_output,
            attention_mask=extended_attention_mask,
            head_mask=head_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_extended_attention_mask,
            past_key_values=past_key_values,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = encoder_outputs[0]
        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

        if not return_dict:
            return (sequence_output, pooled_output) + encoder_outputs[1:]

        return BaseModelOutputWithPoolingAndCrossAttentions(
            last_hidden_state=sequence_output,
            pooler_output=pooled_output,
            past_key_values=encoder_outputs.past_key_values,
            hidden_states=encoder_outputs.hidden_states,
            attentions=encoder_outputs.attentions,
            cross_attentions=encoder_outputs.cross_attentions,
        )

mindnlp.transformers.models.bert.modeling_bert.BertModel.__init__(config, add_pooling_layer=True)

Initializes a BertModel instance.

PARAMETER	DESCRIPTION
self	The instance of the BertModel class.
config	The configuration object for the BertModel. It contains the required parameters for initializing the model. TYPE: object
add_pooling_layer	A flag indicating whether to add a pooling layer to the model. Defaults to True. TYPE: bool DEFAULT: True

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config, add_pooling_layer=True):
    """
    Initializes a BertModel instance.

    Args:
        self: The instance of the BertModel class.
        config (object): The configuration object for the BertModel.
            It contains the required parameters for initializing the model.
        add_pooling_layer (bool): A flag indicating whether to add a pooling layer to the model.
            Defaults to True.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.config = config
    self.embeddings = BertEmbeddings(config)
    self.encoder = BertEncoder(config)

    self.pooler = BertPooler(config) if add_pooling_layer else None

    self.post_init()

mindnlp.transformers.models.bert.modeling_bert.BertModel.forward(input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_values=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

This method forwards a BERT model with the specified input parameters.

PARAMETER	DESCRIPTION
self	The instance of the class.
input_ids	The input tensor containing token indices. TYPE: Optional[Tensor] DEFAULT: None
attention_mask	Mask tensor indicating which tokens should be attended to. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	Tensor indicating token types for different sequences in the input. TYPE: Optional[Tensor] DEFAULT: None
position_ids	Tensor containing position indices. TYPE: Optional[Tensor] DEFAULT: None
head_mask	Mask tensor specifying which heads to prune in the attention layers. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The embedded input tensor. TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	Tensor containing hidden states from the encoder. TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	Mask tensor for encoder attention. TYPE: Optional[Tensor] DEFAULT: None
past_key_values	List of tensors containing past key values. TYPE: Optional[List[Tensor]] DEFAULT: None
use_cache	Flag indicating whether to use cache in the decoder. TYPE: Optional[bool] DEFAULT: None
output_attentions	Flag indicating whether to output attentions. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Flag indicating whether to output hidden states. TYPE: Optional[bool] DEFAULT: None
return_dict	Flag indicating whether to return a dictionary. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If both input_ids and inputs_embeds are specified simultaneously.
ValueError	If neither input_ids nor inputs_embeds are specified.
ValueError	If padding is present without an attention mask.
ValueError	If the function encounters any other invalid input configuration.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
):
    """
    This method forwards a BERT model with the specified input parameters.

    Args:
        self: The instance of the class.
        input_ids (Optional[mindspore.Tensor]): The input tensor containing token indices.
        attention_mask (Optional[mindspore.Tensor]): Mask tensor indicating which tokens should be attended to.
        token_type_ids (Optional[mindspore.Tensor]): Tensor indicating token types for different sequences in the input.
        position_ids (Optional[mindspore.Tensor]): Tensor containing position indices.
        head_mask (Optional[mindspore.Tensor]): Mask tensor specifying which heads to prune in the attention layers.
        inputs_embeds (Optional[mindspore.Tensor]): The embedded input tensor.
        encoder_hidden_states (Optional[mindspore.Tensor]): Tensor containing hidden states from the encoder.
        encoder_attention_mask (Optional[mindspore.Tensor]): Mask tensor for encoder attention.
        past_key_values (Optional[List[mindspore.Tensor]]): List of tensors containing past key values.
        use_cache (Optional[bool]): Flag indicating whether to use cache in the decoder.
        output_attentions (Optional[bool]): Flag indicating whether to output attentions.
        output_hidden_states (Optional[bool]): Flag indicating whether to output hidden states.
        return_dict (Optional[bool]): Flag indicating whether to return a dictionary.

    Returns:
        None.

    Raises:
        ValueError: If both input_ids and inputs_embeds are specified simultaneously.
        ValueError: If neither input_ids nor inputs_embeds are specified.
        ValueError: If padding is present without an attention mask.
        ValueError: If the function encounters any other invalid input configuration.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if self.config.is_decoder:
        use_cache = use_cache if use_cache is not None else self.config.use_cache
    else:
        use_cache = False

    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    if input_ids is not None:
        self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
        input_shape = input_ids.shape
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
    else:
        raise ValueError("You have to specify either input_ids or inputs_embeds")

    batch_size, seq_length = input_shape

    # past_key_values_length
    past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

    if attention_mask is None:
        attention_mask = ops.ones(batch_size, seq_length + past_key_values_length)

    if token_type_ids is None:
        if hasattr(self.embeddings, "token_type_ids"):
            buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
            buffered_token_type_ids_expanded = buffered_token_type_ids.broadcast_to((batch_size, seq_length))
            token_type_ids = buffered_token_type_ids_expanded
        else:
            token_type_ids = ops.zeros(*input_shape, dtype=mindspore.int64)
    # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
    # ourselves in which case we just need to make it broadcastable to all heads.
    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)

    # If a 2D or 3D attention mask is provided for the cross-attention
    # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
    if self.config.is_decoder and encoder_hidden_states is not None:
        encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
        encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
        if encoder_attention_mask is None:
            encoder_attention_mask = ops.ones(*encoder_hidden_shape)
        encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
    else:
        encoder_extended_attention_mask = None

    # Prepare head mask if needed
    # 1.0 in head_mask indicate we keep the head
    # attention_probs has shape bsz x n_heads x N x N
    # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
    # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
    head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

    embedding_output = self.embeddings(
        input_ids=input_ids,
        position_ids=position_ids,
        token_type_ids=token_type_ids,
        inputs_embeds=inputs_embeds,
        past_key_values_length=past_key_values_length,
    )

    encoder_outputs = self.encoder(
        embedding_output,
        attention_mask=extended_attention_mask,
        head_mask=head_mask,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_extended_attention_mask,
        past_key_values=past_key_values,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = encoder_outputs[0]
    pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

    if not return_dict:
        return (sequence_output, pooled_output) + encoder_outputs[1:]

    return BaseModelOutputWithPoolingAndCrossAttentions(
        last_hidden_state=sequence_output,
        pooler_output=pooled_output,
        past_key_values=encoder_outputs.past_key_values,
        hidden_states=encoder_outputs.hidden_states,
        attentions=encoder_outputs.attentions,
        cross_attentions=encoder_outputs.cross_attentions,
    )

mindnlp.transformers.models.bert.modeling_bert.BertModel.get_input_embeddings()

Gets the input embeddings for the BertModel.

PARAMETER	DESCRIPTION
self	The instance of the BertModel class. TYPE: BertModel

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def get_input_embeddings(self):
    """
    Gets the input embeddings for the BertModel.

    Args:
        self (BertModel): The instance of the BertModel class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.embeddings.word_embeddings

mindnlp.transformers.models.bert.modeling_bert.BertModel.set_input_embeddings(new_embeddings)

This method sets the input embeddings for the BertModel.

PARAMETER	DESCRIPTION
self	The instance of the BertModel class. TYPE: BertModel
new_embeddings	The new input embeddings to be set for the BertModel. It can be of any valid object type. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def set_input_embeddings(self, new_embeddings):
    """
    This method sets the input embeddings for the BertModel.

    Args:
        self (BertModel): The instance of the BertModel class.
        new_embeddings (object): The new input embeddings to be set for the BertModel.
            It can be of any valid object type.

    Returns:
        None.

    Raises:
        None.
    """
    self.embeddings.word_embeddings = new_embeddings

mindnlp.transformers.models.bert.modeling_bert.BertOnlyMLMHead

Bases: Module

BertOnlyMLMHead

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertOnlyMLMHead(nn.Module):
    """BertOnlyMLMHead"""
    def __init__(self, config):
        """
        __init__

        This method initializes an instance of the BertOnlyMLMHead class.

        Args:
            self (BertOnlyMLMHead): The instance of the BertOnlyMLMHead class.
            config: The configuration parameters for the BertLMPredictionHead.

        Returns:
            None.

        Raises:
            None
        """
        super().__init__()
        self.predictions = BertLMPredictionHead(config)

    def forward(self, sequence_output: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs the masked language modeling (MLM) head for the BERT model.

        Args:
            self (BertOnlyMLMHead): The instance of the BertOnlyMLMHead class.
            sequence_output (mindspore.Tensor): The output tensor from the BERT model's sequence output layer.
                It should have the shape (batch_size, sequence_length, hidden_size).

        Returns:
            mindspore.Tensor: The prediction scores for the masked language modeling task.
                It has the shape (batch_size, sequence_length, vocab_size).

        Raises:
            TypeError: If the input parameters are not of the expected types.
            ValueError: If the input tensor does not have the expected shape.

        Note:
            The MLM head is responsible for generating prediction scores for the masked tokens in the input sequence.
            The prediction scores are computed by passing the sequence output through the predictions layer.
            The predictions layer maps the hidden states of each token to the vocabulary size, representing the probabilities
            of each token being the correct masked token.

        Example:
            ```python
            >>> head = BertOnlyMLMHead()
            >>> output = head.forward(sequence_output)
            ```
        """
        prediction_scores = self.predictions(sequence_output)
        return prediction_scores

mindnlp.transformers.models.bert.modeling_bert.BertOnlyMLMHead.__init__(config)

init

This method initializes an instance of the BertOnlyMLMHead class.

PARAMETER	DESCRIPTION
self	The instance of the BertOnlyMLMHead class. TYPE: BertOnlyMLMHead
config	The configuration parameters for the BertLMPredictionHead.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    __init__

    This method initializes an instance of the BertOnlyMLMHead class.

    Args:
        self (BertOnlyMLMHead): The instance of the BertOnlyMLMHead class.
        config: The configuration parameters for the BertLMPredictionHead.

    Returns:
        None.

    Raises:
        None
    """
    super().__init__()
    self.predictions = BertLMPredictionHead(config)

mindnlp.transformers.models.bert.modeling_bert.BertOnlyMLMHead.forward(sequence_output)

Constructs the masked language modeling (MLM) head for the BERT model.

PARAMETER	DESCRIPTION
self	The instance of the BertOnlyMLMHead class. TYPE: BertOnlyMLMHead
sequence_output	The output tensor from the BERT model's sequence output layer. It should have the shape (batch_size, sequence_length, hidden_size). TYPE: Tensor

RETURNS	DESCRIPTION
Tensor	mindspore.Tensor: The prediction scores for the masked language modeling task. It has the shape (batch_size, sequence_length, vocab_size).

RAISES	DESCRIPTION
TypeError	If the input parameters are not of the expected types.
ValueError	If the input tensor does not have the expected shape.

Note

The MLM head is responsible for generating prediction scores for the masked tokens in the input sequence. The prediction scores are computed by passing the sequence output through the predictions layer. The predictions layer maps the hidden states of each token to the vocabulary size, representing the probabilities of each token being the correct masked token.

Example

>>> head = BertOnlyMLMHead()
>>> output = head.forward(sequence_output)

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, sequence_output: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs the masked language modeling (MLM) head for the BERT model.

    Args:
        self (BertOnlyMLMHead): The instance of the BertOnlyMLMHead class.
        sequence_output (mindspore.Tensor): The output tensor from the BERT model's sequence output layer.
            It should have the shape (batch_size, sequence_length, hidden_size).

    Returns:
        mindspore.Tensor: The prediction scores for the masked language modeling task.
            It has the shape (batch_size, sequence_length, vocab_size).

    Raises:
        TypeError: If the input parameters are not of the expected types.
        ValueError: If the input tensor does not have the expected shape.

    Note:
        The MLM head is responsible for generating prediction scores for the masked tokens in the input sequence.
        The prediction scores are computed by passing the sequence output through the predictions layer.
        The predictions layer maps the hidden states of each token to the vocabulary size, representing the probabilities
        of each token being the correct masked token.

    Example:
        ```python
        >>> head = BertOnlyMLMHead()
        >>> output = head.forward(sequence_output)
        ```
    """
    prediction_scores = self.predictions(sequence_output)
    return prediction_scores

mindnlp.transformers.models.bert.modeling_bert.BertOnlyNSPHead

Bases: Module

BertOnlyNSPHead

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertOnlyNSPHead(nn.Module):
    """BertOnlyNSPHead"""
    def __init__(self, config):
        """
        Initializes a BertOnlyNSPHead object with the specified configuration.

        Args:
            self (BertOnlyNSPHead): The instance of the BertOnlyNSPHead class.
            config: The configuration object containing parameters for the NSP head.
                Expected to be an instance of a class that includes a 'hidden_size' attribute.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not provided or is not of the expected type.
            AttributeError: If the config object does not have the 'hidden_size' attribute.
        """
        super().__init__()
        self.seq_relationship = nn.Linear(config.hidden_size, 2)

    def forward(self, pooled_output):
        """
        This method forwards a sequence relationship score based on the pooled output.

        Args:
            self (object): The instance of the class.
            pooled_output (object): The pooled output from the BERT model.

        Returns:
            None:
                This method returns None,
                as the forwarded sequence relationship score is directly assigned to the seq_relationship_score variable.

        Raises:
            None.
        """
        seq_relationship_score = self.seq_relationship(pooled_output)
        return seq_relationship_score

mindnlp.transformers.models.bert.modeling_bert.BertOnlyNSPHead.__init__(config)

Initializes a BertOnlyNSPHead object with the specified configuration.

PARAMETER	DESCRIPTION
self	The instance of the BertOnlyNSPHead class. TYPE: BertOnlyNSPHead
config	The configuration object containing parameters for the NSP head. Expected to be an instance of a class that includes a 'hidden_size' attribute.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the config parameter is not provided or is not of the expected type.
AttributeError	If the config object does not have the 'hidden_size' attribute.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes a BertOnlyNSPHead object with the specified configuration.

    Args:
        self (BertOnlyNSPHead): The instance of the BertOnlyNSPHead class.
        config: The configuration object containing parameters for the NSP head.
            Expected to be an instance of a class that includes a 'hidden_size' attribute.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not provided or is not of the expected type.
        AttributeError: If the config object does not have the 'hidden_size' attribute.
    """
    super().__init__()
    self.seq_relationship = nn.Linear(config.hidden_size, 2)

mindnlp.transformers.models.bert.modeling_bert.BertOnlyNSPHead.forward(pooled_output)

This method forwards a sequence relationship score based on the pooled output.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
pooled_output	The pooled output from the BERT model. TYPE: object

RETURNS	DESCRIPTION
None	This method returns None, as the forwarded sequence relationship score is directly assigned to the seq_relationship_score variable.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, pooled_output):
    """
    This method forwards a sequence relationship score based on the pooled output.

    Args:
        self (object): The instance of the class.
        pooled_output (object): The pooled output from the BERT model.

    Returns:
        None:
            This method returns None,
            as the forwarded sequence relationship score is directly assigned to the seq_relationship_score variable.

    Raises:
        None.
    """
    seq_relationship_score = self.seq_relationship(pooled_output)
    return seq_relationship_score

mindnlp.transformers.models.bert.modeling_bert.BertOutput

Bases: Module

Bert Output

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertOutput(nn.Module):
    r"""
    Bert Output
    """
    def __init__(self, config):
        """
        Initializes an instance of the BertOutput class.

        Args:
            self: The object itself.
            config: An instance of a configuration class containing various hyperparameters and settings.
                It is expected to have the following attributes:

                - intermediate_size: An integer representing the size of the intermediate layer.
                - hidden_size: An integer representing the size of the hidden layer.
                - layer_norm_eps: A floating-point number representing the epsilon value for layer normalization.
                - hidden_dropout_prob: A floating-point number representing the dropout probability for the hidden layer.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
        self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        """
        This method forwards the output of a BERT model by applying transformations to the hidden states.

        Args:
            self: The instance of the BertOutput class.
            hidden_states (tensor): The hidden states from the BERT model, typically of shape (batch_size, sequence_length, hidden_size).
                It is the input to the method and represents the encoded information from the input tokens.
            input_tensor (tensor): The input tensor to be added to the hidden_states after transformation.
                It is typically of the same shape as hidden_states and serves as additional input for the transformation.

        Returns:
            tensor: The transformed hidden_states after applying a series of operations including dense layer, dropout, and layer normalization.
                The returned tensor represents the forwarded output of the BERT model.

        Raises:
            ValueError: If the shapes of hidden_states and input_tensor are not compatible for the addition operation.
            RuntimeError: If any runtime error occurs during the transformation process.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertOutput.__init__(config)

Initializes an instance of the BertOutput class.

PARAMETER	DESCRIPTION
self	The object itself.
config	An instance of a configuration class containing various hyperparameters and settings. It is expected to have the following attributes: intermediate_size: An integer representing the size of the intermediate layer. hidden_size: An integer representing the size of the hidden layer. layer_norm_eps: A floating-point number representing the epsilon value for layer normalization. hidden_dropout_prob: A floating-point number representing the dropout probability for the hidden layer.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of the BertOutput class.

    Args:
        self: The object itself.
        config: An instance of a configuration class containing various hyperparameters and settings.
            It is expected to have the following attributes:

            - intermediate_size: An integer representing the size of the intermediate layer.
            - hidden_size: An integer representing the size of the hidden layer.
            - layer_norm_eps: A floating-point number representing the epsilon value for layer normalization.
            - hidden_dropout_prob: A floating-point number representing the dropout probability for the hidden layer.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
    self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.bert.modeling_bert.BertOutput.forward(hidden_states, input_tensor)

This method forwards the output of a BERT model by applying transformations to the hidden states.

PARAMETER	DESCRIPTION
self	The instance of the BertOutput class.
hidden_states	The hidden states from the BERT model, typically of shape (batch_size, sequence_length, hidden_size). It is the input to the method and represents the encoded information from the input tokens. TYPE: tensor
input_tensor	The input tensor to be added to the hidden_states after transformation. It is typically of the same shape as hidden_states and serves as additional input for the transformation. TYPE: tensor

RETURNS	DESCRIPTION
tensor	The transformed hidden_states after applying a series of operations including dense layer, dropout, and layer normalization. The returned tensor represents the forwarded output of the BERT model.

RAISES	DESCRIPTION
ValueError	If the shapes of hidden_states and input_tensor are not compatible for the addition operation.
RuntimeError	If any runtime error occurs during the transformation process.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states, input_tensor):
    """
    This method forwards the output of a BERT model by applying transformations to the hidden states.

    Args:
        self: The instance of the BertOutput class.
        hidden_states (tensor): The hidden states from the BERT model, typically of shape (batch_size, sequence_length, hidden_size).
            It is the input to the method and represents the encoded information from the input tokens.
        input_tensor (tensor): The input tensor to be added to the hidden_states after transformation.
            It is typically of the same shape as hidden_states and serves as additional input for the transformation.

    Returns:
        tensor: The transformed hidden_states after applying a series of operations including dense layer, dropout, and layer normalization.
            The returned tensor represents the forwarded output of the BERT model.

    Raises:
        ValueError: If the shapes of hidden_states and input_tensor are not compatible for the addition operation.
        RuntimeError: If any runtime error occurs during the transformation process.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertPooler

Bases: Module

Bert Pooler

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertPooler(nn.Module):
    r"""
    Bert Pooler
    """
    def __init__(self, config):
        """
        Initializes the BertPooler class.

        Args:
            self: The instance of the BertPooler class.
            config:
                An instance of the configuration class containing the hidden size parameter.

                - Type: object
                - Purpose: To configure the BertPooler with specified hidden size.
                - Restrictions: Must be a valid configuration object.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.activation = nn.Tanh()

    def forward(self, hidden_states):
        """
        Constructs the pooled output tensor from the given hidden states tensor.

        Args:
            self (BertPooler): The instance of the BertPooler class.
            hidden_states (torch.Tensor): The tensor of shape (batch_size, sequence_length, hidden_size)
                containing the hidden states of the input sequence.

        Returns:
            torch.Tensor: The pooled output tensor of shape (batch_size, hidden_size)
                representing the contextualized representation of the entire input sequence.

        Raises:
            None.
        """
        # We "pool" the model by simply taking the hidden state corresponding.
        # to the first token
        first_token_tensor = hidden_states[:, 0]
        pooled_output = self.dense(first_token_tensor)
        pooled_output = self.activation(pooled_output)
        return pooled_output

mindnlp.transformers.models.bert.modeling_bert.BertPooler.__init__(config)

Initializes the BertPooler class.

PARAMETER	DESCRIPTION
self	The instance of the BertPooler class.
config	An instance of the configuration class containing the hidden size parameter. Type: object Purpose: To configure the BertPooler with specified hidden size. Restrictions: Must be a valid configuration object.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes the BertPooler class.

    Args:
        self: The instance of the BertPooler class.
        config:
            An instance of the configuration class containing the hidden size parameter.

            - Type: object
            - Purpose: To configure the BertPooler with specified hidden size.
            - Restrictions: Must be a valid configuration object.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    self.activation = nn.Tanh()

mindnlp.transformers.models.bert.modeling_bert.BertPooler.forward(hidden_states)

Constructs the pooled output tensor from the given hidden states tensor.

PARAMETER	DESCRIPTION
self	The instance of the BertPooler class. TYPE: BertPooler
hidden_states	The tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states of the input sequence. TYPE: Tensor

RETURNS	DESCRIPTION
	torch.Tensor: The pooled output tensor of shape (batch_size, hidden_size) representing the contextualized representation of the entire input sequence.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states):
    """
    Constructs the pooled output tensor from the given hidden states tensor.

    Args:
        self (BertPooler): The instance of the BertPooler class.
        hidden_states (torch.Tensor): The tensor of shape (batch_size, sequence_length, hidden_size)
            containing the hidden states of the input sequence.

    Returns:
        torch.Tensor: The pooled output tensor of shape (batch_size, hidden_size)
            representing the contextualized representation of the entire input sequence.

    Raises:
        None.
    """
    # We "pool" the model by simply taking the hidden state corresponding.
    # to the first token
    first_token_tensor = hidden_states[:, 0]
    pooled_output = self.dense(first_token_tensor)
    pooled_output = self.activation(pooled_output)
    return pooled_output

mindnlp.transformers.models.bert.modeling_bert.BertPreTrainedModel

Bases: PreTrainedModel

BertPretrainedModel

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertPreTrainedModel(PreTrainedModel):
    """BertPretrainedModel"""
    config_class = BertConfig
    base_model_prefix = 'bert'

    def _init_weights(self, cell):
        """Initialize the weights"""
        if isinstance(cell, nn.Linear):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            cell.weight.set_data(initializer(Normal(self.config.initializer_range),
                                                    cell.weight.shape, cell.weight.dtype))
            if cell.bias is not None:
                cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))
        elif isinstance(cell, nn.Embedding):
            weight = np.random.normal(0.0, self.config.initializer_range, cell.weight.shape)
            if cell.padding_idx:
                weight[cell.padding_idx] = 0

            cell.weight.set_data(Tensor(weight, cell.weight.dtype))
        elif isinstance(cell, nn.LayerNorm):
            cell.weight.set_data(initializer('ones', cell.weight.shape, cell.weight.dtype))
            cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))

mindnlp.transformers.models.bert.modeling_bert.BertPreTrainingHeads

Bases: Module

Bert PreTraining Heads

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertPreTrainingHeads(nn.Module):
    r"""
    Bert PreTraining Heads
    """
    def __init__(self, config):
        """
        Initializes the BertPreTrainingHeads class.

        Args:
            self (BertPreTrainingHeads): The instance of the BertPreTrainingHeads class.
            config: A configuration object containing settings for the BertPreTrainingHeads instance.

        Returns:
            None.

        Raises:
            TypeError: If the provided config parameter is not of the expected type.
            ValueError: If the config parameter does not contain the necessary settings.
        """
        super().__init__()
        self.predictions = BertLMPredictionHead(config)
        self.seq_relationship = nn.Linear(config.hidden_size, 2)

    def forward(self, sequence_output, pooled_output):
        """
        Construct the prediction scores and sequence relationship scores for pre-training heads in BERT.

        Args:
            self (BertPreTrainingHeads): An instance of the BertPreTrainingHeads class.
            sequence_output (Tensor): The output sequence tensor of shape (batch_size, sequence_length, hidden_size).
                It represents the contextualized representation of each token in the input sequence.
            pooled_output (Tensor): The pooled output tensor of shape (batch_size, hidden_size).
                It represents the contextualized representation of the entire input sequence.

        Returns:
            tuple:
                A tuple containing two elements:

                - prediction_scores (Tensor): The prediction scores tensor of shape (batch_size, sequence_length, vocab_size).
                It represents the scores for predicting the masked tokens in the input sequence.
                - seq_relationship_score (Tensor): The sequence relationship score tensor of shape (batch_size, 2).
                It represents the scores for predicting the next sentence relationship.

        Raises:
            None.
        """
        prediction_scores = self.predictions(sequence_output)
        seq_relationship_score = self.seq_relationship(pooled_output)
        return prediction_scores, seq_relationship_score

mindnlp.transformers.models.bert.modeling_bert.BertPreTrainingHeads.__init__(config)

Initializes the BertPreTrainingHeads class.

PARAMETER	DESCRIPTION
self	The instance of the BertPreTrainingHeads class. TYPE: BertPreTrainingHeads
config	A configuration object containing settings for the BertPreTrainingHeads instance.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the provided config parameter is not of the expected type.
ValueError	If the config parameter does not contain the necessary settings.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes the BertPreTrainingHeads class.

    Args:
        self (BertPreTrainingHeads): The instance of the BertPreTrainingHeads class.
        config: A configuration object containing settings for the BertPreTrainingHeads instance.

    Returns:
        None.

    Raises:
        TypeError: If the provided config parameter is not of the expected type.
        ValueError: If the config parameter does not contain the necessary settings.
    """
    super().__init__()
    self.predictions = BertLMPredictionHead(config)
    self.seq_relationship = nn.Linear(config.hidden_size, 2)

mindnlp.transformers.models.bert.modeling_bert.BertPreTrainingHeads.forward(sequence_output, pooled_output)

Construct the prediction scores and sequence relationship scores for pre-training heads in BERT.

PARAMETER	DESCRIPTION
self	An instance of the BertPreTrainingHeads class. TYPE: BertPreTrainingHeads
sequence_output	The output sequence tensor of shape (batch_size, sequence_length, hidden_size). It represents the contextualized representation of each token in the input sequence. TYPE: Tensor
pooled_output	The pooled output tensor of shape (batch_size, hidden_size). It represents the contextualized representation of the entire input sequence. TYPE: Tensor

RETURNS	DESCRIPTION
tuple	A tuple containing two elements: prediction_scores (Tensor): The prediction scores tensor of shape (batch_size, sequence_length, vocab_size). It represents the scores for predicting the masked tokens in the input sequence. seq_relationship_score (Tensor): The sequence relationship score tensor of shape (batch_size, 2). It represents the scores for predicting the next sentence relationship.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, sequence_output, pooled_output):
    """
    Construct the prediction scores and sequence relationship scores for pre-training heads in BERT.

    Args:
        self (BertPreTrainingHeads): An instance of the BertPreTrainingHeads class.
        sequence_output (Tensor): The output sequence tensor of shape (batch_size, sequence_length, hidden_size).
            It represents the contextualized representation of each token in the input sequence.
        pooled_output (Tensor): The pooled output tensor of shape (batch_size, hidden_size).
            It represents the contextualized representation of the entire input sequence.

    Returns:
        tuple:
            A tuple containing two elements:

            - prediction_scores (Tensor): The prediction scores tensor of shape (batch_size, sequence_length, vocab_size).
            It represents the scores for predicting the masked tokens in the input sequence.
            - seq_relationship_score (Tensor): The sequence relationship score tensor of shape (batch_size, 2).
            It represents the scores for predicting the next sentence relationship.

    Raises:
        None.
    """
    prediction_scores = self.predictions(sequence_output)
    seq_relationship_score = self.seq_relationship(pooled_output)
    return prediction_scores, seq_relationship_score

mindnlp.transformers.models.bert.modeling_bert.BertPredictionHeadTransform

Bases: Module

Bert Prediction Head Transform

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertPredictionHeadTransform(nn.Module):
    r"""
    Bert Prediction Head Transform
    """
    def __init__(self, config):
        """
        Initializes an instance of the BertPredictionHeadTransform class.

        Args:
            self (BertPredictionHeadTransform): The instance of the class.
            config: A configuration object containing the necessary parameters for the transformation.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.transform_act_fn = ACT2FN[config.hidden_act]
        self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)

    def forward(self, hidden_states):
        """
        Constructs the transformed hidden states for the BertPredictionHeadTransform class.

        Args:
            self (BertPredictionHeadTransform): An instance of the BertPredictionHeadTransform class.
            hidden_states: The input hidden states to be transformed.
                Expected to be of shape [batch_size, sequence_length, hidden_size].

        Returns:
            None

        Raises:
            None
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.transform_act_fn(hidden_states)
        hidden_states = self.LayerNorm(hidden_states)
        return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertPredictionHeadTransform.__init__(config)

Initializes an instance of the BertPredictionHeadTransform class.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: BertPredictionHeadTransform
config	A configuration object containing the necessary parameters for the transformation.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of the BertPredictionHeadTransform class.

    Args:
        self (BertPredictionHeadTransform): The instance of the class.
        config: A configuration object containing the necessary parameters for the transformation.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    self.transform_act_fn = ACT2FN[config.hidden_act]
    self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)

mindnlp.transformers.models.bert.modeling_bert.BertPredictionHeadTransform.forward(hidden_states)

Constructs the transformed hidden states for the BertPredictionHeadTransform class.

PARAMETER	DESCRIPTION
self	An instance of the BertPredictionHeadTransform class. TYPE: BertPredictionHeadTransform
hidden_states	The input hidden states to be transformed. Expected to be of shape [batch_size, sequence_length, hidden_size].

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states):
    """
    Constructs the transformed hidden states for the BertPredictionHeadTransform class.

    Args:
        self (BertPredictionHeadTransform): An instance of the BertPredictionHeadTransform class.
        hidden_states: The input hidden states to be transformed.
            Expected to be of shape [batch_size, sequence_length, hidden_size].

    Returns:
        None

    Raises:
        None
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.transform_act_fn(hidden_states)
    hidden_states = self.LayerNorm(hidden_states)
    return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertSelfAttention

Bases: Module

Self attention layer for BERT.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertSelfAttention(nn.Module):
    """
    Self attention layer for BERT.
    """
    def __init__(self, config, position_embedding_type=None):
        """
        Initializes the BertSelfAttention instance.

        Args:
            self: The instance of the class.
            config: A configuration object containing the model's settings and hyperparameters.
            position_embedding_type (str, optional): The type of position embedding to be used. Defaults to None.

        Returns:
            None.

        Raises:
            ValueError: If the hidden size specified in the config is not a multiple of the number of attention heads.
        """
        super().__init__()
        if config.hidden_size % config.num_attention_heads != 0:
            raise ValueError(
                f"The hidden size {config.hidden_size} is not a multiple of the number of attention "
                f"heads {config.num_attention_heads}"
            )
        self.output_attentions = config.output_attentions

        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.query = nn.Linear(config.hidden_size, self.all_head_size)
        self.key = nn.Linear(config.hidden_size, self.all_head_size)
        self.value = nn.Linear(config.hidden_size, self.all_head_size)

        self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
        self.position_embedding_type = position_embedding_type or getattr(
            config, "position_embedding_type", "absolute"
        )
        if self.position_embedding_type in ('relative_key', 'relative_key_query'):
            self.max_position_embeddings = config.max_position_embeddings
            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)

        self.is_decoder = config.is_decoder

    def transpose_for_scores(self, input_x):
        r"""
        transpose for scores
        """
        new_x_shape = input_x.shape[:-1] + (self.num_attention_heads, self.attention_head_size)
        input_x = input_x.view(*new_x_shape)
        return input_x.transpose(0, 2, 1, 3)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        output_attentions: Optional[bool] = False,
    ):
        """
        Constructs the self-attention mechanism for the Bert model.

        Args:
            self (BertSelfAttention): The instance of the BertSelfAttention class.
            hidden_states (mindspore.Tensor): The input hidden states of the model.
                It has shape (batch_size, sequence_length, hidden_size).
            attention_mask (Optional[mindspore.Tensor]):
                The attention mask tensor to mask certain positions in the input.
                It has shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length).
            head_mask (Optional[mindspore.Tensor]):
                The head mask tensor to mask certain heads of the attention mechanism.
                It has shape (num_attention_heads,) or (batch_size, num_attention_heads) or
                (batch_size, num_attention_heads, sequence_length, sequence_length).
            encoder_hidden_states (Optional[mindspore.Tensor]): The hidden states of the encoder.
                It has shape (batch_size, encoder_sequence_length, hidden_size).
            encoder_attention_mask (Optional[mindspore.Tensor]): The attention mask tensor for the encoder.
                It has shape (batch_size, 1, 1, encoder_sequence_length).
            past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
                The cached key and value tensors from previous steps.
                It is a tuple containing two tensors of shape (batch_size, num_attention_heads, sequence_length, head_size).
            output_attentions (Optional[bool]): Whether to output attention probabilities.

        Returns:
            Tuple[mindspore.Tensor]: A tuple containing the computed context layer and the attention probabilities.
                The context layer has shape (batch_size, sequence_length, hidden_size) and the attention
                probabilities have shape (batch_size, num_attention_heads, sequence_length, sequence_length).

        Raises:
            None

        """
        mixed_query_layer = self.query(hidden_states)

        # If this is instantiated as a cross-attention module, the keys
        # and values come from an encoder; the attention mask needs to be
        # such that the encoder's padding tokens are not attended to.
        is_cross_attention = encoder_hidden_states is not None
        if is_cross_attention and past_key_value is not None:
            # reuse k,v, cross_attentions
            key_layer = past_key_value[0]
            value_layer = past_key_value[1]
            attention_mask = encoder_attention_mask
        elif is_cross_attention:
            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
            attention_mask = encoder_attention_mask
        elif past_key_value is not None:
            key_layer = self.transpose_for_scores(self.key(hidden_states))
            value_layer = self.transpose_for_scores(self.value(hidden_states))
            key_layer = ops.cat([past_key_value[0], key_layer], dim=2)
            value_layer = ops.cat([past_key_value[1], value_layer], dim=2)
        else:
            key_layer = self.transpose_for_scores(self.key(hidden_states))
            value_layer = self.transpose_for_scores(self.value(hidden_states))

        query_layer = self.transpose_for_scores(mixed_query_layer)
        use_cache = past_key_value is not None
        if self.is_decoder:
            # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
            # Further calls to cross_attention layer can then reuse all cross-attention
            # key/value_states (first "if" case)
            # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
            # all previous decoder key/value_states. Further calls to uni-directional self-attention
            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
            # if encoder bi-directional self-attention `past_key_value` is always `None`
            past_key_value = (key_layer, value_layer)

        # Take the dot product between "query" and "key" to get the raw attention scores.
        attention_scores = ops.matmul(query_layer, key_layer.swapaxes(-1, -2))

        if self.position_embedding_type in ('relative_key', 'relative_key_query'):
            query_length, key_length = query_layer.shape[2], key_layer.shape[2]
            if use_cache:
                position_ids_l = Tensor(key_length - 1, dtype=mindspore.int64).view(-1, 1)
            else:
                position_ids_l = ops.arange(query_length, dtype=mindspore.int64).view(-1, 1)
            position_ids_r = ops.arange(key_length, dtype=mindspore.int64).view(1, -1)
            distance = position_ids_l - position_ids_r
            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility

            if self.position_embedding_type == "relative_key":
                relative_position_scores = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding.broadcast_to((query_length, -1, -1)))
                attention_scores = attention_scores + relative_position_scores
            elif self.position_embedding_type == "relative_key_query":
                relative_position_scores_query = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
                relative_position_scores_key = ops.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key

        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
        if attention_mask is not None:
            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
            attention_scores = attention_scores + attention_mask

        # Normalize the attention scores to probabilities.
        attention_probs = ops.softmax(attention_scores, dim=-1)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs = self.dropout(attention_probs)

        # Mask heads if we want to
        if head_mask is not None:
            attention_probs = attention_probs * head_mask

        context_layer = ops.matmul(attention_probs, value_layer)

        context_layer = context_layer.transpose(0, 2, 1, 3)
        new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(new_context_layer_shape)
        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)

        if self.is_decoder:
            outputs = outputs + (past_key_value,)
        return outputs

mindnlp.transformers.models.bert.modeling_bert.BertSelfAttention.__init__(config, position_embedding_type=None)

Initializes the BertSelfAttention instance.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	A configuration object containing the model's settings and hyperparameters.
position_embedding_type	The type of position embedding to be used. Defaults to None. TYPE: str DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the hidden size specified in the config is not a multiple of the number of attention heads.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config, position_embedding_type=None):
    """
    Initializes the BertSelfAttention instance.

    Args:
        self: The instance of the class.
        config: A configuration object containing the model's settings and hyperparameters.
        position_embedding_type (str, optional): The type of position embedding to be used. Defaults to None.

    Returns:
        None.

    Raises:
        ValueError: If the hidden size specified in the config is not a multiple of the number of attention heads.
    """
    super().__init__()
    if config.hidden_size % config.num_attention_heads != 0:
        raise ValueError(
            f"The hidden size {config.hidden_size} is not a multiple of the number of attention "
            f"heads {config.num_attention_heads}"
        )
    self.output_attentions = config.output_attentions

    self.num_attention_heads = config.num_attention_heads
    self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
    self.all_head_size = self.num_attention_heads * self.attention_head_size

    self.query = nn.Linear(config.hidden_size, self.all_head_size)
    self.key = nn.Linear(config.hidden_size, self.all_head_size)
    self.value = nn.Linear(config.hidden_size, self.all_head_size)

    self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
    self.position_embedding_type = position_embedding_type or getattr(
        config, "position_embedding_type", "absolute"
    )
    if self.position_embedding_type in ('relative_key', 'relative_key_query'):
        self.max_position_embeddings = config.max_position_embeddings
        self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)

    self.is_decoder = config.is_decoder

mindnlp.transformers.models.bert.modeling_bert.BertSelfAttention.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, past_key_value=None, output_attentions=False)

Constructs the self-attention mechanism for the Bert model.

PARAMETER	DESCRIPTION
self	The instance of the BertSelfAttention class. TYPE: BertSelfAttention
hidden_states	The input hidden states of the model. It has shape (batch_size, sequence_length, hidden_size). TYPE: Tensor
attention_mask	The attention mask tensor to mask certain positions in the input. It has shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length). TYPE: Optional[Tensor] DEFAULT: None
head_mask	The head mask tensor to mask certain heads of the attention mechanism. It has shape (num_attention_heads,) or (batch_size, num_attention_heads) or (batch_size, num_attention_heads, sequence_length, sequence_length). TYPE: Optional[Tensor] DEFAULT: None
encoder_hidden_states	The hidden states of the encoder. It has shape (batch_size, encoder_sequence_length, hidden_size). TYPE: Optional[Tensor] DEFAULT: None
encoder_attention_mask	The attention mask tensor for the encoder. It has shape (batch_size, 1, 1, encoder_sequence_length). TYPE: Optional[Tensor] DEFAULT: None
past_key_value	The cached key and value tensors from previous steps. It is a tuple containing two tensors of shape (batch_size, num_attention_heads, sequence_length, head_size). TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
output_attentions	Whether to output attention probabilities. TYPE: Optional[bool] DEFAULT: False

RETURNS	DESCRIPTION
	Tuple[mindspore.Tensor]: A tuple containing the computed context layer and the attention probabilities. The context layer has shape (batch_size, sequence_length, hidden_size) and the attention probabilities have shape (batch_size, num_attention_heads, sequence_length, sequence_length).

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    output_attentions: Optional[bool] = False,
):
    """
    Constructs the self-attention mechanism for the Bert model.

    Args:
        self (BertSelfAttention): The instance of the BertSelfAttention class.
        hidden_states (mindspore.Tensor): The input hidden states of the model.
            It has shape (batch_size, sequence_length, hidden_size).
        attention_mask (Optional[mindspore.Tensor]):
            The attention mask tensor to mask certain positions in the input.
            It has shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length).
        head_mask (Optional[mindspore.Tensor]):
            The head mask tensor to mask certain heads of the attention mechanism.
            It has shape (num_attention_heads,) or (batch_size, num_attention_heads) or
            (batch_size, num_attention_heads, sequence_length, sequence_length).
        encoder_hidden_states (Optional[mindspore.Tensor]): The hidden states of the encoder.
            It has shape (batch_size, encoder_sequence_length, hidden_size).
        encoder_attention_mask (Optional[mindspore.Tensor]): The attention mask tensor for the encoder.
            It has shape (batch_size, 1, 1, encoder_sequence_length).
        past_key_value (Optional[Tuple[Tuple[mindspore.Tensor]]]):
            The cached key and value tensors from previous steps.
            It is a tuple containing two tensors of shape (batch_size, num_attention_heads, sequence_length, head_size).
        output_attentions (Optional[bool]): Whether to output attention probabilities.

    Returns:
        Tuple[mindspore.Tensor]: A tuple containing the computed context layer and the attention probabilities.
            The context layer has shape (batch_size, sequence_length, hidden_size) and the attention
            probabilities have shape (batch_size, num_attention_heads, sequence_length, sequence_length).

    Raises:
        None

    """
    mixed_query_layer = self.query(hidden_states)

    # If this is instantiated as a cross-attention module, the keys
    # and values come from an encoder; the attention mask needs to be
    # such that the encoder's padding tokens are not attended to.
    is_cross_attention = encoder_hidden_states is not None
    if is_cross_attention and past_key_value is not None:
        # reuse k,v, cross_attentions
        key_layer = past_key_value[0]
        value_layer = past_key_value[1]
        attention_mask = encoder_attention_mask
    elif is_cross_attention:
        key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
        value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
        attention_mask = encoder_attention_mask
    elif past_key_value is not None:
        key_layer = self.transpose_for_scores(self.key(hidden_states))
        value_layer = self.transpose_for_scores(self.value(hidden_states))
        key_layer = ops.cat([past_key_value[0], key_layer], dim=2)
        value_layer = ops.cat([past_key_value[1], value_layer], dim=2)
    else:
        key_layer = self.transpose_for_scores(self.key(hidden_states))
        value_layer = self.transpose_for_scores(self.value(hidden_states))

    query_layer = self.transpose_for_scores(mixed_query_layer)
    use_cache = past_key_value is not None
    if self.is_decoder:
        # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
        # Further calls to cross_attention layer can then reuse all cross-attention
        # key/value_states (first "if" case)
        # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
        # all previous decoder key/value_states. Further calls to uni-directional self-attention
        # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
        # if encoder bi-directional self-attention `past_key_value` is always `None`
        past_key_value = (key_layer, value_layer)

    # Take the dot product between "query" and "key" to get the raw attention scores.
    attention_scores = ops.matmul(query_layer, key_layer.swapaxes(-1, -2))

    if self.position_embedding_type in ('relative_key', 'relative_key_query'):
        query_length, key_length = query_layer.shape[2], key_layer.shape[2]
        if use_cache:
            position_ids_l = Tensor(key_length - 1, dtype=mindspore.int64).view(-1, 1)
        else:
            position_ids_l = ops.arange(query_length, dtype=mindspore.int64).view(-1, 1)
        position_ids_r = ops.arange(key_length, dtype=mindspore.int64).view(1, -1)
        distance = position_ids_l - position_ids_r
        positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
        positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility

        if self.position_embedding_type == "relative_key":
            relative_position_scores = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding.broadcast_to((query_length, -1, -1)))
            attention_scores = attention_scores + relative_position_scores
        elif self.position_embedding_type == "relative_key_query":
            relative_position_scores_query = ops.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
            relative_position_scores_key = ops.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
            attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key

    attention_scores = attention_scores / math.sqrt(self.attention_head_size)
    if attention_mask is not None:
        # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
        attention_scores = attention_scores + attention_mask

    # Normalize the attention scores to probabilities.
    attention_probs = ops.softmax(attention_scores, dim=-1)

    # This is actually dropping out entire tokens to attend to, which might
    # seem a bit unusual, but is taken from the original Transformer paper.
    attention_probs = self.dropout(attention_probs)

    # Mask heads if we want to
    if head_mask is not None:
        attention_probs = attention_probs * head_mask

    context_layer = ops.matmul(attention_probs, value_layer)

    context_layer = context_layer.transpose(0, 2, 1, 3)
    new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
    context_layer = context_layer.view(new_context_layer_shape)
    outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)

    if self.is_decoder:
        outputs = outputs + (past_key_value,)
    return outputs

mindnlp.transformers.models.bert.modeling_bert.BertSelfAttention.transpose_for_scores(input_x)

transpose for scores

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def transpose_for_scores(self, input_x):
    r"""
    transpose for scores
    """
    new_x_shape = input_x.shape[:-1] + (self.num_attention_heads, self.attention_head_size)
    input_x = input_x.view(*new_x_shape)
    return input_x.transpose(0, 2, 1, 3)

mindnlp.transformers.models.bert.modeling_bert.BertSelfOutput

Bases: Module

Bert Self Output

Source code in mindnlp/transformers/models/bert/modeling_bert.py

class BertSelfOutput(nn.Module):
    r"""
    Bert Self Output
    """
    def __init__(self, config):
        """
        Initializes an instance of the BertSelfOutput class.

        Args:
            self: The instance of the class.
            config:
                An object that holds configuration parameters for the self-attention mechanism.

                - Type: object
                - Purpose: Specifies the configuration settings for the self-attention mechanism.
                - Restrictions: Must be a valid configuration object.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not of the expected type.
            ValueError: If the config parameter does not contain the required configuration settings.
        """
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.LayerNorm  = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        """
        This method 'forward' is a part of the 'BertSelfOutput' class and is responsible
        for processing hidden states in a BERT model.

        Args:
            self:
                The instance of the class.

                - Type: BertSelfOutput.
                - Purpose: Represents the current instance of the class.
                - Restrictions: None.

            hidden_states:
                The hidden states obtained from the previous layer.

                - Type: Tensor.
                - Purpose: Represents the input hidden states to be processed.
                - Restrictions: Should be a valid Tensor object.

            input_tensor:
                The input tensor that needs to be added to the processed hidden states.

                - Type: Tensor.
                - Purpose: Represents the additional input tensor to be combined with the processed hidden states.
                - Restrictions: Should be a valid Tensor object.

        Returns:
            None: This method does not return any value but directly modifies the hidden states.

        Raises:
            None.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.bert.modeling_bert.BertSelfOutput.__init__(config)

Initializes an instance of the BertSelfOutput class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An object that holds configuration parameters for the self-attention mechanism. Type: object Purpose: Specifies the configuration settings for the self-attention mechanism. Restrictions: Must be a valid configuration object.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the config parameter is not of the expected type.
ValueError	If the config parameter does not contain the required configuration settings.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def __init__(self, config):
    """
    Initializes an instance of the BertSelfOutput class.

    Args:
        self: The instance of the class.
        config:
            An object that holds configuration parameters for the self-attention mechanism.

            - Type: object
            - Purpose: Specifies the configuration settings for the self-attention mechanism.
            - Restrictions: Must be a valid configuration object.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not of the expected type.
        ValueError: If the config parameter does not contain the required configuration settings.
    """
    super().__init__()
    self.dense = nn.Linear(config.hidden_size, config.hidden_size)
    self.LayerNorm  = nn.LayerNorm((config.hidden_size,), eps=config.layer_norm_eps)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.bert.modeling_bert.BertSelfOutput.forward(hidden_states, input_tensor)

This method 'forward' is a part of the 'BertSelfOutput' class and is responsible for processing hidden states in a BERT model.

PARAMETER	DESCRIPTION
self	The instance of the class. Type: BertSelfOutput. Purpose: Represents the current instance of the class. Restrictions: None.
hidden_states	The hidden states obtained from the previous layer. Type: Tensor. Purpose: Represents the input hidden states to be processed. Restrictions: Should be a valid Tensor object.
input_tensor	The input tensor that needs to be added to the processed hidden states. Type: Tensor. Purpose: Represents the additional input tensor to be combined with the processed hidden states. Restrictions: Should be a valid Tensor object.

RETURNS	DESCRIPTION
None	This method does not return any value but directly modifies the hidden states.

Source code in mindnlp/transformers/models/bert/modeling_bert.py

def forward(self, hidden_states, input_tensor):
    """
    This method 'forward' is a part of the 'BertSelfOutput' class and is responsible
    for processing hidden states in a BERT model.

    Args:
        self:
            The instance of the class.

            - Type: BertSelfOutput.
            - Purpose: Represents the current instance of the class.
            - Restrictions: None.

        hidden_states:
            The hidden states obtained from the previous layer.

            - Type: Tensor.
            - Purpose: Represents the input hidden states to be processed.
            - Restrictions: Should be a valid Tensor object.

        input_tensor:
            The input tensor that needs to be added to the processed hidden states.

            - Type: Tensor.
            - Purpose: Represents the additional input tensor to be combined with the processed hidden states.
            - Restrictions: Should be a valid Tensor object.

    Returns:
        None: This method does not return any value but directly modifies the hidden states.

    Raises:
        None.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert

MindNLP bert model

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertAttention

Bases: Module

Bert Attention

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertAttention(nn.Module):
    r"""
    Bert Attention
    """
    def __init__(self, config, causal, init_cache=False):
        """
        Initializes an instance of MSBertAttention.

        Args:
            self: The instance of the class itself.
            config (object): The configuration object containing various settings.
            causal (bool): Flag indicating whether the attention mechanism is causal.
            init_cache (bool, optional): Flag indicating whether to initialize cache. Default is False.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.self = MSBertSelfAttention(config, causal, init_cache)
        self.output = MSBertSelfOutput(config)

    def forward(self, hidden_states, attention_mask=None, head_mask=None):
        """
        Constructs the attention mechanism for a multi-head self-attention layer in MSBertAttention.

        Args:
            self (MSBertAttention): The instance of the MSBertAttention class.
            hidden_states (torch.Tensor): The input tensor of shape (batch_size, sequence_length, hidden_size).
                It represents the sequence of hidden states for each token in the input sequence.
            attention_mask (torch.Tensor, optional): An optional tensor of shape (batch_size, sequence_length) indicating
                which tokens should be attended to and which should be ignored. The value 1 indicates to attend to the token,
                while 0 indicates to ignore it. If not provided, all tokens are attended to.
            head_mask (torch.Tensor, optional): An optional tensor of shape (num_heads,) or (num_layers, num_heads) indicating
                which heads or layers to mask. 1 indicates to include the head/layer, while 0 indicates to mask it.
                If not provided, all heads/layers are included.

        Returns:
            Tuple[torch.Tensor]:
                A tuple containing:

                - attention_output (torch.Tensor): The output tensor of shape (batch_size, sequence_length, hidden_size),
                  which represents the attended hidden states for each token in the input sequence.
                - self_outputs[1:] (tuple): A tuple of length `num_layers` containing tensors representing intermediate
                  outputs of the self-attention mechanism.

        Raises:
            None.
        """
        self_outputs = self.self(hidden_states, attention_mask, head_mask)
        attention_output = self.output(self_outputs[0], hidden_states)
        outputs = (attention_output,) + self_outputs[1:]
        return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertAttention.__init__(config, causal, init_cache=False)

Initializes an instance of MSBertAttention.

PARAMETER	DESCRIPTION
self	The instance of the class itself.
config	The configuration object containing various settings. TYPE: object
causal	Flag indicating whether the attention mechanism is causal. TYPE: bool
init_cache	Flag indicating whether to initialize cache. Default is False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config, causal, init_cache=False):
    """
    Initializes an instance of MSBertAttention.

    Args:
        self: The instance of the class itself.
        config (object): The configuration object containing various settings.
        causal (bool): Flag indicating whether the attention mechanism is causal.
        init_cache (bool, optional): Flag indicating whether to initialize cache. Default is False.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.self = MSBertSelfAttention(config, causal, init_cache)
    self.output = MSBertSelfOutput(config)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertAttention.forward(hidden_states, attention_mask=None, head_mask=None)

Constructs the attention mechanism for a multi-head self-attention layer in MSBertAttention.

PARAMETER	DESCRIPTION
self	The instance of the MSBertAttention class. TYPE: MSBertAttention
hidden_states	The input tensor of shape (batch_size, sequence_length, hidden_size). It represents the sequence of hidden states for each token in the input sequence. TYPE: Tensor
attention_mask	An optional tensor of shape (batch_size, sequence_length) indicating which tokens should be attended to and which should be ignored. The value 1 indicates to attend to the token, while 0 indicates to ignore it. If not provided, all tokens are attended to. TYPE: Tensor DEFAULT: None
head_mask	An optional tensor of shape (num_heads,) or (num_layers, num_heads) indicating which heads or layers to mask. 1 indicates to include the head/layer, while 0 indicates to mask it. If not provided, all heads/layers are included. TYPE: Tensor DEFAULT: None

RETURNS	DESCRIPTION
	Tuple[torch.Tensor]: A tuple containing: attention_output (torch.Tensor): The output tensor of shape (batch_size, sequence_length, hidden_size), which represents the attended hidden states for each token in the input sequence. self_outputs[1:] (tuple): A tuple of length num_layers containing tensors representing intermediate outputs of the self-attention mechanism.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states, attention_mask=None, head_mask=None):
    """
    Constructs the attention mechanism for a multi-head self-attention layer in MSBertAttention.

    Args:
        self (MSBertAttention): The instance of the MSBertAttention class.
        hidden_states (torch.Tensor): The input tensor of shape (batch_size, sequence_length, hidden_size).
            It represents the sequence of hidden states for each token in the input sequence.
        attention_mask (torch.Tensor, optional): An optional tensor of shape (batch_size, sequence_length) indicating
            which tokens should be attended to and which should be ignored. The value 1 indicates to attend to the token,
            while 0 indicates to ignore it. If not provided, all tokens are attended to.
        head_mask (torch.Tensor, optional): An optional tensor of shape (num_heads,) or (num_layers, num_heads) indicating
            which heads or layers to mask. 1 indicates to include the head/layer, while 0 indicates to mask it.
            If not provided, all heads/layers are included.

    Returns:
        Tuple[torch.Tensor]:
            A tuple containing:

            - attention_output (torch.Tensor): The output tensor of shape (batch_size, sequence_length, hidden_size),
              which represents the attended hidden states for each token in the input sequence.
            - self_outputs[1:] (tuple): A tuple of length `num_layers` containing tensors representing intermediate
              outputs of the self-attention mechanism.

    Raises:
        None.
    """
    self_outputs = self.self(hidden_states, attention_mask, head_mask)
    attention_output = self.output(self_outputs[0], hidden_states)
    outputs = (attention_output,) + self_outputs[1:]
    return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertEmbeddings

Bases: Module

Embeddings for BERT, include word, position and token_type

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertEmbeddings(nn.Module):
    """
    Embeddings for BERT, include word, position and token_type
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSBertEmbeddings class.

        Args:
            self: The object instance.
            config: An object of the config class containing the configuration parameters for the embeddings.

        Returns:
            None.

        Raises:
            None.

        This method initializes the MSBertEmbeddings object by setting up the word embeddings, position embeddings,
        token type embeddings, layer normalization, and dropout. The configuration parameters are used to determine
        the size of the embeddings and other properties.

        - The 'word_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table
        for word embeddings. It takes the vocabulary size (config.vocab_size) and hidden size (config.hidden_size) as 
        arguments.
        - The 'position_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table
        for position embeddings. It takes the maximum position embeddings (config.max_position_embeddings) and hidden size 
        (config.hidden_size) as arguments.
        - The 'token_type_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table
        for token type embeddings. It takes the token type vocabulary size (config.type_vocab_size) and hidden size 
        (config.hidden_size) as arguments.
        - The 'LayerNorm' attribute is an instance of the nn.LayerNorm class, which applies layer normalization to the
        input embeddings. It takes the hidden size (config.hidden_size) and epsilon (config.layer_norm_eps) as arguments.
        - The 'dropout' attribute is an instance of the nn.Dropout class, which applies dropout regularization to the
        input embeddings. It takes the dropout probability (config.hidden_dropout_prob) as an argument.
        """
        super().__init__()
        self.word_embeddings = nn.Embedding(
            config.vocab_size,
            config.hidden_size,
        )
        self.position_embeddings = nn.Embedding(
            config.max_position_embeddings,
            config.hidden_size,
        )
        self.token_type_embeddings = nn.Embedding(
            config.type_vocab_size,
            config.hidden_size,
        )
        self.LayerNorm = nn.LayerNorm(
            (config.hidden_size,), eps=config.layer_norm_eps
        )
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, input_ids, token_type_ids, position_ids):
        """
        This method forwards the embeddings for MSBert model.

        Args:
            self (object): The object instance of MSBertEmbeddings class.
            input_ids (tensor): The input tensor containing the token ids for the input sequence.
            token_type_ids (tensor): The token type ids to distinguish different sentences in the input sequence.
            position_ids (tensor): The position ids to indicate the position of each token in the input sequence.

        Returns:
            tensor: The forwarded embeddings for the input sequence represented as a tensor.

        Raises:
            None
        """
        words_embeddings = self.word_embeddings(input_ids)
        position_embeddings = self.position_embeddings(position_ids)
        token_type_embeddings = self.token_type_embeddings(token_type_ids)
        embeddings = words_embeddings + position_embeddings + token_type_embeddings
        embeddings = self.LayerNorm(embeddings)
        embeddings = self.dropout(embeddings)
        return embeddings

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertEmbeddings.__init__(config)

Initializes an instance of the MSBertEmbeddings class.

PARAMETER	DESCRIPTION
self	The object instance.
config	An object of the config class containing the configuration parameters for the embeddings.

RETURNS	DESCRIPTION
	None.

This method initializes the MSBertEmbeddings object by setting up the word embeddings, position embeddings, token type embeddings, layer normalization, and dropout. The configuration parameters are used to determine the size of the embeddings and other properties.

• The 'word_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table for word embeddings. It takes the vocabulary size (config.vocab_size) and hidden size (config.hidden_size) as arguments.
• The 'position_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table for position embeddings. It takes the maximum position embeddings (config.max_position_embeddings) and hidden size (config.hidden_size) as arguments.
• The 'token_type_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table for token type embeddings. It takes the token type vocabulary size (config.type_vocab_size) and hidden size (config.hidden_size) as arguments.
• The 'LayerNorm' attribute is an instance of the nn.LayerNorm class, which applies layer normalization to the input embeddings. It takes the hidden size (config.hidden_size) and epsilon (config.layer_norm_eps) as arguments.
• The 'dropout' attribute is an instance of the nn.Dropout class, which applies dropout regularization to the input embeddings. It takes the dropout probability (config.hidden_dropout_prob) as an argument.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertEmbeddings class.

    Args:
        self: The object instance.
        config: An object of the config class containing the configuration parameters for the embeddings.

    Returns:
        None.

    Raises:
        None.

    This method initializes the MSBertEmbeddings object by setting up the word embeddings, position embeddings,
    token type embeddings, layer normalization, and dropout. The configuration parameters are used to determine
    the size of the embeddings and other properties.

    - The 'word_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table
    for word embeddings. It takes the vocabulary size (config.vocab_size) and hidden size (config.hidden_size) as 
    arguments.
    - The 'position_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table
    for position embeddings. It takes the maximum position embeddings (config.max_position_embeddings) and hidden size 
    (config.hidden_size) as arguments.
    - The 'token_type_embeddings' attribute is an instance of the nn.Embedding class, which represents a lookup table
    for token type embeddings. It takes the token type vocabulary size (config.type_vocab_size) and hidden size 
    (config.hidden_size) as arguments.
    - The 'LayerNorm' attribute is an instance of the nn.LayerNorm class, which applies layer normalization to the
    input embeddings. It takes the hidden size (config.hidden_size) and epsilon (config.layer_norm_eps) as arguments.
    - The 'dropout' attribute is an instance of the nn.Dropout class, which applies dropout regularization to the
    input embeddings. It takes the dropout probability (config.hidden_dropout_prob) as an argument.
    """
    super().__init__()
    self.word_embeddings = nn.Embedding(
        config.vocab_size,
        config.hidden_size,
    )
    self.position_embeddings = nn.Embedding(
        config.max_position_embeddings,
        config.hidden_size,
    )
    self.token_type_embeddings = nn.Embedding(
        config.type_vocab_size,
        config.hidden_size,
    )
    self.LayerNorm = nn.LayerNorm(
        (config.hidden_size,), eps=config.layer_norm_eps
    )
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertEmbeddings.forward(input_ids, token_type_ids, position_ids)

This method forwards the embeddings for MSBert model.

PARAMETER	DESCRIPTION
self	The object instance of MSBertEmbeddings class. TYPE: object
input_ids	The input tensor containing the token ids for the input sequence. TYPE: tensor
token_type_ids	The token type ids to distinguish different sentences in the input sequence. TYPE: tensor
position_ids	The position ids to indicate the position of each token in the input sequence. TYPE: tensor

RETURNS	DESCRIPTION
tensor	The forwarded embeddings for the input sequence represented as a tensor.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, input_ids, token_type_ids, position_ids):
    """
    This method forwards the embeddings for MSBert model.

    Args:
        self (object): The object instance of MSBertEmbeddings class.
        input_ids (tensor): The input tensor containing the token ids for the input sequence.
        token_type_ids (tensor): The token type ids to distinguish different sentences in the input sequence.
        position_ids (tensor): The position ids to indicate the position of each token in the input sequence.

    Returns:
        tensor: The forwarded embeddings for the input sequence represented as a tensor.

    Raises:
        None
    """
    words_embeddings = self.word_embeddings(input_ids)
    position_embeddings = self.position_embeddings(position_ids)
    token_type_embeddings = self.token_type_embeddings(token_type_ids)
    embeddings = words_embeddings + position_embeddings + token_type_embeddings
    embeddings = self.LayerNorm(embeddings)
    embeddings = self.dropout(embeddings)
    return embeddings

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertEncoder

Bases: Module

Bert Encoder

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertEncoder(nn.Module):
    r"""
    Bert Encoder
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSBertEncoder class.

        Args:
            self (MSBertEncoder): The instance of the class itself.
            config:
                An object containing the configuration parameters for the MSBertEncoder.

                - output_attentions (bool): Whether to output attentions weights.
                - output_hidden_states (bool): Whether to output all hidden states.
                - layer (nn.ModuleList): List of MSBertLayer instances.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.output_attentions = config.output_attentions
        self.output_hidden_states = config.output_hidden_states
        self.layer = nn.ModuleList(
            [MSBertLayer(config) for _ in range(config.num_hidden_layers)]
        )

    def _set_recompute(self):
        """
        Sets the recompute flag for each layer in the MSBertEncoder.

        Args:
            self: An instance of the MSBertEncoder class.

        Returns:
            None.

        Raises:
            None.

        Description:
            This method iterates over each layer within the MSBertEncoder instance and sets the recompute flag
            for each layer by calling the 'recompute()' method of the layer.
            The recompute flag is used to indicate whether the layer needs to be recomputed during the
            forward pass of the encoder.
            By setting the recompute flag, it allows for dynamic computation of the layer based on the input.

        Example:
            ```python
            >>> encoder = MSBertEncoder()
            >>> encoder._set_recompute()
            ```

        Note:
            This method is typically called internally within the MSBertEncoder class
            and does not need to be called externally.
        """
        for layer in self.layer:
            layer.recompute()

    def forward(self, hidden_states, attention_mask=None, head_mask=None,
                encoder_hidden_states = None,
                encoder_attention_mask = None):
        """
        Constructs the MSBertEncoder.

        Args:
            self: An instance of the MSBertEncoder class.
            hidden_states (Tensor): The input hidden states of the encoder.
                Shape: (batch_size, sequence_length, hidden_size)
            attention_mask (Tensor, optional): The attention mask for the input hidden states.
                If provided, the attention mask should have the same shape as the hidden states.
                Each element of the mask should be 0 or 1, where 0 indicates the position is padded/invalid and 1
                indicates the position is not padded/valid.
                Defaults to None.
            head_mask (Tensor, optional): The head mask for the attention mechanism.
                If provided, the head mask should have the same shape as the number of layers in the encoder.
                Each element of the mask should be 0 or 1, where 0 indicates the head is masked and 1 indicates the head is not masked.
                Defaults to None.
            encoder_hidden_states (Tensor, optional): The hidden states of the encoder.
                Shape: (batch_size, sequence_length, hidden_size)
                Defaults to None.
            encoder_attention_mask (Tensor, optional): The attention mask for the encoder hidden states.
                If provided, the attention mask should have the same shape as the encoder hidden states.
                Each element of the mask should be 0 or 1, where 0 indicates the position is padded/invalid
                and 1 indicates the position is not padded/valid.
                Defaults to None.

        Returns:
            outputs (Tuple):
                A tuple containing the following elements:

                - hidden_states (Tensor): The output hidden states of the encoder.
                    Shape: (batch_size, sequence_length, hidden_size)
                - all_hidden_states (Tuple[Tensor]): A tuple of hidden states of all layers.
                    Each element of the tuple has the shape (batch_size, sequence_length, hidden_size).
                    This will be included if the 'output_hidden_states' flag is set to True.
                - all_attentions (Tuple[Tensor]): A tuple of attention scores of all layers.
                    Each element of the tuple has the shape (batch_size, num_heads, sequence_length, sequence_length).
                    This will be included if the 'output_attentions' flag is set to True.

        Raises:
            None.
        """
        all_hidden_states = ()
        all_attentions = ()
        for i, layer_module in enumerate(self.layer):
            if self.output_hidden_states:
                all_hidden_states += (hidden_states,)

            layer_outputs = layer_module(
                hidden_states,
                attention_mask,
                head_mask[i] if head_mask is not None else None,
                encoder_hidden_states,
                encoder_attention_mask
                )
            hidden_states = layer_outputs[0]

            if self.output_attentions:
                all_attentions += (layer_outputs[1],)

        if self.output_hidden_states:
            all_hidden_states += (hidden_states,)

        outputs = (hidden_states,)
        if self.output_hidden_states:
            outputs += (all_hidden_states,)
        if self.output_attentions:
            outputs += (all_attentions,)
        return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertEncoder.__init__(config)

Initializes an instance of the MSBertEncoder class.

PARAMETER	DESCRIPTION
self	The instance of the class itself. TYPE: MSBertEncoder
config	An object containing the configuration parameters for the MSBertEncoder. output_attentions (bool): Whether to output attentions weights. output_hidden_states (bool): Whether to output all hidden states. layer (nn.ModuleList): List of MSBertLayer instances.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertEncoder class.

    Args:
        self (MSBertEncoder): The instance of the class itself.
        config:
            An object containing the configuration parameters for the MSBertEncoder.

            - output_attentions (bool): Whether to output attentions weights.
            - output_hidden_states (bool): Whether to output all hidden states.
            - layer (nn.ModuleList): List of MSBertLayer instances.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.output_attentions = config.output_attentions
    self.output_hidden_states = config.output_hidden_states
    self.layer = nn.ModuleList(
        [MSBertLayer(config) for _ in range(config.num_hidden_layers)]
    )

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertEncoder.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None)

Constructs the MSBertEncoder.

PARAMETER	DESCRIPTION
self	An instance of the MSBertEncoder class.
hidden_states	The input hidden states of the encoder. Shape: (batch_size, sequence_length, hidden_size) TYPE: Tensor
attention_mask	The attention mask for the input hidden states. If provided, the attention mask should have the same shape as the hidden states. Each element of the mask should be 0 or 1, where 0 indicates the position is padded/invalid and 1 indicates the position is not padded/valid. Defaults to None. TYPE: Tensor DEFAULT: None
head_mask	The head mask for the attention mechanism. If provided, the head mask should have the same shape as the number of layers in the encoder. Each element of the mask should be 0 or 1, where 0 indicates the head is masked and 1 indicates the head is not masked. Defaults to None. TYPE: Tensor DEFAULT: None
encoder_hidden_states	The hidden states of the encoder. Shape: (batch_size, sequence_length, hidden_size) Defaults to None. TYPE: Tensor DEFAULT: None
encoder_attention_mask	The attention mask for the encoder hidden states. If provided, the attention mask should have the same shape as the encoder hidden states. Each element of the mask should be 0 or 1, where 0 indicates the position is padded/invalid and 1 indicates the position is not padded/valid. Defaults to None. TYPE: Tensor DEFAULT: None

RETURNS

DESCRIPTION

outputs

A tuple containing the following elements: hidden_states (Tensor): The output hidden states of the encoder. Shape: (batch_size, sequence_length, hidden_size) all_hidden_states (Tuple[Tensor]): A tuple of hidden states of all layers. Each element of the tuple has the shape (batch_size, sequence_length, hidden_size). This will be included if the 'output_hidden_states' flag is set to True. all_attentions (Tuple[Tensor]): A tuple of attention scores of all layers. Each element of the tuple has the shape (batch_size, num_heads, sequence_length, sequence_length). This will be included if the 'output_attentions' flag is set to True. TYPE: Tuple

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states, attention_mask=None, head_mask=None,
            encoder_hidden_states = None,
            encoder_attention_mask = None):
    """
    Constructs the MSBertEncoder.

    Args:
        self: An instance of the MSBertEncoder class.
        hidden_states (Tensor): The input hidden states of the encoder.
            Shape: (batch_size, sequence_length, hidden_size)
        attention_mask (Tensor, optional): The attention mask for the input hidden states.
            If provided, the attention mask should have the same shape as the hidden states.
            Each element of the mask should be 0 or 1, where 0 indicates the position is padded/invalid and 1
            indicates the position is not padded/valid.
            Defaults to None.
        head_mask (Tensor, optional): The head mask for the attention mechanism.
            If provided, the head mask should have the same shape as the number of layers in the encoder.
            Each element of the mask should be 0 or 1, where 0 indicates the head is masked and 1 indicates the head is not masked.
            Defaults to None.
        encoder_hidden_states (Tensor, optional): The hidden states of the encoder.
            Shape: (batch_size, sequence_length, hidden_size)
            Defaults to None.
        encoder_attention_mask (Tensor, optional): The attention mask for the encoder hidden states.
            If provided, the attention mask should have the same shape as the encoder hidden states.
            Each element of the mask should be 0 or 1, where 0 indicates the position is padded/invalid
            and 1 indicates the position is not padded/valid.
            Defaults to None.

    Returns:
        outputs (Tuple):
            A tuple containing the following elements:

            - hidden_states (Tensor): The output hidden states of the encoder.
                Shape: (batch_size, sequence_length, hidden_size)
            - all_hidden_states (Tuple[Tensor]): A tuple of hidden states of all layers.
                Each element of the tuple has the shape (batch_size, sequence_length, hidden_size).
                This will be included if the 'output_hidden_states' flag is set to True.
            - all_attentions (Tuple[Tensor]): A tuple of attention scores of all layers.
                Each element of the tuple has the shape (batch_size, num_heads, sequence_length, sequence_length).
                This will be included if the 'output_attentions' flag is set to True.

    Raises:
        None.
    """
    all_hidden_states = ()
    all_attentions = ()
    for i, layer_module in enumerate(self.layer):
        if self.output_hidden_states:
            all_hidden_states += (hidden_states,)

        layer_outputs = layer_module(
            hidden_states,
            attention_mask,
            head_mask[i] if head_mask is not None else None,
            encoder_hidden_states,
            encoder_attention_mask
            )
        hidden_states = layer_outputs[0]

        if self.output_attentions:
            all_attentions += (layer_outputs[1],)

    if self.output_hidden_states:
        all_hidden_states += (hidden_states,)

    outputs = (hidden_states,)
    if self.output_hidden_states:
        outputs += (all_hidden_states,)
    if self.output_attentions:
        outputs += (all_attentions,)
    return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertForPretraining

Bases: MSBertPreTrainedModel

Bert For Pretraining

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertForPretraining(MSBertPreTrainedModel):
    r"""
    Bert For Pretraining
    """
    def __init__(self, config, *args, **kwargs):
        """
        __init__

        Initialize the MSBertForPretraining class.

        Args:
            self: The instance of the MSBertForPretraining class.
            config: The configuration for the MSBertForPretraining,
                containing various parameters and settings for model initialization.
                It should be an instance of the configuration class specific to the MSBertForPretraining model.

        Returns:
            None.

        Raises:
            None
        """
        super().__init__(config, *args, **kwargs)
        self.bert = MSBertModel(config)
        self.cls = MSBertPreTrainingHeads(config)
        self.vocab_size = config.vocab_size

        self.cls.predictions.decoder.weight = (
            self.bert.embeddings.word_embeddings.weight
        )

    def forward(
        self,
        input_ids,
        attention_mask=None,
        token_type_ids=None,
        position_ids=None,
        head_mask=None,
        masked_lm_positions=None,
    ):
        """
        This method forwards the pretraining model for MSBertForPretraining.

        Args:
            self (MSBertForPretraining): The instance of the MSBertForPretraining class.
            input_ids (Tensor): The input tensor containing the token ids.
            attention_mask (Tensor, optional): A tensor representing the attention mask. Default is None.
            token_type_ids (Tensor, optional): A tensor representing the token type ids. Default is None.
            position_ids (Tensor, optional): A tensor representing the position ids. Default is None.
            head_mask (Tensor, optional): A tensor representing the head mask. Default is None.
            masked_lm_positions (List[int]): A list of integer positions of masked language model tokens.

        Returns:
            Tuple[Tensor, Tensor]: A tuple containing the prediction scores and sequence relationship score.

        Raises:
            None
        """
        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
        )
        # ic(outputs) # [shape(batch_size, 128, 256), shape(batch_size, 256)]

        sequence_output, pooled_output = outputs[:2]
        prediction_scores, seq_relationship_score = self.cls(
            sequence_output, pooled_output, masked_lm_positions
        )

        outputs = (
            prediction_scores,
            seq_relationship_score,
        ) + outputs[2:]
        # ic(outputs) # [shape(batch_size, 128, 256), shape(batch_size, 256)]

        return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertForPretraining.__init__(config, *args, **kwargs)

init

Initialize the MSBertForPretraining class.

PARAMETER	DESCRIPTION
self	The instance of the MSBertForPretraining class.
config	The configuration for the MSBertForPretraining, containing various parameters and settings for model initialization. It should be an instance of the configuration class specific to the MSBertForPretraining model.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config, *args, **kwargs):
    """
    __init__

    Initialize the MSBertForPretraining class.

    Args:
        self: The instance of the MSBertForPretraining class.
        config: The configuration for the MSBertForPretraining,
            containing various parameters and settings for model initialization.
            It should be an instance of the configuration class specific to the MSBertForPretraining model.

    Returns:
        None.

    Raises:
        None
    """
    super().__init__(config, *args, **kwargs)
    self.bert = MSBertModel(config)
    self.cls = MSBertPreTrainingHeads(config)
    self.vocab_size = config.vocab_size

    self.cls.predictions.decoder.weight = (
        self.bert.embeddings.word_embeddings.weight
    )

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertForPretraining.forward(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_positions=None)

This method forwards the pretraining model for MSBertForPretraining.

PARAMETER	DESCRIPTION
self	The instance of the MSBertForPretraining class. TYPE: MSBertForPretraining
input_ids	The input tensor containing the token ids. TYPE: Tensor
attention_mask	A tensor representing the attention mask. Default is None. TYPE: Tensor DEFAULT: None
token_type_ids	A tensor representing the token type ids. Default is None. TYPE: Tensor DEFAULT: None
position_ids	A tensor representing the position ids. Default is None. TYPE: Tensor DEFAULT: None
head_mask	A tensor representing the head mask. Default is None. TYPE: Tensor DEFAULT: None
masked_lm_positions	A list of integer positions of masked language model tokens. TYPE: List[int] DEFAULT: None

RETURNS	DESCRIPTION
	Tuple[Tensor, Tensor]: A tuple containing the prediction scores and sequence relationship score.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(
    self,
    input_ids,
    attention_mask=None,
    token_type_ids=None,
    position_ids=None,
    head_mask=None,
    masked_lm_positions=None,
):
    """
    This method forwards the pretraining model for MSBertForPretraining.

    Args:
        self (MSBertForPretraining): The instance of the MSBertForPretraining class.
        input_ids (Tensor): The input tensor containing the token ids.
        attention_mask (Tensor, optional): A tensor representing the attention mask. Default is None.
        token_type_ids (Tensor, optional): A tensor representing the token type ids. Default is None.
        position_ids (Tensor, optional): A tensor representing the position ids. Default is None.
        head_mask (Tensor, optional): A tensor representing the head mask. Default is None.
        masked_lm_positions (List[int]): A list of integer positions of masked language model tokens.

    Returns:
        Tuple[Tensor, Tensor]: A tuple containing the prediction scores and sequence relationship score.

    Raises:
        None
    """
    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
    )
    # ic(outputs) # [shape(batch_size, 128, 256), shape(batch_size, 256)]

    sequence_output, pooled_output = outputs[:2]
    prediction_scores, seq_relationship_score = self.cls(
        sequence_output, pooled_output, masked_lm_positions
    )

    outputs = (
        prediction_scores,
        seq_relationship_score,
    ) + outputs[2:]
    # ic(outputs) # [shape(batch_size, 128, 256), shape(batch_size, 256)]

    return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertForSequenceClassification

Bases: MSBertPreTrainedModel

Bert Model for classification tasks

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertForSequenceClassification(MSBertPreTrainedModel):
    """Bert Model for classification tasks"""
    def __init__(self, config):
        """
        Initializes an instance of the MSBertForSequenceClassification class.

        Args:
            self: The instance of the class.
            config (object): A configuration object containing the settings for the model.
                It should include the following attributes:

                - num_labels (int): The number of labels for sequence classification.
                - classifier_dropout (float, optional): The dropout probability for the classifier layer.
                If not provided, the value will default to config.hidden_dropout_prob.

        Returns:
            None: This method initializes the instance with the provided configuration.

        Raises:
            TypeError: If the config parameter is not provided or is not of the expected type.
            ValueError: If the num_labels attribute is not present in the config object.
            AttributeError: If the config object does not contain the necessary attributes for model configuration.
            RuntimeError: If an error occurs during model initialization.
        """
        super().__init__(config)
        self.num_labels = config.num_labels
        self.config = config

        self.bert = MSBertModel(config)
        classifier_dropout = (
            config.classifier_dropout
            if config.classifier_dropout is not None
            else config.hidden_dropout_prob
        )
        self.classifier = nn.Linear(config.hidden_size, self.num_labels)
        self.dropout = nn.Dropout(p=classifier_dropout)

    def forward(
        self,
        input_ids,
        attention_mask=None,
        token_type_ids=None,
        position_ids=None,
        head_mask=None,
        **kwargs
    ):
        """
        Constructs the MSBertForSequenceClassification model for a given input.

        Args:
            self (MSBertForSequenceClassification): The instance of the MSBertForSequenceClassification class.
            input_ids (Tensor): The input tensor containing the indices of input tokens.
            attention_mask (Tensor, optional): An optional tensor containing the attention mask for the input.
            token_type_ids (Tensor, optional): An optional tensor containing the token type ids.
            position_ids (Tensor, optional): An optional tensor containing the position ids.
            head_mask (Tensor, optional): An optional tensor containing the head mask.

        Returns:
            tuple: A tuple containing the logits for the classification and additional outputs from the model.

        Raises:
            None
        """
        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
        )
        pooled_output = outputs[1]

        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)

        output = (logits,) + outputs[2:]

        return output

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertForSequenceClassification.__init__(config)

Initializes an instance of the MSBertForSequenceClassification class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	A configuration object containing the settings for the model. It should include the following attributes: num_labels (int): The number of labels for sequence classification. classifier_dropout (float, optional): The dropout probability for the classifier layer. If not provided, the value will default to config.hidden_dropout_prob. TYPE: object

RETURNS	DESCRIPTION
None	This method initializes the instance with the provided configuration.

RAISES	DESCRIPTION
TypeError	If the config parameter is not provided or is not of the expected type.
ValueError	If the num_labels attribute is not present in the config object.
AttributeError	If the config object does not contain the necessary attributes for model configuration.
RuntimeError	If an error occurs during model initialization.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertForSequenceClassification class.

    Args:
        self: The instance of the class.
        config (object): A configuration object containing the settings for the model.
            It should include the following attributes:

            - num_labels (int): The number of labels for sequence classification.
            - classifier_dropout (float, optional): The dropout probability for the classifier layer.
            If not provided, the value will default to config.hidden_dropout_prob.

    Returns:
        None: This method initializes the instance with the provided configuration.

    Raises:
        TypeError: If the config parameter is not provided or is not of the expected type.
        ValueError: If the num_labels attribute is not present in the config object.
        AttributeError: If the config object does not contain the necessary attributes for model configuration.
        RuntimeError: If an error occurs during model initialization.
    """
    super().__init__(config)
    self.num_labels = config.num_labels
    self.config = config

    self.bert = MSBertModel(config)
    classifier_dropout = (
        config.classifier_dropout
        if config.classifier_dropout is not None
        else config.hidden_dropout_prob
    )
    self.classifier = nn.Linear(config.hidden_size, self.num_labels)
    self.dropout = nn.Dropout(p=classifier_dropout)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertForSequenceClassification.forward(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, **kwargs)

Constructs the MSBertForSequenceClassification model for a given input.

PARAMETER	DESCRIPTION
self	The instance of the MSBertForSequenceClassification class. TYPE: MSBertForSequenceClassification
input_ids	The input tensor containing the indices of input tokens. TYPE: Tensor
attention_mask	An optional tensor containing the attention mask for the input. TYPE: Tensor DEFAULT: None
token_type_ids	An optional tensor containing the token type ids. TYPE: Tensor DEFAULT: None
position_ids	An optional tensor containing the position ids. TYPE: Tensor DEFAULT: None
head_mask	An optional tensor containing the head mask. TYPE: Tensor DEFAULT: None

RETURNS	DESCRIPTION
tuple	A tuple containing the logits for the classification and additional outputs from the model.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(
    self,
    input_ids,
    attention_mask=None,
    token_type_ids=None,
    position_ids=None,
    head_mask=None,
    **kwargs
):
    """
    Constructs the MSBertForSequenceClassification model for a given input.

    Args:
        self (MSBertForSequenceClassification): The instance of the MSBertForSequenceClassification class.
        input_ids (Tensor): The input tensor containing the indices of input tokens.
        attention_mask (Tensor, optional): An optional tensor containing the attention mask for the input.
        token_type_ids (Tensor, optional): An optional tensor containing the token type ids.
        position_ids (Tensor, optional): An optional tensor containing the position ids.
        head_mask (Tensor, optional): An optional tensor containing the head mask.

    Returns:
        tuple: A tuple containing the logits for the classification and additional outputs from the model.

    Raises:
        None
    """
    outputs = self.bert(
        input_ids,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
    )
    pooled_output = outputs[1]

    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    output = (logits,) + outputs[2:]

    return output

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertIntermediate

Bases: Module

Bert Intermediate

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertIntermediate(nn.Module):
    r"""
    Bert Intermediate
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSBertIntermediate class.

        Args:
            self: The instance of the MSBertIntermediate class.
            config: An object representing the configuration for the MSBertIntermediate model.
                It contains the following attributes:

                - hidden_size (int): The size of the hidden layer.
                - intermediate_size (int): The size of the intermediate layer.
                - hidden_act (str): The activation function for the hidden layer.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not provided or is not of the correct type.
            ValueError: If the config object does not contain the required attributes.
        """
        super().__init__()
        self.dense = nn.Linear(
            config.hidden_size,
            config.intermediate_size,
        )
        self.intermediate_act_fn = ACT2FN[config.hidden_act]

    def forward(self, hidden_states):
        """
        Constructs the intermediate layer of the MSBert model.

        Args:
            self: An instance of the MSBertIntermediate class.
            hidden_states (Tensor): The input hidden states.
                Should be a tensor of shape (batch_size, sequence_length, hidden_size).

        Returns:
            Tensor: The output hidden states after passing through the intermediate layer.
                Has the same shape as the input hidden states.

        Raises:
            None.

        This method takes in the input hidden states and applies the intermediate layer transformations.
        It first passes the hidden states through a dense layer, then applies an activation function.
        The resulting hidden states are returned as the output.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.intermediate_act_fn(hidden_states)
        return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertIntermediate.__init__(config)

Initializes an instance of the MSBertIntermediate class.

PARAMETER	DESCRIPTION
self	The instance of the MSBertIntermediate class.
config	An object representing the configuration for the MSBertIntermediate model. It contains the following attributes: hidden_size (int): The size of the hidden layer. intermediate_size (int): The size of the intermediate layer. hidden_act (str): The activation function for the hidden layer.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the config parameter is not provided or is not of the correct type.
ValueError	If the config object does not contain the required attributes.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertIntermediate class.

    Args:
        self: The instance of the MSBertIntermediate class.
        config: An object representing the configuration for the MSBertIntermediate model.
            It contains the following attributes:

            - hidden_size (int): The size of the hidden layer.
            - intermediate_size (int): The size of the intermediate layer.
            - hidden_act (str): The activation function for the hidden layer.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not provided or is not of the correct type.
        ValueError: If the config object does not contain the required attributes.
    """
    super().__init__()
    self.dense = nn.Linear(
        config.hidden_size,
        config.intermediate_size,
    )
    self.intermediate_act_fn = ACT2FN[config.hidden_act]

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertIntermediate.forward(hidden_states)

Constructs the intermediate layer of the MSBert model.

PARAMETER	DESCRIPTION
self	An instance of the MSBertIntermediate class.
hidden_states	The input hidden states. Should be a tensor of shape (batch_size, sequence_length, hidden_size). TYPE: Tensor

RETURNS	DESCRIPTION
Tensor	The output hidden states after passing through the intermediate layer. Has the same shape as the input hidden states.

This method takes in the input hidden states and applies the intermediate layer transformations. It first passes the hidden states through a dense layer, then applies an activation function. The resulting hidden states are returned as the output.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states):
    """
    Constructs the intermediate layer of the MSBert model.

    Args:
        self: An instance of the MSBertIntermediate class.
        hidden_states (Tensor): The input hidden states.
            Should be a tensor of shape (batch_size, sequence_length, hidden_size).

    Returns:
        Tensor: The output hidden states after passing through the intermediate layer.
            Has the same shape as the input hidden states.

    Raises:
        None.

    This method takes in the input hidden states and applies the intermediate layer transformations.
    It first passes the hidden states through a dense layer, then applies an activation function.
    The resulting hidden states are returned as the output.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.intermediate_act_fn(hidden_states)
    return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertLMPredictionHead

Bases: Module

Bert LM Prediction Head

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertLMPredictionHead(nn.Module):
    r"""
    Bert LM Prediction Head
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSBertLMPredictionHead class.

        Args:
            self: The object instance.
            config:
                An instance of the configuration class that contains the model's configuration settings.

                - Type: Any
                - Purpose: This parameter is used to configure the MSBertLMPredictionHead instance.
                - Restrictions: None

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.transform = MSBertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(
            config.hidden_size,
            config.vocab_size,
            bias=False,
        )

        self.bias = Parameter(initializer("zeros", config.vocab_size), "bias")

    def forward(self, hidden_states, masked_lm_positions):
        """
        Constructs the MSBertLMPredictionHead.

        This method takes in the hidden states and masked language model positions,
        and applies a series of operations to compute the final hidden states for the MSBertLMPredictionHead.
        The resulting hidden states are then transformed and decoded to produce the final output.

        Args:
            self (MSBertLMPredictionHead): An instance of the MSBertLMPredictionHead class.
            hidden_states (Tensor): A tensor of shape (batch_size, seq_len, hidden_size) containing the hidden states.
            masked_lm_positions (Tensor): A tensor of shape (batch_size, num_masked_lm_positions)
                containing the positions of the masked language model tokens. If None, no masking is applied.

        Returns:
            Tensor:
                A tensor of shape (batch_size, seq_len, hidden_size) containing
                the final hidden states for the MSBertLMPredictionHead.

        Raises:
            None.
        """
        batch_size, seq_len, hidden_size = hidden_states.shape
        if masked_lm_positions is not None:
            flat_offsets = ops.arange(batch_size) * seq_len
            flat_position = (masked_lm_positions + flat_offsets.reshape(-1, 1)).reshape(
                -1
            )
            flat_sequence_tensor = hidden_states.reshape(-1, hidden_size)
            hidden_states = ops.gather(flat_sequence_tensor, flat_position, 0)
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states) + self.bias
        return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertLMPredictionHead.__init__(config)

Initializes an instance of the MSBertLMPredictionHead class.

PARAMETER	DESCRIPTION
self	The object instance.
config	An instance of the configuration class that contains the model's configuration settings. Type: Any Purpose: This parameter is used to configure the MSBertLMPredictionHead instance. Restrictions: None

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertLMPredictionHead class.

    Args:
        self: The object instance.
        config:
            An instance of the configuration class that contains the model's configuration settings.

            - Type: Any
            - Purpose: This parameter is used to configure the MSBertLMPredictionHead instance.
            - Restrictions: None

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.transform = MSBertPredictionHeadTransform(config)

    # The output weights are the same as the input embeddings, but there is
    # an output-only bias for each token.
    self.decoder = nn.Linear(
        config.hidden_size,
        config.vocab_size,
        bias=False,
    )

    self.bias = Parameter(initializer("zeros", config.vocab_size), "bias")

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertLMPredictionHead.forward(hidden_states, masked_lm_positions)

Constructs the MSBertLMPredictionHead.

This method takes in the hidden states and masked language model positions, and applies a series of operations to compute the final hidden states for the MSBertLMPredictionHead. The resulting hidden states are then transformed and decoded to produce the final output.

PARAMETER	DESCRIPTION
self	An instance of the MSBertLMPredictionHead class. TYPE: MSBertLMPredictionHead
hidden_states	A tensor of shape (batch_size, seq_len, hidden_size) containing the hidden states. TYPE: Tensor
masked_lm_positions	A tensor of shape (batch_size, num_masked_lm_positions) containing the positions of the masked language model tokens. If None, no masking is applied. TYPE: Tensor

RETURNS	DESCRIPTION
Tensor	A tensor of shape (batch_size, seq_len, hidden_size) containing the final hidden states for the MSBertLMPredictionHead.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states, masked_lm_positions):
    """
    Constructs the MSBertLMPredictionHead.

    This method takes in the hidden states and masked language model positions,
    and applies a series of operations to compute the final hidden states for the MSBertLMPredictionHead.
    The resulting hidden states are then transformed and decoded to produce the final output.

    Args:
        self (MSBertLMPredictionHead): An instance of the MSBertLMPredictionHead class.
        hidden_states (Tensor): A tensor of shape (batch_size, seq_len, hidden_size) containing the hidden states.
        masked_lm_positions (Tensor): A tensor of shape (batch_size, num_masked_lm_positions)
            containing the positions of the masked language model tokens. If None, no masking is applied.

    Returns:
        Tensor:
            A tensor of shape (batch_size, seq_len, hidden_size) containing
            the final hidden states for the MSBertLMPredictionHead.

    Raises:
        None.
    """
    batch_size, seq_len, hidden_size = hidden_states.shape
    if masked_lm_positions is not None:
        flat_offsets = ops.arange(batch_size) * seq_len
        flat_position = (masked_lm_positions + flat_offsets.reshape(-1, 1)).reshape(
            -1
        )
        flat_sequence_tensor = hidden_states.reshape(-1, hidden_size)
        hidden_states = ops.gather(flat_sequence_tensor, flat_position, 0)
    hidden_states = self.transform(hidden_states)
    hidden_states = self.decoder(hidden_states) + self.bias
    return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertLayer

Bases: Module

Bert Layer

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertLayer(nn.Module):
    r"""
    Bert Layer
    """
    def __init__(self, config, init_cache=False):
        """
        Initializes an instance of the MSBertLayer class.

        Args:
            self: The instance of the class.
            config (object): The configuration object containing various settings and parameters.
            init_cache (bool, optional): Whether to initialize the cache. Defaults to False.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.attention = MSBertAttention(config, causal=config.is_decoder, init_cache=init_cache)
        self.intermediate = MSBertIntermediate(config)
        self.output = MSBertOutput(config)
        if config.add_cross_attention:
            self.crossattention = MSBertAttention(config, causal=False, init_cache=init_cache)

    def forward(self, hidden_states, attention_mask=None, head_mask=None,
                encoder_hidden_states = None,
                encoder_attention_mask = None):
        """
        Constructs the MSBertLayer.

        Args:
            self: The instance of the MSBertLayer class.
            hidden_states: The input hidden states (tensor) of shape (batch_size, sequence_length, hidden_size).
            attention_mask:
                Optional attention mask (tensor) of shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length).
                Defaults to None.
            head_mask: Optional head mask (tensor) of shape (num_heads,) or (num_layers, num_heads). Defaults to None.
            encoder_hidden_states:
                Optional encoder hidden states (tensor) of shape (batch_size, sequence_length, hidden_size).
                Defaults to None.
            encoder_attention_mask:
                Optional encoder attention mask (tensor) of shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length).
                Defaults to None.

        Returns:
            tuple:
                A tuple containing the layer output (tensor) of shape (batch_size, sequence_length, hidden_size)
                and any additional attention outputs.

        Raises:
            None.
        """
        attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
        attention_output = attention_outputs[0]

        # Cross-Attention Block
        if encoder_hidden_states is not None:
            cross_attention_outputs = self.crossattention(
                attention_output,
                attention_mask=encoder_attention_mask,
                head_mask=head_mask,
            )
            attention_output = cross_attention_outputs[0]

        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)
        outputs = (layer_output,) + attention_outputs[1:]
        return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertLayer.__init__(config, init_cache=False)

Initializes an instance of the MSBertLayer class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object containing various settings and parameters. TYPE: object
init_cache	Whether to initialize the cache. Defaults to False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config, init_cache=False):
    """
    Initializes an instance of the MSBertLayer class.

    Args:
        self: The instance of the class.
        config (object): The configuration object containing various settings and parameters.
        init_cache (bool, optional): Whether to initialize the cache. Defaults to False.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.attention = MSBertAttention(config, causal=config.is_decoder, init_cache=init_cache)
    self.intermediate = MSBertIntermediate(config)
    self.output = MSBertOutput(config)
    if config.add_cross_attention:
        self.crossattention = MSBertAttention(config, causal=False, init_cache=init_cache)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertLayer.forward(hidden_states, attention_mask=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None)

Constructs the MSBertLayer.

PARAMETER	DESCRIPTION
self	The instance of the MSBertLayer class.
hidden_states	The input hidden states (tensor) of shape (batch_size, sequence_length, hidden_size).
attention_mask	Optional attention mask (tensor) of shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length). Defaults to None. DEFAULT: None
head_mask	Optional head mask (tensor) of shape (num_heads,) or (num_layers, num_heads). Defaults to None. DEFAULT: None
encoder_hidden_states	Optional encoder hidden states (tensor) of shape (batch_size, sequence_length, hidden_size). Defaults to None. DEFAULT: None
encoder_attention_mask	Optional encoder attention mask (tensor) of shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length). Defaults to None. DEFAULT: None

RETURNS	DESCRIPTION
tuple	A tuple containing the layer output (tensor) of shape (batch_size, sequence_length, hidden_size) and any additional attention outputs.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states, attention_mask=None, head_mask=None,
            encoder_hidden_states = None,
            encoder_attention_mask = None):
    """
    Constructs the MSBertLayer.

    Args:
        self: The instance of the MSBertLayer class.
        hidden_states: The input hidden states (tensor) of shape (batch_size, sequence_length, hidden_size).
        attention_mask:
            Optional attention mask (tensor) of shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length).
            Defaults to None.
        head_mask: Optional head mask (tensor) of shape (num_heads,) or (num_layers, num_heads). Defaults to None.
        encoder_hidden_states:
            Optional encoder hidden states (tensor) of shape (batch_size, sequence_length, hidden_size).
            Defaults to None.
        encoder_attention_mask:
            Optional encoder attention mask (tensor) of shape (batch_size, sequence_length) or (batch_size, 1, 1, sequence_length).
            Defaults to None.

    Returns:
        tuple:
            A tuple containing the layer output (tensor) of shape (batch_size, sequence_length, hidden_size)
            and any additional attention outputs.

    Raises:
        None.
    """
    attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
    attention_output = attention_outputs[0]

    # Cross-Attention Block
    if encoder_hidden_states is not None:
        cross_attention_outputs = self.crossattention(
            attention_output,
            attention_mask=encoder_attention_mask,
            head_mask=head_mask,
        )
        attention_output = cross_attention_outputs[0]

    intermediate_output = self.intermediate(attention_output)
    layer_output = self.output(intermediate_output, attention_output)
    outputs = (layer_output,) + attention_outputs[1:]
    return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertModel

Bases: MSBertPreTrainedModel

Bert Model

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertModel(MSBertPreTrainedModel):
    r"""
    Bert Model
    """
    def __init__(self, config, add_pooling_layer=True):
        """
        Initializes the MSBertModel class with the provided configuration and optional pooling layer.

        Args:
            self (MSBertModel): The current instance of the MSBertModel class.
            config (object): The configuration object containing settings for the model.
            add_pooling_layer (bool): Flag indicating whether to add a pooling layer to the model.

        Returns:
            None.

        Raises:
            None
        """
        super().__init__(config)
        self.embeddings = MSBertEmbeddings(config)
        self.encoder = MSBertEncoder(config)
        self.pooler = MSBertPooler(config) if add_pooling_layer else None
        self.num_hidden_layers = config.num_hidden_layers

    def get_input_embeddings(self):
        """
        This method returns the input embeddings of the MSBertModel.

        Args:
            self: The instance of the MSBertModel class.

        Returns:
            word_embeddings:
                This method returns the input embeddings of the MSBertModel.

        Raises:
            None.
        """
        return self.embeddings.word_embeddings

    def set_input_embeddings(self, new_embeddings):
        """
        Set the input embeddings for the MSBertModel.

        Args:
            self (MSBertModel): The MSBertModel instance.
            new_embeddings (object): The new input embeddings to be set. This could be of any type, such as a tensor or an array.

        Returns:
            None.

        Raises:
            None
        """
        self.embeddings.word_embeddings = new_embeddings

    def forward(
        self,
        input_ids,
        attention_mask=None,
        token_type_ids=None,
        position_ids=None,
        head_mask=None,
        encoder_hidden_states = None,
        encoder_attention_mask = None
    ):
        """
        Construct method in the MSBertModel class.

        Args:
            self: MSBertModel object.
            input_ids (Tensor): The input tensor containing the token ids for the input sequence.
            attention_mask (Tensor, optional):
                A mask tensor indicating which tokens should be attended to and which should be ignored.
            token_type_ids (Tensor, optional): A tensor indicating the token types for each token in the input sequence.
            position_ids (Tensor, optional): A tensor specifying the position ids for each token in the input sequence.
            head_mask (Tensor, optional): A mask tensor applied to the attention scores in the self-attention mechanism.
            encoder_hidden_states (Tensor, optional): Hidden states from the encoder.
            encoder_attention_mask (Tensor, optional): A mask tensor indicating which encoder tokens should be attended to in the self-attention mechanism.

        Returns:
            Tuple:
                A tuple containing the following:

                - sequence_output (Tensor): The output tensor from the encoder for each token in the input sequence.
                - pooled_output (Tensor): The pooled output tensor from the pooler layer, if available.
                - Additional encoder outputs.

        Raises:
            ValueError: If the dimensions of the head_mask tensor are incompatible.
        """
        if attention_mask is None:
            attention_mask = ops.ones_like(input_ids)
        if token_type_ids is None:
            token_type_ids = ops.zeros_like(input_ids)
        if position_ids is None:
            position_ids = ops.broadcast_to(ops.arange(ops.atleast_2d(input_ids).shape[-1]), input_ids.shape)

        if head_mask is not None:
            if head_mask.ndim == 1:
                head_mask = (
                    head_mask.expand_dims(0)
                    .expand_dims(0)
                    .expand_dims(-1)
                    .expand_dims(-1)
                )
                head_mask = ops.broadcast_to(
                    head_mask, (self.num_hidden_layers, -1, -1, -1, -1)
                )
            elif head_mask.ndim == 2:
                head_mask = head_mask.expand_dims(1).expand_dims(-1).expand_dims(-1)
        else:
            head_mask = [None] * self.num_hidden_layers

        embedding_output = self.embeddings(
            input_ids, position_ids=position_ids, token_type_ids=token_type_ids
        )
        encoder_outputs = self.encoder(
            embedding_output,
            attention_mask,
            head_mask=head_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
        )
        sequence_output = encoder_outputs[0]
        pooled_output = (
            self.pooler(sequence_output) if self.pooler is not None else None
        )

        outputs = (
            sequence_output,
            pooled_output,
        ) + encoder_outputs[1:]
        # add hidden_states and attentions if they are here
        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertModel.__init__(config, add_pooling_layer=True)

Initializes the MSBertModel class with the provided configuration and optional pooling layer.

PARAMETER	DESCRIPTION
self	The current instance of the MSBertModel class. TYPE: MSBertModel
config	The configuration object containing settings for the model. TYPE: object
add_pooling_layer	Flag indicating whether to add a pooling layer to the model. TYPE: bool DEFAULT: True

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config, add_pooling_layer=True):
    """
    Initializes the MSBertModel class with the provided configuration and optional pooling layer.

    Args:
        self (MSBertModel): The current instance of the MSBertModel class.
        config (object): The configuration object containing settings for the model.
        add_pooling_layer (bool): Flag indicating whether to add a pooling layer to the model.

    Returns:
        None.

    Raises:
        None
    """
    super().__init__(config)
    self.embeddings = MSBertEmbeddings(config)
    self.encoder = MSBertEncoder(config)
    self.pooler = MSBertPooler(config) if add_pooling_layer else None
    self.num_hidden_layers = config.num_hidden_layers

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertModel.forward(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, encoder_hidden_states=None, encoder_attention_mask=None)

Construct method in the MSBertModel class.

PARAMETER	DESCRIPTION
self	MSBertModel object.
input_ids	The input tensor containing the token ids for the input sequence. TYPE: Tensor
attention_mask	A mask tensor indicating which tokens should be attended to and which should be ignored. TYPE: Tensor DEFAULT: None
token_type_ids	A tensor indicating the token types for each token in the input sequence. TYPE: Tensor DEFAULT: None
position_ids	A tensor specifying the position ids for each token in the input sequence. TYPE: Tensor DEFAULT: None
head_mask	A mask tensor applied to the attention scores in the self-attention mechanism. TYPE: Tensor DEFAULT: None
encoder_hidden_states	Hidden states from the encoder. TYPE: Tensor DEFAULT: None
encoder_attention_mask	A mask tensor indicating which encoder tokens should be attended to in the self-attention mechanism. TYPE: Tensor DEFAULT: None

RETURNS	DESCRIPTION
Tuple	A tuple containing the following: sequence_output (Tensor): The output tensor from the encoder for each token in the input sequence. pooled_output (Tensor): The pooled output tensor from the pooler layer, if available. Additional encoder outputs.

RAISES	DESCRIPTION
ValueError	If the dimensions of the head_mask tensor are incompatible.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(
    self,
    input_ids,
    attention_mask=None,
    token_type_ids=None,
    position_ids=None,
    head_mask=None,
    encoder_hidden_states = None,
    encoder_attention_mask = None
):
    """
    Construct method in the MSBertModel class.

    Args:
        self: MSBertModel object.
        input_ids (Tensor): The input tensor containing the token ids for the input sequence.
        attention_mask (Tensor, optional):
            A mask tensor indicating which tokens should be attended to and which should be ignored.
        token_type_ids (Tensor, optional): A tensor indicating the token types for each token in the input sequence.
        position_ids (Tensor, optional): A tensor specifying the position ids for each token in the input sequence.
        head_mask (Tensor, optional): A mask tensor applied to the attention scores in the self-attention mechanism.
        encoder_hidden_states (Tensor, optional): Hidden states from the encoder.
        encoder_attention_mask (Tensor, optional): A mask tensor indicating which encoder tokens should be attended to in the self-attention mechanism.

    Returns:
        Tuple:
            A tuple containing the following:

            - sequence_output (Tensor): The output tensor from the encoder for each token in the input sequence.
            - pooled_output (Tensor): The pooled output tensor from the pooler layer, if available.
            - Additional encoder outputs.

    Raises:
        ValueError: If the dimensions of the head_mask tensor are incompatible.
    """
    if attention_mask is None:
        attention_mask = ops.ones_like(input_ids)
    if token_type_ids is None:
        token_type_ids = ops.zeros_like(input_ids)
    if position_ids is None:
        position_ids = ops.broadcast_to(ops.arange(ops.atleast_2d(input_ids).shape[-1]), input_ids.shape)

    if head_mask is not None:
        if head_mask.ndim == 1:
            head_mask = (
                head_mask.expand_dims(0)
                .expand_dims(0)
                .expand_dims(-1)
                .expand_dims(-1)
            )
            head_mask = ops.broadcast_to(
                head_mask, (self.num_hidden_layers, -1, -1, -1, -1)
            )
        elif head_mask.ndim == 2:
            head_mask = head_mask.expand_dims(1).expand_dims(-1).expand_dims(-1)
    else:
        head_mask = [None] * self.num_hidden_layers

    embedding_output = self.embeddings(
        input_ids, position_ids=position_ids, token_type_ids=token_type_ids
    )
    encoder_outputs = self.encoder(
        embedding_output,
        attention_mask,
        head_mask=head_mask,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_attention_mask,
    )
    sequence_output = encoder_outputs[0]
    pooled_output = (
        self.pooler(sequence_output) if self.pooler is not None else None
    )

    outputs = (
        sequence_output,
        pooled_output,
    ) + encoder_outputs[1:]
    # add hidden_states and attentions if they are here
    return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertModel.get_input_embeddings()

This method returns the input embeddings of the MSBertModel.

PARAMETER	DESCRIPTION
self	The instance of the MSBertModel class.

RETURNS	DESCRIPTION
word_embeddings	This method returns the input embeddings of the MSBertModel.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def get_input_embeddings(self):
    """
    This method returns the input embeddings of the MSBertModel.

    Args:
        self: The instance of the MSBertModel class.

    Returns:
        word_embeddings:
            This method returns the input embeddings of the MSBertModel.

    Raises:
        None.
    """
    return self.embeddings.word_embeddings

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertModel.set_input_embeddings(new_embeddings)

Set the input embeddings for the MSBertModel.

PARAMETER	DESCRIPTION
self	The MSBertModel instance. TYPE: MSBertModel
new_embeddings	The new input embeddings to be set. This could be of any type, such as a tensor or an array. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def set_input_embeddings(self, new_embeddings):
    """
    Set the input embeddings for the MSBertModel.

    Args:
        self (MSBertModel): The MSBertModel instance.
        new_embeddings (object): The new input embeddings to be set. This could be of any type, such as a tensor or an array.

    Returns:
        None.

    Raises:
        None
    """
    self.embeddings.word_embeddings = new_embeddings

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertOutput

Bases: Module

Bert Output

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertOutput(nn.Module):
    r"""
    Bert Output
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSBertOutput class.

        Args:
            self: The instance of the class.
            config: An object of type 'config' that contains the configuration parameters for the MSBertOutput.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.dense = nn.Linear(
            config.intermediate_size,
            config.hidden_size,
        )
        self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=1e-12)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        """
        This method forwards the output of the MSBert model.

        Args:
            self: The instance of the MSBertOutput class.
            hidden_states (tensor): The hidden states from the MSBert model.
                This tensor contains the encoded information from the input.
            input_tensor (tensor): The input tensor to be added to the hidden states.
                This tensor represents the original input to the MSBert model.

        Returns:
            tensor: The forwarded output tensor representing the final hidden states.
            This tensor is the result of processing the hidden states and input tensor.

        Raises:
            None.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertOutput.__init__(config)

Initializes an instance of the MSBertOutput class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An object of type 'config' that contains the configuration parameters for the MSBertOutput.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertOutput class.

    Args:
        self: The instance of the class.
        config: An object of type 'config' that contains the configuration parameters for the MSBertOutput.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.dense = nn.Linear(
        config.intermediate_size,
        config.hidden_size,
    )
    self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=1e-12)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertOutput.forward(hidden_states, input_tensor)

This method forwards the output of the MSBert model.

PARAMETER	DESCRIPTION
self	The instance of the MSBertOutput class.
hidden_states	The hidden states from the MSBert model. This tensor contains the encoded information from the input. TYPE: tensor
input_tensor	The input tensor to be added to the hidden states. This tensor represents the original input to the MSBert model. TYPE: tensor

RETURNS	DESCRIPTION
tensor	The forwarded output tensor representing the final hidden states.
	This tensor is the result of processing the hidden states and input tensor.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states, input_tensor):
    """
    This method forwards the output of the MSBert model.

    Args:
        self: The instance of the MSBertOutput class.
        hidden_states (tensor): The hidden states from the MSBert model.
            This tensor contains the encoded information from the input.
        input_tensor (tensor): The input tensor to be added to the hidden states.
            This tensor represents the original input to the MSBert model.

    Returns:
        tensor: The forwarded output tensor representing the final hidden states.
        This tensor is the result of processing the hidden states and input tensor.

    Raises:
        None.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPooler

Bases: Module

Bert Pooler

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertPooler(nn.Module):
    r"""
    Bert Pooler
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSBertPooler class.

        Args:
            self (MSBertPooler): The instance of the MSBertPooler class.
            config:
                An object containing configuration parameters.

                - Type: Any
                - Purpose: Holds the configuration settings for the MSBertPooler.
                - Restrictions: Must be compatible with the expected configuration format.

        Returns:
            None.

        Raises:
            None
        """
        super().__init__()
        self.dense = nn.Linear(
            config.hidden_size,
            config.hidden_size,
        )
        self.activation = nn.Tanh()

    def forward(self, hidden_states):
        """
        This method forwards a pooled output from the given hidden states.

        Args:
            self (MSBertPooler): The instance of the MSBertPooler class.
            hidden_states (torch.Tensor): A tensor containing the hidden states.
                It is expected to have a shape of (batch_size, sequence_length, hidden_size).

        Returns:
            torch.Tensor: The pooled output tensor obtained by applying dense
                and activation functions to the first token tensor from the hidden_states.

        Raises:
            TypeError: If the input hidden_states is not a torch.Tensor.
            ValueError: If the hidden_states tensor does not have the expected shape of
                (batch_size, sequence_length, hidden_size).
        """
        # We "pool" the model by simply taking the hidden state corresponding.
        # to the first token
        first_token_tensor = hidden_states[:, 0]
        pooled_output = self.dense(first_token_tensor)
        pooled_output = self.activation(pooled_output)
        return pooled_output

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPooler.__init__(config)

Initializes an instance of the MSBertPooler class.

PARAMETER	DESCRIPTION
self	The instance of the MSBertPooler class. TYPE: MSBertPooler
config	An object containing configuration parameters. Type: Any Purpose: Holds the configuration settings for the MSBertPooler. Restrictions: Must be compatible with the expected configuration format.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertPooler class.

    Args:
        self (MSBertPooler): The instance of the MSBertPooler class.
        config:
            An object containing configuration parameters.

            - Type: Any
            - Purpose: Holds the configuration settings for the MSBertPooler.
            - Restrictions: Must be compatible with the expected configuration format.

    Returns:
        None.

    Raises:
        None
    """
    super().__init__()
    self.dense = nn.Linear(
        config.hidden_size,
        config.hidden_size,
    )
    self.activation = nn.Tanh()

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPooler.forward(hidden_states)

This method forwards a pooled output from the given hidden states.

PARAMETER	DESCRIPTION
self	The instance of the MSBertPooler class. TYPE: MSBertPooler
hidden_states	A tensor containing the hidden states. It is expected to have a shape of (batch_size, sequence_length, hidden_size). TYPE: Tensor

RETURNS	DESCRIPTION
	torch.Tensor: The pooled output tensor obtained by applying dense and activation functions to the first token tensor from the hidden_states.

RAISES	DESCRIPTION
TypeError	If the input hidden_states is not a torch.Tensor.
ValueError	If the hidden_states tensor does not have the expected shape of (batch_size, sequence_length, hidden_size).

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states):
    """
    This method forwards a pooled output from the given hidden states.

    Args:
        self (MSBertPooler): The instance of the MSBertPooler class.
        hidden_states (torch.Tensor): A tensor containing the hidden states.
            It is expected to have a shape of (batch_size, sequence_length, hidden_size).

    Returns:
        torch.Tensor: The pooled output tensor obtained by applying dense
            and activation functions to the first token tensor from the hidden_states.

    Raises:
        TypeError: If the input hidden_states is not a torch.Tensor.
        ValueError: If the hidden_states tensor does not have the expected shape of
            (batch_size, sequence_length, hidden_size).
    """
    # We "pool" the model by simply taking the hidden state corresponding.
    # to the first token
    first_token_tensor = hidden_states[:, 0]
    pooled_output = self.dense(first_token_tensor)
    pooled_output = self.activation(pooled_output)
    return pooled_output

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPreTrainedModel

Bases: PreTrainedModel

BertPretrainedModel

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertPreTrainedModel(PreTrainedModel):
    """BertPretrainedModel"""
    config_class = BertConfig
    base_model_prefix = "bert"
    supports_recompute = True

    def _init_weights(self, cell):
        """Initialize the weights"""
        if isinstance(cell, nn.Linear):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            cell.weight.set_data(
                initializer(
                    Normal(self.config.initializer_range),
                    cell.weight.shape,
                    cell.weight.dtype,
                )
            )
            if cell.bias:
                cell.bias.set_data(
                    initializer("zeros", cell.bias.shape, cell.bias.dtype)
                )
        elif isinstance(cell, nn.Embedding):
            weight = initializer(
                Normal(self.config.initializer_range),
                cell.weight.shape,
                cell.weight.dtype,
            )
            if cell.padding_idx is not None:
                weight[cell.padding_idx] = 0
            cell.weight.set_data(weight)
        elif isinstance(cell, nn.LayerNorm):
            cell.weight.set_data(initializer("ones", cell.weight.shape, cell.weight.dtype))
            cell.bias.set_data(initializer("zeros", cell.bias.shape, cell.bias.dtype))

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPreTrainingHeads

Bases: Module

Bert PreTraining Heads

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertPreTrainingHeads(nn.Module):
    r"""
    Bert PreTraining Heads
    """
    def __init__(self, config):
        """
        Initialize the MSBertPreTrainingHeads class.

        Args:
            self (object): The instance of the class.
            config (object):
                An object containing configuration settings.

                - Type: Custom class
                - Purpose: Provides configuration parameters for the pre-training heads.
                - Restrictions: Must be compatible with the MSBertLMPredictionHead and nn.Linear classes.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.predictions = MSBertLMPredictionHead(config)
        self.seq_relationship = nn.Linear(config.hidden_size, 2)

    def forward(self, sequence_output, pooled_output, masked_lm_positions):
        """
        Construct method in the MSBertPreTrainingHeads class.

        Args:
            self (object): The instance of the class.
            sequence_output (tensor): The output tensor from the pre-trained model for the input sequence.
            pooled_output (tensor): The output tensor obtained by applying pooling to the sequence_output.
            masked_lm_positions (tensor): The positions of the masked language model tokens in the input sequence.

        Returns:
            Tuple: A tuple containing the prediction_scores (tensor) and seq_relationship_score (tensor)
                calculated based on the inputs.

        Raises:
            None: This method does not raise any exceptions.
        """
        prediction_scores = self.predictions(sequence_output, masked_lm_positions)
        seq_relationship_score = self.seq_relationship(pooled_output)
        return prediction_scores, seq_relationship_score

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPreTrainingHeads.__init__(config)

Initialize the MSBertPreTrainingHeads class.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
config	An object containing configuration settings. Type: Custom class Purpose: Provides configuration parameters for the pre-training heads. Restrictions: Must be compatible with the MSBertLMPredictionHead and nn.Linear classes. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initialize the MSBertPreTrainingHeads class.

    Args:
        self (object): The instance of the class.
        config (object):
            An object containing configuration settings.

            - Type: Custom class
            - Purpose: Provides configuration parameters for the pre-training heads.
            - Restrictions: Must be compatible with the MSBertLMPredictionHead and nn.Linear classes.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.predictions = MSBertLMPredictionHead(config)
    self.seq_relationship = nn.Linear(config.hidden_size, 2)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPreTrainingHeads.forward(sequence_output, pooled_output, masked_lm_positions)

Construct method in the MSBertPreTrainingHeads class.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
sequence_output	The output tensor from the pre-trained model for the input sequence. TYPE: tensor
pooled_output	The output tensor obtained by applying pooling to the sequence_output. TYPE: tensor
masked_lm_positions	The positions of the masked language model tokens in the input sequence. TYPE: tensor

RETURNS	DESCRIPTION
Tuple	A tuple containing the prediction_scores (tensor) and seq_relationship_score (tensor) calculated based on the inputs.

RAISES	DESCRIPTION
None	This method does not raise any exceptions.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, sequence_output, pooled_output, masked_lm_positions):
    """
    Construct method in the MSBertPreTrainingHeads class.

    Args:
        self (object): The instance of the class.
        sequence_output (tensor): The output tensor from the pre-trained model for the input sequence.
        pooled_output (tensor): The output tensor obtained by applying pooling to the sequence_output.
        masked_lm_positions (tensor): The positions of the masked language model tokens in the input sequence.

    Returns:
        Tuple: A tuple containing the prediction_scores (tensor) and seq_relationship_score (tensor)
            calculated based on the inputs.

    Raises:
        None: This method does not raise any exceptions.
    """
    prediction_scores = self.predictions(sequence_output, masked_lm_positions)
    seq_relationship_score = self.seq_relationship(pooled_output)
    return prediction_scores, seq_relationship_score

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPredictionHeadTransform

Bases: Module

Bert Prediction Head Transform

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertPredictionHeadTransform(nn.Module):
    r"""
    Bert Prediction Head Transform
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSBertPredictionHeadTransform class.

        Args:
            self: An instance of the MSBertPredictionHeadTransform class.
            config: An object containing configuration settings for the transformation.
                It is expected to have the following attributes:

                - hidden_size (int): The size of the hidden layer.
                - hidden_act (str): The activation function to be used for the hidden layer.
                - layer_norm_eps (float): The epsilon value for LayerNorm.

        Returns:
            None: This method initializes the dense layer, activation function, and LayerNorm parameters for the transformation.

        Raises:
            TypeError: If the config parameter is not provided.
            ValueError: If the config parameter is missing any required attributes.
            KeyError: If the hidden activation function specified in the config is not found in the ACT2FN dictionary.
        """
        super().__init__()
        self.dense = nn.Linear(
            config.hidden_size,
            config.hidden_size,
        )
        self.transform_act_fn = ACT2FN[config.hidden_act]
        self.LayerNorm = nn.LayerNorm(
            (config.hidden_size,), eps=config.layer_norm_eps
        )

    def forward(self, hidden_states):
        """
        This method 'forward' is part of the 'MSBertPredictionHeadTransform' class and is used to perform transformations on hidden states.

        Args:
            self:
                The instance of the 'MSBertPredictionHeadTransform' class.

                - Type: MSBertPredictionHeadTransform
                - Purpose: Represents the current instance of the class.
                - Restrictions: None

            hidden_states:
                The input hidden states that need to undergo transformations.

                - Type: Any
                - Purpose: Represents the hidden states to be processed.
                - Restrictions: Should be compatible with the operations performed within the method.

        Returns:
            hidden_states:

                - Type: None
                - Purpose: To return the processed hidden states for further usage.

        Raises:
            None.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.transform_act_fn(hidden_states)
        hidden_states = self.LayerNorm(hidden_states)
        return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPredictionHeadTransform.__init__(config)

Initializes an instance of the MSBertPredictionHeadTransform class.

PARAMETER	DESCRIPTION
self	An instance of the MSBertPredictionHeadTransform class.
config	An object containing configuration settings for the transformation. It is expected to have the following attributes: hidden_size (int): The size of the hidden layer. hidden_act (str): The activation function to be used for the hidden layer. layer_norm_eps (float): The epsilon value for LayerNorm.

RETURNS	DESCRIPTION
None	This method initializes the dense layer, activation function, and LayerNorm parameters for the transformation.

RAISES	DESCRIPTION
TypeError	If the config parameter is not provided.
ValueError	If the config parameter is missing any required attributes.
KeyError	If the hidden activation function specified in the config is not found in the ACT2FN dictionary.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertPredictionHeadTransform class.

    Args:
        self: An instance of the MSBertPredictionHeadTransform class.
        config: An object containing configuration settings for the transformation.
            It is expected to have the following attributes:

            - hidden_size (int): The size of the hidden layer.
            - hidden_act (str): The activation function to be used for the hidden layer.
            - layer_norm_eps (float): The epsilon value for LayerNorm.

    Returns:
        None: This method initializes the dense layer, activation function, and LayerNorm parameters for the transformation.

    Raises:
        TypeError: If the config parameter is not provided.
        ValueError: If the config parameter is missing any required attributes.
        KeyError: If the hidden activation function specified in the config is not found in the ACT2FN dictionary.
    """
    super().__init__()
    self.dense = nn.Linear(
        config.hidden_size,
        config.hidden_size,
    )
    self.transform_act_fn = ACT2FN[config.hidden_act]
    self.LayerNorm = nn.LayerNorm(
        (config.hidden_size,), eps=config.layer_norm_eps
    )

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertPredictionHeadTransform.forward(hidden_states)

This method 'forward' is part of the 'MSBertPredictionHeadTransform' class and is used to perform transformations on hidden states.

PARAMETER	DESCRIPTION
self	The instance of the 'MSBertPredictionHeadTransform' class. Type: MSBertPredictionHeadTransform Purpose: Represents the current instance of the class. Restrictions: None
hidden_states	The input hidden states that need to undergo transformations. Type: Any Purpose: Represents the hidden states to be processed. Restrictions: Should be compatible with the operations performed within the method.

RETURNS	DESCRIPTION
hidden_states	Type: None Purpose: To return the processed hidden states for further usage.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states):
    """
    This method 'forward' is part of the 'MSBertPredictionHeadTransform' class and is used to perform transformations on hidden states.

    Args:
        self:
            The instance of the 'MSBertPredictionHeadTransform' class.

            - Type: MSBertPredictionHeadTransform
            - Purpose: Represents the current instance of the class.
            - Restrictions: None

        hidden_states:
            The input hidden states that need to undergo transformations.

            - Type: Any
            - Purpose: Represents the hidden states to be processed.
            - Restrictions: Should be compatible with the operations performed within the method.

    Returns:
        hidden_states:

            - Type: None
            - Purpose: To return the processed hidden states for further usage.

    Raises:
        None.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.transform_act_fn(hidden_states)
    hidden_states = self.LayerNorm(hidden_states)
    return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertSelfAttention

Bases: Module

Self attention layer for BERT.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertSelfAttention(nn.Module):
    """
    Self attention layer for BERT.
    """
    def __init__(self, config, causal, init_cache=False):
        """Initializes an instance of the MSBertSelfAttention class.

        Args:
            self: The instance of the class.
            config:
                A configuration object containing various parameters.

                - Type: Object
                - Purpose: Specifies the configuration parameters for the attention mechanism.
                - Restrictions: None

            causal:
                A boolean value indicating whether the attention mechanism is causal or not.

                - Type: bool
                - Purpose: Determines if the attention mechanism is restricted to attend to previous positions only.
                - Restrictions: None

            init_cache:
                A boolean value indicating whether to initialize the cache or not.

                - Type: bool
                - Purpose: Determines if the cache for attention weights and values should be initialized.
                - Restrictions: None

        Returns:
            None.

        Raises:
            ValueError: If the hidden size is not a multiple of the number of attention heads.

        Notes:
            - This method is called when creating an instance of the MSBertSelfAttention class.
            - The attention mechanism is responsible for computing self-attention weights and values based on the input.
            - The method initializes various instance variables and parameters required for the attention mechanism.
            - If the hidden size is not divisible by the number of attention heads, a ValueError is raised.
            - The method also initializes the cache variables if `init_cache` is True, otherwise sets them to None.
            - The method creates dense layers for query, key, and value projections.
            - The method initializes dropout and softmax layers for attention probabilities computation.
            - The method creates a causal mask if `causal` is True, otherwise uses a mask of ones.
        """
        super().__init__()
        if config.hidden_size % config.num_attention_heads != 0:
            raise ValueError(
                f"The hidden size {config.hidden_size} is not a multiple of the number of attention "
                f"heads {config.num_attention_heads}"
            )
        self.output_attentions = config.output_attentions

        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.query = nn.Linear(
            config.hidden_size,
            self.all_head_size,
        )
        self.key = nn.Linear(
            config.hidden_size,
            self.all_head_size,
        )
        self.value = nn.Linear(
            config.hidden_size,
            self.all_head_size,
        )

        self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
        self.softmax = nn.Softmax(-1)

        self.causal = causal
        self.init_cache = init_cache

        self.causal_mask = make_causal_mask(
            ops.ones((1, config.max_position_embeddings), dtype=mstype.bool_),
            dtype=mstype.bool_,
        )

        if not init_cache:
            self.cache_key = None
            self.cache_value = None
            self.cache_index = None
        else:
            self.cache_key = Parameter(
                initializer(
                    "zeros",
                    (
                        config.max_length,
                        config.max_batch_size,
                        config.num_attention_heads,
                        config.attention_head_size,
                    ),
                )
            )
            self.cache_value = Parameter(
                initializer(
                    "zeros",
                    (
                        config.max_length,
                        config.max_batch_size,
                        config.num_attention_heads,
                        config.attention_head_size,
                    ),
                )
            )
            self.cache_index = Parameter(Tensor(0, mstype.int32))

    def _concatenate_to_cache(self, key, value, query, attention_mask):
        """
        Concatenates the given key, value, query, and attention mask to the cache in the MSBertSelfAttention class.

        Args:
            self (MSBertSelfAttention): An instance of the MSBertSelfAttention class.
            key (Tensor): The key tensor to be concatenated to the cache.
                Shape: (batch_size, num_updated_cache_vectors, hidden_size).
            value (Tensor): The value tensor to be concatenated to the cache.
                Shape: (batch_size, num_updated_cache_vectors, hidden_size).
            query (Tensor): The query tensor. Shape: (batch_size, sequence_length, hidden_size).
            attention_mask (Tensor): The attention mask tensor. Shape: (batch_size, sequence_length).

        Returns:
            tuple: A tuple containing the updated key, value, and attention mask tensors.

        Raises:
            None.
        """
        if self.init_cache:
            batch_size = query.shape[0]
            num_updated_cache_vectors = query.shape[1]
            max_length = self.cache_key.shape[0]
            indices = ops.arange(
                self.cache_index, self.cache_index + num_updated_cache_vectors
            )
            key = ops.scatter_update(self.cache_key, indices, key.swapaxes(0, 1))
            value = ops.scatter_update(self.cache_value, indices, value.swapaxes(0, 1))

            self.cache_index += num_updated_cache_vectors

            pad_mask = ops.broadcast_to(
                ops.arange(max_length) < self.cache_index,
                (batch_size, 1, num_updated_cache_vectors, max_length),
            )
            attention_mask = ops.logical_and(attention_mask, pad_mask)

        return key, value, attention_mask

    def transpose_for_scores(self, input_x):
        r"""
        transpose for scores
        """
        new_x_shape = input_x.shape[:-1] + (
            self.num_attention_heads,
            self.attention_head_size,
        )
        input_x = input_x.view(*new_x_shape)
        return input_x.transpose(0, 2, 1, 3)

    def forward(self, hidden_states, attention_mask=None, head_mask=None):
        """
        Constructs the self-attention layer for the MSBert model.

        Args:
            self (MSBertSelfAttention): The instance of the MSBertSelfAttention class.
            hidden_states (Tensor):
                The input tensor of shape (batch_size, seq_length, hidden_size) representing the hidden states.
            attention_mask (Tensor, optional):
                The attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length)
                to mask out certain positions from the attention computation.
                Defaults to None.
            head_mask (Tensor, optional):
                The tensor of shape (num_attention_heads,) representing the mask for the attention heads.
                Defaults to None.

        Returns:
            outputs (tuple): A tuple containing the context layer tensor of shape (batch_size, seq_length, hidden_size)
                and the attention probabilities tensor of shape (batch_size, num_attention_heads, eq_length, seq_length)
                if self.output_attentions is True, else only the context layer tensor is returned.

        Raises:
            None.
        """
        batch_size = hidden_states.shape[0]

        mixed_query_layer = self.query(hidden_states)
        mixed_key_layer = self.key(hidden_states)
        mixed_value_layer = self.value(hidden_states)
        query_states = self.transpose_for_scores(mixed_query_layer)
        key_states = self.transpose_for_scores(mixed_key_layer)
        value_states = self.transpose_for_scores(mixed_value_layer)

        if self.causal:
            query_length, key_length = query_states.shape[1], key_states.shape[1]
            if self.has_variable("cache", "cached_key"):
                mask_shift = self.variables["cache"]["cache_index"]
                max_decoder_length = self.variables["cache"]["cached_key"].shape[1]
                causal_mask = ops.slice(
                    self.causal_mask,
                    (0, 0, mask_shift, 0),
                    (1, 1, query_length, max_decoder_length),
                )
            else:
                causal_mask = self.causal_mask[:, :, :query_length, :key_length]
            causal_mask = ops.broadcast_to(
                causal_mask, (batch_size,) + causal_mask.shape[1:]
            )
        else:
            causal_mask = None

        if attention_mask is not None and self.causal:
            attention_mask = ops.broadcast_to(
                attention_mask.expand_dims(-2).expand_dims(-3), causal_mask.shape
            )
            attention_mask = ops.logical_and(attention_mask, causal_mask)
        elif self.causal:
            attention_mask = causal_mask
        elif attention_mask is not None:
            attention_mask = attention_mask.expand_dims(-2).expand_dims(-3)

        if self.causal and self.init_cache:
            key_states, value_states, attention_mask = self._concatenate_to_cache(
                key_states, value_states, query_states, attention_mask
            )

        # Convert the boolean attention mask to an attention bias.
        if attention_mask is not None:
            # attention mask in the form of attention bias
            # attention_bias = ops.select(
            #     attention_mask > 0,
            #     ops.full(attention_mask.shape, 0.0).astype(hidden_states.dtype),
            #     ops.full(attention_mask.shape, finfo(hidden_states.dtype, "min")).astype(
            #         hidden_states.dtype
            #     ),
            # )
            attention_bias = ops.select(
                attention_mask > 0,
                ops.zeros_like(attention_mask).astype(hidden_states.dtype),
                (ops.ones_like(attention_mask) * finfo(hidden_states.dtype, "min")).astype(
                    hidden_states.dtype
                ),
            )
        else:
            attention_bias = None

        # Take the dot product between "query" snd "key" to get the raw attention scores.
        attention_scores = ops.matmul(query_states, key_states.swapaxes(-1, -2))
        attention_scores = attention_scores / ops.sqrt(
            Tensor(self.attention_head_size, mstype.float32)
        )
        # Apply the attention mask is (precommputed for all layers in BertModel forward() function)
        attention_scores = attention_scores + attention_bias

        # Normalize the attention scores to probabilities.
        attention_probs = self.softmax(attention_scores)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs = self.dropout(attention_probs)

        if head_mask is not None:
            attention_probs = attention_probs * head_mask

        context_layer = ops.matmul(attention_probs, value_states)
        context_layer = context_layer.transpose(0, 2, 1, 3)
        new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(*new_context_layer_shape)

        outputs = (
            (context_layer, attention_probs)
            if self.output_attentions
            else (context_layer,)
        )
        return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertSelfAttention.__init__(config, causal, init_cache=False)

Initializes an instance of the MSBertSelfAttention class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	A configuration object containing various parameters. Type: Object Purpose: Specifies the configuration parameters for the attention mechanism. Restrictions: None
causal	A boolean value indicating whether the attention mechanism is causal or not. Type: bool Purpose: Determines if the attention mechanism is restricted to attend to previous positions only. Restrictions: None
init_cache	A boolean value indicating whether to initialize the cache or not. Type: bool Purpose: Determines if the cache for attention weights and values should be initialized. Restrictions: None DEFAULT: False

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the hidden size is not a multiple of the number of attention heads.

Notes

• This method is called when creating an instance of the MSBertSelfAttention class.
• The attention mechanism is responsible for computing self-attention weights and values based on the input.
• The method initializes various instance variables and parameters required for the attention mechanism.
• If the hidden size is not divisible by the number of attention heads, a ValueError is raised.
• The method also initializes the cache variables if init_cache is True, otherwise sets them to None.
• The method creates dense layers for query, key, and value projections.
• The method initializes dropout and softmax layers for attention probabilities computation.
• The method creates a causal mask if causal is True, otherwise uses a mask of ones.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config, causal, init_cache=False):
    """Initializes an instance of the MSBertSelfAttention class.

    Args:
        self: The instance of the class.
        config:
            A configuration object containing various parameters.

            - Type: Object
            - Purpose: Specifies the configuration parameters for the attention mechanism.
            - Restrictions: None

        causal:
            A boolean value indicating whether the attention mechanism is causal or not.

            - Type: bool
            - Purpose: Determines if the attention mechanism is restricted to attend to previous positions only.
            - Restrictions: None

        init_cache:
            A boolean value indicating whether to initialize the cache or not.

            - Type: bool
            - Purpose: Determines if the cache for attention weights and values should be initialized.
            - Restrictions: None

    Returns:
        None.

    Raises:
        ValueError: If the hidden size is not a multiple of the number of attention heads.

    Notes:
        - This method is called when creating an instance of the MSBertSelfAttention class.
        - The attention mechanism is responsible for computing self-attention weights and values based on the input.
        - The method initializes various instance variables and parameters required for the attention mechanism.
        - If the hidden size is not divisible by the number of attention heads, a ValueError is raised.
        - The method also initializes the cache variables if `init_cache` is True, otherwise sets them to None.
        - The method creates dense layers for query, key, and value projections.
        - The method initializes dropout and softmax layers for attention probabilities computation.
        - The method creates a causal mask if `causal` is True, otherwise uses a mask of ones.
    """
    super().__init__()
    if config.hidden_size % config.num_attention_heads != 0:
        raise ValueError(
            f"The hidden size {config.hidden_size} is not a multiple of the number of attention "
            f"heads {config.num_attention_heads}"
        )
    self.output_attentions = config.output_attentions

    self.num_attention_heads = config.num_attention_heads
    self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
    self.all_head_size = self.num_attention_heads * self.attention_head_size

    self.query = nn.Linear(
        config.hidden_size,
        self.all_head_size,
    )
    self.key = nn.Linear(
        config.hidden_size,
        self.all_head_size,
    )
    self.value = nn.Linear(
        config.hidden_size,
        self.all_head_size,
    )

    self.dropout = nn.Dropout(p=config.attention_probs_dropout_prob)
    self.softmax = nn.Softmax(-1)

    self.causal = causal
    self.init_cache = init_cache

    self.causal_mask = make_causal_mask(
        ops.ones((1, config.max_position_embeddings), dtype=mstype.bool_),
        dtype=mstype.bool_,
    )

    if not init_cache:
        self.cache_key = None
        self.cache_value = None
        self.cache_index = None
    else:
        self.cache_key = Parameter(
            initializer(
                "zeros",
                (
                    config.max_length,
                    config.max_batch_size,
                    config.num_attention_heads,
                    config.attention_head_size,
                ),
            )
        )
        self.cache_value = Parameter(
            initializer(
                "zeros",
                (
                    config.max_length,
                    config.max_batch_size,
                    config.num_attention_heads,
                    config.attention_head_size,
                ),
            )
        )
        self.cache_index = Parameter(Tensor(0, mstype.int32))

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertSelfAttention.forward(hidden_states, attention_mask=None, head_mask=None)

Constructs the self-attention layer for the MSBert model.

PARAMETER	DESCRIPTION
self	The instance of the MSBertSelfAttention class. TYPE: MSBertSelfAttention
hidden_states	The input tensor of shape (batch_size, seq_length, hidden_size) representing the hidden states. TYPE: Tensor
attention_mask	The attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length) to mask out certain positions from the attention computation. Defaults to None. TYPE: Tensor DEFAULT: None
head_mask	The tensor of shape (num_attention_heads,) representing the mask for the attention heads. Defaults to None. TYPE: Tensor DEFAULT: None

RETURNS	DESCRIPTION
outputs	A tuple containing the context layer tensor of shape (batch_size, seq_length, hidden_size) and the attention probabilities tensor of shape (batch_size, num_attention_heads, eq_length, seq_length) if self.output_attentions is True, else only the context layer tensor is returned. TYPE: tuple

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states, attention_mask=None, head_mask=None):
    """
    Constructs the self-attention layer for the MSBert model.

    Args:
        self (MSBertSelfAttention): The instance of the MSBertSelfAttention class.
        hidden_states (Tensor):
            The input tensor of shape (batch_size, seq_length, hidden_size) representing the hidden states.
        attention_mask (Tensor, optional):
            The attention mask tensor of shape (batch_size, seq_length) or (batch_size, seq_length, seq_length)
            to mask out certain positions from the attention computation.
            Defaults to None.
        head_mask (Tensor, optional):
            The tensor of shape (num_attention_heads,) representing the mask for the attention heads.
            Defaults to None.

    Returns:
        outputs (tuple): A tuple containing the context layer tensor of shape (batch_size, seq_length, hidden_size)
            and the attention probabilities tensor of shape (batch_size, num_attention_heads, eq_length, seq_length)
            if self.output_attentions is True, else only the context layer tensor is returned.

    Raises:
        None.
    """
    batch_size = hidden_states.shape[0]

    mixed_query_layer = self.query(hidden_states)
    mixed_key_layer = self.key(hidden_states)
    mixed_value_layer = self.value(hidden_states)
    query_states = self.transpose_for_scores(mixed_query_layer)
    key_states = self.transpose_for_scores(mixed_key_layer)
    value_states = self.transpose_for_scores(mixed_value_layer)

    if self.causal:
        query_length, key_length = query_states.shape[1], key_states.shape[1]
        if self.has_variable("cache", "cached_key"):
            mask_shift = self.variables["cache"]["cache_index"]
            max_decoder_length = self.variables["cache"]["cached_key"].shape[1]
            causal_mask = ops.slice(
                self.causal_mask,
                (0, 0, mask_shift, 0),
                (1, 1, query_length, max_decoder_length),
            )
        else:
            causal_mask = self.causal_mask[:, :, :query_length, :key_length]
        causal_mask = ops.broadcast_to(
            causal_mask, (batch_size,) + causal_mask.shape[1:]
        )
    else:
        causal_mask = None

    if attention_mask is not None and self.causal:
        attention_mask = ops.broadcast_to(
            attention_mask.expand_dims(-2).expand_dims(-3), causal_mask.shape
        )
        attention_mask = ops.logical_and(attention_mask, causal_mask)
    elif self.causal:
        attention_mask = causal_mask
    elif attention_mask is not None:
        attention_mask = attention_mask.expand_dims(-2).expand_dims(-3)

    if self.causal and self.init_cache:
        key_states, value_states, attention_mask = self._concatenate_to_cache(
            key_states, value_states, query_states, attention_mask
        )

    # Convert the boolean attention mask to an attention bias.
    if attention_mask is not None:
        # attention mask in the form of attention bias
        # attention_bias = ops.select(
        #     attention_mask > 0,
        #     ops.full(attention_mask.shape, 0.0).astype(hidden_states.dtype),
        #     ops.full(attention_mask.shape, finfo(hidden_states.dtype, "min")).astype(
        #         hidden_states.dtype
        #     ),
        # )
        attention_bias = ops.select(
            attention_mask > 0,
            ops.zeros_like(attention_mask).astype(hidden_states.dtype),
            (ops.ones_like(attention_mask) * finfo(hidden_states.dtype, "min")).astype(
                hidden_states.dtype
            ),
        )
    else:
        attention_bias = None

    # Take the dot product between "query" snd "key" to get the raw attention scores.
    attention_scores = ops.matmul(query_states, key_states.swapaxes(-1, -2))
    attention_scores = attention_scores / ops.sqrt(
        Tensor(self.attention_head_size, mstype.float32)
    )
    # Apply the attention mask is (precommputed for all layers in BertModel forward() function)
    attention_scores = attention_scores + attention_bias

    # Normalize the attention scores to probabilities.
    attention_probs = self.softmax(attention_scores)

    # This is actually dropping out entire tokens to attend to, which might
    # seem a bit unusual, but is taken from the original Transformer paper.
    attention_probs = self.dropout(attention_probs)

    if head_mask is not None:
        attention_probs = attention_probs * head_mask

    context_layer = ops.matmul(attention_probs, value_states)
    context_layer = context_layer.transpose(0, 2, 1, 3)
    new_context_layer_shape = context_layer.shape[:-2] + (self.all_head_size,)
    context_layer = context_layer.view(*new_context_layer_shape)

    outputs = (
        (context_layer, attention_probs)
        if self.output_attentions
        else (context_layer,)
    )
    return outputs

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertSelfAttention.transpose_for_scores(input_x)

transpose for scores

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def transpose_for_scores(self, input_x):
    r"""
    transpose for scores
    """
    new_x_shape = input_x.shape[:-1] + (
        self.num_attention_heads,
        self.attention_head_size,
    )
    input_x = input_x.view(*new_x_shape)
    return input_x.transpose(0, 2, 1, 3)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertSelfOutput

Bases: Module

Bert Self Output

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

class MSBertSelfOutput(nn.Module):
    r"""
    Bert Self Output
    """
    def __init__(self, config):
        """
        Initializes an instance of the MSBertSelfOutput class.

        Args:
            self: The instance of the MSBertSelfOutput class.
            config: An object containing configuration parameters for the MSBertSelfOutput class.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not of the expected type.
            ValueError: If the config parameter does not contain the required configuration parameters.
            RuntimeError: If there is an issue with initializing the dense, LayerNorm, or dropout attributes.
        """
        super().__init__()
        self.dense = nn.Linear(
            config.hidden_size,
            config.hidden_size,
        )
        self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=1e-12)
        self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        """
        This method 'forward' is a part of the 'MSBertSelfOutput' class and is responsible for
        processing the hidden states and input tensor.

        Args:
            self: The instance of the class.

            hidden_states (tensor): The hidden states to be processed. It is expected to be a tensor.

            input_tensor (tensor): The input tensor to be incorporated into the hidden states. It is expected to be a tensor.

        Returns:
            tensor: The processed hidden states with the input tensor incorporated.

        Raises:
            This method does not raise any exceptions.
        """
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertSelfOutput.__init__(config)

Initializes an instance of the MSBertSelfOutput class.

PARAMETER	DESCRIPTION
self	The instance of the MSBertSelfOutput class.
config	An object containing configuration parameters for the MSBertSelfOutput class.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the config parameter is not of the expected type.
ValueError	If the config parameter does not contain the required configuration parameters.
RuntimeError	If there is an issue with initializing the dense, LayerNorm, or dropout attributes.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def __init__(self, config):
    """
    Initializes an instance of the MSBertSelfOutput class.

    Args:
        self: The instance of the MSBertSelfOutput class.
        config: An object containing configuration parameters for the MSBertSelfOutput class.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not of the expected type.
        ValueError: If the config parameter does not contain the required configuration parameters.
        RuntimeError: If there is an issue with initializing the dense, LayerNorm, or dropout attributes.
    """
    super().__init__()
    self.dense = nn.Linear(
        config.hidden_size,
        config.hidden_size,
    )
    self.LayerNorm = nn.LayerNorm((config.hidden_size,), eps=1e-12)
    self.dropout = nn.Dropout(p=config.hidden_dropout_prob)

mindnlp.transformers.models.bert.modeling_graph_bert.MSBertSelfOutput.forward(hidden_states, input_tensor)

This method 'forward' is a part of the 'MSBertSelfOutput' class and is responsible for processing the hidden states and input tensor.

PARAMETER	DESCRIPTION
self	The instance of the class.
hidden_states	The hidden states to be processed. It is expected to be a tensor. TYPE: tensor
input_tensor	The input tensor to be incorporated into the hidden states. It is expected to be a tensor. TYPE: tensor

RETURNS	DESCRIPTION
tensor	The processed hidden states with the input tensor incorporated.

Source code in mindnlp/transformers/models/bert/modeling_graph_bert.py

def forward(self, hidden_states, input_tensor):
    """
    This method 'forward' is a part of the 'MSBertSelfOutput' class and is responsible for
    processing the hidden states and input tensor.

    Args:
        self: The instance of the class.

        hidden_states (tensor): The hidden states to be processed. It is expected to be a tensor.

        input_tensor (tensor): The input tensor to be incorporated into the hidden states. It is expected to be a tensor.

    Returns:
        tensor: The processed hidden states with the input tensor incorporated.

    Raises:
        This method does not raise any exceptions.
    """
    hidden_states = self.dense(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.LayerNorm(hidden_states + input_tensor)
    return hidden_states

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer

Bases: PreTrainedTokenizer

Construct a BERT tokenizer. Based on WordPiece.

This tokenizer inherits from [PreTrainedTokenizer] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	File containing the vocabulary. TYPE: `str`
do_lower_case	Whether or not to lowercase the input when tokenizing. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
do_basic_tokenize	Whether or not to do basic tokenization before WordPiece. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
never_split	Collection of tokens which will never be split during tokenization. Only has an effect when do_basic_tokenize=True TYPE: `Iterable`, *optional* DEFAULT: None
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"[UNK]"` DEFAULT: '[UNK]'
sep_token	The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. TYPE: `str`, *optional*, defaults to `"[SEP]"` DEFAULT: '[SEP]'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"[PAD]"` DEFAULT: '[PAD]'
cls_token	The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. TYPE: `str`, *optional*, defaults to `"[CLS]"` DEFAULT: '[CLS]'
mask_token	The token used for masking values. This is the token used when training this model with masked language modeling. This is the token which the model will try to predict. TYPE: `str`, *optional*, defaults to `"[MASK]"` DEFAULT: '[MASK]'
tokenize_chinese_chars	Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see this issue). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
strip_accents	Whether or not to strip all accents. If this option is not specified, then it will be determined by the value for lowercase (as in the original BERT). TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/bert/tokenization_bert.py

class BertTokenizer(PreTrainedTokenizer):
    r"""
    Construct a BERT tokenizer. Based on WordPiece.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            File containing the vocabulary.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        do_basic_tokenize (`bool`, *optional*, defaults to `True`):
            Whether or not to do basic tokenization before WordPiece.
        never_split (`Iterable`, *optional*):
            Collection of tokens which will never be split during tokenization. Only has an effect when
            `do_basic_tokenize=True`
        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters.

            This should likely be deactivated for Japanese (see this
            [issue](https://github.com/huggingface/transformers/issues/328)).
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase` (as in the original BERT).
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

    def __init__(
        self,
        vocab_file,
        do_lower_case=True,
        do_basic_tokenize=True,
        never_split=None,
        unk_token="[UNK]",
        sep_token="[SEP]",
        pad_token="[PAD]",
        cls_token="[CLS]",
        mask_token="[MASK]",
        tokenize_chinese_chars=True,
        strip_accents=None,
        **kwargs,
    ):
        """
        This method initializes a BertTokenizer object.

        Args:
            self: The instance of the class.
            vocab_file (str): The path to the vocabulary file.
            do_lower_case (bool, optional): Whether to convert tokens to lowercase. Default is True.
            do_basic_tokenize (bool, optional): Whether to perform basic tokenization. Default is True.
            never_split (list, optional): List of tokens that should not be split further.
            unk_token (str, optional): The unknown token representation. Default is '[UNK]'.
            sep_token (str, optional): The separator token. Default is '[SEP]'.
            pad_token (str, optional): The padding token. Default is '[PAD]'.
            cls_token (str, optional): The classification token. Default is '[CLS]'.
            mask_token (str, optional): The masking token. Default is '[MASK]'.
            tokenize_chinese_chars (bool, optional): Whether to tokenize Chinese characters. Default is True.
            strip_accents (str, optional): Method to strip accents. None by default.

        Returns:
            None.

        Raises:
            ValueError: If the vocab_file path is invalid or the file does not exist.
            Exception: Any unexpected errors that may occur during the initialization process.
        """
        if not os.path.isfile(vocab_file):
            raise ValueError(
                f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
                " model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
            )
        self.vocab = load_vocab(vocab_file)
        self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])
        self.do_basic_tokenize = do_basic_tokenize
        if do_basic_tokenize:
            self.basic_tokenizer = BasicTokenizer(
                do_lower_case=do_lower_case,
                never_split=never_split,
                tokenize_chinese_chars=tokenize_chinese_chars,
                strip_accents=strip_accents,
            )

        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))

        super().__init__(
            do_lower_case=do_lower_case,
            do_basic_tokenize=do_basic_tokenize,
            never_split=never_split,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            tokenize_chinese_chars=tokenize_chinese_chars,
            strip_accents=strip_accents,
            **kwargs,
        )

    @property
    def do_lower_case(self):
        """
        This method 'do_lower_case' is a property in the class 'BertTokenizer' and returns the value of
        the 'do_lower_case' property of the 'basic_tokenizer' attribute.

        Args:
            self: The instance of the BertTokenizer class.

        Returns:
            None.

        Raises:
            This method does not raise any exceptions.
        """
        return self.basic_tokenizer.do_lower_case

    @property
    def vocab_size(self):
        """
        Method to retrieve the size of the vocabulary used by the BertTokenizer.

        Args:
            self (BertTokenizer): The instance of the BertTokenizer class.
                This parameter is used to access the vocabulary stored within the BertTokenizer instance.

        Returns:
            int: The number of unique tokens in the vocabulary of the BertTokenizer.
                This value represents the size of the vocabulary used by the tokenizer.

        Raises:
            None.
        """
        return len(self.vocab)

    def get_vocab(self):
        """
        Retrieve the vocabulary of the BertTokenizer including any added tokens.

        Args:
            self (BertTokenizer): An instance of the BertTokenizer class.
                It represents the tokenizer object.

        Returns:
            dict: A dictionary containing the vocabulary of the BertTokenizer, including any added tokens.

        Raises:
            None.
        """
        return dict(self.vocab, **self.added_tokens_encoder)

    def _tokenize(self, text, split_special_tokens=False):
        """
        This method _tokenize in the class BertTokenizer tokenizes the input text based on the
        specified tokenizer configurations.

        Args:
            self (object): The instance of the BertTokenizer class.
            text (str): The input text to be tokenized.
            split_special_tokens (bool):
                A flag indicating whether special tokens should be split or not. Default is False.
                If set to True, special tokens will be split.

        Returns:
            list: A list of tokens resulting from tokenizing the input text.
                If the basic tokenization is enabled, the tokens are split further using the wordpiece tokenizer.

        Raises:
            None
        """
        split_tokens = []
        if self.do_basic_tokenize:
            for token in self.basic_tokenizer.tokenize(
                text, never_split=self.all_special_tokens if not split_special_tokens else None
            ):
                # If the token is part of the never_split set
                if token in self.basic_tokenizer.never_split:
                    split_tokens.append(token)
                else:
                    split_tokens += self.wordpiece_tokenizer.tokenize(token)
        else:
            split_tokens = self.wordpiece_tokenizer.tokenize(text)
        return split_tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.vocab.get(token, self.vocab.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        return self.ids_to_tokens.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        out_string = " ".join(tokens).replace(" ##", "").strip()
        return out_string

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A BERT sequence has the following format:

        - single sequence: `[CLS] X [SEP]`
        - pair of sequences: `[CLS] A [SEP] B [SEP]`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + token_ids_1 + sep

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is not None:
            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence
        pair mask has the following format:

        ```
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |
        ```

        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary of the tokenizer to a file.

        Args:
            self: An instance of the BertTokenizer class.
            save_directory (str): The directory where the vocabulary file will be saved. It can be an existing directory or a file path.
            filename_prefix (Optional[str]): An optional prefix to be added to the vocabulary file name. Default is None.

        Returns:
            Tuple[str]: A tuple containing the path to the saved vocabulary file.

        Raises:
            OSError: If there is an issue with accessing or writing to the save_directory.
            UnicodeEncodeError: If there is an issue encoding the vocabulary file with 'utf-8'.

        The method saves the vocabulary of the tokenizer to a file in the specified save_directory. If save_directory is a directory, the vocabulary file will be saved with the default name (or with the
        filename_prefix if provided) in the directory. If save_directory is a file path, the vocabulary file will be saved with the same name as the file in the specified path.

        The vocabulary is saved in a newline-separated format, where each line contains a token from the vocabulary. The tokens are sorted based on their token_index in the vocabulary dictionary. If the token
        indices are not consecutive, a warning message is logged.

        Example:
            ```python
            >>> tokenizer = BertTokenizer()
            >>> save_directory = '/path/to/save'
            >>> filename_prefix = 'my-vocab'
            >>> saved_file = tokenizer.save_vocabulary(save_directory, filename_prefix)
            ```
        """
        index = 0
        if os.path.isdir(save_directory):
            vocab_file = os.path.join(
                save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
            )
        else:
            vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory
        with open(vocab_file, "w", encoding="utf-8") as writer:
            for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                        " Please check that the vocabulary is not corrupted!"
                    )
                    index = token_index
                writer.write(token + "\n")
                index += 1
        return (vocab_file,)

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.do_lower_case property

This method 'do_lower_case' is a property in the class 'BertTokenizer' and returns the value of the 'do_lower_case' property of the 'basic_tokenizer' attribute.

PARAMETER	DESCRIPTION
self	The instance of the BertTokenizer class.

RETURNS	DESCRIPTION
	None.

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.vocab_size property

Method to retrieve the size of the vocabulary used by the BertTokenizer.

PARAMETER	DESCRIPTION
self	The instance of the BertTokenizer class. This parameter is used to access the vocabulary stored within the BertTokenizer instance. TYPE: BertTokenizer

RETURNS	DESCRIPTION
int	The number of unique tokens in the vocabulary of the BertTokenizer. This value represents the size of the vocabulary used by the tokenizer.

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.__init__(vocab_file, do_lower_case=True, do_basic_tokenize=True, never_split=None, unk_token='[UNK]', sep_token='[SEP]', pad_token='[PAD]', cls_token='[CLS]', mask_token='[MASK]', tokenize_chinese_chars=True, strip_accents=None, **kwargs)

This method initializes a BertTokenizer object.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_file	The path to the vocabulary file. TYPE: str
do_lower_case	Whether to convert tokens to lowercase. Default is True. TYPE: bool DEFAULT: True
do_basic_tokenize	Whether to perform basic tokenization. Default is True. TYPE: bool DEFAULT: True
never_split	List of tokens that should not be split further. TYPE: list DEFAULT: None
unk_token	The unknown token representation. Default is '[UNK]'. TYPE: str DEFAULT: '[UNK]'
sep_token	The separator token. Default is '[SEP]'. TYPE: str DEFAULT: '[SEP]'
pad_token	The padding token. Default is '[PAD]'. TYPE: str DEFAULT: '[PAD]'
cls_token	The classification token. Default is '[CLS]'. TYPE: str DEFAULT: '[CLS]'
mask_token	The masking token. Default is '[MASK]'. TYPE: str DEFAULT: '[MASK]'
tokenize_chinese_chars	Whether to tokenize Chinese characters. Default is True. TYPE: bool DEFAULT: True
strip_accents	Method to strip accents. None by default. TYPE: str DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the vocab_file path is invalid or the file does not exist.
Exception	Any unexpected errors that may occur during the initialization process.

Source code in mindnlp/transformers/models/bert/tokenization_bert.py

def __init__(
    self,
    vocab_file,
    do_lower_case=True,
    do_basic_tokenize=True,
    never_split=None,
    unk_token="[UNK]",
    sep_token="[SEP]",
    pad_token="[PAD]",
    cls_token="[CLS]",
    mask_token="[MASK]",
    tokenize_chinese_chars=True,
    strip_accents=None,
    **kwargs,
):
    """
    This method initializes a BertTokenizer object.

    Args:
        self: The instance of the class.
        vocab_file (str): The path to the vocabulary file.
        do_lower_case (bool, optional): Whether to convert tokens to lowercase. Default is True.
        do_basic_tokenize (bool, optional): Whether to perform basic tokenization. Default is True.
        never_split (list, optional): List of tokens that should not be split further.
        unk_token (str, optional): The unknown token representation. Default is '[UNK]'.
        sep_token (str, optional): The separator token. Default is '[SEP]'.
        pad_token (str, optional): The padding token. Default is '[PAD]'.
        cls_token (str, optional): The classification token. Default is '[CLS]'.
        mask_token (str, optional): The masking token. Default is '[MASK]'.
        tokenize_chinese_chars (bool, optional): Whether to tokenize Chinese characters. Default is True.
        strip_accents (str, optional): Method to strip accents. None by default.

    Returns:
        None.

    Raises:
        ValueError: If the vocab_file path is invalid or the file does not exist.
        Exception: Any unexpected errors that may occur during the initialization process.
    """
    if not os.path.isfile(vocab_file):
        raise ValueError(
            f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
            " model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
        )
    self.vocab = load_vocab(vocab_file)
    self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])
    self.do_basic_tokenize = do_basic_tokenize
    if do_basic_tokenize:
        self.basic_tokenizer = BasicTokenizer(
            do_lower_case=do_lower_case,
            never_split=never_split,
            tokenize_chinese_chars=tokenize_chinese_chars,
            strip_accents=strip_accents,
        )

    self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))

    super().__init__(
        do_lower_case=do_lower_case,
        do_basic_tokenize=do_basic_tokenize,
        never_split=never_split,
        unk_token=unk_token,
        sep_token=sep_token,
        pad_token=pad_token,
        cls_token=cls_token,
        mask_token=mask_token,
        tokenize_chinese_chars=tokenize_chinese_chars,
        strip_accents=strip_accents,
        **kwargs,
    )

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A BERT sequence has the following format:

• single sequence: [CLS] X [SEP]
• pair of sequences: [CLS] A [SEP] B [SEP]

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp/transformers/models/bert/tokenization_bert.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. A BERT sequence has the following format:

    - single sequence: `[CLS] X [SEP]`
    - pair of sequences: `[CLS] A [SEP] B [SEP]`

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    if token_ids_1 is None:
        return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
    cls = [self.cls_token_id]
    sep = [self.sep_token_id]
    return cls + token_ids_0 + sep + token_ids_1 + sep

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (string) in a single string.

Source code in mindnlp/transformers/models/bert/tokenization_bert.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (string) in a single string."""
    out_string = " ".join(tokens).replace(" ##", "").strip()
    return out_string

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format:

0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
| first sequence    | second sequence |

If token_ids_1 is None, this method only returns the first portion of the mask (0s).

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of token type IDs according to the given sequence(s).

Source code in mindnlp/transformers/models/bert/tokenization_bert.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence
    pair mask has the following format:

    ```
    0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
    | first sequence    | second sequence |
    ```

    If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
    """
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]
    if token_ids_1 is None:
        return len(cls + token_ids_0 + sep) * [0]
    return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/models/bert/tokenization_bert.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    if token_ids_1 is not None:
        return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
    return [1] + ([0] * len(token_ids_0)) + [1]

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.get_vocab()

Retrieve the vocabulary of the BertTokenizer including any added tokens.

PARAMETER	DESCRIPTION
self	An instance of the BertTokenizer class. It represents the tokenizer object. TYPE: BertTokenizer

RETURNS	DESCRIPTION
dict	A dictionary containing the vocabulary of the BertTokenizer, including any added tokens.

Source code in mindnlp/transformers/models/bert/tokenization_bert.py

def get_vocab(self):
    """
    Retrieve the vocabulary of the BertTokenizer including any added tokens.

    Args:
        self (BertTokenizer): An instance of the BertTokenizer class.
            It represents the tokenizer object.

    Returns:
        dict: A dictionary containing the vocabulary of the BertTokenizer, including any added tokens.

    Raises:
        None.
    """
    return dict(self.vocab, **self.added_tokens_encoder)

mindnlp.transformers.models.bert.tokenization_bert.BertTokenizer.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary of the tokenizer to a file.

PARAMETER	DESCRIPTION
self	An instance of the BertTokenizer class.
save_directory	The directory where the vocabulary file will be saved. It can be an existing directory or a file path. TYPE: str
filename_prefix	An optional prefix to be added to the vocabulary file name. Default is None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the path to the saved vocabulary file.

RAISES	DESCRIPTION
OSError	If there is an issue with accessing or writing to the save_directory.
UnicodeEncodeError	If there is an issue encoding the vocabulary file with 'utf-8'.

The method saves the vocabulary of the tokenizer to a file in the specified save_directory. If save_directory is a directory, the vocabulary file will be saved with the default name (or with the filename_prefix if provided) in the directory. If save_directory is a file path, the vocabulary file will be saved with the same name as the file in the specified path.

The vocabulary is saved in a newline-separated format, where each line contains a token from the vocabulary. The tokens are sorted based on their token_index in the vocabulary dictionary. If the token indices are not consecutive, a warning message is logged.

Example

>>> tokenizer = BertTokenizer()
>>> save_directory = '/path/to/save'
>>> filename_prefix = 'my-vocab'
>>> saved_file = tokenizer.save_vocabulary(save_directory, filename_prefix)

Source code in mindnlp/transformers/models/bert/tokenization_bert.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary of the tokenizer to a file.

    Args:
        self: An instance of the BertTokenizer class.
        save_directory (str): The directory where the vocabulary file will be saved. It can be an existing directory or a file path.
        filename_prefix (Optional[str]): An optional prefix to be added to the vocabulary file name. Default is None.

    Returns:
        Tuple[str]: A tuple containing the path to the saved vocabulary file.

    Raises:
        OSError: If there is an issue with accessing or writing to the save_directory.
        UnicodeEncodeError: If there is an issue encoding the vocabulary file with 'utf-8'.

    The method saves the vocabulary of the tokenizer to a file in the specified save_directory. If save_directory is a directory, the vocabulary file will be saved with the default name (or with the
    filename_prefix if provided) in the directory. If save_directory is a file path, the vocabulary file will be saved with the same name as the file in the specified path.

    The vocabulary is saved in a newline-separated format, where each line contains a token from the vocabulary. The tokens are sorted based on their token_index in the vocabulary dictionary. If the token
    indices are not consecutive, a warning message is logged.

    Example:
        ```python
        >>> tokenizer = BertTokenizer()
        >>> save_directory = '/path/to/save'
        >>> filename_prefix = 'my-vocab'
        >>> saved_file = tokenizer.save_vocabulary(save_directory, filename_prefix)
        ```
    """
    index = 0
    if os.path.isdir(save_directory):
        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )
    else:
        vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory
    with open(vocab_file, "w", encoding="utf-8") as writer:
        for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):
            if index != token_index:
                logger.warning(
                    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                    " Please check that the vocabulary is not corrupted!"
                )
                index = token_index
            writer.write(token + "\n")
            index += 1
    return (vocab_file,)

mindnlp.transformers.models.bert.tokenization_bert_fast.BertTokenizerFast

Bases: PreTrainedTokenizerFast

Construct a "fast" BERT tokenizer (backed by HuggingFace's tokenizers library). Based on WordPiece.

This tokenizer inherits from [PreTrainedTokenizerFast] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	File containing the vocabulary. TYPE: `str` DEFAULT: None
do_lower_case	Whether or not to lowercase the input when tokenizing. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"[UNK]"` DEFAULT: '[UNK]'
sep_token	The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. TYPE: `str`, *optional*, defaults to `"[SEP]"` DEFAULT: '[SEP]'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"[PAD]"` DEFAULT: '[PAD]'
cls_token	The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. TYPE: `str`, *optional*, defaults to `"[CLS]"` DEFAULT: '[CLS]'
mask_token	The token used for masking values. This is the token used when training this model with masked language modeling. This is the token which the model will try to predict. TYPE: `str`, *optional*, defaults to `"[MASK]"` DEFAULT: '[MASK]'
clean_text	Whether or not to clean the text before tokenization by removing any control characters and replacing all whitespaces by the classic one. TYPE: `bool`, *optional*, defaults to `True`
tokenize_chinese_chars	Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see this issue). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
strip_accents	Whether or not to strip all accents. If this option is not specified, then it will be determined by the value for lowercase (as in the original BERT). TYPE: `bool`, *optional* DEFAULT: None
wordpieces_prefix	The prefix for subwords. TYPE: `str`, *optional*, defaults to `"##"`

Source code in mindnlp/transformers/models/bert/tokenization_bert_fast.py

class BertTokenizerFast(PreTrainedTokenizerFast):
    r"""
    Construct a "fast" BERT tokenizer (backed by HuggingFace's *tokenizers* library). Based on WordPiece.

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            File containing the vocabulary.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        clean_text (`bool`, *optional*, defaults to `True`):
            Whether or not to clean the text before tokenization by removing any control characters and replacing all
            whitespaces by the classic one.
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see [this
            issue](https://github.com/huggingface/transformers/issues/328)).
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase` (as in the original BERT).
        wordpieces_prefix (`str`, *optional*, defaults to `"##"`):
            The prefix for subwords.
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    slow_tokenizer_class = BertTokenizer

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        do_lower_case=True,
        unk_token="[UNK]",
        sep_token="[SEP]",
        pad_token="[PAD]",
        cls_token="[CLS]",
        mask_token="[MASK]",
        tokenize_chinese_chars=True,
        strip_accents=None,
        **kwargs,
    ):
        """
        Initialize the BertTokenizerFast class.

        Args:
            self: The instance of the class.
            vocab_file (str): The file path to the vocabulary file. Defaults to None.
            tokenizer_file (str): The file path to the tokenizer file. Defaults to None.
            do_lower_case (bool): Flag indicating whether to convert tokens to lowercase. Defaults to True.
            unk_token (str): The special token for unknown tokens. Defaults to '[UNK]'.
            sep_token (str): The special token for separating sequences. Defaults to '[SEP]'.
            pad_token (str): The special token for padding sequences. Defaults to '[PAD]'.
            cls_token (str): The special token for classifying sequences. Defaults to '[CLS]'.
            mask_token (str): The special token for masking tokens. Defaults to '[MASK]'.
            tokenize_chinese_chars (bool): Flag indicating whether to tokenize Chinese characters. Defaults to True.
            strip_accents (str or None): Flag indicating whether to strip accents. Defaults to None.
            **kwargs: Additional keyword arguments.

        Returns:
            None.

        Raises:
            Exception: If an error occurs during the initialization process.
        """
        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            do_lower_case=do_lower_case,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            tokenize_chinese_chars=tokenize_chinese_chars,
            strip_accents=strip_accents,
            **kwargs,
        )

        normalizer_state = json.loads(self.backend_tokenizer.normalizer.__getstate__())
        if (
            normalizer_state.get("lowercase", do_lower_case) != do_lower_case
            or normalizer_state.get("strip_accents", strip_accents) != strip_accents
            or normalizer_state.get("handle_chinese_chars", tokenize_chinese_chars) != tokenize_chinese_chars
        ):
            normalizer_class = getattr(normalizers, normalizer_state.pop("type"))
            normalizer_state["lowercase"] = do_lower_case
            normalizer_state["strip_accents"] = strip_accents
            normalizer_state["handle_chinese_chars"] = tokenize_chinese_chars
            self.backend_tokenizer.normalizer = normalizer_class(**normalizer_state)

        self.do_lower_case = do_lower_case

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A BERT sequence has the following format:

        - single sequence: `[CLS] X [SEP]`
        - pair of sequences: `[CLS] A [SEP] B [SEP]`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]

        if token_ids_1 is not None:
            output += token_ids_1 + [self.sep_token_id]

        return output

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence
        pair mask has the following format:

        ```
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |
        ```

        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary of the BertTokenizerFast model to the specified directory.

        Args:
            self (BertTokenizerFast): The instance of the BertTokenizerFast class.
            save_directory (str): The directory where the vocabulary files will be saved.
            filename_prefix (Optional[str]): An optional prefix for the saved vocabulary files. Defaults to None.

        Returns:
            Tuple[str]: A tuple containing the names of the saved files.

        Raises:
            This method does not explicitly raise any exceptions.
        """
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

mindnlp.transformers.models.bert.tokenization_bert_fast.BertTokenizerFast.__init__(vocab_file=None, tokenizer_file=None, do_lower_case=True, unk_token='[UNK]', sep_token='[SEP]', pad_token='[PAD]', cls_token='[CLS]', mask_token='[MASK]', tokenize_chinese_chars=True, strip_accents=None, **kwargs)

Initialize the BertTokenizerFast class.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_file	The file path to the vocabulary file. Defaults to None. TYPE: str DEFAULT: None
tokenizer_file	The file path to the tokenizer file. Defaults to None. TYPE: str DEFAULT: None
do_lower_case	Flag indicating whether to convert tokens to lowercase. Defaults to True. TYPE: bool DEFAULT: True
unk_token	The special token for unknown tokens. Defaults to '[UNK]'. TYPE: str DEFAULT: '[UNK]'
sep_token	The special token for separating sequences. Defaults to '[SEP]'. TYPE: str DEFAULT: '[SEP]'
pad_token	The special token for padding sequences. Defaults to '[PAD]'. TYPE: str DEFAULT: '[PAD]'
cls_token	The special token for classifying sequences. Defaults to '[CLS]'. TYPE: str DEFAULT: '[CLS]'
mask_token	The special token for masking tokens. Defaults to '[MASK]'. TYPE: str DEFAULT: '[MASK]'
tokenize_chinese_chars	Flag indicating whether to tokenize Chinese characters. Defaults to True. TYPE: bool DEFAULT: True
strip_accents	Flag indicating whether to strip accents. Defaults to None. TYPE: str or None DEFAULT: None
**kwargs	Additional keyword arguments. DEFAULT: {}

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
Exception	If an error occurs during the initialization process.

Source code in mindnlp/transformers/models/bert/tokenization_bert_fast.py

def __init__(
    self,
    vocab_file=None,
    tokenizer_file=None,
    do_lower_case=True,
    unk_token="[UNK]",
    sep_token="[SEP]",
    pad_token="[PAD]",
    cls_token="[CLS]",
    mask_token="[MASK]",
    tokenize_chinese_chars=True,
    strip_accents=None,
    **kwargs,
):
    """
    Initialize the BertTokenizerFast class.

    Args:
        self: The instance of the class.
        vocab_file (str): The file path to the vocabulary file. Defaults to None.
        tokenizer_file (str): The file path to the tokenizer file. Defaults to None.
        do_lower_case (bool): Flag indicating whether to convert tokens to lowercase. Defaults to True.
        unk_token (str): The special token for unknown tokens. Defaults to '[UNK]'.
        sep_token (str): The special token for separating sequences. Defaults to '[SEP]'.
        pad_token (str): The special token for padding sequences. Defaults to '[PAD]'.
        cls_token (str): The special token for classifying sequences. Defaults to '[CLS]'.
        mask_token (str): The special token for masking tokens. Defaults to '[MASK]'.
        tokenize_chinese_chars (bool): Flag indicating whether to tokenize Chinese characters. Defaults to True.
        strip_accents (str or None): Flag indicating whether to strip accents. Defaults to None.
        **kwargs: Additional keyword arguments.

    Returns:
        None.

    Raises:
        Exception: If an error occurs during the initialization process.
    """
    super().__init__(
        vocab_file,
        tokenizer_file=tokenizer_file,
        do_lower_case=do_lower_case,
        unk_token=unk_token,
        sep_token=sep_token,
        pad_token=pad_token,
        cls_token=cls_token,
        mask_token=mask_token,
        tokenize_chinese_chars=tokenize_chinese_chars,
        strip_accents=strip_accents,
        **kwargs,
    )

    normalizer_state = json.loads(self.backend_tokenizer.normalizer.__getstate__())
    if (
        normalizer_state.get("lowercase", do_lower_case) != do_lower_case
        or normalizer_state.get("strip_accents", strip_accents) != strip_accents
        or normalizer_state.get("handle_chinese_chars", tokenize_chinese_chars) != tokenize_chinese_chars
    ):
        normalizer_class = getattr(normalizers, normalizer_state.pop("type"))
        normalizer_state["lowercase"] = do_lower_case
        normalizer_state["strip_accents"] = strip_accents
        normalizer_state["handle_chinese_chars"] = tokenize_chinese_chars
        self.backend_tokenizer.normalizer = normalizer_class(**normalizer_state)

    self.do_lower_case = do_lower_case

mindnlp.transformers.models.bert.tokenization_bert_fast.BertTokenizerFast.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A BERT sequence has the following format:

• single sequence: [CLS] X [SEP]
• pair of sequences: [CLS] A [SEP] B [SEP]

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp/transformers/models/bert/tokenization_bert_fast.py

def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. A BERT sequence has the following format:

    - single sequence: `[CLS] X [SEP]`
    - pair of sequences: `[CLS] A [SEP] B [SEP]`

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]

    if token_ids_1 is not None:
        output += token_ids_1 + [self.sep_token_id]

    return output

mindnlp.transformers.models.bert.tokenization_bert_fast.BertTokenizerFast.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format:

0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
| first sequence    | second sequence |

If token_ids_1 is None, this method only returns the first portion of the mask (0s).

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of token type IDs according to the given sequence(s).

Source code in mindnlp/transformers/models/bert/tokenization_bert_fast.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence
    pair mask has the following format:

    ```
    0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
    | first sequence    | second sequence |
    ```

    If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
    """
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]
    if token_ids_1 is None:
        return len(cls + token_ids_0 + sep) * [0]
    return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

mindnlp.transformers.models.bert.tokenization_bert_fast.BertTokenizerFast.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary of the BertTokenizerFast model to the specified directory.

PARAMETER	DESCRIPTION
self	The instance of the BertTokenizerFast class. TYPE: BertTokenizerFast
save_directory	The directory where the vocabulary files will be saved. TYPE: str
filename_prefix	An optional prefix for the saved vocabulary files. Defaults to None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the names of the saved files.

Source code in mindnlp/transformers/models/bert/tokenization_bert_fast.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary of the BertTokenizerFast model to the specified directory.

    Args:
        self (BertTokenizerFast): The instance of the BertTokenizerFast class.
        save_directory (str): The directory where the vocabulary files will be saved.
        filename_prefix (Optional[str]): An optional prefix for the saved vocabulary files. Defaults to None.

    Returns:
        Tuple[str]: A tuple containing the names of the saved files.

    Raises:
        This method does not explicitly raise any exceptions.
    """
    files = self._tokenizer.model.save(save_directory, name=filename_prefix)
    return tuple(files)