t5

mindnlp.transformers.models.t5.modeling_t5

MindSpore T5 model.

mindnlp.transformers.models.t5.modeling_t5.T5Attention

Bases: Module

T5Attention

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5Attention(nn.Module):
    """T5Attention"""
    def __init__(self, config: T5Config, has_relative_attention_bias=False):
        """
        Initializes an instance of the T5Attention class.

        Args:
            self: The object itself.
            config (T5Config):
                An instance of the T5Config class that holds the configuration parameters for the attention mechanism.
            has_relative_attention_bias (bool):
                A boolean value indicating whether the attention mechanism has relative attention bias.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.is_decoder = config.is_decoder
        self.has_relative_attention_bias = has_relative_attention_bias
        self.relative_attention_num_buckets = config.relative_attention_num_buckets
        self.relative_attention_max_distance = config.relative_attention_max_distance
        self.d_model = config.d_model
        self.key_value_proj_dim = config.d_kv
        self.n_heads = config.num_heads
        self.dropout = config.dropout_rate
        self.inner_dim = self.n_heads * self.key_value_proj_dim

        # Mesh TensorFlow initialization to avoid scaling before softmax
        self.q = nn.Linear(self.d_model, self.inner_dim, bias=False)
        self.k = nn.Linear(self.d_model, self.inner_dim, bias=False)
        self.v = nn.Linear(self.d_model, self.inner_dim, bias=False)
        self.o = nn.Linear(self.inner_dim, self.d_model, bias=False)

        if self.has_relative_attention_bias:
            self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads)
        self.pruned_heads = set()

    def prune_heads(self, heads):
        """
        Prunes the attention heads in the T5Attention class.

        Args:
            self (T5Attention): An instance of the T5Attention class.
            heads (list): A list of attention heads to be pruned.

        Returns:
            None.

        Raises:
            None.
        """
        if len(heads) == 0:
            return
        heads, index = find_pruneable_heads_and_indices(
            heads, self.n_heads, self.key_value_proj_dim, self.pruned_heads
        )
        # Prune linear layers
        self.q = prune_linear_layer(self.q, index)
        self.k = prune_linear_layer(self.k, index)
        self.v = prune_linear_layer(self.v, index)
        self.o = prune_linear_layer(self.o, index, dim=1)
        # Update hyper params
        self.n_heads = self.n_heads - len(heads)
        self.inner_dim = self.key_value_proj_dim * self.n_heads
        self.pruned_heads = self.pruned_heads.union(heads)

    @staticmethod
    def _relative_position_bucket(relative_position, bidirectional=True, num_buckets=32, max_distance=128):
        """
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        """
        relative_buckets = 0
        if bidirectional:
            num_buckets //= 2
            relative_buckets += (relative_position > 0).to(mindspore.int64) * num_buckets
            relative_position = ops.abs(relative_position)
        else:
            relative_position = -ops.minimum(relative_position, ops.zeros_like(relative_position))
        # now relative_position is in the range [0, inf)

        # half of the buckets are for exact increments in positions
        max_exact = num_buckets // 2
        is_small = relative_position < max_exact

        # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance
        relative_position_if_large = max_exact + (
            ops.log(relative_position.float() / max_exact)
            / math.log(max_distance / max_exact)
            * (num_buckets - max_exact)
        ).to(mindspore.int64)
        relative_position_if_large = ops.minimum(
            relative_position_if_large, ops.full_like(relative_position_if_large, num_buckets - 1)
        )

        relative_buckets += ops.where(is_small, relative_position, relative_position_if_large)
        return relative_buckets

    def compute_bias(self, query_length, key_length):
        """Compute binned relative position bias"""
        context_position = ops.arange(query_length, dtype=mindspore.int64)[:, None]
        memory_position = ops.arange(key_length, dtype=mindspore.int64)[None, :]
        relative_position = memory_position - context_position  # shape (query_length, key_length)
        relative_position_bucket = self._relative_position_bucket(
            relative_position,  # shape (query_length, key_length)
            bidirectional=(not self.is_decoder),
            num_buckets=self.relative_attention_num_buckets,
            max_distance=self.relative_attention_max_distance,
        )
        values = self.relative_attention_bias(relative_position_bucket)  # shape (query_length, key_length, num_heads)
        values = values.permute([2, 0, 1]).unsqueeze(0)  # shape (1, num_heads, query_length, key_length)
        return values

    def forward(
        self,
        hidden_states,
        mask=None,
        key_value_states=None,
        position_bias=None,
        past_key_value=None,
        layer_head_mask=None,
        query_length=None,
        use_cache=False,
        output_attentions=False,
    ):
        """
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        """
        # Input is (batch_size, seq_length, dim)
        # Mask is (batch_size, key_length) (non-causal) or (batch_size, key_length, key_length)
        # past_key_value[0] is (batch_size, n_heads, q_len - 1, dim_per_head)
        batch_size, seq_length = hidden_states.shape[:2]

        real_seq_length = seq_length

        if past_key_value is not None:
            if len(past_key_value) != 2:
                raise ValueError(
                    f"past_key_value should have 2 past states: keys and values. Got { len(past_key_value)} past states"
                )
            real_seq_length += past_key_value[0].shape[2] if query_length is None else query_length

        key_length = real_seq_length if key_value_states is None else key_value_states.shape[1]

        def shape(states):
            """projection"""
            return states.view(batch_size, -1, self.n_heads, self.key_value_proj_dim).swapaxes(1, 2)

        def unshape(states):
            """reshape"""
            return states.swapaxes(1, 2).view(batch_size, -1, self.inner_dim)

        def project(hidden_states, proj_layer, key_value_states, past_key_value):
            """projects hidden states correctly to key/query states"""
            if key_value_states is None:
                # self-attn
                # (batch_size, n_heads, seq_length, dim_per_head)
                hidden_states = shape(proj_layer(hidden_states))
            elif past_key_value is None:
                # cross-attn
                # (batch_size, n_heads, seq_length, dim_per_head)
                hidden_states = shape(proj_layer(key_value_states))

            if past_key_value is not None:
                if key_value_states is None:
                    # self-attn
                    # (batch_size, n_heads, key_length, dim_per_head)
                    hidden_states = ops.cat([past_key_value, hidden_states], dim=2)
                elif past_key_value.shape[2] != key_value_states.shape[1]:
                    # checking that the `sequence_length` of the `past_key_value` is the same as
                    # the provided `key_value_states` to support prefix tuning
                    # cross-attn
                    # (batch_size, n_heads, seq_length, dim_per_head)
                    hidden_states = shape(proj_layer(key_value_states))
                else:
                    # cross-attn
                    hidden_states = past_key_value
            return hidden_states

        # get query states
        query_states = shape(self.q(hidden_states))  # (batch_size, n_heads, seq_length, dim_per_head)
        # get key/value states
        key_states = project(
            hidden_states, self.k, key_value_states, past_key_value[0] if past_key_value is not None else None
        )
        value_states = project(
            hidden_states, self.v, key_value_states, past_key_value[1] if past_key_value is not None else None
        )

        # compute scores
        scores = ops.matmul(
            query_states, key_states.swapaxes(3, 2)
        )  # equivalent of torch.einsum("bnqd,bnkd->bnqk", query_states, key_states), compatible with onnx op>9
        if position_bias is None:
            if not self.has_relative_attention_bias:
                position_bias = ops.zeros(
                    1, self.n_heads, real_seq_length, key_length, dtype=scores.dtype
                )
            else:
                position_bias = self.compute_bias(real_seq_length, key_length)
            # if key and values are already calculated
            # we want only the last query position bias
            if past_key_value is not None:
                position_bias = position_bias[:, :, -hidden_states.shape[1] :, :]

            if mask is not None:
                position_bias = position_bias + mask  # (batch_size, n_heads, seq_length, key_length)

        if self.pruned_heads:
            mask = ops.ones(position_bias.shape[1])
            mask[list(self.pruned_heads)] = 0
            position_bias_masked = position_bias[:, mask.bool()]
        else:
            position_bias_masked = position_bias

        scores += position_bias_masked
        attn_weights = ops.softmax(scores.float() + 1e-10, dim=-1).astype(
            scores.dtype
        )  # (batch_size, n_heads, seq_length, key_length)
        attn_weights = F.dropout(
            attn_weights, p=self.dropout, training=self.training
        )  # (batch_size, n_heads, seq_length, key_length)

        # Mask heads if we want to
        if layer_head_mask is not None:
            attn_weights = attn_weights * layer_head_mask

        attn_output = unshape(ops.matmul(attn_weights, value_states))  # (batch_size, seq_length, dim)
        attn_output = self.o(attn_output)

        present_key_value_state = (key_states, value_states) if (self.is_decoder and use_cache) else None
        outputs = (attn_output,) + (present_key_value_state,) + (position_bias,)

        if output_attentions:
            outputs = outputs + (attn_weights,)
        return outputs

mindnlp.transformers.models.t5.modeling_t5.T5Attention.__init__(config, has_relative_attention_bias=False)

Initializes an instance of the T5Attention class.

PARAMETER	DESCRIPTION
self	The object itself.
config	An instance of the T5Config class that holds the configuration parameters for the attention mechanism. TYPE: T5Config
has_relative_attention_bias	A boolean value indicating whether the attention mechanism has relative attention bias. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config, has_relative_attention_bias=False):
    """
    Initializes an instance of the T5Attention class.

    Args:
        self: The object itself.
        config (T5Config):
            An instance of the T5Config class that holds the configuration parameters for the attention mechanism.
        has_relative_attention_bias (bool):
            A boolean value indicating whether the attention mechanism has relative attention bias.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.is_decoder = config.is_decoder
    self.has_relative_attention_bias = has_relative_attention_bias
    self.relative_attention_num_buckets = config.relative_attention_num_buckets
    self.relative_attention_max_distance = config.relative_attention_max_distance
    self.d_model = config.d_model
    self.key_value_proj_dim = config.d_kv
    self.n_heads = config.num_heads
    self.dropout = config.dropout_rate
    self.inner_dim = self.n_heads * self.key_value_proj_dim

    # Mesh TensorFlow initialization to avoid scaling before softmax
    self.q = nn.Linear(self.d_model, self.inner_dim, bias=False)
    self.k = nn.Linear(self.d_model, self.inner_dim, bias=False)
    self.v = nn.Linear(self.d_model, self.inner_dim, bias=False)
    self.o = nn.Linear(self.inner_dim, self.d_model, bias=False)

    if self.has_relative_attention_bias:
        self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads)
    self.pruned_heads = set()

mindnlp.transformers.models.t5.modeling_t5.T5Attention.compute_bias(query_length, key_length)

Compute binned relative position bias

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def compute_bias(self, query_length, key_length):
    """Compute binned relative position bias"""
    context_position = ops.arange(query_length, dtype=mindspore.int64)[:, None]
    memory_position = ops.arange(key_length, dtype=mindspore.int64)[None, :]
    relative_position = memory_position - context_position  # shape (query_length, key_length)
    relative_position_bucket = self._relative_position_bucket(
        relative_position,  # shape (query_length, key_length)
        bidirectional=(not self.is_decoder),
        num_buckets=self.relative_attention_num_buckets,
        max_distance=self.relative_attention_max_distance,
    )
    values = self.relative_attention_bias(relative_position_bucket)  # shape (query_length, key_length, num_heads)
    values = values.permute([2, 0, 1]).unsqueeze(0)  # shape (1, num_heads, query_length, key_length)
    return values

mindnlp.transformers.models.t5.modeling_t5.T5Attention.forward(hidden_states, mask=None, key_value_states=None, position_bias=None, past_key_value=None, layer_head_mask=None, query_length=None, use_cache=False, output_attentions=False)

Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    hidden_states,
    mask=None,
    key_value_states=None,
    position_bias=None,
    past_key_value=None,
    layer_head_mask=None,
    query_length=None,
    use_cache=False,
    output_attentions=False,
):
    """
    Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
    """
    # Input is (batch_size, seq_length, dim)
    # Mask is (batch_size, key_length) (non-causal) or (batch_size, key_length, key_length)
    # past_key_value[0] is (batch_size, n_heads, q_len - 1, dim_per_head)
    batch_size, seq_length = hidden_states.shape[:2]

    real_seq_length = seq_length

    if past_key_value is not None:
        if len(past_key_value) != 2:
            raise ValueError(
                f"past_key_value should have 2 past states: keys and values. Got { len(past_key_value)} past states"
            )
        real_seq_length += past_key_value[0].shape[2] if query_length is None else query_length

    key_length = real_seq_length if key_value_states is None else key_value_states.shape[1]

    def shape(states):
        """projection"""
        return states.view(batch_size, -1, self.n_heads, self.key_value_proj_dim).swapaxes(1, 2)

    def unshape(states):
        """reshape"""
        return states.swapaxes(1, 2).view(batch_size, -1, self.inner_dim)

    def project(hidden_states, proj_layer, key_value_states, past_key_value):
        """projects hidden states correctly to key/query states"""
        if key_value_states is None:
            # self-attn
            # (batch_size, n_heads, seq_length, dim_per_head)
            hidden_states = shape(proj_layer(hidden_states))
        elif past_key_value is None:
            # cross-attn
            # (batch_size, n_heads, seq_length, dim_per_head)
            hidden_states = shape(proj_layer(key_value_states))

        if past_key_value is not None:
            if key_value_states is None:
                # self-attn
                # (batch_size, n_heads, key_length, dim_per_head)
                hidden_states = ops.cat([past_key_value, hidden_states], dim=2)
            elif past_key_value.shape[2] != key_value_states.shape[1]:
                # checking that the `sequence_length` of the `past_key_value` is the same as
                # the provided `key_value_states` to support prefix tuning
                # cross-attn
                # (batch_size, n_heads, seq_length, dim_per_head)
                hidden_states = shape(proj_layer(key_value_states))
            else:
                # cross-attn
                hidden_states = past_key_value
        return hidden_states

    # get query states
    query_states = shape(self.q(hidden_states))  # (batch_size, n_heads, seq_length, dim_per_head)
    # get key/value states
    key_states = project(
        hidden_states, self.k, key_value_states, past_key_value[0] if past_key_value is not None else None
    )
    value_states = project(
        hidden_states, self.v, key_value_states, past_key_value[1] if past_key_value is not None else None
    )

    # compute scores
    scores = ops.matmul(
        query_states, key_states.swapaxes(3, 2)
    )  # equivalent of torch.einsum("bnqd,bnkd->bnqk", query_states, key_states), compatible with onnx op>9
    if position_bias is None:
        if not self.has_relative_attention_bias:
            position_bias = ops.zeros(
                1, self.n_heads, real_seq_length, key_length, dtype=scores.dtype
            )
        else:
            position_bias = self.compute_bias(real_seq_length, key_length)
        # if key and values are already calculated
        # we want only the last query position bias
        if past_key_value is not None:
            position_bias = position_bias[:, :, -hidden_states.shape[1] :, :]

        if mask is not None:
            position_bias = position_bias + mask  # (batch_size, n_heads, seq_length, key_length)

    if self.pruned_heads:
        mask = ops.ones(position_bias.shape[1])
        mask[list(self.pruned_heads)] = 0
        position_bias_masked = position_bias[:, mask.bool()]
    else:
        position_bias_masked = position_bias

    scores += position_bias_masked
    attn_weights = ops.softmax(scores.float() + 1e-10, dim=-1).astype(
        scores.dtype
    )  # (batch_size, n_heads, seq_length, key_length)
    attn_weights = F.dropout(
        attn_weights, p=self.dropout, training=self.training
    )  # (batch_size, n_heads, seq_length, key_length)

    # Mask heads if we want to
    if layer_head_mask is not None:
        attn_weights = attn_weights * layer_head_mask

    attn_output = unshape(ops.matmul(attn_weights, value_states))  # (batch_size, seq_length, dim)
    attn_output = self.o(attn_output)

    present_key_value_state = (key_states, value_states) if (self.is_decoder and use_cache) else None
    outputs = (attn_output,) + (present_key_value_state,) + (position_bias,)

    if output_attentions:
        outputs = outputs + (attn_weights,)
    return outputs

mindnlp.transformers.models.t5.modeling_t5.T5Attention.prune_heads(heads)

Prunes the attention heads in the T5Attention class.

PARAMETER	DESCRIPTION
self	An instance of the T5Attention class. TYPE: T5Attention
heads	A list of attention heads to be pruned. TYPE: list

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def prune_heads(self, heads):
    """
    Prunes the attention heads in the T5Attention class.

    Args:
        self (T5Attention): An instance of the T5Attention class.
        heads (list): A list of attention heads to be pruned.

    Returns:
        None.

    Raises:
        None.
    """
    if len(heads) == 0:
        return
    heads, index = find_pruneable_heads_and_indices(
        heads, self.n_heads, self.key_value_proj_dim, self.pruned_heads
    )
    # Prune linear layers
    self.q = prune_linear_layer(self.q, index)
    self.k = prune_linear_layer(self.k, index)
    self.v = prune_linear_layer(self.v, index)
    self.o = prune_linear_layer(self.o, index, dim=1)
    # Update hyper params
    self.n_heads = self.n_heads - len(heads)
    self.inner_dim = self.key_value_proj_dim * self.n_heads
    self.pruned_heads = self.pruned_heads.union(heads)

mindnlp.transformers.models.t5.modeling_t5.T5Block

Bases: Module

T5Block

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5Block(nn.Module):
    """T5Block"""
    def __init__(self, config, has_relative_attention_bias=False):
        """
        Initializes a new instance of the T5Block class.

        Args:
            self: The object itself.
            config (object): The configuration object containing the settings for the T5Block.
            has_relative_attention_bias (bool, optional): Specifies whether the attention bias is relative or not.
                Default is False.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.is_decoder = config.is_decoder
        self.layer = nn.ModuleList()
        self.layer.append(T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias))
        if self.is_decoder:
            self.layer.append(T5LayerCrossAttention(config))

        self.layer.append(T5LayerFF(config))

    def forward(
        self,
        hidden_states,
        attention_mask=None,
        position_bias=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        encoder_decoder_position_bias=None,
        layer_head_mask=None,
        cross_attn_layer_head_mask=None,
        past_key_value=None,
        use_cache=False,
        output_attentions=False,
        # return_dict=True,
    ):
        """
        Constructs a T5Block.

        Args:
            self (T5Block): The T5Block instance.
            hidden_states (Tensor): The input hidden states.
            attention_mask (Tensor, optional): The attention mask tensor. Defaults to None.
            position_bias (Tensor, optional): The position bias tensor. Defaults to None.
            encoder_hidden_states (Tensor, optional): The encoder hidden states tensor. Defaults to None.
            encoder_attention_mask (Tensor, optional): The encoder attention mask tensor. Defaults to None.
            encoder_decoder_position_bias (Tensor, optional): The encoder-decoder position bias tensor. Defaults to None.
            layer_head_mask (Tensor, optional): The layer head mask tensor. Defaults to None.
            cross_attn_layer_head_mask (Tensor, optional): The cross-attention layer head mask tensor. Defaults to None.
            past_key_value (Tuple[Tensor], optional): The past key-value states. Defaults to None.
            use_cache (bool, optional): Whether to use cache. Defaults to False.
            output_attentions (bool, optional): Whether to output attentions. Defaults to False.

        Returns:
            Tuple:
                A tuple containing the following elements:

                - hidden_states (Tensor): The output hidden states.
                - present_key_value_state (Tuple[Tensor], optional): The present key-value state. None if not available.
                - attention_outputs (Tuple[Tensor], optional): The attention outputs. None if not available.

        Raises:
            ValueError: If the number of past states is not as expected.
            Warning: If `past_key_values` is passed to the encoder.
        """
        if past_key_value is not None:
            if not self.is_decoder:
                logging.warning("`past_key_values` is passed to the encoder. Please make sure this is intended.")
            expected_num_past_key_values = 2 if encoder_hidden_states is None else 4

            if len(past_key_value) != expected_num_past_key_values:
                raise ValueError(
                    f"There should be {expected_num_past_key_values} past states. "
                    f"{'2 (past / key) for cross attention. ' if expected_num_past_key_values == 4 else ''}"
                    f"Got {len(past_key_value)} past key / value states"
                )

            self_attn_past_key_value = past_key_value[:2]
            cross_attn_past_key_value = past_key_value[2:]
        else:
            self_attn_past_key_value, cross_attn_past_key_value = None, None
        self_attention_outputs = self.layer[0](
            hidden_states,
            attention_mask=attention_mask,
            position_bias=position_bias,
            layer_head_mask=layer_head_mask,
            past_key_value=self_attn_past_key_value,
            use_cache=use_cache,
            output_attentions=output_attentions,
        )
        hidden_states, present_key_value_state = self_attention_outputs[:2]
        attention_outputs = self_attention_outputs[2:]  # Keep self-attention outputs and relative position weights

        # clamp inf values to enable fp16 training
        if hidden_states.dtype == mindspore.float16 and ops.isinf(hidden_states).any():
            clamp_value = mindspore.tensor(np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max) - 1000
            hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

        do_cross_attention = self.is_decoder and encoder_hidden_states is not None
        if do_cross_attention:
            # the actual query length is unknown for cross attention
            # if using past key value states. Need to inject it here
            if present_key_value_state is not None:
                query_length = present_key_value_state[0].shape[2]
            else:
                query_length = None

            cross_attention_outputs = self.layer[1](
                hidden_states,
                key_value_states=encoder_hidden_states,
                attention_mask=encoder_attention_mask,
                position_bias=encoder_decoder_position_bias,
                layer_head_mask=cross_attn_layer_head_mask,
                past_key_value=cross_attn_past_key_value,
                query_length=query_length,
                use_cache=use_cache,
                output_attentions=output_attentions,
            )
            hidden_states = cross_attention_outputs[0]

            # clamp inf values to enable fp16 training
            if hidden_states.dtype == mindspore.float16 and ops.isinf(hidden_states).any():
                clamp_value = mindspore.tensor(np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max) - 1000
                hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

            # Combine self attn and cross attn key value states
            if present_key_value_state is not None:
                present_key_value_state = present_key_value_state + cross_attention_outputs[1]

            # Keep cross-attention outputs and relative position weights
            attention_outputs = attention_outputs + cross_attention_outputs[2:]

        # Apply Feed Forward layer
        hidden_states = self.layer[-1](hidden_states)

        # clamp inf values to enable fp16 training
        if hidden_states.dtype == mindspore.float16 and ops.isinf(hidden_states).any():
            clamp_value = mindspore.tensor(np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max) - 1000
            hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

        outputs = (hidden_states,)

        if use_cache:
            outputs = outputs + (present_key_value_state,) + attention_outputs
        else:
            outputs = outputs + attention_outputs

        return outputs

mindnlp.transformers.models.t5.modeling_t5.T5Block.__init__(config, has_relative_attention_bias=False)

Initializes a new instance of the T5Block class.

PARAMETER	DESCRIPTION
self	The object itself.
config	The configuration object containing the settings for the T5Block. TYPE: object
has_relative_attention_bias	Specifies whether the attention bias is relative or not. Default is False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config, has_relative_attention_bias=False):
    """
    Initializes a new instance of the T5Block class.

    Args:
        self: The object itself.
        config (object): The configuration object containing the settings for the T5Block.
        has_relative_attention_bias (bool, optional): Specifies whether the attention bias is relative or not.
            Default is False.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.is_decoder = config.is_decoder
    self.layer = nn.ModuleList()
    self.layer.append(T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias))
    if self.is_decoder:
        self.layer.append(T5LayerCrossAttention(config))

    self.layer.append(T5LayerFF(config))

mindnlp.transformers.models.t5.modeling_t5.T5Block.forward(hidden_states, attention_mask=None, position_bias=None, encoder_hidden_states=None, encoder_attention_mask=None, encoder_decoder_position_bias=None, layer_head_mask=None, cross_attn_layer_head_mask=None, past_key_value=None, use_cache=False, output_attentions=False)

Constructs a T5Block.

PARAMETER	DESCRIPTION
self	The T5Block instance. TYPE: T5Block
hidden_states	The input hidden states. TYPE: Tensor
attention_mask	The attention mask tensor. Defaults to None. TYPE: Tensor DEFAULT: None
position_bias	The position bias tensor. Defaults to None. TYPE: Tensor DEFAULT: None
encoder_hidden_states	The encoder hidden states tensor. Defaults to None. TYPE: Tensor DEFAULT: None
encoder_attention_mask	The encoder attention mask tensor. Defaults to None. TYPE: Tensor DEFAULT: None
encoder_decoder_position_bias	The encoder-decoder position bias tensor. Defaults to None. TYPE: Tensor DEFAULT: None
layer_head_mask	The layer head mask tensor. Defaults to None. TYPE: Tensor DEFAULT: None
cross_attn_layer_head_mask	The cross-attention layer head mask tensor. Defaults to None. TYPE: Tensor DEFAULT: None
past_key_value	The past key-value states. Defaults to None. TYPE: Tuple[Tensor] DEFAULT: None
use_cache	Whether to use cache. Defaults to False. TYPE: bool DEFAULT: False
output_attentions	Whether to output attentions. Defaults to False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
Tuple	A tuple containing the following elements: hidden_states (Tensor): The output hidden states. present_key_value_state (Tuple[Tensor], optional): The present key-value state. None if not available. attention_outputs (Tuple[Tensor], optional): The attention outputs. None if not available.

RAISES	DESCRIPTION
ValueError	If the number of past states is not as expected.
Warning	If past_key_values is passed to the encoder.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    hidden_states,
    attention_mask=None,
    position_bias=None,
    encoder_hidden_states=None,
    encoder_attention_mask=None,
    encoder_decoder_position_bias=None,
    layer_head_mask=None,
    cross_attn_layer_head_mask=None,
    past_key_value=None,
    use_cache=False,
    output_attentions=False,
    # return_dict=True,
):
    """
    Constructs a T5Block.

    Args:
        self (T5Block): The T5Block instance.
        hidden_states (Tensor): The input hidden states.
        attention_mask (Tensor, optional): The attention mask tensor. Defaults to None.
        position_bias (Tensor, optional): The position bias tensor. Defaults to None.
        encoder_hidden_states (Tensor, optional): The encoder hidden states tensor. Defaults to None.
        encoder_attention_mask (Tensor, optional): The encoder attention mask tensor. Defaults to None.
        encoder_decoder_position_bias (Tensor, optional): The encoder-decoder position bias tensor. Defaults to None.
        layer_head_mask (Tensor, optional): The layer head mask tensor. Defaults to None.
        cross_attn_layer_head_mask (Tensor, optional): The cross-attention layer head mask tensor. Defaults to None.
        past_key_value (Tuple[Tensor], optional): The past key-value states. Defaults to None.
        use_cache (bool, optional): Whether to use cache. Defaults to False.
        output_attentions (bool, optional): Whether to output attentions. Defaults to False.

    Returns:
        Tuple:
            A tuple containing the following elements:

            - hidden_states (Tensor): The output hidden states.
            - present_key_value_state (Tuple[Tensor], optional): The present key-value state. None if not available.
            - attention_outputs (Tuple[Tensor], optional): The attention outputs. None if not available.

    Raises:
        ValueError: If the number of past states is not as expected.
        Warning: If `past_key_values` is passed to the encoder.
    """
    if past_key_value is not None:
        if not self.is_decoder:
            logging.warning("`past_key_values` is passed to the encoder. Please make sure this is intended.")
        expected_num_past_key_values = 2 if encoder_hidden_states is None else 4

        if len(past_key_value) != expected_num_past_key_values:
            raise ValueError(
                f"There should be {expected_num_past_key_values} past states. "
                f"{'2 (past / key) for cross attention. ' if expected_num_past_key_values == 4 else ''}"
                f"Got {len(past_key_value)} past key / value states"
            )

        self_attn_past_key_value = past_key_value[:2]
        cross_attn_past_key_value = past_key_value[2:]
    else:
        self_attn_past_key_value, cross_attn_past_key_value = None, None
    self_attention_outputs = self.layer[0](
        hidden_states,
        attention_mask=attention_mask,
        position_bias=position_bias,
        layer_head_mask=layer_head_mask,
        past_key_value=self_attn_past_key_value,
        use_cache=use_cache,
        output_attentions=output_attentions,
    )
    hidden_states, present_key_value_state = self_attention_outputs[:2]
    attention_outputs = self_attention_outputs[2:]  # Keep self-attention outputs and relative position weights

    # clamp inf values to enable fp16 training
    if hidden_states.dtype == mindspore.float16 and ops.isinf(hidden_states).any():
        clamp_value = mindspore.tensor(np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max) - 1000
        hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

    do_cross_attention = self.is_decoder and encoder_hidden_states is not None
    if do_cross_attention:
        # the actual query length is unknown for cross attention
        # if using past key value states. Need to inject it here
        if present_key_value_state is not None:
            query_length = present_key_value_state[0].shape[2]
        else:
            query_length = None

        cross_attention_outputs = self.layer[1](
            hidden_states,
            key_value_states=encoder_hidden_states,
            attention_mask=encoder_attention_mask,
            position_bias=encoder_decoder_position_bias,
            layer_head_mask=cross_attn_layer_head_mask,
            past_key_value=cross_attn_past_key_value,
            query_length=query_length,
            use_cache=use_cache,
            output_attentions=output_attentions,
        )
        hidden_states = cross_attention_outputs[0]

        # clamp inf values to enable fp16 training
        if hidden_states.dtype == mindspore.float16 and ops.isinf(hidden_states).any():
            clamp_value = mindspore.tensor(np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max) - 1000
            hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

        # Combine self attn and cross attn key value states
        if present_key_value_state is not None:
            present_key_value_state = present_key_value_state + cross_attention_outputs[1]

        # Keep cross-attention outputs and relative position weights
        attention_outputs = attention_outputs + cross_attention_outputs[2:]

    # Apply Feed Forward layer
    hidden_states = self.layer[-1](hidden_states)

    # clamp inf values to enable fp16 training
    if hidden_states.dtype == mindspore.float16 and ops.isinf(hidden_states).any():
        clamp_value = mindspore.tensor(np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max) - 1000
        hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

    outputs = (hidden_states,)

    if use_cache:
        outputs = outputs + (present_key_value_state,) + attention_outputs
    else:
        outputs = outputs + attention_outputs

    return outputs

mindnlp.transformers.models.t5.modeling_t5.T5ClassificationHead

Bases: Module

Head for sentence-level classification tasks.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5ClassificationHead(nn.Module):
    """Head for sentence-level classification tasks."""
    def __init__(self, config: T5Config):
        """
        Initializes a T5ClassificationHead instance.

        Args:
            self: The T5ClassificationHead instance.
            config (T5Config): The configuration for the T5 model. It specifies the model's architecture and parameters.

        Returns:
            None.

        Raises:
            TypeError: If the config parameter is not of type T5Config.
            ValueError: If the config parameters are not valid or if there are any issues during initialization.
        """
        super().__init__()
        self.dense = nn.Linear(config.d_model, config.d_model)
        self.dropout = nn.Dropout(p=config.classifier_dropout)
        self.out_proj = nn.Linear(config.d_model, config.num_labels)

    def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
        """
        Constructs the T5 classification head.

        Args:
            self: The T5ClassificationHead object.
            hidden_states (mindspore.Tensor): The input hidden states tensor.
                This tensor contains the hidden states from the T5 model.
                Shape of the tensor should be (batch_size, sequence_length, hidden_size).

        Returns:
            mindspore.Tensor: The output tensor after passing through the T5 classification head.
                Shape of the tensor is (batch_size, sequence_length, num_labels).

        Raises:
            None.
        """
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.dense(hidden_states)
        hidden_states = ops.tanh(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.out_proj(hidden_states)
        return hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5ClassificationHead.__init__(config)

Initializes a T5ClassificationHead instance.

PARAMETER	DESCRIPTION
self	The T5ClassificationHead instance.
config	The configuration for the T5 model. It specifies the model's architecture and parameters. TYPE: T5Config

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the config parameter is not of type T5Config.
ValueError	If the config parameters are not valid or if there are any issues during initialization.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    Initializes a T5ClassificationHead instance.

    Args:
        self: The T5ClassificationHead instance.
        config (T5Config): The configuration for the T5 model. It specifies the model's architecture and parameters.

    Returns:
        None.

    Raises:
        TypeError: If the config parameter is not of type T5Config.
        ValueError: If the config parameters are not valid or if there are any issues during initialization.
    """
    super().__init__()
    self.dense = nn.Linear(config.d_model, config.d_model)
    self.dropout = nn.Dropout(p=config.classifier_dropout)
    self.out_proj = nn.Linear(config.d_model, config.num_labels)

mindnlp.transformers.models.t5.modeling_t5.T5ClassificationHead.forward(hidden_states)

Constructs the T5 classification head.

PARAMETER	DESCRIPTION
self	The T5ClassificationHead object.
hidden_states	The input hidden states tensor. This tensor contains the hidden states from the T5 model. Shape of the tensor should be (batch_size, sequence_length, hidden_size). TYPE: Tensor

RETURNS	DESCRIPTION
Tensor	mindspore.Tensor: The output tensor after passing through the T5 classification head. Shape of the tensor is (batch_size, sequence_length, num_labels).

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(self, hidden_states: mindspore.Tensor) -> mindspore.Tensor:
    """
    Constructs the T5 classification head.

    Args:
        self: The T5ClassificationHead object.
        hidden_states (mindspore.Tensor): The input hidden states tensor.
            This tensor contains the hidden states from the T5 model.
            Shape of the tensor should be (batch_size, sequence_length, hidden_size).

    Returns:
        mindspore.Tensor: The output tensor after passing through the T5 classification head.
            Shape of the tensor is (batch_size, sequence_length, num_labels).

    Raises:
        None.
    """
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.dense(hidden_states)
    hidden_states = ops.tanh(hidden_states)
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.out_proj(hidden_states)
    return hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5DenseActDense

Bases: Module

T5DenseActDense

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5DenseActDense(nn.Module):
    """T5DenseActDense"""
    def __init__(self, config: T5Config):
        """
        Initializes an instance of the T5DenseActDense class.

        Args:
            self: The instance of the class.
            config (T5Config):
                The configuration object containing the model's settings.

                - The 'config' parameter is of type T5Config, which specifies the configuration for the T5 model.
                - It is used to set up the parameters for the dense layers and the dropout rate.
                - This parameter is required and has no default value.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.wi = nn.Linear(config.d_model, config.d_ff, bias=False)
        self.wo = nn.Linear(config.d_ff, config.d_model, bias=False)
        self.dropout = nn.Dropout(p=config.dropout_rate)
        self.act = ACT2FN[config.dense_act_fn]

    def forward(self, hidden_states):
        """
        This method forwards the hidden states by applying a series of transformations including linear mapping,
        activation function, dropout, and additional conversion based on weight data types.

        Args:
            self (T5DenseActDense): The instance of the T5DenseActDense class.
            hidden_states (Tensor): The input hidden states to be processed by the method.

        Returns:
            None.

        Raises:
            TypeError:
                If the data type of weights in self.wo does not match the data type of hidden_states or mindspore.int8.
        """
        hidden_states = self.wi(hidden_states)
        hidden_states = self.act(hidden_states)
        hidden_states = self.dropout(hidden_states)
        if self.wo.weight.dtype not in (hidden_states.dtype, mindspore.int8):
            hidden_states = hidden_states.astype(self.wo.weight.dtype)
        hidden_states = self.wo(hidden_states)
        return hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5DenseActDense.__init__(config)

Initializes an instance of the T5DenseActDense class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object containing the model's settings. The 'config' parameter is of type T5Config, which specifies the configuration for the T5 model. It is used to set up the parameters for the dense layers and the dropout rate. This parameter is required and has no default value. TYPE: T5Config

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    Initializes an instance of the T5DenseActDense class.

    Args:
        self: The instance of the class.
        config (T5Config):
            The configuration object containing the model's settings.

            - The 'config' parameter is of type T5Config, which specifies the configuration for the T5 model.
            - It is used to set up the parameters for the dense layers and the dropout rate.
            - This parameter is required and has no default value.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.wi = nn.Linear(config.d_model, config.d_ff, bias=False)
    self.wo = nn.Linear(config.d_ff, config.d_model, bias=False)
    self.dropout = nn.Dropout(p=config.dropout_rate)
    self.act = ACT2FN[config.dense_act_fn]

mindnlp.transformers.models.t5.modeling_t5.T5DenseActDense.forward(hidden_states)

This method forwards the hidden states by applying a series of transformations including linear mapping, activation function, dropout, and additional conversion based on weight data types.

PARAMETER	DESCRIPTION
self	The instance of the T5DenseActDense class. TYPE: T5DenseActDense
hidden_states	The input hidden states to be processed by the method. TYPE: Tensor

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the data type of weights in self.wo does not match the data type of hidden_states or mindspore.int8.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(self, hidden_states):
    """
    This method forwards the hidden states by applying a series of transformations including linear mapping,
    activation function, dropout, and additional conversion based on weight data types.

    Args:
        self (T5DenseActDense): The instance of the T5DenseActDense class.
        hidden_states (Tensor): The input hidden states to be processed by the method.

    Returns:
        None.

    Raises:
        TypeError:
            If the data type of weights in self.wo does not match the data type of hidden_states or mindspore.int8.
    """
    hidden_states = self.wi(hidden_states)
    hidden_states = self.act(hidden_states)
    hidden_states = self.dropout(hidden_states)
    if self.wo.weight.dtype not in (hidden_states.dtype, mindspore.int8):
        hidden_states = hidden_states.astype(self.wo.weight.dtype)
    hidden_states = self.wo(hidden_states)
    return hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5DenseGatedActDense

Bases: Module

T5DenseGatedActDense

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5DenseGatedActDense(nn.Module):
    """T5DenseGatedActDense"""
    def __init__(self, config: T5Config):
        """
        Initializes an instance of the T5DenseGatedActDense class.

        Args:
            self: An instance of the T5DenseGatedActDense class.
            config (T5Config): The configuration object for the T5 model.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.wi_0 = nn.Linear(config.d_model, config.d_ff, bias=False)
        self.wi_1 = nn.Linear(config.d_model, config.d_ff, bias=False)
        self.wo = nn.Linear(config.d_ff, config.d_model, bias=False)
        self.dropout = nn.Dropout(p=config.dropout_rate)
        self.act = ACT2FN[config.dense_act_fn]

    def forward(self, hidden_states):
        """
        Constructs the hidden states of the T5DenseGatedActDense model.

        Args:
            self: The instance of the T5DenseGatedActDense class.
            hidden_states (Tensor): The input hidden states.
                It should have the shape (batch_size, sequence_length, hidden_size).

        Returns:
            None

        Raises:
            None
        """
        hidden_gelu = self.act(self.wi_0(hidden_states))
        hidden_linear = self.wi_1(hidden_states)
        hidden_states = hidden_gelu * hidden_linear
        hidden_states = self.dropout(hidden_states)

        if self.wo.weight.dtype not in (hidden_states.dtype, mindspore.int8):
            hidden_states = hidden_states.astype(self.wo.weight.dtype)

        hidden_states = self.wo(hidden_states)
        return hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5DenseGatedActDense.__init__(config)

Initializes an instance of the T5DenseGatedActDense class.

PARAMETER	DESCRIPTION
self	An instance of the T5DenseGatedActDense class.
config	The configuration object for the T5 model. TYPE: T5Config

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    Initializes an instance of the T5DenseGatedActDense class.

    Args:
        self: An instance of the T5DenseGatedActDense class.
        config (T5Config): The configuration object for the T5 model.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.wi_0 = nn.Linear(config.d_model, config.d_ff, bias=False)
    self.wi_1 = nn.Linear(config.d_model, config.d_ff, bias=False)
    self.wo = nn.Linear(config.d_ff, config.d_model, bias=False)
    self.dropout = nn.Dropout(p=config.dropout_rate)
    self.act = ACT2FN[config.dense_act_fn]

mindnlp.transformers.models.t5.modeling_t5.T5DenseGatedActDense.forward(hidden_states)

Constructs the hidden states of the T5DenseGatedActDense model.

PARAMETER	DESCRIPTION
self	The instance of the T5DenseGatedActDense class.
hidden_states	The input hidden states. It should have the shape (batch_size, sequence_length, hidden_size). TYPE: Tensor

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(self, hidden_states):
    """
    Constructs the hidden states of the T5DenseGatedActDense model.

    Args:
        self: The instance of the T5DenseGatedActDense class.
        hidden_states (Tensor): The input hidden states.
            It should have the shape (batch_size, sequence_length, hidden_size).

    Returns:
        None

    Raises:
        None
    """
    hidden_gelu = self.act(self.wi_0(hidden_states))
    hidden_linear = self.wi_1(hidden_states)
    hidden_states = hidden_gelu * hidden_linear
    hidden_states = self.dropout(hidden_states)

    if self.wo.weight.dtype not in (hidden_states.dtype, mindspore.int8):
        hidden_states = hidden_states.astype(self.wo.weight.dtype)

    hidden_states = self.wo(hidden_states)
    return hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5EncoderModel

Bases: T5PreTrainedModel

T5EncoderModel

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5EncoderModel(T5PreTrainedModel):
    """T5EncoderModel"""
    _tied_weights_keys = ["encoder.embed_tokens.weight"]
    _keys_to_ignore_on_load_unexpected = [r"decoder"]

    def __init__(self, config: T5Config):
        """
        Initializes a T5EncoderModel instance.

        Args:
            self: The T5EncoderModel instance itself.
            config (T5Config): An instance of T5Config containing the configuration parameters for the model.
                It specifies the configuration settings such as vocab_size and d_model.
                This parameter is required for configuring the T5EncoderModel.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.shared = nn.Embedding(config.vocab_size, config.d_model)

        encoder_config = copy.deepcopy(config)
        encoder_config.use_cache = False
        encoder_config.is_encoder_decoder = False
        self.encoder = T5Stack(encoder_config)
        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        Retrieve the input embeddings.

        This method is used to obtain the input embeddings for the T5EncoderModel class.

        Args:
            self: An instance of the T5EncoderModel class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.shared

    def set_input_embeddings(self, new_embeddings):
        """
        Sets the input embeddings for the T5EncoderModel.

        Args:
            self (T5EncoderModel): The instance of the T5EncoderModel class.
            new_embeddings (torch.Tensor): The new input embeddings to be set.

        Returns:
            None

        Raises:
            None
        """
        self.shared = new_embeddings
        self.encoder.set_input_embeddings(new_embeddings)

    def _tie_weights(self):
        """
        Ties the weights of the word embeddings in the T5EncoderModel.

        Args:
            self (T5EncoderModel): An instance of the T5EncoderModel class.

        Returns:
            None.

        Raises:
            None.
        """
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.encoder.embed_tokens, self.shared)

    def get_encoder(self):
        """
        Get the encoder of the T5EncoderModel.

        Args:
            self (T5EncoderModel): An instance of the T5EncoderModel class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.encoder

    def _prune_heads(self, heads_to_prune):
        """
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        """
        for layer, heads in heads_to_prune.items():
            self.encoder.block[layer].layer[0].SelfAttention.prune_heads(heads)

    def forward(
        self,
        input_ids = None,
        attention_mask = None,
        head_mask = None,
        inputs_embeds = None,
        output_attentions = None,
        output_hidden_states = None,
        return_dict = None,
    ):
        """
        Constructs the T5EncoderModel.

        Args:
            self: The T5EncoderModel object.
            input_ids (optional): A tensor of shape (batch_size, sequence_length) containing the input token IDs.
                Defaults to None.
            attention_mask (optional): A tensor of shape (batch_size, sequence_length) containing the attention mask.
                Defaults to None.
            head_mask (optional): A tensor of shape (num_heads,) containing the head mask. Defaults to None.
            inputs_embeds (optional): A tensor of shape (batch_size, sequence_length, embedding_size)
                containing the input embeddings. Defaults to None.
            output_attentions (optional): A boolean indicating whether to return the attentions. Defaults to None.
            output_hidden_states (optional): A boolean indicating whether to return the hidden states. Defaults to None.
            return_dict (optional): A boolean indicating whether to return a dictionary. If not provided,
                it is determined by self.config.use_return_dict. Defaults to None.

        Returns:
            encoder_outputs: A tuple containing the encoder outputs.
                It typically consists of the following elements:

                - last_hidden_state: A tensor of shape (batch_size, sequence_length, hidden_size) containing the last
                hidden state of the encoder.
                - hidden_states: A tuple of tensors containing all the hidden states of the encoder. Each tensor has
                a shape of (batch_size, sequence_length, hidden_size).
                - attentions: A tuple of tensors containing the attentions of the encoder. Each tensor has a shape of
                (batch_size, num_heads, sequence_length, sequence_length).

        Raises:
            None.
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        encoder_outputs = self.encoder(
            input_ids=input_ids,
            attention_mask=attention_mask,
            inputs_embeds=inputs_embeds,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        return encoder_outputs

mindnlp.transformers.models.t5.modeling_t5.T5EncoderModel.__init__(config)

Initializes a T5EncoderModel instance.

PARAMETER	DESCRIPTION
self	The T5EncoderModel instance itself.
config	An instance of T5Config containing the configuration parameters for the model. It specifies the configuration settings such as vocab_size and d_model. This parameter is required for configuring the T5EncoderModel. TYPE: T5Config

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    Initializes a T5EncoderModel instance.

    Args:
        self: The T5EncoderModel instance itself.
        config (T5Config): An instance of T5Config containing the configuration parameters for the model.
            It specifies the configuration settings such as vocab_size and d_model.
            This parameter is required for configuring the T5EncoderModel.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.shared = nn.Embedding(config.vocab_size, config.d_model)

    encoder_config = copy.deepcopy(config)
    encoder_config.use_cache = False
    encoder_config.is_encoder_decoder = False
    self.encoder = T5Stack(encoder_config)
    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.t5.modeling_t5.T5EncoderModel.forward(input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Constructs the T5EncoderModel.

PARAMETER	DESCRIPTION
self	The T5EncoderModel object.
input_ids	A tensor of shape (batch_size, sequence_length) containing the input token IDs. Defaults to None. TYPE: optional DEFAULT: None
attention_mask	A tensor of shape (batch_size, sequence_length) containing the attention mask. Defaults to None. TYPE: optional DEFAULT: None
head_mask	A tensor of shape (num_heads,) containing the head mask. Defaults to None. TYPE: optional DEFAULT: None
inputs_embeds	A tensor of shape (batch_size, sequence_length, embedding_size) containing the input embeddings. Defaults to None. TYPE: optional DEFAULT: None
output_attentions	A boolean indicating whether to return the attentions. Defaults to None. TYPE: optional DEFAULT: None
output_hidden_states	A boolean indicating whether to return the hidden states. Defaults to None. TYPE: optional DEFAULT: None
return_dict	A boolean indicating whether to return a dictionary. If not provided, it is determined by self.config.use_return_dict. Defaults to None. TYPE: optional DEFAULT: None

RETURNS

DESCRIPTION

encoder_outputs

A tuple containing the encoder outputs. It typically consists of the following elements: last_hidden_state: A tensor of shape (batch_size, sequence_length, hidden_size) containing the last hidden state of the encoder. hidden_states: A tuple of tensors containing all the hidden states of the encoder. Each tensor has a shape of (batch_size, sequence_length, hidden_size). attentions: A tuple of tensors containing the attentions of the encoder. Each tensor has a shape of (batch_size, num_heads, sequence_length, sequence_length).

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    input_ids = None,
    attention_mask = None,
    head_mask = None,
    inputs_embeds = None,
    output_attentions = None,
    output_hidden_states = None,
    return_dict = None,
):
    """
    Constructs the T5EncoderModel.

    Args:
        self: The T5EncoderModel object.
        input_ids (optional): A tensor of shape (batch_size, sequence_length) containing the input token IDs.
            Defaults to None.
        attention_mask (optional): A tensor of shape (batch_size, sequence_length) containing the attention mask.
            Defaults to None.
        head_mask (optional): A tensor of shape (num_heads,) containing the head mask. Defaults to None.
        inputs_embeds (optional): A tensor of shape (batch_size, sequence_length, embedding_size)
            containing the input embeddings. Defaults to None.
        output_attentions (optional): A boolean indicating whether to return the attentions. Defaults to None.
        output_hidden_states (optional): A boolean indicating whether to return the hidden states. Defaults to None.
        return_dict (optional): A boolean indicating whether to return a dictionary. If not provided,
            it is determined by self.config.use_return_dict. Defaults to None.

    Returns:
        encoder_outputs: A tuple containing the encoder outputs.
            It typically consists of the following elements:

            - last_hidden_state: A tensor of shape (batch_size, sequence_length, hidden_size) containing the last
            hidden state of the encoder.
            - hidden_states: A tuple of tensors containing all the hidden states of the encoder. Each tensor has
            a shape of (batch_size, sequence_length, hidden_size).
            - attentions: A tuple of tensors containing the attentions of the encoder. Each tensor has a shape of
            (batch_size, num_heads, sequence_length, sequence_length).

    Raises:
        None.
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    encoder_outputs = self.encoder(
        input_ids=input_ids,
        attention_mask=attention_mask,
        inputs_embeds=inputs_embeds,
        head_mask=head_mask,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    return encoder_outputs

mindnlp.transformers.models.t5.modeling_t5.T5EncoderModel.get_encoder()

Get the encoder of the T5EncoderModel.

PARAMETER	DESCRIPTION
self	An instance of the T5EncoderModel class. TYPE: T5EncoderModel

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_encoder(self):
    """
    Get the encoder of the T5EncoderModel.

    Args:
        self (T5EncoderModel): An instance of the T5EncoderModel class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.encoder

mindnlp.transformers.models.t5.modeling_t5.T5EncoderModel.get_input_embeddings()

Retrieve the input embeddings.

This method is used to obtain the input embeddings for the T5EncoderModel class.

PARAMETER	DESCRIPTION
self	An instance of the T5EncoderModel class.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_input_embeddings(self):
    """
    Retrieve the input embeddings.

    This method is used to obtain the input embeddings for the T5EncoderModel class.

    Args:
        self: An instance of the T5EncoderModel class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.shared

mindnlp.transformers.models.t5.modeling_t5.T5EncoderModel.set_input_embeddings(new_embeddings)

Sets the input embeddings for the T5EncoderModel.

PARAMETER	DESCRIPTION
self	The instance of the T5EncoderModel class. TYPE: T5EncoderModel
new_embeddings	The new input embeddings to be set. TYPE: Tensor

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def set_input_embeddings(self, new_embeddings):
    """
    Sets the input embeddings for the T5EncoderModel.

    Args:
        self (T5EncoderModel): The instance of the T5EncoderModel class.
        new_embeddings (torch.Tensor): The new input embeddings to be set.

    Returns:
        None

    Raises:
        None
    """
    self.shared = new_embeddings
    self.encoder.set_input_embeddings(new_embeddings)

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration

Bases: T5PreTrainedModel

T5ForConditionalGeneration

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5ForConditionalGeneration(T5PreTrainedModel):
    """T5ForConditionalGeneration"""
    _keys_to_ignore_on_load_unexpected = [
        "decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight",
    ]
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight", "lm_head.weight"]

    def __init__(self, config: T5Config):
        """
        Initializes an instance of the T5ForConditionalGeneration class.

        Args:
            self: The object instance.
            config (T5Config): The configuration object for the T5 model.
                It contains various parameters to customize the model's behavior, such as the model dimension,
                vocabulary size, and number of decoder layers.

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.model_dim = config.d_model

        self.shared = nn.Embedding(config.vocab_size, config.d_model)

        encoder_config = copy.deepcopy(config)
        encoder_config.is_decoder = False
        encoder_config.use_cache = False
        encoder_config.is_encoder_decoder = False
        self.encoder = T5Stack(encoder_config)

        decoder_config = copy.deepcopy(config)
        decoder_config.is_decoder = True
        decoder_config.is_encoder_decoder = False
        decoder_config.num_layers = config.num_decoder_layers
        self.decoder = T5Stack(decoder_config)

        self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)

        self.post_init()

    def get_input_embeddings(self):
        """
        Returns the input embeddings for the T5 model.

        Args:
            self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.shared

    def set_input_embeddings(self, new_embeddings):
        """
        Set input embeddings for the T5 model.

        Args:
            self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.
            new_embeddings (tensor): The new input embeddings to be set for the model.
                It should be a tensor of shape (vocab_size, hidden_size).

        Returns:
            None.

        Raises:
            TypeError: If the new_embeddings parameter is not a tensor.
            ValueError: If the shape of the new_embeddings tensor does not match the required shape
                (vocab_size, hidden_size).
        """
        self.shared = new_embeddings
        # self.encoder.set_input_embeddings(new_embeddings)
        # self.decoder.set_input_embeddings(new_embeddings)

    def _tie_weights(self):
        """
        Method _tie_weights in the class T5ForConditionalGeneration ties or clones weights for word embeddings.

        Args:
            self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.
                It represents the current object and is used to access attributes and methods within the class.

        Returns:
            None.

        Raises:
            None.
        """
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.decoder.embed_tokens, self.shared)

    def set_output_embeddings(self, new_embeddings):
        """
        Set the output embeddings for the T5 model.

        Args:
            self (T5ForConditionalGeneration): The T5 model instance.
            new_embeddings (torch.Tensor): The new embeddings to set as the output embeddings for the model.

        Returns:
            None: This method updates the output embeddings of the T5 model in place.

        Raises:
            TypeError: If the new_embeddings parameter is not a torch.Tensor.
            ValueError: If the shape of the new_embeddings does not match the expected shape for model output embeddings.
        """
        self.lm_head = new_embeddings

    def get_output_embeddings(self):
        """
        Returns the output embeddings for the T5 model.

        Args:
            self (T5ForConditionalGeneration): An instance of the T5ForConditionalGeneration class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.lm_head

    def get_encoder(self):
        """
        This method is part of the 'T5ForConditionalGeneration' class and is used to retrieve the encoder.

        Args:
            self (T5ForConditionalGeneration): An instance of the 'T5ForConditionalGeneration' class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.encoder

    def get_decoder(self):
        """
        Returns the decoder used by the T5 model for conditional generation.

        Args:
            self (T5ForConditionalGeneration): The current instance of the T5ForConditionalGeneration class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.decoder

    def forward(
        self,
        input_ids = None,
        attention_mask = None,
        decoder_input_ids = None,
        decoder_attention_mask = None,
        head_mask = None,
        decoder_head_mask = None,
        cross_attn_head_mask = None,
        encoder_outputs = None,
        past_key_values = None,
        inputs_embeds = None,
        decoder_inputs_embeds = None,
        labels = None,
        use_cache = None,
        output_attentions = None,
        output_hidden_states = None,
        return_dict = None,
    ):
        """Constructs the T5 model for conditional generation.

        Args:
            self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.
            input_ids (torch.Tensor, optional): The input sequence tensor of shape (batch_size, sequence_length).
                Defaults to None.
            attention_mask (torch.Tensor, optional): The attention mask tensor of shape (batch_size, sequence_length).
                Defaults to None.
            decoder_input_ids (torch.Tensor, optional): The decoder input sequence tensor of shape
                (batch_size, decoder_sequence_length).  Defaults to None.
            decoder_attention_mask (torch.Tensor, optional): The decoder attention mask tensor of shape
                (batch_size, decoder_sequence_length). Defaults to None.
            head_mask (torch.Tensor, optional): The head mask tensor of shape (num_layers, num_heads).
                Defaults to None.
            decoder_head_mask (torch.Tensor, optional): The decoder head mask tensor of shape (num_layers, num_heads).
                Defaults to None.
            cross_attn_head_mask (torch.Tensor, optional): The cross-attention head mask tensor of shape
                (num_layers, num_heads). Defaults to None.
            encoder_outputs (tuple, optional): The encoder outputs returned by the encoder model.
                Defaults to None.
            past_key_values (tuple, optional): The past key values returned by the decoder model.
                Defaults to None.
            inputs_embeds (torch.Tensor, optional): The input embeddings tensor of shape
                (batch_size, sequence_length, hidden_size). Defaults to None.
            decoder_inputs_embeds (torch.Tensor, optional): The decoder input embeddings tensor of shape
                (batch_size, decoder_sequence_length, hidden_size). Defaults to None.
            labels (torch.Tensor, optional): The labels tensor of shape (batch_size, sequence_length).
                Defaults to None.
            use_cache (bool, optional): Whether to use cache for the model.
                Defaults to None.
            output_attentions (bool, optional): Whether to output attentions.
                Defaults to None.
            output_hidden_states (bool, optional): Whether to output hidden states.
                Defaults to None.
            return_dict (bool, optional): Whether to return a dictionary as the output.
                Defaults to None.

        Returns:
            None.

        Raises:
            None.
        """
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask
        if head_mask is not None and decoder_head_mask is None:
            if self.config.num_layers == self.config.num_decoder_layers:
                decoder_head_mask = head_mask

        # Encode if needed (training, first prediction pass)
        if encoder_outputs is None:
            # Convert encoder inputs in embeddings if needed
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                head_mask=head_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )

        hidden_states = encoder_outputs[0]
        if labels is not None and decoder_input_ids is None and decoder_inputs_embeds is None:
            # get decoder inputs from shifting lm labels to the right
            decoder_input_ids = self._shift_right(labels)
        # Decode
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            inputs_embeds=decoder_inputs_embeds,
            past_key_values=past_key_values,
            encoder_hidden_states=hidden_states,
            encoder_attention_mask=attention_mask,
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = decoder_outputs[0]

        if self.config.tie_word_embeddings:
            # Rescale output before projecting on vocab
            sequence_output = sequence_output * (self.model_dim**-0.5)

        lm_logits = self.lm_head(sequence_output)

        loss = None
        if labels is not None:
            loss = F.cross_entropy(lm_logits.view(-1, lm_logits.shape[-1]), labels.view(-1), ignore_index=-100)
            # TODO(thom): Add z_loss

        if not return_dict:
            output = (lm_logits,) + decoder_outputs[1:] + encoder_outputs
            return ((loss,) + output) if loss is not None else output

        return Seq2SeqLMOutput(
            loss=loss,
            logits=lm_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    def prepare_inputs_for_generation(
        self,
        input_ids,
        past_key_values=None,
        attention_mask=None,
        head_mask=None,
        decoder_head_mask=None,
        decoder_attention_mask=None,
        cross_attn_head_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        """
        Prepare inputs for generation.

        Args:
            self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.
            input_ids (torch.Tensor): The input tensor of shape (batch_size, sequence_length) containing input IDs.
            past_key_values (tuple, optional): The tuple of past key values for the transformer decoder. Default is None.
            attention_mask (torch.Tensor, optional): The attention mask tensor of shape (batch_size, sequence_length)
                indicating which tokens to attend to. Default is None.
            head_mask (torch.Tensor, optional): The head mask tensor of shape (num_layers, num_heads) indicating
                which heads to mask. Default is None.
            decoder_head_mask (torch.Tensor, optional): The decoder head mask tensor of shape (num_layers, num_heads)
                indicating which decoder heads to mask. Default is None.
            decoder_attention_mask (torch.Tensor, optional): The decoder attention mask tensor of shape
                (batch_size, sequence_length) indicating which tokens to attend to in the decoder. Default is None.
            cross_attn_head_mask (torch.Tensor, optional): The cross-attention head mask tensor of shape
                (num_layers, num_heads) indicating which cross-attention heads to mask. Default is None.
            use_cache (bool, optional): Whether to use cache. Default is None.
            encoder_outputs (torch.Tensor, optional): The encoder outputs tensor of shape
                (batch_size, sequence_length, hidden_size) containing the hidden states of the encoder. Default is None.

        Returns:
            dict:
                A dictionary containing the prepared inputs for generation with the following keys:

                - 'decoder_input_ids' (torch.Tensor): The decoder input tensor of shape (batch_size, sequence_length)
                containing input IDs.
                - 'past_key_values' (tuple): The tuple of past key values for the transformer decoder.
                - 'encoder_outputs' (torch.Tensor): The encoder outputs tensor of shape
                (batch_size, sequence_length, hidden_size) containing the hidden states of the encoder.
                - 'attention_mask' (torch.Tensor): The attention mask tensor of shape (batch_size, sequence_length)
                indicating which tokens to attend to.
                - 'head_mask' (torch.Tensor): The head mask tensor of shape (num_layers, num_heads) indicating
                which heads to mask.
                - 'decoder_head_mask' (torch.Tensor): The decoder head mask tensor of shape (num_layers, num_heads)
                indicating which decoder heads to mask.
                - 'decoder_attention_mask' (torch.Tensor): The decoder attention mask tensor of shape
                (batch_size, sequence_length) indicating which tokens to attend to in the decoder.
                - 'cross_attn_head_mask' (torch.Tensor): The cross-attention head mask tensor of shape
                (num_layers, num_heads) indicating which cross-attention heads to mask.
                - 'use_cache' (bool): Whether to use cache.

        Raises:
            None.
        """
        # cut decoder_input_ids if past_key_values is used
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]

        return {
            "decoder_input_ids": input_ids,
            "past_key_values": past_key_values,
            "encoder_outputs": encoder_outputs,
            "attention_mask": attention_mask,
            "head_mask": head_mask,
            "decoder_head_mask": decoder_head_mask,
            "decoder_attention_mask": decoder_attention_mask,
            "cross_attn_head_mask": cross_attn_head_mask,
            "use_cache": use_cache,
        }

    def prepare_decoder_input_ids_from_labels(self, labels: mindspore.Tensor):
        """
        Prepare decoder input ids from labels.

        This method is used to prepare the input ids for the decoder by shifting the given labels sequence to the right.

        Args:
            self (T5ForConditionalGeneration): An instance of the T5ForConditionalGeneration class.
            labels (mindspore.Tensor): The labels tensor containing the sequence of labels.

        Returns:
            None: This method modifies the decoder input ids in-place.

        Raises:
            None.

        """
        return self._shift_right(labels)

    def _reorder_cache(self, past_key_values, beam_idx):
        """
        This method '_reorder_cache' is defined within the class 'T5ForConditionalGeneration' and is used to reorder
        the cache for decoding during the T5 model's conditional generation.

        Args:
            self (object): The instance of the class.
            past_key_values (tuple): The past key value states generated during the model's previous decoding steps.
                If set to None, a warning is logged to consider setting `use_cache=True` to speed up decoding.
            beam_idx (tensor): The indices of the beam to reorder the cache.

        Returns:
            tuple: The reordered past key value states for the decoder. If 'past_key_values' is None, it returns None.

        Raises:
            ValueError: If the shape of the reordered layer past states and the original layer past states mismatch.
            ValueError: If the length of the reordered layer past states and the original layer past states mismatch.
        """
        # if decoder past is not included in output
        # speedy decoding is disabled and no need to reorder
        if past_key_values is None:
            logger.warning("You might want to consider setting `use_cache=True` to speed up decoding")
            return past_key_values

        reordered_decoder_past = ()
        for layer_past_states in past_key_values:
            # get the correct batch idx from layer past batch dim
            # batch dim of `past` is at 2nd position
            reordered_layer_past_states = ()
            for layer_past_state in layer_past_states:
                # need to set correct `past` for each of the four key / value states
                reordered_layer_past_states = reordered_layer_past_states + (
                    layer_past_state.index_select(0, beam_idx),
                )

            if reordered_layer_past_states[0].shape != layer_past_states[0].shape:
                raise ValueError(
                    f"reordered_layer_past_states[0] shape {reordered_layer_past_states[0].shape} and layer_past_states[0] shape {layer_past_states[0].shape} mismatched"
                )
            if len(reordered_layer_past_states) != len(layer_past_states):
                raise ValueError(
                    f"length of reordered_layer_past_states {len(reordered_layer_past_states)} and length of layer_past_states {len(layer_past_states)} mismatched"
                )

            reordered_decoder_past = reordered_decoder_past + (reordered_layer_past_states,)
        return reordered_decoder_past

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.__init__(config)

Initializes an instance of the T5ForConditionalGeneration class.

PARAMETER	DESCRIPTION
self	The object instance.
config	The configuration object for the T5 model. It contains various parameters to customize the model's behavior, such as the model dimension, vocabulary size, and number of decoder layers. TYPE: T5Config

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    Initializes an instance of the T5ForConditionalGeneration class.

    Args:
        self: The object instance.
        config (T5Config): The configuration object for the T5 model.
            It contains various parameters to customize the model's behavior, such as the model dimension,
            vocabulary size, and number of decoder layers.

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.model_dim = config.d_model

    self.shared = nn.Embedding(config.vocab_size, config.d_model)

    encoder_config = copy.deepcopy(config)
    encoder_config.is_decoder = False
    encoder_config.use_cache = False
    encoder_config.is_encoder_decoder = False
    self.encoder = T5Stack(encoder_config)

    decoder_config = copy.deepcopy(config)
    decoder_config.is_decoder = True
    decoder_config.is_encoder_decoder = False
    decoder_config.num_layers = config.num_decoder_layers
    self.decoder = T5Stack(decoder_config)

    self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)

    self.post_init()

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.forward(input_ids=None, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, head_mask=None, decoder_head_mask=None, cross_attn_head_mask=None, encoder_outputs=None, past_key_values=None, inputs_embeds=None, decoder_inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Constructs the T5 model for conditional generation.

PARAMETER	DESCRIPTION
self	The instance of the T5ForConditionalGeneration class. TYPE: T5ForConditionalGeneration
input_ids	The input sequence tensor of shape (batch_size, sequence_length). Defaults to None. TYPE: Tensor DEFAULT: None
attention_mask	The attention mask tensor of shape (batch_size, sequence_length). Defaults to None. TYPE: Tensor DEFAULT: None
decoder_input_ids	The decoder input sequence tensor of shape (batch_size, decoder_sequence_length). Defaults to None. TYPE: Tensor DEFAULT: None
decoder_attention_mask	The decoder attention mask tensor of shape (batch_size, decoder_sequence_length). Defaults to None. TYPE: Tensor DEFAULT: None
head_mask	The head mask tensor of shape (num_layers, num_heads). Defaults to None. TYPE: Tensor DEFAULT: None
decoder_head_mask	The decoder head mask tensor of shape (num_layers, num_heads). Defaults to None. TYPE: Tensor DEFAULT: None
cross_attn_head_mask	The cross-attention head mask tensor of shape (num_layers, num_heads). Defaults to None. TYPE: Tensor DEFAULT: None
encoder_outputs	The encoder outputs returned by the encoder model. Defaults to None. TYPE: tuple DEFAULT: None
past_key_values	The past key values returned by the decoder model. Defaults to None. TYPE: tuple DEFAULT: None
inputs_embeds	The input embeddings tensor of shape (batch_size, sequence_length, hidden_size). Defaults to None. TYPE: Tensor DEFAULT: None
decoder_inputs_embeds	The decoder input embeddings tensor of shape (batch_size, decoder_sequence_length, hidden_size). Defaults to None. TYPE: Tensor DEFAULT: None
labels	The labels tensor of shape (batch_size, sequence_length). Defaults to None. TYPE: Tensor DEFAULT: None
use_cache	Whether to use cache for the model. Defaults to None. TYPE: bool DEFAULT: None
output_attentions	Whether to output attentions. Defaults to None. TYPE: bool DEFAULT: None
output_hidden_states	Whether to output hidden states. Defaults to None. TYPE: bool DEFAULT: None
return_dict	Whether to return a dictionary as the output. Defaults to None. TYPE: bool DEFAULT: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    input_ids = None,
    attention_mask = None,
    decoder_input_ids = None,
    decoder_attention_mask = None,
    head_mask = None,
    decoder_head_mask = None,
    cross_attn_head_mask = None,
    encoder_outputs = None,
    past_key_values = None,
    inputs_embeds = None,
    decoder_inputs_embeds = None,
    labels = None,
    use_cache = None,
    output_attentions = None,
    output_hidden_states = None,
    return_dict = None,
):
    """Constructs the T5 model for conditional generation.

    Args:
        self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.
        input_ids (torch.Tensor, optional): The input sequence tensor of shape (batch_size, sequence_length).
            Defaults to None.
        attention_mask (torch.Tensor, optional): The attention mask tensor of shape (batch_size, sequence_length).
            Defaults to None.
        decoder_input_ids (torch.Tensor, optional): The decoder input sequence tensor of shape
            (batch_size, decoder_sequence_length).  Defaults to None.
        decoder_attention_mask (torch.Tensor, optional): The decoder attention mask tensor of shape
            (batch_size, decoder_sequence_length). Defaults to None.
        head_mask (torch.Tensor, optional): The head mask tensor of shape (num_layers, num_heads).
            Defaults to None.
        decoder_head_mask (torch.Tensor, optional): The decoder head mask tensor of shape (num_layers, num_heads).
            Defaults to None.
        cross_attn_head_mask (torch.Tensor, optional): The cross-attention head mask tensor of shape
            (num_layers, num_heads). Defaults to None.
        encoder_outputs (tuple, optional): The encoder outputs returned by the encoder model.
            Defaults to None.
        past_key_values (tuple, optional): The past key values returned by the decoder model.
            Defaults to None.
        inputs_embeds (torch.Tensor, optional): The input embeddings tensor of shape
            (batch_size, sequence_length, hidden_size). Defaults to None.
        decoder_inputs_embeds (torch.Tensor, optional): The decoder input embeddings tensor of shape
            (batch_size, decoder_sequence_length, hidden_size). Defaults to None.
        labels (torch.Tensor, optional): The labels tensor of shape (batch_size, sequence_length).
            Defaults to None.
        use_cache (bool, optional): Whether to use cache for the model.
            Defaults to None.
        output_attentions (bool, optional): Whether to output attentions.
            Defaults to None.
        output_hidden_states (bool, optional): Whether to output hidden states.
            Defaults to None.
        return_dict (bool, optional): Whether to return a dictionary as the output.
            Defaults to None.

    Returns:
        None.

    Raises:
        None.
    """
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask
    if head_mask is not None and decoder_head_mask is None:
        if self.config.num_layers == self.config.num_decoder_layers:
            decoder_head_mask = head_mask

    # Encode if needed (training, first prediction pass)
    if encoder_outputs is None:
        # Convert encoder inputs in embeddings if needed
        encoder_outputs = self.encoder(
            input_ids=input_ids,
            attention_mask=attention_mask,
            inputs_embeds=inputs_embeds,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

    hidden_states = encoder_outputs[0]
    if labels is not None and decoder_input_ids is None and decoder_inputs_embeds is None:
        # get decoder inputs from shifting lm labels to the right
        decoder_input_ids = self._shift_right(labels)
    # Decode
    decoder_outputs = self.decoder(
        input_ids=decoder_input_ids,
        attention_mask=decoder_attention_mask,
        inputs_embeds=decoder_inputs_embeds,
        past_key_values=past_key_values,
        encoder_hidden_states=hidden_states,
        encoder_attention_mask=attention_mask,
        head_mask=decoder_head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = decoder_outputs[0]

    if self.config.tie_word_embeddings:
        # Rescale output before projecting on vocab
        sequence_output = sequence_output * (self.model_dim**-0.5)

    lm_logits = self.lm_head(sequence_output)

    loss = None
    if labels is not None:
        loss = F.cross_entropy(lm_logits.view(-1, lm_logits.shape[-1]), labels.view(-1), ignore_index=-100)
        # TODO(thom): Add z_loss

    if not return_dict:
        output = (lm_logits,) + decoder_outputs[1:] + encoder_outputs
        return ((loss,) + output) if loss is not None else output

    return Seq2SeqLMOutput(
        loss=loss,
        logits=lm_logits,
        past_key_values=decoder_outputs.past_key_values,
        decoder_hidden_states=decoder_outputs.hidden_states,
        decoder_attentions=decoder_outputs.attentions,
        cross_attentions=decoder_outputs.cross_attentions,
        encoder_last_hidden_state=encoder_outputs.last_hidden_state,
        encoder_hidden_states=encoder_outputs.hidden_states,
        encoder_attentions=encoder_outputs.attentions,
    )

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.get_decoder()

Returns the decoder used by the T5 model for conditional generation.

PARAMETER	DESCRIPTION
self	The current instance of the T5ForConditionalGeneration class. TYPE: T5ForConditionalGeneration

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_decoder(self):
    """
    Returns the decoder used by the T5 model for conditional generation.

    Args:
        self (T5ForConditionalGeneration): The current instance of the T5ForConditionalGeneration class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.decoder

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.get_encoder()

This method is part of the 'T5ForConditionalGeneration' class and is used to retrieve the encoder.

PARAMETER	DESCRIPTION
self	An instance of the 'T5ForConditionalGeneration' class. TYPE: T5ForConditionalGeneration

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_encoder(self):
    """
    This method is part of the 'T5ForConditionalGeneration' class and is used to retrieve the encoder.

    Args:
        self (T5ForConditionalGeneration): An instance of the 'T5ForConditionalGeneration' class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.encoder

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.get_input_embeddings()

Returns the input embeddings for the T5 model.

PARAMETER	DESCRIPTION
self	The instance of the T5ForConditionalGeneration class. TYPE: T5ForConditionalGeneration

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_input_embeddings(self):
    """
    Returns the input embeddings for the T5 model.

    Args:
        self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.shared

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.get_output_embeddings()

Returns the output embeddings for the T5 model.

PARAMETER	DESCRIPTION
self	An instance of the T5ForConditionalGeneration class. TYPE: T5ForConditionalGeneration

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_output_embeddings(self):
    """
    Returns the output embeddings for the T5 model.

    Args:
        self (T5ForConditionalGeneration): An instance of the T5ForConditionalGeneration class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.lm_head

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.prepare_decoder_input_ids_from_labels(labels)

Prepare decoder input ids from labels.

This method is used to prepare the input ids for the decoder by shifting the given labels sequence to the right.

PARAMETER	DESCRIPTION
self	An instance of the T5ForConditionalGeneration class. TYPE: T5ForConditionalGeneration
labels	The labels tensor containing the sequence of labels. TYPE: Tensor

RETURNS	DESCRIPTION
None	This method modifies the decoder input ids in-place.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def prepare_decoder_input_ids_from_labels(self, labels: mindspore.Tensor):
    """
    Prepare decoder input ids from labels.

    This method is used to prepare the input ids for the decoder by shifting the given labels sequence to the right.

    Args:
        self (T5ForConditionalGeneration): An instance of the T5ForConditionalGeneration class.
        labels (mindspore.Tensor): The labels tensor containing the sequence of labels.

    Returns:
        None: This method modifies the decoder input ids in-place.

    Raises:
        None.

    """
    return self._shift_right(labels)

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.prepare_inputs_for_generation(input_ids, past_key_values=None, attention_mask=None, head_mask=None, decoder_head_mask=None, decoder_attention_mask=None, cross_attn_head_mask=None, use_cache=None, encoder_outputs=None, **kwargs)

Prepare inputs for generation.

PARAMETER	DESCRIPTION
self	The instance of the T5ForConditionalGeneration class. TYPE: T5ForConditionalGeneration
input_ids	The input tensor of shape (batch_size, sequence_length) containing input IDs. TYPE: Tensor
past_key_values	The tuple of past key values for the transformer decoder. Default is None. TYPE: tuple DEFAULT: None
attention_mask	The attention mask tensor of shape (batch_size, sequence_length) indicating which tokens to attend to. Default is None. TYPE: Tensor DEFAULT: None
head_mask	The head mask tensor of shape (num_layers, num_heads) indicating which heads to mask. Default is None. TYPE: Tensor DEFAULT: None
decoder_head_mask	The decoder head mask tensor of shape (num_layers, num_heads) indicating which decoder heads to mask. Default is None. TYPE: Tensor DEFAULT: None
decoder_attention_mask	The decoder attention mask tensor of shape (batch_size, sequence_length) indicating which tokens to attend to in the decoder. Default is None. TYPE: Tensor DEFAULT: None
cross_attn_head_mask	The cross-attention head mask tensor of shape (num_layers, num_heads) indicating which cross-attention heads to mask. Default is None. TYPE: Tensor DEFAULT: None
use_cache	Whether to use cache. Default is None. TYPE: bool DEFAULT: None
encoder_outputs	The encoder outputs tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states of the encoder. Default is None. TYPE: Tensor DEFAULT: None

RETURNS

DESCRIPTION

dict

A dictionary containing the prepared inputs for generation with the following keys: 'decoder_input_ids' (torch.Tensor): The decoder input tensor of shape (batch_size, sequence_length) containing input IDs. 'past_key_values' (tuple): The tuple of past key values for the transformer decoder. 'encoder_outputs' (torch.Tensor): The encoder outputs tensor of shape (batch_size, sequence_length, hidden_size) containing the hidden states of the encoder. 'attention_mask' (torch.Tensor): The attention mask tensor of shape (batch_size, sequence_length) indicating which tokens to attend to. 'head_mask' (torch.Tensor): The head mask tensor of shape (num_layers, num_heads) indicating which heads to mask. 'decoder_head_mask' (torch.Tensor): The decoder head mask tensor of shape (num_layers, num_heads) indicating which decoder heads to mask. 'decoder_attention_mask' (torch.Tensor): The decoder attention mask tensor of shape (batch_size, sequence_length) indicating which tokens to attend to in the decoder. 'cross_attn_head_mask' (torch.Tensor): The cross-attention head mask tensor of shape (num_layers, num_heads) indicating which cross-attention heads to mask. 'use_cache' (bool): Whether to use cache.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def prepare_inputs_for_generation(
    self,
    input_ids,
    past_key_values=None,
    attention_mask=None,
    head_mask=None,
    decoder_head_mask=None,
    decoder_attention_mask=None,
    cross_attn_head_mask=None,
    use_cache=None,
    encoder_outputs=None,
    **kwargs,
):
    """
    Prepare inputs for generation.

    Args:
        self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.
        input_ids (torch.Tensor): The input tensor of shape (batch_size, sequence_length) containing input IDs.
        past_key_values (tuple, optional): The tuple of past key values for the transformer decoder. Default is None.
        attention_mask (torch.Tensor, optional): The attention mask tensor of shape (batch_size, sequence_length)
            indicating which tokens to attend to. Default is None.
        head_mask (torch.Tensor, optional): The head mask tensor of shape (num_layers, num_heads) indicating
            which heads to mask. Default is None.
        decoder_head_mask (torch.Tensor, optional): The decoder head mask tensor of shape (num_layers, num_heads)
            indicating which decoder heads to mask. Default is None.
        decoder_attention_mask (torch.Tensor, optional): The decoder attention mask tensor of shape
            (batch_size, sequence_length) indicating which tokens to attend to in the decoder. Default is None.
        cross_attn_head_mask (torch.Tensor, optional): The cross-attention head mask tensor of shape
            (num_layers, num_heads) indicating which cross-attention heads to mask. Default is None.
        use_cache (bool, optional): Whether to use cache. Default is None.
        encoder_outputs (torch.Tensor, optional): The encoder outputs tensor of shape
            (batch_size, sequence_length, hidden_size) containing the hidden states of the encoder. Default is None.

    Returns:
        dict:
            A dictionary containing the prepared inputs for generation with the following keys:

            - 'decoder_input_ids' (torch.Tensor): The decoder input tensor of shape (batch_size, sequence_length)
            containing input IDs.
            - 'past_key_values' (tuple): The tuple of past key values for the transformer decoder.
            - 'encoder_outputs' (torch.Tensor): The encoder outputs tensor of shape
            (batch_size, sequence_length, hidden_size) containing the hidden states of the encoder.
            - 'attention_mask' (torch.Tensor): The attention mask tensor of shape (batch_size, sequence_length)
            indicating which tokens to attend to.
            - 'head_mask' (torch.Tensor): The head mask tensor of shape (num_layers, num_heads) indicating
            which heads to mask.
            - 'decoder_head_mask' (torch.Tensor): The decoder head mask tensor of shape (num_layers, num_heads)
            indicating which decoder heads to mask.
            - 'decoder_attention_mask' (torch.Tensor): The decoder attention mask tensor of shape
            (batch_size, sequence_length) indicating which tokens to attend to in the decoder.
            - 'cross_attn_head_mask' (torch.Tensor): The cross-attention head mask tensor of shape
            (num_layers, num_heads) indicating which cross-attention heads to mask.
            - 'use_cache' (bool): Whether to use cache.

    Raises:
        None.
    """
    # cut decoder_input_ids if past_key_values is used
    if past_key_values is not None:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = input_ids.shape[1] - 1

        input_ids = input_ids[:, remove_prefix_length:]

    return {
        "decoder_input_ids": input_ids,
        "past_key_values": past_key_values,
        "encoder_outputs": encoder_outputs,
        "attention_mask": attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
        "decoder_attention_mask": decoder_attention_mask,
        "cross_attn_head_mask": cross_attn_head_mask,
        "use_cache": use_cache,
    }

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.set_input_embeddings(new_embeddings)

Set input embeddings for the T5 model.

PARAMETER	DESCRIPTION
self	The instance of the T5ForConditionalGeneration class. TYPE: T5ForConditionalGeneration
new_embeddings	The new input embeddings to be set for the model. It should be a tensor of shape (vocab_size, hidden_size). TYPE: tensor

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the new_embeddings parameter is not a tensor.
ValueError	If the shape of the new_embeddings tensor does not match the required shape (vocab_size, hidden_size).

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def set_input_embeddings(self, new_embeddings):
    """
    Set input embeddings for the T5 model.

    Args:
        self (T5ForConditionalGeneration): The instance of the T5ForConditionalGeneration class.
        new_embeddings (tensor): The new input embeddings to be set for the model.
            It should be a tensor of shape (vocab_size, hidden_size).

    Returns:
        None.

    Raises:
        TypeError: If the new_embeddings parameter is not a tensor.
        ValueError: If the shape of the new_embeddings tensor does not match the required shape
            (vocab_size, hidden_size).
    """
    self.shared = new_embeddings

mindnlp.transformers.models.t5.modeling_t5.T5ForConditionalGeneration.set_output_embeddings(new_embeddings)

Set the output embeddings for the T5 model.

PARAMETER	DESCRIPTION
self	The T5 model instance. TYPE: T5ForConditionalGeneration
new_embeddings	The new embeddings to set as the output embeddings for the model. TYPE: Tensor

RETURNS	DESCRIPTION
None	This method updates the output embeddings of the T5 model in place.

RAISES	DESCRIPTION
TypeError	If the new_embeddings parameter is not a torch.Tensor.
ValueError	If the shape of the new_embeddings does not match the expected shape for model output embeddings.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def set_output_embeddings(self, new_embeddings):
    """
    Set the output embeddings for the T5 model.

    Args:
        self (T5ForConditionalGeneration): The T5 model instance.
        new_embeddings (torch.Tensor): The new embeddings to set as the output embeddings for the model.

    Returns:
        None: This method updates the output embeddings of the T5 model in place.

    Raises:
        TypeError: If the new_embeddings parameter is not a torch.Tensor.
        ValueError: If the shape of the new_embeddings does not match the expected shape for model output embeddings.
    """
    self.lm_head = new_embeddings

mindnlp.transformers.models.t5.modeling_t5.T5ForQuestionAnswering

Bases: T5PreTrainedModel

This class represents a T5 model for question answering tasks. It is designed specifically for question answering applications where the model takes input text and outputs answers to questions posed about the input. The model architecture includes an encoder and a decoder, both based on the T5Stack structure. The T5ForQuestionAnswering class provides methods for setting input embeddings, tying weights, accessing the encoder and decoder components, and forwarding the model for inference or training.

The forwardor initializes the T5ForQuestionAnswering model with a T5Config object, setting up the model dimensions, shared embeddings, encoder, decoder, and other necessary components. The model can be fine-tuned for specific question answering tasks by adjusting configurations and utilizing the provided methods.

The forward method executes the forward pass of the model, taking input tensors and generating outputs for question answering. It handles input embeddings, attention masks, decoder inputs, and various optional arguments to control the model's behavior during inference or training. The method returns the model's output, including predicted start and end positions for answering questions, loss values, and other relevant information.

Overall, the T5ForQuestionAnswering class encapsulates a T5 model tailored for question answering tasks, providing a convenient interface for utilizing and fine-tuning the model for specific applications.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5ForQuestionAnswering(T5PreTrainedModel):

    """
    This class represents a T5 model for question answering tasks. It is designed specifically for question answering
    applications where the model takes input text and outputs answers to questions posed about the input.
    The model architecture includes an encoder and a decoder, both based on the T5Stack structure.
    The T5ForQuestionAnswering class provides methods for setting input embeddings, tying weights, accessing the encoder
    and decoder components, and forwarding the model for inference or training.

    The forwardor initializes the T5ForQuestionAnswering model with a T5Config object, setting up the model dimensions,
    shared embeddings, encoder, decoder, and other necessary components. The model can be fine-tuned for specific
    question answering tasks by adjusting configurations and utilizing the provided methods.

    The forward method executes the forward pass of the model, taking input tensors and generating outputs for
    question answering. It handles input embeddings, attention masks, decoder inputs, and various optional arguments
    to control the model's behavior during inference or training. The method returns the model's output,
    including predicted start and end positions for answering questions, loss values, and other relevant information.

    Overall, the T5ForQuestionAnswering class encapsulates a T5 model tailored for question answering tasks,
    providing a convenient interface for utilizing and fine-tuning the model for specific applications.
    """
    _keys_to_ignore_on_load_unexpected = ["decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight"]
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"]

    def __init__(self, config: T5Config):
        """
        Initializes an instance of the T5ForQuestionAnswering class.

        Args:
            self: The instance of the class.
            config (T5Config):
                The configuration object that defines the model's parameters.

                - The config parameter must be an instance of the T5Config class.
                - It is used to set up the model's architecture and hyperparameters.
                - This parameter is required.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.model_dim = config.d_model

        self.shared = nn.Embedding(config.vocab_size, config.d_model)

        encoder_config = copy.deepcopy(config)
        encoder_config.is_decoder = False
        encoder_config.use_cache = False
        encoder_config.is_encoder_decoder = False
        self.encoder = T5Stack(encoder_config)

        decoder_config = copy.deepcopy(config)
        decoder_config.is_decoder = True
        decoder_config.is_encoder_decoder = False
        decoder_config.num_layers = config.num_decoder_layers
        self.decoder = T5Stack(decoder_config)

        self.num_labels = config.num_labels
        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        '''
        Description:
            This method returns the shared input embeddings of the T5 model for question answering.

        Args:
            self: The instance of the T5ForQuestionAnswering class.

        Returns:
            None

        Raises:
            None
        '''
        return self.shared

    def set_input_embeddings(self, new_embeddings):
        """
        Method to set new input embeddings for the T5 model used for Question Answering.

        Args:
            self (T5ForQuestionAnswering): The instance of the T5ForQuestionAnswering class.
                This parameter is automatically passed and refers to the current instance of the class.
            new_embeddings (object): The new input embeddings to be set for the model.
                This parameter represents the embeddings that will replace the existing ones in the model.

        Returns:
            None.

        Raises:
            None.
        """
        self.shared = new_embeddings
        # self.encoder.set_input_embeddings(new_embeddings)
        # self.decoder.set_input_embeddings(new_embeddings)

    def _tie_weights(self):
        """
        Ties the weights of the word embeddings in the T5ForQuestionAnswering model.

        Args:
            self: An instance of the T5ForQuestionAnswering class.

        Returns:
            None.

        Raises:
            None.
        """
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.decoder.embed_tokens, self.shared)

    def get_encoder(self):
        """
        Returns the encoder used for T5 question answering.

        Args:
            self: An instance of the T5ForQuestionAnswering class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.encoder

    def get_decoder(self):
        """
        Returns the decoder for the T5 model used for question answering.

        Args:
            self: An instance of the T5ForQuestionAnswering class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.decoder

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        decoder_head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_head_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        start_positions: Optional[mindspore.Tensor] = None,
        end_positions: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], Seq2SeqQuestionAnsweringModelOutput]:
        r"""
        Args:
            start_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for position (index) of the start of the labelled span for computing the token classification loss.
                Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence
                are not taken into account for computing the loss.
            end_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for position (index) of the end of the labelled span for computing the token classification loss.
                Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence
                are not taken into account for computing the loss.

        Returns:
            `Union[Tuple[mindspore.Tensor], Seq2SeqQuestionAnsweringModelOutput]`
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        if start_positions is not None and end_positions is not None:
            use_cache = False

        # Copied from models.bart.modeling_bart.BartModel.forward
        #   different to other models, T5 automatically creates decoder_input_ids from
        #   input_ids if no decoder_input_ids are provided
        if decoder_input_ids is None and decoder_inputs_embeds is None:
            if input_ids is None:
                raise ValueError(
                    "If no `decoder_input_ids` or `decoder_inputs_embeds` are "
                    "passed, `input_ids` cannot be `None`. Please pass either "
                    "`input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`."
                )
            decoder_input_ids = self._shift_right(input_ids)

        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask
        if head_mask is not None and decoder_head_mask is None:
            if self.config.num_layers == self.config.num_decoder_layers:
                warnings.warn("""
                The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently,
                `decoder_head_mask` is set to copy `head_mask`, but this feature is deprecated and will be removed in future versions.
                If you do not want to use any `decoder_head_mask` now, please set `decoder_head_mask = ops.ones(num_layers,
                num_heads)`.
                """, FutureWarning)
                decoder_head_mask = head_mask

        # Encode if needed (training, first prediction pass)
        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                head_mask=head_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        hidden_states = encoder_outputs[0]

        # Decode
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            inputs_embeds=decoder_inputs_embeds,
            past_key_values=None,
            encoder_hidden_states=hidden_states,
            encoder_attention_mask=attention_mask,
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        sequence_output = decoder_outputs[0]

        logits = self.qa_outputs(sequence_output)
        start_logits, end_logits = logits.split(1, axis=-1)
        start_logits = start_logits.squeeze(-1)
        end_logits = end_logits.squeeze(-1)

        total_loss = None
        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, split add a dimension
            if len(start_positions.shape) > 1:
                start_positions = start_positions.squeeze(-1)
            if len(end_positions.shape) > 1:
                end_positions = end_positions.squeeze(-1)
            # sometimes the start/end positions are outside our model inputs, we ignore these terms
            ignored_index = start_logits.shape[1]
            start_positions = start_positions.clamp(0, ignored_index)
            end_positions = end_positions.clamp(0, ignored_index)

            start_loss = F.cross_entropy(start_logits, start_positions, ignore_index=ignored_index)
            end_loss = F.cross_entropy(end_logits, end_positions, ignore_index=ignored_index)
            total_loss = (start_loss + end_loss) / 2

        if not return_dict:
            output = (start_logits, end_logits) + decoder_outputs[1:] + encoder_outputs
            return ((total_loss,) + output) if total_loss is not None else output

        return Seq2SeqQuestionAnsweringModelOutput(
            loss=total_loss,
            start_logits=start_logits,
            end_logits=end_logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

mindnlp.transformers.models.t5.modeling_t5.T5ForQuestionAnswering.__init__(config)

Initializes an instance of the T5ForQuestionAnswering class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object that defines the model's parameters. The config parameter must be an instance of the T5Config class. It is used to set up the model's architecture and hyperparameters. This parameter is required. TYPE: T5Config

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    Initializes an instance of the T5ForQuestionAnswering class.

    Args:
        self: The instance of the class.
        config (T5Config):
            The configuration object that defines the model's parameters.

            - The config parameter must be an instance of the T5Config class.
            - It is used to set up the model's architecture and hyperparameters.
            - This parameter is required.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.model_dim = config.d_model

    self.shared = nn.Embedding(config.vocab_size, config.d_model)

    encoder_config = copy.deepcopy(config)
    encoder_config.is_decoder = False
    encoder_config.use_cache = False
    encoder_config.is_encoder_decoder = False
    self.encoder = T5Stack(encoder_config)

    decoder_config = copy.deepcopy(config)
    decoder_config.is_decoder = True
    decoder_config.is_encoder_decoder = False
    decoder_config.num_layers = config.num_decoder_layers
    self.decoder = T5Stack(decoder_config)

    self.num_labels = config.num_labels
    self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.t5.modeling_t5.T5ForQuestionAnswering.forward(input_ids=None, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, head_mask=None, decoder_head_mask=None, cross_attn_head_mask=None, encoder_outputs=None, start_positions=None, end_positions=None, inputs_embeds=None, decoder_inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
start_positions	Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (sequence_length). Position outside of the sequence are not taken into account for computing the loss. TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None
end_positions	Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (sequence_length). Position outside of the sequence are not taken into account for computing the loss. TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple[Tensor], Seq2SeqQuestionAnsweringModelOutput]	Union[Tuple[mindspore.Tensor], Seq2SeqQuestionAnsweringModelOutput]

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    decoder_input_ids: Optional[mindspore.Tensor] = None,
    decoder_attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    decoder_head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_head_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    start_positions: Optional[mindspore.Tensor] = None,
    end_positions: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], Seq2SeqQuestionAnsweringModelOutput]:
    r"""
    Args:
        start_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence
            are not taken into account for computing the loss.

    Returns:
        `Union[Tuple[mindspore.Tensor], Seq2SeqQuestionAnsweringModelOutput]`
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    if start_positions is not None and end_positions is not None:
        use_cache = False

    # Copied from models.bart.modeling_bart.BartModel.forward
    #   different to other models, T5 automatically creates decoder_input_ids from
    #   input_ids if no decoder_input_ids are provided
    if decoder_input_ids is None and decoder_inputs_embeds is None:
        if input_ids is None:
            raise ValueError(
                "If no `decoder_input_ids` or `decoder_inputs_embeds` are "
                "passed, `input_ids` cannot be `None`. Please pass either "
                "`input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`."
            )
        decoder_input_ids = self._shift_right(input_ids)

    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask
    if head_mask is not None and decoder_head_mask is None:
        if self.config.num_layers == self.config.num_decoder_layers:
            warnings.warn("""
            The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently,
            `decoder_head_mask` is set to copy `head_mask`, but this feature is deprecated and will be removed in future versions.
            If you do not want to use any `decoder_head_mask` now, please set `decoder_head_mask = ops.ones(num_layers,
            num_heads)`.
            """, FutureWarning)
            decoder_head_mask = head_mask

    # Encode if needed (training, first prediction pass)
    if encoder_outputs is None:
        encoder_outputs = self.encoder(
            input_ids=input_ids,
            attention_mask=attention_mask,
            inputs_embeds=inputs_embeds,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
    elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
        encoder_outputs = BaseModelOutput(
            last_hidden_state=encoder_outputs[0],
            hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
            attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
        )

    hidden_states = encoder_outputs[0]

    # Decode
    decoder_outputs = self.decoder(
        input_ids=decoder_input_ids,
        attention_mask=decoder_attention_mask,
        inputs_embeds=decoder_inputs_embeds,
        past_key_values=None,
        encoder_hidden_states=hidden_states,
        encoder_attention_mask=attention_mask,
        head_mask=decoder_head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    sequence_output = decoder_outputs[0]

    logits = self.qa_outputs(sequence_output)
    start_logits, end_logits = logits.split(1, axis=-1)
    start_logits = start_logits.squeeze(-1)
    end_logits = end_logits.squeeze(-1)

    total_loss = None
    if start_positions is not None and end_positions is not None:
        # If we are on multi-GPU, split add a dimension
        if len(start_positions.shape) > 1:
            start_positions = start_positions.squeeze(-1)
        if len(end_positions.shape) > 1:
            end_positions = end_positions.squeeze(-1)
        # sometimes the start/end positions are outside our model inputs, we ignore these terms
        ignored_index = start_logits.shape[1]
        start_positions = start_positions.clamp(0, ignored_index)
        end_positions = end_positions.clamp(0, ignored_index)

        start_loss = F.cross_entropy(start_logits, start_positions, ignore_index=ignored_index)
        end_loss = F.cross_entropy(end_logits, end_positions, ignore_index=ignored_index)
        total_loss = (start_loss + end_loss) / 2

    if not return_dict:
        output = (start_logits, end_logits) + decoder_outputs[1:] + encoder_outputs
        return ((total_loss,) + output) if total_loss is not None else output

    return Seq2SeqQuestionAnsweringModelOutput(
        loss=total_loss,
        start_logits=start_logits,
        end_logits=end_logits,
        past_key_values=decoder_outputs.past_key_values,
        decoder_hidden_states=decoder_outputs.hidden_states,
        decoder_attentions=decoder_outputs.attentions,
        cross_attentions=decoder_outputs.cross_attentions,
        encoder_last_hidden_state=encoder_outputs.last_hidden_state,
        encoder_hidden_states=encoder_outputs.hidden_states,
        encoder_attentions=encoder_outputs.attentions,
    )

mindnlp.transformers.models.t5.modeling_t5.T5ForQuestionAnswering.get_decoder()

Returns the decoder for the T5 model used for question answering.

PARAMETER	DESCRIPTION
self	An instance of the T5ForQuestionAnswering class.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_decoder(self):
    """
    Returns the decoder for the T5 model used for question answering.

    Args:
        self: An instance of the T5ForQuestionAnswering class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.decoder

mindnlp.transformers.models.t5.modeling_t5.T5ForQuestionAnswering.get_encoder()

Returns the encoder used for T5 question answering.

PARAMETER	DESCRIPTION
self	An instance of the T5ForQuestionAnswering class.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_encoder(self):
    """
    Returns the encoder used for T5 question answering.

    Args:
        self: An instance of the T5ForQuestionAnswering class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.encoder

mindnlp.transformers.models.t5.modeling_t5.T5ForQuestionAnswering.get_input_embeddings()

Description

This method returns the shared input embeddings of the T5 model for question answering.

PARAMETER	DESCRIPTION
self	The instance of the T5ForQuestionAnswering class.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_input_embeddings(self):
    '''
    Description:
        This method returns the shared input embeddings of the T5 model for question answering.

    Args:
        self: The instance of the T5ForQuestionAnswering class.

    Returns:
        None

    Raises:
        None
    '''
    return self.shared

mindnlp.transformers.models.t5.modeling_t5.T5ForQuestionAnswering.set_input_embeddings(new_embeddings)

Method to set new input embeddings for the T5 model used for Question Answering.

PARAMETER	DESCRIPTION
self	The instance of the T5ForQuestionAnswering class. This parameter is automatically passed and refers to the current instance of the class. TYPE: T5ForQuestionAnswering
new_embeddings	The new input embeddings to be set for the model. This parameter represents the embeddings that will replace the existing ones in the model. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def set_input_embeddings(self, new_embeddings):
    """
    Method to set new input embeddings for the T5 model used for Question Answering.

    Args:
        self (T5ForQuestionAnswering): The instance of the T5ForQuestionAnswering class.
            This parameter is automatically passed and refers to the current instance of the class.
        new_embeddings (object): The new input embeddings to be set for the model.
            This parameter represents the embeddings that will replace the existing ones in the model.

    Returns:
        None.

    Raises:
        None.
    """
    self.shared = new_embeddings

mindnlp.transformers.models.t5.modeling_t5.T5ForSequenceClassification

Bases: T5PreTrainedModel

T5ForSequenceClassification class implements a T5 model for sequence classification tasks. It inherits from the T5PreTrainedModel class.

This class includes methods for initializing the model with a T5 configuration, forwarding the model for sequence classification tasks, and computing the loss based on the provided labels.

The init method initializes the T5ForSequenceClassification instance with a T5 configuration. The forward method forwards the model for sequence classification tasks and returns the computed loss and logits.

The forward method takes various input arguments such as input_ids, attention_mask, decoder_input_ids, labels, and other optional parameters to customize the behavior of the model during inference.

If labels are provided, the model computes the loss based on the problem type specified in the T5 configuration. The loss can be computed for regression, single-label classification, or multi-label classification tasks.

This class provides flexibility in handling different types of sequence classification tasks and supports customization through the T5 configuration settings.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5ForSequenceClassification(T5PreTrainedModel):

    """
    T5ForSequenceClassification class implements a T5 model for sequence classification tasks.
    It inherits from the T5PreTrainedModel class.

    This class includes methods for initializing the model with a T5 configuration, forwarding the model for
    sequence classification tasks, and computing the loss based on the provided labels.

    The __init__ method initializes the T5ForSequenceClassification instance with a T5 configuration.
    The forward method forwards the model for sequence classification tasks and returns the computed loss and logits.

    The forward method takes various input arguments such as input_ids, attention_mask, decoder_input_ids, labels,
    and other optional parameters to customize the behavior of the model during inference.

    If labels are provided, the model computes the loss based on the problem type specified in the T5 configuration.
    The loss can be computed for regression, single-label classification, or multi-label classification tasks.

    This class provides flexibility in handling different types of sequence classification tasks and supports
    customization through the T5 configuration settings.

    """
    _keys_to_ignore_on_load_unexpected = ["decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight"]
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"]

    def __init__(self, config: T5Config):
        """
        Initializes an instance of the T5ForSequenceClassification class.

        Args:
            self: An instance of the T5ForSequenceClassification class.
            config (T5Config): The configuration object that contains the model's hyperparameters and settings.

        Returns:
            None

        Raises:
            None

        Description:
            This method initializes an instance of the T5ForSequenceClassification class by setting up the necessary
            components for sequence classification tasks. It takes in the self parameter, which refers to the instance
            of the class itself, and the config parameter, which is an instance of the T5Config class.

            The config parameter is of type T5Config and represents the configuration object that contains various
            hyperparameters and settings for the T5 model. It is used to initialize the transformer and
            classification_head attributes of the T5ForSequenceClassification instance.

            The transformer attribute is of type T5Model and is responsible for the main transformer model used for
            sequence classification. It is initialized with the provided config object.

            The classification_head attribute is of type T5ClassificationHead and represents the classification head
            that is added on top of the transformer model. It is also initialized with the provided config object.

            After initializing the transformer and classification_head attributes, the post_init method is called to
            perform any additional setup or customization required.

        Note:
            This method is automatically called when creating a new instance of the T5ForSequenceClassification class.
        """
        super().__init__(config)
        self.transformer = T5Model(config)
        self.classification_head = T5ClassificationHead(config)
        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        decoder_head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_head_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[List[mindspore.Tensor]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
                config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:
            Union[Tuple, Seq2SeqSequenceClassifierOutput]
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        if labels is not None:
            use_cache = False

        if input_ids is None and inputs_embeds is not None:
            raise NotImplementedError(
                f"Passing input embeddings is currently not supported for {self.__class__.__name__}"
            )

        # Copied from models.bart.modeling_bart.BartModel.forward different to other models, T5 automatically creates
        # decoder_input_ids from input_ids if no decoder_input_ids are provided
        if decoder_input_ids is None and decoder_inputs_embeds is None:
            if input_ids is None:
                raise ValueError(
                    "If no `decoder_input_ids` or `decoder_inputs_embeds` are "
                    "passed, `input_ids` cannot be `None`. Please pass either "
                    "`input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`."
                )
            decoder_input_ids = self._shift_right(input_ids)

        outputs = self.transformer(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            decoder_attention_mask=decoder_attention_mask,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            encoder_outputs=encoder_outputs,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        sequence_output = outputs[0]

        eos_mask = input_ids.eq(self.config.eos_token_id)

        # if len(ops.unique_consecutive(eos_mask.sum(1))) > 1:
        #     raise ValueError("All examples must have the same number of <eos> tokens.")
        batch_size, _, hidden_size = sequence_output.shape
        sentence_representation = sequence_output[eos_mask].view(batch_size, -1, hidden_size)[:, -1, :]
        logits = self.classification_head(sentence_representation)

        loss = None
        if labels is not None:
            if self.config.problem_type is None:
                if self.config.num_labels == 1:
                    self.config.problem_type = "regression"
                elif self.config.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                    self.config.problem_type = "single_label_classification"
                else:
                    self.config.problem_type = "multi_label_classification"

            if self.config.problem_type == "regression":
                if self.config.num_labels == 1:
                    loss = F.mse_loss(logits.squeeze(), labels.squeeze())
                else:
                    loss = F.mse_loss(logits, labels)
            elif self.config.problem_type == "single_label_classification":
                loss = F.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1))
            elif self.config.problem_type == "multi_label_classification":
                loss = F.binary_cross_entropy_with_logits(logits, labels)
        if not return_dict:
            output = (logits,) + outputs[1:]
            return ((loss,) + output) if loss is not None else output

        return Seq2SeqSequenceClassifierOutput(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            decoder_hidden_states=outputs.decoder_hidden_states,
            decoder_attentions=outputs.decoder_attentions,
            cross_attentions=outputs.cross_attentions,
            encoder_last_hidden_state=outputs.encoder_last_hidden_state,
            encoder_hidden_states=outputs.encoder_hidden_states,
            encoder_attentions=outputs.encoder_attentions,
        )

mindnlp.transformers.models.t5.modeling_t5.T5ForSequenceClassification.__init__(config)

Initializes an instance of the T5ForSequenceClassification class.

PARAMETER	DESCRIPTION
self	An instance of the T5ForSequenceClassification class.
config	The configuration object that contains the model's hyperparameters and settings. TYPE: T5Config

RETURNS	DESCRIPTION
	None

Description

This method initializes an instance of the T5ForSequenceClassification class by setting up the necessary components for sequence classification tasks. It takes in the self parameter, which refers to the instance of the class itself, and the config parameter, which is an instance of the T5Config class.

The config parameter is of type T5Config and represents the configuration object that contains various hyperparameters and settings for the T5 model. It is used to initialize the transformer and classification_head attributes of the T5ForSequenceClassification instance.

The transformer attribute is of type T5Model and is responsible for the main transformer model used for sequence classification. It is initialized with the provided config object.

The classification_head attribute is of type T5ClassificationHead and represents the classification head that is added on top of the transformer model. It is also initialized with the provided config object.

After initializing the transformer and classification_head attributes, the post_init method is called to perform any additional setup or customization required.

Note

This method is automatically called when creating a new instance of the T5ForSequenceClassification class.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    Initializes an instance of the T5ForSequenceClassification class.

    Args:
        self: An instance of the T5ForSequenceClassification class.
        config (T5Config): The configuration object that contains the model's hyperparameters and settings.

    Returns:
        None

    Raises:
        None

    Description:
        This method initializes an instance of the T5ForSequenceClassification class by setting up the necessary
        components for sequence classification tasks. It takes in the self parameter, which refers to the instance
        of the class itself, and the config parameter, which is an instance of the T5Config class.

        The config parameter is of type T5Config and represents the configuration object that contains various
        hyperparameters and settings for the T5 model. It is used to initialize the transformer and
        classification_head attributes of the T5ForSequenceClassification instance.

        The transformer attribute is of type T5Model and is responsible for the main transformer model used for
        sequence classification. It is initialized with the provided config object.

        The classification_head attribute is of type T5ClassificationHead and represents the classification head
        that is added on top of the transformer model. It is also initialized with the provided config object.

        After initializing the transformer and classification_head attributes, the post_init method is called to
        perform any additional setup or customization required.

    Note:
        This method is automatically called when creating a new instance of the T5ForSequenceClassification class.
    """
    super().__init__(config)
    self.transformer = T5Model(config)
    self.classification_head = T5ClassificationHead(config)
    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.t5.modeling_t5.T5ForSequenceClassification.forward(input_ids=None, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, head_mask=None, decoder_head_mask=None, cross_attn_head_mask=None, encoder_outputs=None, inputs_embeds=None, decoder_inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the sequence classification/regression loss. Indices should be in [0, ..., config.num_labels - 1]. If config.num_labels > 1 a classification loss is computed (Cross-Entropy). TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple, Seq2SeqSequenceClassifierOutput]	Union[Tuple, Seq2SeqSequenceClassifierOutput]

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    decoder_input_ids: Optional[mindspore.Tensor] = None,
    decoder_attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    decoder_head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_head_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[List[mindspore.Tensor]] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

    Returns:
        Union[Tuple, Seq2SeqSequenceClassifierOutput]
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    if labels is not None:
        use_cache = False

    if input_ids is None and inputs_embeds is not None:
        raise NotImplementedError(
            f"Passing input embeddings is currently not supported for {self.__class__.__name__}"
        )

    # Copied from models.bart.modeling_bart.BartModel.forward different to other models, T5 automatically creates
    # decoder_input_ids from input_ids if no decoder_input_ids are provided
    if decoder_input_ids is None and decoder_inputs_embeds is None:
        if input_ids is None:
            raise ValueError(
                "If no `decoder_input_ids` or `decoder_inputs_embeds` are "
                "passed, `input_ids` cannot be `None`. Please pass either "
                "`input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`."
            )
        decoder_input_ids = self._shift_right(input_ids)

    outputs = self.transformer(
        input_ids,
        attention_mask=attention_mask,
        decoder_input_ids=decoder_input_ids,
        decoder_attention_mask=decoder_attention_mask,
        head_mask=head_mask,
        decoder_head_mask=decoder_head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        encoder_outputs=encoder_outputs,
        inputs_embeds=inputs_embeds,
        decoder_inputs_embeds=decoder_inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )
    sequence_output = outputs[0]

    eos_mask = input_ids.eq(self.config.eos_token_id)

    # if len(ops.unique_consecutive(eos_mask.sum(1))) > 1:
    #     raise ValueError("All examples must have the same number of <eos> tokens.")
    batch_size, _, hidden_size = sequence_output.shape
    sentence_representation = sequence_output[eos_mask].view(batch_size, -1, hidden_size)[:, -1, :]
    logits = self.classification_head(sentence_representation)

    loss = None
    if labels is not None:
        if self.config.problem_type is None:
            if self.config.num_labels == 1:
                self.config.problem_type = "regression"
            elif self.config.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                self.config.problem_type = "single_label_classification"
            else:
                self.config.problem_type = "multi_label_classification"

        if self.config.problem_type == "regression":
            if self.config.num_labels == 1:
                loss = F.mse_loss(logits.squeeze(), labels.squeeze())
            else:
                loss = F.mse_loss(logits, labels)
        elif self.config.problem_type == "single_label_classification":
            loss = F.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1))
        elif self.config.problem_type == "multi_label_classification":
            loss = F.binary_cross_entropy_with_logits(logits, labels)
    if not return_dict:
        output = (logits,) + outputs[1:]
        return ((loss,) + output) if loss is not None else output

    return Seq2SeqSequenceClassifierOutput(
        loss=loss,
        logits=logits,
        past_key_values=outputs.past_key_values,
        decoder_hidden_states=outputs.decoder_hidden_states,
        decoder_attentions=outputs.decoder_attentions,
        cross_attentions=outputs.cross_attentions,
        encoder_last_hidden_state=outputs.encoder_last_hidden_state,
        encoder_hidden_states=outputs.encoder_hidden_states,
        encoder_attentions=outputs.encoder_attentions,
    )

mindnlp.transformers.models.t5.modeling_t5.T5LayerCrossAttention

Bases: Module

T5LayerCrossAttention

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5LayerCrossAttention(nn.Module):
    """T5LayerCrossAttention"""
    def __init__(self, config):
        """
        Initializes an instance of the T5LayerCrossAttention class.

        Args:
            self: The object instance.
            config: An instance of the configuration class that contains the model's hyperparameters and settings.
                It is of type 'Any' and is used to configure the behavior of the cross-attention layer.
                The configuration object must have the following attributes:

                - d_model: An integer representing the dimensionality of the model's hidden states.
                - layer_norm_epsilon: A small float value used to stabilize the layer normalization process.
                - dropout_rate: A float value between 0 and 1, denoting the dropout rate for the layer.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.EncDecAttention = T5Attention(config, has_relative_attention_bias=False)
        self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
        self.dropout = nn.Dropout(p=config.dropout_rate)

    def forward(
        self,
        hidden_states,
        key_value_states,
        attention_mask=None,
        position_bias=None,
        layer_head_mask=None,
        past_key_value=None,
        use_cache=False,
        query_length=None,
        output_attentions=False,
    ):
        """
        This method forwards the T5 layer cross-attention mechanism.

        Args:
            self: Reference to the current instance of the class.
            hidden_states: Tensor representing the input hidden states.
            key_value_states: Tensor representing the key-value states for the attention mechanism.
            attention_mask: Optional tensor specifying the attention mask. Defaults to None.
            position_bias: Optional tensor providing positional bias information. Defaults to None.
            layer_head_mask: Optional tensor masking specific attention heads. Defaults to None.
            past_key_value: Optional tensor containing cached key-value states from previous steps. Defaults to None.
            use_cache: Boolean indicating whether to use cache for key-value states. Defaults to False.
            query_length: Optional integer specifying the length of the query. Defaults to None.
            output_attentions: Boolean indicating whether to output attentions. Defaults to False.

        Returns:
            Tuple containing the layer output and additional attention outputs.

        Raises:
            None
        """
        normed_hidden_states = self.layer_norm(hidden_states)
        attention_output = self.EncDecAttention(
            normed_hidden_states,
            mask=attention_mask,
            key_value_states=key_value_states,
            position_bias=position_bias,
            layer_head_mask=layer_head_mask,
            past_key_value=past_key_value,
            use_cache=use_cache,
            query_length=query_length,
            output_attentions=output_attentions,
        )
        layer_output = hidden_states + self.dropout(attention_output[0])
        outputs = (layer_output,) + attention_output[1:]  # add attentions if we output them
        return outputs

mindnlp.transformers.models.t5.modeling_t5.T5LayerCrossAttention.__init__(config)

Initializes an instance of the T5LayerCrossAttention class.

PARAMETER	DESCRIPTION
self	The object instance.
config	An instance of the configuration class that contains the model's hyperparameters and settings. It is of type 'Any' and is used to configure the behavior of the cross-attention layer. The configuration object must have the following attributes: d_model: An integer representing the dimensionality of the model's hidden states. layer_norm_epsilon: A small float value used to stabilize the layer normalization process. dropout_rate: A float value between 0 and 1, denoting the dropout rate for the layer.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config):
    """
    Initializes an instance of the T5LayerCrossAttention class.

    Args:
        self: The object instance.
        config: An instance of the configuration class that contains the model's hyperparameters and settings.
            It is of type 'Any' and is used to configure the behavior of the cross-attention layer.
            The configuration object must have the following attributes:

            - d_model: An integer representing the dimensionality of the model's hidden states.
            - layer_norm_epsilon: A small float value used to stabilize the layer normalization process.
            - dropout_rate: A float value between 0 and 1, denoting the dropout rate for the layer.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.EncDecAttention = T5Attention(config, has_relative_attention_bias=False)
    self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
    self.dropout = nn.Dropout(p=config.dropout_rate)

mindnlp.transformers.models.t5.modeling_t5.T5LayerCrossAttention.forward(hidden_states, key_value_states, attention_mask=None, position_bias=None, layer_head_mask=None, past_key_value=None, use_cache=False, query_length=None, output_attentions=False)

This method forwards the T5 layer cross-attention mechanism.

PARAMETER	DESCRIPTION
self	Reference to the current instance of the class.
hidden_states	Tensor representing the input hidden states.
key_value_states	Tensor representing the key-value states for the attention mechanism.
attention_mask	Optional tensor specifying the attention mask. Defaults to None. DEFAULT: None
position_bias	Optional tensor providing positional bias information. Defaults to None. DEFAULT: None
layer_head_mask	Optional tensor masking specific attention heads. Defaults to None. DEFAULT: None
past_key_value	Optional tensor containing cached key-value states from previous steps. Defaults to None. DEFAULT: None
use_cache	Boolean indicating whether to use cache for key-value states. Defaults to False. DEFAULT: False
query_length	Optional integer specifying the length of the query. Defaults to None. DEFAULT: None
output_attentions	Boolean indicating whether to output attentions. Defaults to False. DEFAULT: False

RETURNS	DESCRIPTION
	Tuple containing the layer output and additional attention outputs.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    hidden_states,
    key_value_states,
    attention_mask=None,
    position_bias=None,
    layer_head_mask=None,
    past_key_value=None,
    use_cache=False,
    query_length=None,
    output_attentions=False,
):
    """
    This method forwards the T5 layer cross-attention mechanism.

    Args:
        self: Reference to the current instance of the class.
        hidden_states: Tensor representing the input hidden states.
        key_value_states: Tensor representing the key-value states for the attention mechanism.
        attention_mask: Optional tensor specifying the attention mask. Defaults to None.
        position_bias: Optional tensor providing positional bias information. Defaults to None.
        layer_head_mask: Optional tensor masking specific attention heads. Defaults to None.
        past_key_value: Optional tensor containing cached key-value states from previous steps. Defaults to None.
        use_cache: Boolean indicating whether to use cache for key-value states. Defaults to False.
        query_length: Optional integer specifying the length of the query. Defaults to None.
        output_attentions: Boolean indicating whether to output attentions. Defaults to False.

    Returns:
        Tuple containing the layer output and additional attention outputs.

    Raises:
        None
    """
    normed_hidden_states = self.layer_norm(hidden_states)
    attention_output = self.EncDecAttention(
        normed_hidden_states,
        mask=attention_mask,
        key_value_states=key_value_states,
        position_bias=position_bias,
        layer_head_mask=layer_head_mask,
        past_key_value=past_key_value,
        use_cache=use_cache,
        query_length=query_length,
        output_attentions=output_attentions,
    )
    layer_output = hidden_states + self.dropout(attention_output[0])
    outputs = (layer_output,) + attention_output[1:]  # add attentions if we output them
    return outputs

mindnlp.transformers.models.t5.modeling_t5.T5LayerFF

Bases: Module

T5LayerFF

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5LayerFF(nn.Module):
    """T5LayerFF"""
    def __init__(self, config: T5Config):
        """
        Initializes an instance of the T5LayerFF class.

        Args:
            self: The instance of the T5LayerFF class.
            config (T5Config): The configuration object for the T5 model.
                It contains various parameters and settings for the model.

        Returns:
            None

        Raises:
            None.
        """
        super().__init__()
        if config.is_gated_act:
            self.DenseReluDense = T5DenseGatedActDense(config)
        else:
            self.DenseReluDense = T5DenseActDense(config)

        self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
        self.dropout = nn.Dropout(p=config.dropout_rate)

    def forward(self, hidden_states):
        """
        Constructs the forward pass of the T5LayerFF class.

        Args:
            self (T5LayerFF): An instance of the T5LayerFF class.
            hidden_states (Tensor): The hidden states input tensor.
                Shape (batch_size, sequence_length, hidden_size).

        Returns:
            None

        Raises:
            None
        """
        forwarded_states = self.layer_norm(hidden_states)
        forwarded_states = self.DenseReluDense(forwarded_states)
        hidden_states = hidden_states + self.dropout(forwarded_states)
        return hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5LayerFF.__init__(config)

Initializes an instance of the T5LayerFF class.

PARAMETER	DESCRIPTION
self	The instance of the T5LayerFF class.
config	The configuration object for the T5 model. It contains various parameters and settings for the model. TYPE: T5Config

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    Initializes an instance of the T5LayerFF class.

    Args:
        self: The instance of the T5LayerFF class.
        config (T5Config): The configuration object for the T5 model.
            It contains various parameters and settings for the model.

    Returns:
        None

    Raises:
        None.
    """
    super().__init__()
    if config.is_gated_act:
        self.DenseReluDense = T5DenseGatedActDense(config)
    else:
        self.DenseReluDense = T5DenseActDense(config)

    self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
    self.dropout = nn.Dropout(p=config.dropout_rate)

mindnlp.transformers.models.t5.modeling_t5.T5LayerFF.forward(hidden_states)

Constructs the forward pass of the T5LayerFF class.

PARAMETER	DESCRIPTION
self	An instance of the T5LayerFF class. TYPE: T5LayerFF
hidden_states	The hidden states input tensor. Shape (batch_size, sequence_length, hidden_size). TYPE: Tensor

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(self, hidden_states):
    """
    Constructs the forward pass of the T5LayerFF class.

    Args:
        self (T5LayerFF): An instance of the T5LayerFF class.
        hidden_states (Tensor): The hidden states input tensor.
            Shape (batch_size, sequence_length, hidden_size).

    Returns:
        None

    Raises:
        None
    """
    forwarded_states = self.layer_norm(hidden_states)
    forwarded_states = self.DenseReluDense(forwarded_states)
    hidden_states = hidden_states + self.dropout(forwarded_states)
    return hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5LayerNorm

Bases: Module

T5LayerNorm

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5LayerNorm(nn.Module):
    """T5LayerNorm"""
    def __init__(self, hidden_size, eps=1e-6):
        """
        Construct a layernorm module in the T5 style. No bias and no subtraction of mean.
        """
        super().__init__()
        self.weight = Parameter(initializer('zeros', (hidden_size,), mindspore.float32), 'weight')
        self.variance_epsilon = eps

    def forward(self, hidden_states):
        """
        This method 'forward' is a part of the class 'T5LayerNorm' and is used to perform layer normalization
        on the input hidden states.

        Args:
            self (T5LayerNorm): The instance of the T5LayerNorm class.
            hidden_states (numpy.ndarray): The input hidden states to be normalized.
                It is expected to be an array of numerical values.

        Returns:
            None.

        Raises:
            ValueError: If the input hidden_states is not a valid numerical array.
            TypeError: If the input hidden_states or self.weight is not of the expected data type.
            RuntimeError: If there is an issue with the normalization process.
        """
        variance = hidden_states.astype(mindspore.float32).pow(2).mean(-1, keep_dims=True)
        hidden_states = hidden_states / ops.sqrt(variance + self.variance_epsilon)
        # convert into half-precision if necessary
        if self.weight.dtype in [mindspore.float16, mindspore.bfloat16]:
            hidden_states = hidden_states.astype(self.weight.dtype)
        return self.weight * hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5LayerNorm.__init__(hidden_size, eps=1e-06)

Construct a layernorm module in the T5 style. No bias and no subtraction of mean.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, hidden_size, eps=1e-6):
    """
    Construct a layernorm module in the T5 style. No bias and no subtraction of mean.
    """
    super().__init__()
    self.weight = Parameter(initializer('zeros', (hidden_size,), mindspore.float32), 'weight')
    self.variance_epsilon = eps

mindnlp.transformers.models.t5.modeling_t5.T5LayerNorm.forward(hidden_states)

This method 'forward' is a part of the class 'T5LayerNorm' and is used to perform layer normalization on the input hidden states.

PARAMETER	DESCRIPTION
self	The instance of the T5LayerNorm class. TYPE: T5LayerNorm
hidden_states	The input hidden states to be normalized. It is expected to be an array of numerical values. TYPE: ndarray

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the input hidden_states is not a valid numerical array.
TypeError	If the input hidden_states or self.weight is not of the expected data type.
RuntimeError	If there is an issue with the normalization process.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(self, hidden_states):
    """
    This method 'forward' is a part of the class 'T5LayerNorm' and is used to perform layer normalization
    on the input hidden states.

    Args:
        self (T5LayerNorm): The instance of the T5LayerNorm class.
        hidden_states (numpy.ndarray): The input hidden states to be normalized.
            It is expected to be an array of numerical values.

    Returns:
        None.

    Raises:
        ValueError: If the input hidden_states is not a valid numerical array.
        TypeError: If the input hidden_states or self.weight is not of the expected data type.
        RuntimeError: If there is an issue with the normalization process.
    """
    variance = hidden_states.astype(mindspore.float32).pow(2).mean(-1, keep_dims=True)
    hidden_states = hidden_states / ops.sqrt(variance + self.variance_epsilon)
    # convert into half-precision if necessary
    if self.weight.dtype in [mindspore.float16, mindspore.bfloat16]:
        hidden_states = hidden_states.astype(self.weight.dtype)
    return self.weight * hidden_states

mindnlp.transformers.models.t5.modeling_t5.T5LayerSelfAttention

Bases: Module

T5LayerSelfAttention

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5LayerSelfAttention(nn.Module):
    """T5LayerSelfAttention"""
    def __init__(self, config, has_relative_attention_bias=False):
        """Initialize the T5LayerSelfAttention.

        Args:
            self (T5LayerSelfAttention): An instance of the T5LayerSelfAttention class.
            config (Config): An object containing the configuration parameters.
            has_relative_attention_bias (bool, optional): A flag indicating whether the attention bias is relative or not.
                Defaults to False.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.SelfAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias)
        self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
        self.dropout = nn.Dropout(p=config.dropout_rate)

    def forward(
        self,
        hidden_states,
        attention_mask=None,
        position_bias=None,
        layer_head_mask=None,
        past_key_value=None,
        use_cache=False,
        output_attentions=False,
    ):
        """
        This method 'forward' in the class 'T5LayerSelfAttention' forwards the output of a T5 self-attention layer.

        Args:
            self: The instance of the class.
            hidden_states (Tensor): The hidden states of the input sequence.
            attention_mask (Optional[Tensor]): An optional tensor for masking out certain positions in the input
                sequence during attention calculation.
            position_bias (Optional[Tensor]): An optional tensor providing additional bias to attention scores
                based on position.
            layer_head_mask (Optional[Tensor]): An optional tensor for masking out certain heads in the attention
                calculation.
            past_key_value (Optional[Tuple[Tensor]]): An optional tuple of key and value tensors from the previous
                time steps for faster decoding.
            use_cache (bool): A flag indicating whether to use caching for faster decoding.
            output_attentions (bool): A flag indicating whether to output attention weights.

        Returns:
            Tuple[Tensor]: A tuple containing the updated hidden states after self-attention and any additional outputs
                from the attention mechanism.

        Raises:
            None
        """
        normed_hidden_states = self.layer_norm(hidden_states)
        attention_output = self.SelfAttention(
            normed_hidden_states,
            mask=attention_mask,
            position_bias=position_bias,
            layer_head_mask=layer_head_mask,
            past_key_value=past_key_value,
            use_cache=use_cache,
            output_attentions=output_attentions,
        )
        hidden_states = hidden_states + self.dropout(attention_output[0])
        outputs = (hidden_states,) + attention_output[1:]  # add attentions if we output them
        return outputs

mindnlp.transformers.models.t5.modeling_t5.T5LayerSelfAttention.__init__(config, has_relative_attention_bias=False)

Initialize the T5LayerSelfAttention.

PARAMETER	DESCRIPTION
self	An instance of the T5LayerSelfAttention class. TYPE: T5LayerSelfAttention
config	An object containing the configuration parameters. TYPE: Config
has_relative_attention_bias	A flag indicating whether the attention bias is relative or not. Defaults to False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config, has_relative_attention_bias=False):
    """Initialize the T5LayerSelfAttention.

    Args:
        self (T5LayerSelfAttention): An instance of the T5LayerSelfAttention class.
        config (Config): An object containing the configuration parameters.
        has_relative_attention_bias (bool, optional): A flag indicating whether the attention bias is relative or not.
            Defaults to False.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.SelfAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias)
    self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
    self.dropout = nn.Dropout(p=config.dropout_rate)

mindnlp.transformers.models.t5.modeling_t5.T5LayerSelfAttention.forward(hidden_states, attention_mask=None, position_bias=None, layer_head_mask=None, past_key_value=None, use_cache=False, output_attentions=False)

This method 'forward' in the class 'T5LayerSelfAttention' forwards the output of a T5 self-attention layer.

PARAMETER	DESCRIPTION
self	The instance of the class.
hidden_states	The hidden states of the input sequence. TYPE: Tensor
attention_mask	An optional tensor for masking out certain positions in the input sequence during attention calculation. TYPE: Optional[Tensor] DEFAULT: None
position_bias	An optional tensor providing additional bias to attention scores based on position. TYPE: Optional[Tensor] DEFAULT: None
layer_head_mask	An optional tensor for masking out certain heads in the attention calculation. TYPE: Optional[Tensor] DEFAULT: None
past_key_value	An optional tuple of key and value tensors from the previous time steps for faster decoding. TYPE: Optional[Tuple[Tensor]] DEFAULT: None
use_cache	A flag indicating whether to use caching for faster decoding. TYPE: bool DEFAULT: False
output_attentions	A flag indicating whether to output attention weights. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	Tuple[Tensor]: A tuple containing the updated hidden states after self-attention and any additional outputs from the attention mechanism.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    hidden_states,
    attention_mask=None,
    position_bias=None,
    layer_head_mask=None,
    past_key_value=None,
    use_cache=False,
    output_attentions=False,
):
    """
    This method 'forward' in the class 'T5LayerSelfAttention' forwards the output of a T5 self-attention layer.

    Args:
        self: The instance of the class.
        hidden_states (Tensor): The hidden states of the input sequence.
        attention_mask (Optional[Tensor]): An optional tensor for masking out certain positions in the input
            sequence during attention calculation.
        position_bias (Optional[Tensor]): An optional tensor providing additional bias to attention scores
            based on position.
        layer_head_mask (Optional[Tensor]): An optional tensor for masking out certain heads in the attention
            calculation.
        past_key_value (Optional[Tuple[Tensor]]): An optional tuple of key and value tensors from the previous
            time steps for faster decoding.
        use_cache (bool): A flag indicating whether to use caching for faster decoding.
        output_attentions (bool): A flag indicating whether to output attention weights.

    Returns:
        Tuple[Tensor]: A tuple containing the updated hidden states after self-attention and any additional outputs
            from the attention mechanism.

    Raises:
        None
    """
    normed_hidden_states = self.layer_norm(hidden_states)
    attention_output = self.SelfAttention(
        normed_hidden_states,
        mask=attention_mask,
        position_bias=position_bias,
        layer_head_mask=layer_head_mask,
        past_key_value=past_key_value,
        use_cache=use_cache,
        output_attentions=output_attentions,
    )
    hidden_states = hidden_states + self.dropout(attention_output[0])
    outputs = (hidden_states,) + attention_output[1:]  # add attentions if we output them
    return outputs

mindnlp.transformers.models.t5.modeling_t5.T5Model

Bases: T5PreTrainedModel

T5Model

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5Model(T5PreTrainedModel):
    """T5Model"""
    _keys_to_ignore_on_load_unexpected = [
        "decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight",
    ]
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"]

    def __init__(self, config: T5Config):
        """
        __init__ method in the T5Model class initializes a new instance of the class.

        Args:
            self: A reference to the instance of the class.
            config (T5Config): An instance of T5Config class containing configuration parameters for the T5 model.
                It includes parameters such as vocab_size, d_model, is_decoder, use_cache, is_encoder_decoder,
                and num_decoder_layers.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.shared = nn.Embedding(config.vocab_size, config.d_model)

        encoder_config = copy.deepcopy(config)
        encoder_config.is_decoder = False
        encoder_config.use_cache = False
        encoder_config.is_encoder_decoder = False
        self.encoder = T5Stack(encoder_config)

        decoder_config = copy.deepcopy(config)
        decoder_config.is_decoder = True
        decoder_config.is_encoder_decoder = False
        decoder_config.num_layers = config.num_decoder_layers
        self.decoder = T5Stack(decoder_config)

        self.post_init()

    def get_input_embeddings(self):
        """
        Get the input embeddings for the T5Model.

        Args:
            self: The instance of the T5Model class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.shared

    def set_input_embeddings(self, new_embeddings):
        """
        Sets the input embeddings for the T5Model.

        Args:
            self (T5Model): The instance of the T5Model class.
            new_embeddings: The new input embeddings to be set for the model.
                This should be a tensor of shape (vocab_size, hidden_size).

        Returns:
            None.

        Raises:
            None.
        """
        self.shared = new_embeddings
        # self.encoder.set_input_embeddings(new_embeddings)
        # self.decoder.set_input_embeddings(new_embeddings)

    def _tie_weights(self):
        """
        Tie the weights of the T5Model if specified in the configuration.

        Args:
            self (T5Model):
                The instance of the T5Model class.

                - This parameter represents the T5Model object on which the method is called.

        Returns:
            None.

        Raises:
            None
        """
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.decoder.embed_tokens, self.shared)

    def get_encoder(self):
        """
        This method returns the encoder for the T5Model.

        Args:
            self: The instance of the T5Model class.

        Returns:
            encoder:
                Returns the encoder associated with the T5Model.

        Raises:
            None.
        """
        return self.encoder

    def get_decoder(self):
        """
        Method to retrieve the decoder of the T5Model.

        Args:
            self (T5Model): The T5Model instance on which the method is called.

        Returns:
            decoder: The method returns the decoder attribute of the T5Model instance.

        Raises:
            This method does not raise any exceptions.
        """
        return self.decoder

    def _prune_heads(self, heads_to_prune):
        """
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        """
        for layer, heads in heads_to_prune.items():
            self.encoder.layer[layer].attention.prune_heads(heads)

    def forward(
        self,
        input_ids = None,
        attention_mask = None,
        decoder_input_ids = None,
        decoder_attention_mask = None,
        head_mask = None,
        decoder_head_mask = None,
        cross_attn_head_mask = None,
        encoder_outputs = None,
        past_key_values = None,
        inputs_embeds = None,
        decoder_inputs_embeds = None,
        use_cache = None,
        output_attentions = None,
        output_hidden_states = None,
        return_dict = None,
    ):
        """
        Constructs the T5 model for sequence-to-sequence tasks.

        Args:
            self (T5Model): The instance of the T5Model class.
            input_ids (torch.Tensor, optional): The input sequence tensor IDs. Default: None.
            attention_mask (torch.Tensor, optional): The attention mask tensor. Default: None.
            decoder_input_ids (torch.Tensor, optional): The decoder input sequence tensor IDs. Default: None.
            decoder_attention_mask (torch.Tensor, optional): The decoder attention mask tensor. Default: None.
            head_mask (torch.Tensor, optional): The head mask tensor. Default: None.
            decoder_head_mask (torch.Tensor, optional): The decoder head mask tensor. Default: None.
            cross_attn_head_mask (torch.Tensor, optional): The cross-attention head mask tensor. Default: None.
            encoder_outputs (tuple, optional): The encoder outputs. Default: None.
            past_key_values (tuple, optional): The past key values. Default: None.
            inputs_embeds (torch.Tensor, optional): The input embeddings tensor. Default: None.
            decoder_inputs_embeds (torch.Tensor, optional): The decoder input embeddings tensor. Default: None.
            use_cache (bool, optional): Whether to use cache. Default: None.
            output_attentions (bool, optional): Whether to output attentions. Default: None.
            output_hidden_states (bool, optional): Whether to output hidden states. Default: None.
            return_dict (bool, optional): Whether to return a dictionary. Default: None.

        Returns:
            None

        Raises:
            None
        """
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask
        if head_mask is not None and decoder_head_mask is None:
            if self.config.num_layers == self.config.num_decoder_layers:
                # warnings.warn(__HEAD_MASK_WARNING_MSG, FutureWarning)
                decoder_head_mask = head_mask

        # Encode if needed (training, first prediction pass)
        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                head_mask=head_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )

        hidden_states = encoder_outputs[0]

        # Decode
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            inputs_embeds=decoder_inputs_embeds,
            past_key_values=past_key_values,
            encoder_hidden_states=hidden_states,
            encoder_attention_mask=attention_mask,
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        if not return_dict:
            return decoder_outputs + encoder_outputs

        return Seq2SeqModelOutput(
            last_hidden_state=decoder_outputs.last_hidden_state,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

mindnlp.transformers.models.t5.modeling_t5.T5Model.__init__(config)

init method in the T5Model class initializes a new instance of the class.

PARAMETER	DESCRIPTION
self	A reference to the instance of the class.
config	An instance of T5Config class containing configuration parameters for the T5 model. It includes parameters such as vocab_size, d_model, is_decoder, use_cache, is_encoder_decoder, and num_decoder_layers. TYPE: T5Config

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config: T5Config):
    """
    __init__ method in the T5Model class initializes a new instance of the class.

    Args:
        self: A reference to the instance of the class.
        config (T5Config): An instance of T5Config class containing configuration parameters for the T5 model.
            It includes parameters such as vocab_size, d_model, is_decoder, use_cache, is_encoder_decoder,
            and num_decoder_layers.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.shared = nn.Embedding(config.vocab_size, config.d_model)

    encoder_config = copy.deepcopy(config)
    encoder_config.is_decoder = False
    encoder_config.use_cache = False
    encoder_config.is_encoder_decoder = False
    self.encoder = T5Stack(encoder_config)

    decoder_config = copy.deepcopy(config)
    decoder_config.is_decoder = True
    decoder_config.is_encoder_decoder = False
    decoder_config.num_layers = config.num_decoder_layers
    self.decoder = T5Stack(decoder_config)

    self.post_init()

mindnlp.transformers.models.t5.modeling_t5.T5Model.forward(input_ids=None, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, head_mask=None, decoder_head_mask=None, cross_attn_head_mask=None, encoder_outputs=None, past_key_values=None, inputs_embeds=None, decoder_inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Constructs the T5 model for sequence-to-sequence tasks.

PARAMETER	DESCRIPTION
self	The instance of the T5Model class. TYPE: T5Model
input_ids	The input sequence tensor IDs. Default: None. TYPE: Tensor DEFAULT: None
attention_mask	The attention mask tensor. Default: None. TYPE: Tensor DEFAULT: None
decoder_input_ids	The decoder input sequence tensor IDs. Default: None. TYPE: Tensor DEFAULT: None
decoder_attention_mask	The decoder attention mask tensor. Default: None. TYPE: Tensor DEFAULT: None
head_mask	The head mask tensor. Default: None. TYPE: Tensor DEFAULT: None
decoder_head_mask	The decoder head mask tensor. Default: None. TYPE: Tensor DEFAULT: None
cross_attn_head_mask	The cross-attention head mask tensor. Default: None. TYPE: Tensor DEFAULT: None
encoder_outputs	The encoder outputs. Default: None. TYPE: tuple DEFAULT: None
past_key_values	The past key values. Default: None. TYPE: tuple DEFAULT: None
inputs_embeds	The input embeddings tensor. Default: None. TYPE: Tensor DEFAULT: None
decoder_inputs_embeds	The decoder input embeddings tensor. Default: None. TYPE: Tensor DEFAULT: None
use_cache	Whether to use cache. Default: None. TYPE: bool DEFAULT: None
output_attentions	Whether to output attentions. Default: None. TYPE: bool DEFAULT: None
output_hidden_states	Whether to output hidden states. Default: None. TYPE: bool DEFAULT: None
return_dict	Whether to return a dictionary. Default: None. TYPE: bool DEFAULT: None

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    input_ids = None,
    attention_mask = None,
    decoder_input_ids = None,
    decoder_attention_mask = None,
    head_mask = None,
    decoder_head_mask = None,
    cross_attn_head_mask = None,
    encoder_outputs = None,
    past_key_values = None,
    inputs_embeds = None,
    decoder_inputs_embeds = None,
    use_cache = None,
    output_attentions = None,
    output_hidden_states = None,
    return_dict = None,
):
    """
    Constructs the T5 model for sequence-to-sequence tasks.

    Args:
        self (T5Model): The instance of the T5Model class.
        input_ids (torch.Tensor, optional): The input sequence tensor IDs. Default: None.
        attention_mask (torch.Tensor, optional): The attention mask tensor. Default: None.
        decoder_input_ids (torch.Tensor, optional): The decoder input sequence tensor IDs. Default: None.
        decoder_attention_mask (torch.Tensor, optional): The decoder attention mask tensor. Default: None.
        head_mask (torch.Tensor, optional): The head mask tensor. Default: None.
        decoder_head_mask (torch.Tensor, optional): The decoder head mask tensor. Default: None.
        cross_attn_head_mask (torch.Tensor, optional): The cross-attention head mask tensor. Default: None.
        encoder_outputs (tuple, optional): The encoder outputs. Default: None.
        past_key_values (tuple, optional): The past key values. Default: None.
        inputs_embeds (torch.Tensor, optional): The input embeddings tensor. Default: None.
        decoder_inputs_embeds (torch.Tensor, optional): The decoder input embeddings tensor. Default: None.
        use_cache (bool, optional): Whether to use cache. Default: None.
        output_attentions (bool, optional): Whether to output attentions. Default: None.
        output_hidden_states (bool, optional): Whether to output hidden states. Default: None.
        return_dict (bool, optional): Whether to return a dictionary. Default: None.

    Returns:
        None

    Raises:
        None
    """
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask
    if head_mask is not None and decoder_head_mask is None:
        if self.config.num_layers == self.config.num_decoder_layers:
            # warnings.warn(__HEAD_MASK_WARNING_MSG, FutureWarning)
            decoder_head_mask = head_mask

    # Encode if needed (training, first prediction pass)
    if encoder_outputs is None:
        encoder_outputs = self.encoder(
            input_ids=input_ids,
            attention_mask=attention_mask,
            inputs_embeds=inputs_embeds,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

    hidden_states = encoder_outputs[0]

    # Decode
    decoder_outputs = self.decoder(
        input_ids=decoder_input_ids,
        attention_mask=decoder_attention_mask,
        inputs_embeds=decoder_inputs_embeds,
        past_key_values=past_key_values,
        encoder_hidden_states=hidden_states,
        encoder_attention_mask=attention_mask,
        head_mask=decoder_head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    if not return_dict:
        return decoder_outputs + encoder_outputs

    return Seq2SeqModelOutput(
        last_hidden_state=decoder_outputs.last_hidden_state,
        past_key_values=decoder_outputs.past_key_values,
        decoder_hidden_states=decoder_outputs.hidden_states,
        decoder_attentions=decoder_outputs.attentions,
        cross_attentions=decoder_outputs.cross_attentions,
        encoder_last_hidden_state=encoder_outputs.last_hidden_state,
        encoder_hidden_states=encoder_outputs.hidden_states,
        encoder_attentions=encoder_outputs.attentions,
    )

mindnlp.transformers.models.t5.modeling_t5.T5Model.get_decoder()

Method to retrieve the decoder of the T5Model.

PARAMETER	DESCRIPTION
self	The T5Model instance on which the method is called. TYPE: T5Model

RETURNS	DESCRIPTION
decoder	The method returns the decoder attribute of the T5Model instance.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_decoder(self):
    """
    Method to retrieve the decoder of the T5Model.

    Args:
        self (T5Model): The T5Model instance on which the method is called.

    Returns:
        decoder: The method returns the decoder attribute of the T5Model instance.

    Raises:
        This method does not raise any exceptions.
    """
    return self.decoder

mindnlp.transformers.models.t5.modeling_t5.T5Model.get_encoder()

This method returns the encoder for the T5Model.

PARAMETER	DESCRIPTION
self	The instance of the T5Model class.

RETURNS	DESCRIPTION
encoder	Returns the encoder associated with the T5Model.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_encoder(self):
    """
    This method returns the encoder for the T5Model.

    Args:
        self: The instance of the T5Model class.

    Returns:
        encoder:
            Returns the encoder associated with the T5Model.

    Raises:
        None.
    """
    return self.encoder

mindnlp.transformers.models.t5.modeling_t5.T5Model.get_input_embeddings()

Get the input embeddings for the T5Model.

PARAMETER	DESCRIPTION
self	The instance of the T5Model class.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_input_embeddings(self):
    """
    Get the input embeddings for the T5Model.

    Args:
        self: The instance of the T5Model class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.shared

mindnlp.transformers.models.t5.modeling_t5.T5Model.set_input_embeddings(new_embeddings)

Sets the input embeddings for the T5Model.

PARAMETER	DESCRIPTION
self	The instance of the T5Model class. TYPE: T5Model
new_embeddings	The new input embeddings to be set for the model. This should be a tensor of shape (vocab_size, hidden_size).

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def set_input_embeddings(self, new_embeddings):
    """
    Sets the input embeddings for the T5Model.

    Args:
        self (T5Model): The instance of the T5Model class.
        new_embeddings: The new input embeddings to be set for the model.
            This should be a tensor of shape (vocab_size, hidden_size).

    Returns:
        None.

    Raises:
        None.
    """
    self.shared = new_embeddings

mindnlp.transformers.models.t5.modeling_t5.T5PreTrainedModel

Bases: PreTrainedModel

An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5PreTrainedModel(PreTrainedModel):
    """
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    """
    config_class = T5Config
    base_model_prefix = "transformer"

    is_parallelizable = True
    supports_gradient_checkpointing = True
    _no_split_modules = ["T5Block"]
    _keep_in_fp32_modules = ["wo"]

    @property
    def dummy_inputs(self):
        """
        Method: dummy_inputs

        Description:
            This method generates dummy input data for the T5PreTrainedModel.

        Args:
            self: An instance of the T5PreTrainedModel class.

        Returns:
            `dict`:

                - Type: None
                - Purpose: This method returns a dictionary containing dummy input data for the model.

                The dictionary includes the following keys:

                - 'decoder_input_ids': Tensor containing dummy input IDs.
                - 'input_ids': Tensor containing dummy input IDs.
                - 'decoder_attention_mask': Tensor containing dummy mask data.

        Raises:
            This method does not raise any exceptions.
        """
        input_ids = mindspore.tensor(DUMMY_INPUTS)
        input_mask = mindspore.tensor(DUMMY_MASK)
        dummy_inputs = {
            "decoder_input_ids": input_ids,
            "input_ids": input_ids,
            "decoder_attention_mask": input_mask,
        }
        return dummy_inputs

    def _init_weights(self, cell):
        """Initialize the weights"""
        factor = self.config.initializer_factor  # Used for testing weights initialization
        if isinstance(cell, T5LayerNorm):
            cell.weight.set_data(initializer(Constant(factor * 1.0), cell.weight.shape, cell.weight.dtype))
        elif isinstance(
            cell,
            (T5Model, T5ForConditionalGeneration, T5EncoderModel, T5ForQuestionAnswering),
        ):
            # Mesh TensorFlow embeddings initialization
            # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L1624
            cell.shared.weight.set_data(initializer(Normal(factor * 1.0),
                                                cell.shared.weight.shape, cell.shared.weight.dtype))
            if hasattr(cell, "lm_head") and not self.config.tie_word_embeddings:
                cell.lm_head.weight.set_data(initializer(Normal(factor * 1.0), cell.lm_head.weight.shape, cell.lm_head.weight.dtype))
            if hasattr(cell, "qa_outputs"):
                cell.qa_outputs.weight.set_data(initializer(Normal(factor * ((self.config.d_model) ** -0.5)),
                                                            cell.qa_outputs.weight.shape, cell.qa_outputs.weight.dtype))
                cell.qa_outputs.bias.set_data(initializer('zeros', cell.qa_outputs.bias.shape, cell.qa_outputs.bias.dtype))
        elif isinstance(cell, T5ClassificationHead):
            cell.dense.weight.set_data(initializer(Normal(factor * ((self.config.d_model) ** -0.5)),
                                                cell.dense.weight.shape, cell.dense.weight.dtype))

            if hasattr(cell.dense, "bias") and cell.dense.bias is not None:
                cell.dense.bias.set_data(initializer('zeros', cell.dense.bias.shape, cell.dense.bias.dtype))
            cell.out_proj.weight.set_data(initializer(Normal(factor * ((self.config.d_model) ** -0.5)),
                                                cell.out_proj.weight.shape, cell.out_proj.weight.dtype))

            if hasattr(cell.out_proj, "bias") and cell.out_proj.bias is not None:
                cell.out_proj.bias.set_data(initializer('zeros', cell.out_proj.bias.shape, cell.out_proj.bias.dtype))
        elif isinstance(cell, T5DenseActDense):
            # Mesh TensorFlow FF initialization
            # See https://github.com/tensorflow/mesh/blob/master/mesh_tensorflow/transformer/transformer_layers.py#L56
            # and https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L89
            cell.wi.weight.set_data(initializer(Normal(factor * ((self.config.d_model) ** -0.5)),
                                                cell.wi.weight.shape, cell.wi.weight.dtype))
            if hasattr(cell.wi, "bias") and cell.wi.bias is not None:
                cell.wi.bias.set_data(initializer('zeros', cell.wi.bias.shape, cell.wi.bias.dtype))

            cell.wo.weight.set_data(initializer(Normal(factor * ((self.config.d_ff) ** -0.5)),
                                                cell.wo.weight.shape, cell.wo.weight.dtype))

            if hasattr(cell.wo, "bias") and cell.wo.bias is not None:
                cell.wo.bias.set_data(initializer('zeros', cell.wo.bias.shape, cell.wo.bias.dtype))
        elif isinstance(cell, T5DenseGatedActDense):
            cell.wi_0.weight.set_data(initializer(Normal(factor * ((self.config.d_model) ** -0.5)),
                                                cell.wi_0.weight.shape, cell.wi_0.weight.dtype))
            if hasattr(cell.wi_0, "bias") and cell.wi_0.bias is not None:
                cell.wi_0.bias.set_data(initializer('zeros', cell.wi_0.bias.shape, cell.wi_0.bias.dtype))

            cell.wi_1.weight.set_data(initializer(Normal(factor * ((self.config.d_model) ** -0.5)),
                                                cell.wi_1.weight.shape, cell.wi_1.weight.dtype))
            if hasattr(cell.wi_1, "bias") and cell.wi_1.bias is not None:
                cell.wi_1.bias.set_data(initializer('zeros', cell.wi_1.bias.shape, cell.wi_1.bias.dtype))

            cell.wo.weight.set_data(initializer(Normal(factor * ((self.config.d_ff) ** -0.5)),
                                                cell.wo.weight.shape, cell.wo.weight.dtype))

            if hasattr(cell.wo, "bias") and cell.wo.bias is not None:
                cell.wo.bias.set_data(initializer('zeros', cell.wo.bias.shape, cell.wo.bias.dtype))
        elif isinstance(cell, T5Attention):
            # Mesh TensorFlow attention initialization to avoid scaling before softmax
            # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/attention.py#L136
            d_model = self.config.d_model
            key_value_proj_dim = self.config.d_kv
            n_heads = self.config.num_heads

            cell.q.weight.set_data(initializer(Normal(factor * ((d_model * key_value_proj_dim) ** -0.5)),
                                                cell.q.weight.shape, cell.q.weight.dtype))
            cell.k.weight.set_data(initializer(Normal(factor * (d_model**-0.5)),
                                                cell.k.weight.shape, cell.k.weight.dtype))
            cell.v.weight.set_data(initializer(Normal(factor * (d_model**-0.5)),
                                                cell.v.weight.shape, cell.v.weight.dtype))
            cell.o.weight.set_data(initializer(Normal(factor * ((n_heads * key_value_proj_dim) ** -0.5)),
                                                cell.o.weight.shape, cell.o.weight.dtype))
            if cell.has_relative_attention_bias:
                cell.relative_attention_bias.weight.set_data(initializer(Normal(factor * (d_model**-0.5)),
                                                    cell.relative_attention_bias.weight.shape, cell.relative_attention_bias.weight.dtype))

    def _shift_right(self, input_ids):
        """
        Shifts the input IDs to the right by one position, inserting the decoder start token ID at the beginning.

        Args:
            self (T5PreTrainedModel): An instance of the T5PreTrainedModel class.
            input_ids (torch.Tensor): A tensor of shape (batch_size, sequence_length) containing the input IDs.

        Returns:
            torch.Tensor: A tensor of shape (batch_size, sequence_length) representing the shifted input IDs.

        Raises:
            ValueError: If `self.model.config.decoder_start_token_id` is not defined
                or if `self.model.config.pad_token_id` is not defined.
        """
        decoder_start_token_id = self.config.decoder_start_token_id
        pad_token_id = self.config.pad_token_id

        if decoder_start_token_id is None:
            raise ValueError(
                "self.model.config.decoder_start_token_id has to be defined. In T5 it is usually set to the pad_token_id. "
                "See T5 docs for more information."
            )

        # shift inputs to the right
        shifted_input_ids = input_ids.new_zeros(input_ids.shape)
        shifted_input_ids[..., 1:] = input_ids[..., :-1].copy()
        shifted_input_ids[..., 0] = decoder_start_token_id

        if pad_token_id is None:
            raise ValueError("self.model.config.pad_token_id has to be defined.")
        # replace possible -100 values in labels by `pad_token_id`
        shifted_input_ids = shifted_input_ids.masked_fill(shifted_input_ids == -100, pad_token_id)

        return shifted_input_ids

mindnlp.transformers.models.t5.modeling_t5.T5PreTrainedModel.dummy_inputs property

Description

This method generates dummy input data for the T5PreTrainedModel.

PARAMETER	DESCRIPTION
self	An instance of the T5PreTrainedModel class.

RETURNS	DESCRIPTION
	dict: Type: None Purpose: This method returns a dictionary containing dummy input data for the model. The dictionary includes the following keys: 'decoder_input_ids': Tensor containing dummy input IDs. 'input_ids': Tensor containing dummy input IDs. 'decoder_attention_mask': Tensor containing dummy mask data.

mindnlp.transformers.models.t5.modeling_t5.T5Stack

Bases: T5PreTrainedModel

T5Stack

Source code in mindnlp/transformers/models/t5/modeling_t5.py

class T5Stack(T5PreTrainedModel):
    """T5Stack"""
    def __init__(self, config):
        """
        Initializes an instance of the T5Stack class.

        Args:
            self: The instance of the T5Stack class.
            config: An object containing the configuration parameters for the T5Stack.
                It should have the following attributes:

                - vocab_size (int): The size of the vocabulary.
                - d_model (int): The dimensionality of the model.
                - is_decoder (bool): Indicates whether the T5Stack is used as a decoder.
                num_layers (int): The number of layers in the T5Stack.
                - layer_norm_epsilon (float): The epsilon value for layer normalization.
                - dropout_rate (float): The dropout rate.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)

        self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model)
        self.is_decoder = config.is_decoder

        self.block = nn.ModuleList(
            [T5Block(config, has_relative_attention_bias=bool(i == 0)) for i in range(config.num_layers)]
        )
        self.final_layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
        self.dropout = nn.Dropout(p=config.dropout_rate)

        self.post_init()

    def get_input_embeddings(self):
        """Return the input embeddings of the T5Stack.

        Args:
            self: An instance of the T5Stack class.

        Returns:
            embed_tokens: This method returns the input embeddings of the T5Stack.
                The input embeddings are the embedded tokens used as input for the T5 model.

        Raises:
            None.
        """
        return self.embed_tokens

    def set_input_embeddings(self, new_embeddings):
        """
        Method to set new input embeddings for the T5Stack model.

        Args:
            self (T5Stack): The instance of the T5Stack class.
            new_embeddings (object): The new embeddings to set for the input.
                It should be compatible with the model's input format.

        Returns:
            None: This method updates the input embeddings of the T5Stack model in place.

        Raises:
            None.
        """
        self.embed_tokens = new_embeddings

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        inputs_embeds=None,
        head_mask=None,
        cross_attn_head_mask=None,
        past_key_values=None,
        use_cache=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        """
        Constructs the T5Stack model.

        Args:
            self (T5Stack): The instance of the T5Stack class.
            input_ids (Tensor, optional): The input token IDs. Default: None.
            attention_mask (Tensor, optional): The attention mask tensor. Default: None.
            encoder_hidden_states (Tensor, optional): The hidden states of the encoder. Default: None.
            encoder_attention_mask (Tensor, optional): The attention mask for encoder hidden states. Default: None.
            inputs_embeds (Tensor, optional): The embedded inputs. Default: None.
            head_mask (list, optional): The mask for attention heads. Default: None.
            cross_attn_head_mask (list, optional): The mask for cross-attention heads. Default: None.
            past_key_values (list, optional): The past key values for caching. Default: None.
            use_cache (bool, optional): Whether to use caching. Default: None.
            output_attentions (bool, optional): Whether to output attentions. Default: None.
            output_hidden_states (bool, optional): Whether to output hidden states. Default: None.
            return_dict (bool, optional): Whether to return a dictionary. Default: None.

        Returns:
            None

        Raises:
            ValueError: If both input_ids and inputs_embeds are specified at the same time.
            ValueError: If neither input_ids nor inputs_embeds are specified.
            AssertionError: If the model is not initialized with valid token embeddings.
            AssertionError: If use_cache is set to True and the model is not used as a decoder.

        """
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if input_ids is not None and inputs_embeds is not None:
            err_msg_prefix = "decoder_" if self.is_decoder else ""
            raise ValueError(
                f"You cannot specify both {err_msg_prefix}input_ids and {err_msg_prefix}inputs_embeds at the same time"
            )

        if input_ids is not None:
            input_shape = input_ids.shape
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
        else:
            err_msg_prefix = "decoder_" if self.is_decoder else ""
            raise ValueError(f"You have to specify either {err_msg_prefix}input_ids or {err_msg_prefix}inputs_embeds")

        if inputs_embeds is None:
            assert self.embed_tokens is not None, "You have to initialize the model with valid token embeddings"
            inputs_embeds = self.embed_tokens(input_ids.astype(mindspore.int64))

        batch_size, seq_length = input_shape

        # required mask seq length can be calculated via length of past
        mask_seq_length = past_key_values[0][0].shape[2] + seq_length if past_key_values is not None else seq_length

        if use_cache is True:
            assert self.is_decoder, f"`use_cache` can only be set to `True` if {self} is used as a decoder"

        if attention_mask is None:
            attention_mask = ops.ones(batch_size, mask_seq_length, dtype=mindspore.float32)
        if self.is_decoder and encoder_attention_mask is None and encoder_hidden_states is not None:
            encoder_seq_length = encoder_hidden_states.shape[1]
            encoder_attention_mask = ops.ones(
                batch_size, encoder_seq_length, dtype=mindspore.int64
            )

        # initialize past_key_values with `None` if past does not exist
        if past_key_values is None:
            past_key_values = [None] * len(self.block)

        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
        # If a 2D or 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.is_decoder and encoder_hidden_states is not None:
            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
            if encoder_attention_mask is None:
                encoder_attention_mask = ops.ones(*encoder_hidden_shape)
            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
        else:
            encoder_extended_attention_mask = None

        # Prepare head mask if needed
        head_mask = self.get_head_mask(head_mask, self.config.num_layers)
        cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers)
        present_key_value_states = () if use_cache else None
        all_hidden_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None
        all_cross_attentions = () if (output_attentions and self.is_decoder) else None
        position_bias = None
        encoder_decoder_position_bias = None

        hidden_states = self.dropout(inputs_embeds)

        for i, (layer_module, past_key_value) in enumerate(zip(self.block, past_key_values)):
            layer_head_mask = head_mask[i]
            cross_attn_layer_head_mask = cross_attn_head_mask[i]
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_outputs = layer_module(
                hidden_states,
                attention_mask=extended_attention_mask,
                position_bias=position_bias,
                encoder_hidden_states=encoder_hidden_states,
                encoder_attention_mask=encoder_extended_attention_mask,
                encoder_decoder_position_bias=encoder_decoder_position_bias,
                layer_head_mask=layer_head_mask,
                cross_attn_layer_head_mask=cross_attn_layer_head_mask,
                past_key_value=past_key_value,
                use_cache=use_cache,
                output_attentions=output_attentions,
            )

            # layer_outputs is a tuple with:
            # hidden-states, key-value-states, (self-attention position bias), \
            # (self-attention weights), (cross-attention position bias), (cross-attention weights)
            if use_cache is False:
                layer_outputs = layer_outputs[:1] + (None,) + layer_outputs[1:]

            hidden_states, present_key_value_state = layer_outputs[:2]
            # We share the position biases between the layers - the first layer store them
            # layer_outputs = hidden-states, key-value-states (self-attention position bias), (self-attention weights),
            # (cross-attention position bias), (cross-attention weights)
            position_bias = layer_outputs[2]
            if self.is_decoder and encoder_hidden_states is not None:
                encoder_decoder_position_bias = layer_outputs[4 if output_attentions else 3]
            # append next layer key value states
            if use_cache:
                present_key_value_states = present_key_value_states + (present_key_value_state,)

            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[3],)
                if self.is_decoder:
                    all_cross_attentions = all_cross_attentions + (layer_outputs[5],)

        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.dropout(hidden_states)

        # Add last layer
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        if not return_dict:
            return tuple(
                v
                for v in [
                    hidden_states,
                    present_key_value_states,
                    all_hidden_states,
                    all_attentions,
                    all_cross_attentions,
                ]
                if v is not None
            )

        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=present_key_value_states,
            hidden_states=all_hidden_states,
            attentions=all_attentions,
            cross_attentions=all_cross_attentions,
        )

mindnlp.transformers.models.t5.modeling_t5.T5Stack.__init__(config)

Initializes an instance of the T5Stack class.

PARAMETER	DESCRIPTION
self	The instance of the T5Stack class.
config	An object containing the configuration parameters for the T5Stack. It should have the following attributes: vocab_size (int): The size of the vocabulary. d_model (int): The dimensionality of the model. is_decoder (bool): Indicates whether the T5Stack is used as a decoder. num_layers (int): The number of layers in the T5Stack. layer_norm_epsilon (float): The epsilon value for layer normalization. dropout_rate (float): The dropout rate.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def __init__(self, config):
    """
    Initializes an instance of the T5Stack class.

    Args:
        self: The instance of the T5Stack class.
        config: An object containing the configuration parameters for the T5Stack.
            It should have the following attributes:

            - vocab_size (int): The size of the vocabulary.
            - d_model (int): The dimensionality of the model.
            - is_decoder (bool): Indicates whether the T5Stack is used as a decoder.
            num_layers (int): The number of layers in the T5Stack.
            - layer_norm_epsilon (float): The epsilon value for layer normalization.
            - dropout_rate (float): The dropout rate.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)

    self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model)
    self.is_decoder = config.is_decoder

    self.block = nn.ModuleList(
        [T5Block(config, has_relative_attention_bias=bool(i == 0)) for i in range(config.num_layers)]
    )
    self.final_layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
    self.dropout = nn.Dropout(p=config.dropout_rate)

    self.post_init()

mindnlp.transformers.models.t5.modeling_t5.T5Stack.forward(input_ids=None, attention_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, inputs_embeds=None, head_mask=None, cross_attn_head_mask=None, past_key_values=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Constructs the T5Stack model.

PARAMETER	DESCRIPTION
self	The instance of the T5Stack class. TYPE: T5Stack
input_ids	The input token IDs. Default: None. TYPE: Tensor DEFAULT: None
attention_mask	The attention mask tensor. Default: None. TYPE: Tensor DEFAULT: None
encoder_hidden_states	The hidden states of the encoder. Default: None. TYPE: Tensor DEFAULT: None
encoder_attention_mask	The attention mask for encoder hidden states. Default: None. TYPE: Tensor DEFAULT: None
inputs_embeds	The embedded inputs. Default: None. TYPE: Tensor DEFAULT: None
head_mask	The mask for attention heads. Default: None. TYPE: list DEFAULT: None
cross_attn_head_mask	The mask for cross-attention heads. Default: None. TYPE: list DEFAULT: None
past_key_values	The past key values for caching. Default: None. TYPE: list DEFAULT: None
use_cache	Whether to use caching. Default: None. TYPE: bool DEFAULT: None
output_attentions	Whether to output attentions. Default: None. TYPE: bool DEFAULT: None
output_hidden_states	Whether to output hidden states. Default: None. TYPE: bool DEFAULT: None
return_dict	Whether to return a dictionary. Default: None. TYPE: bool DEFAULT: None

RETURNS	DESCRIPTION
	None

RAISES	DESCRIPTION
ValueError	If both input_ids and inputs_embeds are specified at the same time.
ValueError	If neither input_ids nor inputs_embeds are specified.
AssertionError	If the model is not initialized with valid token embeddings.
AssertionError	If use_cache is set to True and the model is not used as a decoder.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def forward(
    self,
    input_ids=None,
    attention_mask=None,
    encoder_hidden_states=None,
    encoder_attention_mask=None,
    inputs_embeds=None,
    head_mask=None,
    cross_attn_head_mask=None,
    past_key_values=None,
    use_cache=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
):
    """
    Constructs the T5Stack model.

    Args:
        self (T5Stack): The instance of the T5Stack class.
        input_ids (Tensor, optional): The input token IDs. Default: None.
        attention_mask (Tensor, optional): The attention mask tensor. Default: None.
        encoder_hidden_states (Tensor, optional): The hidden states of the encoder. Default: None.
        encoder_attention_mask (Tensor, optional): The attention mask for encoder hidden states. Default: None.
        inputs_embeds (Tensor, optional): The embedded inputs. Default: None.
        head_mask (list, optional): The mask for attention heads. Default: None.
        cross_attn_head_mask (list, optional): The mask for cross-attention heads. Default: None.
        past_key_values (list, optional): The past key values for caching. Default: None.
        use_cache (bool, optional): Whether to use caching. Default: None.
        output_attentions (bool, optional): Whether to output attentions. Default: None.
        output_hidden_states (bool, optional): Whether to output hidden states. Default: None.
        return_dict (bool, optional): Whether to return a dictionary. Default: None.

    Returns:
        None

    Raises:
        ValueError: If both input_ids and inputs_embeds are specified at the same time.
        ValueError: If neither input_ids nor inputs_embeds are specified.
        AssertionError: If the model is not initialized with valid token embeddings.
        AssertionError: If use_cache is set to True and the model is not used as a decoder.

    """
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if input_ids is not None and inputs_embeds is not None:
        err_msg_prefix = "decoder_" if self.is_decoder else ""
        raise ValueError(
            f"You cannot specify both {err_msg_prefix}input_ids and {err_msg_prefix}inputs_embeds at the same time"
        )

    if input_ids is not None:
        input_shape = input_ids.shape
        input_ids = input_ids.view(-1, input_shape[-1])
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
    else:
        err_msg_prefix = "decoder_" if self.is_decoder else ""
        raise ValueError(f"You have to specify either {err_msg_prefix}input_ids or {err_msg_prefix}inputs_embeds")

    if inputs_embeds is None:
        assert self.embed_tokens is not None, "You have to initialize the model with valid token embeddings"
        inputs_embeds = self.embed_tokens(input_ids.astype(mindspore.int64))

    batch_size, seq_length = input_shape

    # required mask seq length can be calculated via length of past
    mask_seq_length = past_key_values[0][0].shape[2] + seq_length if past_key_values is not None else seq_length

    if use_cache is True:
        assert self.is_decoder, f"`use_cache` can only be set to `True` if {self} is used as a decoder"

    if attention_mask is None:
        attention_mask = ops.ones(batch_size, mask_seq_length, dtype=mindspore.float32)
    if self.is_decoder and encoder_attention_mask is None and encoder_hidden_states is not None:
        encoder_seq_length = encoder_hidden_states.shape[1]
        encoder_attention_mask = ops.ones(
            batch_size, encoder_seq_length, dtype=mindspore.int64
        )

    # initialize past_key_values with `None` if past does not exist
    if past_key_values is None:
        past_key_values = [None] * len(self.block)

    # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
    # ourselves in which case we just need to make it broadcastable to all heads.
    extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
    # If a 2D or 3D attention mask is provided for the cross-attention
    # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
    if self.is_decoder and encoder_hidden_states is not None:
        encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.shape
        encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
        if encoder_attention_mask is None:
            encoder_attention_mask = ops.ones(*encoder_hidden_shape)
        encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
    else:
        encoder_extended_attention_mask = None

    # Prepare head mask if needed
    head_mask = self.get_head_mask(head_mask, self.config.num_layers)
    cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers)
    present_key_value_states = () if use_cache else None
    all_hidden_states = () if output_hidden_states else None
    all_attentions = () if output_attentions else None
    all_cross_attentions = () if (output_attentions and self.is_decoder) else None
    position_bias = None
    encoder_decoder_position_bias = None

    hidden_states = self.dropout(inputs_embeds)

    for i, (layer_module, past_key_value) in enumerate(zip(self.block, past_key_values)):
        layer_head_mask = head_mask[i]
        cross_attn_layer_head_mask = cross_attn_head_mask[i]
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        layer_outputs = layer_module(
            hidden_states,
            attention_mask=extended_attention_mask,
            position_bias=position_bias,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_extended_attention_mask,
            encoder_decoder_position_bias=encoder_decoder_position_bias,
            layer_head_mask=layer_head_mask,
            cross_attn_layer_head_mask=cross_attn_layer_head_mask,
            past_key_value=past_key_value,
            use_cache=use_cache,
            output_attentions=output_attentions,
        )

        # layer_outputs is a tuple with:
        # hidden-states, key-value-states, (self-attention position bias), \
        # (self-attention weights), (cross-attention position bias), (cross-attention weights)
        if use_cache is False:
            layer_outputs = layer_outputs[:1] + (None,) + layer_outputs[1:]

        hidden_states, present_key_value_state = layer_outputs[:2]
        # We share the position biases between the layers - the first layer store them
        # layer_outputs = hidden-states, key-value-states (self-attention position bias), (self-attention weights),
        # (cross-attention position bias), (cross-attention weights)
        position_bias = layer_outputs[2]
        if self.is_decoder and encoder_hidden_states is not None:
            encoder_decoder_position_bias = layer_outputs[4 if output_attentions else 3]
        # append next layer key value states
        if use_cache:
            present_key_value_states = present_key_value_states + (present_key_value_state,)

        if output_attentions:
            all_attentions = all_attentions + (layer_outputs[3],)
            if self.is_decoder:
                all_cross_attentions = all_cross_attentions + (layer_outputs[5],)

    hidden_states = self.final_layer_norm(hidden_states)
    hidden_states = self.dropout(hidden_states)

    # Add last layer
    if output_hidden_states:
        all_hidden_states = all_hidden_states + (hidden_states,)

    if not return_dict:
        return tuple(
            v
            for v in [
                hidden_states,
                present_key_value_states,
                all_hidden_states,
                all_attentions,
                all_cross_attentions,
            ]
            if v is not None
        )

    return BaseModelOutputWithPastAndCrossAttentions(
        last_hidden_state=hidden_states,
        past_key_values=present_key_value_states,
        hidden_states=all_hidden_states,
        attentions=all_attentions,
        cross_attentions=all_cross_attentions,
    )

mindnlp.transformers.models.t5.modeling_t5.T5Stack.get_input_embeddings()

Return the input embeddings of the T5Stack.

PARAMETER	DESCRIPTION
self	An instance of the T5Stack class.

RETURNS	DESCRIPTION
embed_tokens	This method returns the input embeddings of the T5Stack. The input embeddings are the embedded tokens used as input for the T5 model.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def get_input_embeddings(self):
    """Return the input embeddings of the T5Stack.

    Args:
        self: An instance of the T5Stack class.

    Returns:
        embed_tokens: This method returns the input embeddings of the T5Stack.
            The input embeddings are the embedded tokens used as input for the T5 model.

    Raises:
        None.
    """
    return self.embed_tokens

mindnlp.transformers.models.t5.modeling_t5.T5Stack.set_input_embeddings(new_embeddings)

Method to set new input embeddings for the T5Stack model.

PARAMETER	DESCRIPTION
self	The instance of the T5Stack class. TYPE: T5Stack
new_embeddings	The new embeddings to set for the input. It should be compatible with the model's input format. TYPE: object

RETURNS	DESCRIPTION
None	This method updates the input embeddings of the T5Stack model in place.

Source code in mindnlp/transformers/models/t5/modeling_t5.py

def set_input_embeddings(self, new_embeddings):
    """
    Method to set new input embeddings for the T5Stack model.

    Args:
        self (T5Stack): The instance of the T5Stack class.
        new_embeddings (object): The new embeddings to set for the input.
            It should be compatible with the model's input format.

    Returns:
        None: This method updates the input embeddings of the T5Stack model in place.

    Raises:
        None.
    """
    self.embed_tokens = new_embeddings

mindnlp.transformers.models.t5.configuration_t5

T5 model configuration

mindnlp.transformers.models.t5.configuration_t5.T5Config

Bases: PretrainedConfig

This is the configuration class to store the configuration of a [T5Model] or a [TFT5Model]. It is used to instantiate a T5 model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the T5 t5-small architecture.

Configuration objects inherit from [PretrainedConfig] and can be used to control the model outputs. Read the documentation from [PretrainedConfig] for more information.

PARAMETER	DESCRIPTION
vocab_size	Vocabulary size of the T5 model. Defines the number of different tokens that can be represented by the inputs_ids passed when calling [T5Model] or [TFT5Model]. TYPE: `int`, *optional*, defaults to 32128 DEFAULT: 32128
d_model	Size of the encoder layers and the pooler layer. TYPE: `int`, *optional*, defaults to 512 DEFAULT: 512
d_kv	Size of the key, query, value projections per attention head. The inner_dim of the projection layer will be defined as num_heads * d_kv. TYPE: `int`, *optional*, defaults to 64 DEFAULT: 64
d_ff	Size of the intermediate feed forward layer in each T5Block. TYPE: `int`, *optional*, defaults to 2048 DEFAULT: 2048
num_layers	Number of hidden layers in the Transformer encoder. TYPE: `int`, *optional*, defaults to 6 DEFAULT: 6
num_decoder_layers	Number of hidden layers in the Transformer decoder. Will use the same value as num_layers if not set. TYPE: `int`, *optional* DEFAULT: None
num_heads	Number of attention heads for each attention layer in the Transformer encoder. TYPE: `int`, *optional*, defaults to 8 DEFAULT: 8
relative_attention_num_buckets	The number of buckets to use for each attention layer. TYPE: `int`, *optional*, defaults to 32 DEFAULT: 32
relative_attention_max_distance	The maximum distance of the longer sequences for the bucket separation. TYPE: `int`, *optional*, defaults to 128 DEFAULT: 128
dropout_rate	The ratio for all dropout layers. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.1
classifier_dropout	The dropout ratio for classifier. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
layer_norm_eps	The epsilon used by the layer normalization layers. TYPE: `float`, *optional*, defaults to 1e-6
initializer_factor	A factor for initializing all weight matrices (should be kept to 1, used internally for initialization testing). TYPE: `float`, *optional*, defaults to 1 DEFAULT: 1.0
feed_forward_proj	Type of feed forward layer to be used. Should be one of "relu" or "gated-gelu". T5v1.1 uses the "gated-gelu" feed forward projection. Original T5 uses "relu". TYPE: `string`, *optional*, defaults to `"relu"` DEFAULT: 'relu'
use_cache	Whether or not the model should return the last key/values attentions (not used by all models). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True

Source code in mindnlp/transformers/models/t5/configuration_t5.py

class T5Config(PretrainedConfig):
    r"""
    This is the configuration class to store the configuration of a [`T5Model`] or a [`TFT5Model`]. It is used to
    instantiate a T5 model according to the specified arguments, defining the model architecture. Instantiating a
    configuration with the defaults will yield a similar configuration to that of the T5
    [t5-small](https://hf-mirror.com/t5-small) architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Arguments:
        vocab_size (`int`, *optional*, defaults to 32128):
            Vocabulary size of the T5 model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`T5Model`] or [`TFT5Model`].
        d_model (`int`, *optional*, defaults to 512):
            Size of the encoder layers and the pooler layer.
        d_kv (`int`, *optional*, defaults to 64):
            Size of the key, query, value projections per attention head. The `inner_dim` of the projection layer will
            be defined as `num_heads * d_kv`.
        d_ff (`int`, *optional*, defaults to 2048):
            Size of the intermediate feed forward layer in each `T5Block`.
        num_layers (`int`, *optional*, defaults to 6):
            Number of hidden layers in the Transformer encoder.
        num_decoder_layers (`int`, *optional*):
            Number of hidden layers in the Transformer decoder. Will use the same value as `num_layers` if not set.
        num_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer encoder.
        relative_attention_num_buckets (`int`, *optional*, defaults to 32):
            The number of buckets to use for each attention layer.
        relative_attention_max_distance (`int`, *optional*, defaults to 128):
            The maximum distance of the longer sequences for the bucket separation.
        dropout_rate (`float`, *optional*, defaults to 0.1):
            The ratio for all dropout layers.
        classifier_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for classifier.
        layer_norm_eps (`float`, *optional*, defaults to 1e-6):
            The epsilon used by the layer normalization layers.
        initializer_factor (`float`, *optional*, defaults to 1):
            A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
            testing).
        feed_forward_proj (`string`, *optional*, defaults to `"relu"`):
            Type of feed forward layer to be used. Should be one of `"relu"` or `"gated-gelu"`. T5v1.1 uses the
            `"gated-gelu"` feed forward projection. Original T5 uses `"relu"`.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models).
    """
    model_type = "t5"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}

    def __init__(
        self,
        vocab_size=32128,
        d_model=512,
        d_kv=64,
        d_ff=2048,
        num_layers=6,
        num_decoder_layers=None,
        num_heads=8,
        relative_attention_num_buckets=32,
        relative_attention_max_distance=128,
        dropout_rate=0.1,
        layer_norm_epsilon=1e-6,
        initializer_factor=1.0,
        feed_forward_proj="relu",
        is_encoder_decoder=True,
        use_cache=True,
        pad_token_id=0,
        eos_token_id=1,
        classifier_dropout=0.0,
        **kwargs,
    ):
        """
        Initializes a new instance of the T5Config class.

        Args:
            vocab_size (int, optional): The size of the vocabulary. Defaults to 32128.
            d_model (int, optional): The dimension of the model. Defaults to 512.
            d_kv (int, optional): The dimension of the key and value. Defaults to 64.
            d_ff (int, optional): The dimension of the feed forward layer. Defaults to 2048.
            num_layers (int, optional): The number of layers. Defaults to 6.
            num_decoder_layers (int, optional): The number of decoder layers. Defaults to the value of num_layers
                if not provided.
            num_heads (int, optional): The number of attention heads. Defaults to 8.
            relative_attention_num_buckets (int, optional): The number of buckets for relative attention. Defaults to 32.
            relative_attention_max_distance (int, optional): The maximum distance for relative attention. Defaults to 128.
            dropout_rate (float, optional): The dropout rate. Defaults to 0.1.
            layer_norm_epsilon (float, optional): The epsilon value for layer normalization. Defaults to 1e-06.
            initializer_factor (float, optional): The factor for initializer. Defaults to 1.0.
            feed_forward_proj (str, optional): The type of feed forward projection. Defaults to 'relu'.
            is_encoder_decoder (bool, optional): Indicates if the model is an encoder-decoder. Defaults to True.
            use_cache (bool, optional): Indicates if cache is used. Defaults to True.
            pad_token_id (int, optional): The token id for padding. Defaults to 0.
            eos_token_id (int, optional): The token id for end of sequence. Defaults to 1.
            classifier_dropout (float, optional): The dropout rate for the classifier. Defaults to 0.0.

        Returns:
            None.

        Raises:
            ValueError: If the feed_forward_proj is not a valid activation function of the dense layer.
        """
        self.vocab_size = vocab_size
        self.d_model = d_model
        self.d_kv = d_kv
        self.d_ff = d_ff
        self.num_layers = num_layers
        self.num_decoder_layers = (
            num_decoder_layers if num_decoder_layers is not None else self.num_layers
        )  # default = symmetry
        self.num_heads = num_heads
        self.relative_attention_num_buckets = relative_attention_num_buckets
        self.relative_attention_max_distance = relative_attention_max_distance
        self.dropout_rate = dropout_rate
        self.classifier_dropout = classifier_dropout
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_factor = initializer_factor
        self.feed_forward_proj = feed_forward_proj
        self.use_cache = use_cache

        act_info = self.feed_forward_proj.split("-")
        self.dense_act_fn = act_info[-1]
        self.is_gated_act = act_info[0] == "gated"

        if len(act_info) > 1 and act_info[0] != "gated" or len(act_info) > 2:
            raise ValueError(
                f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer. "
                "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
                "'gated-gelu' or 'relu'"
            )

        # for backwards compatibility
        if feed_forward_proj == "gated-gelu":
            self.dense_act_fn = "gelu_new"

        super().__init__(
            pad_token_id=pad_token_id,
            eos_token_id=eos_token_id,
            is_encoder_decoder=is_encoder_decoder,
            **kwargs,
        )

mindnlp.transformers.models.t5.configuration_t5.T5Config.__init__(vocab_size=32128, d_model=512, d_kv=64, d_ff=2048, num_layers=6, num_decoder_layers=None, num_heads=8, relative_attention_num_buckets=32, relative_attention_max_distance=128, dropout_rate=0.1, layer_norm_epsilon=1e-06, initializer_factor=1.0, feed_forward_proj='relu', is_encoder_decoder=True, use_cache=True, pad_token_id=0, eos_token_id=1, classifier_dropout=0.0, **kwargs)

Initializes a new instance of the T5Config class.

PARAMETER	DESCRIPTION
vocab_size	The size of the vocabulary. Defaults to 32128. TYPE: int DEFAULT: 32128
d_model	The dimension of the model. Defaults to 512. TYPE: int DEFAULT: 512
d_kv	The dimension of the key and value. Defaults to 64. TYPE: int DEFAULT: 64
d_ff	The dimension of the feed forward layer. Defaults to 2048. TYPE: int DEFAULT: 2048
num_layers	The number of layers. Defaults to 6. TYPE: int DEFAULT: 6
num_decoder_layers	The number of decoder layers. Defaults to the value of num_layers if not provided. TYPE: int DEFAULT: None
num_heads	The number of attention heads. Defaults to 8. TYPE: int DEFAULT: 8
relative_attention_num_buckets	The number of buckets for relative attention. Defaults to 32. TYPE: int DEFAULT: 32
relative_attention_max_distance	The maximum distance for relative attention. Defaults to 128. TYPE: int DEFAULT: 128
dropout_rate	The dropout rate. Defaults to 0.1. TYPE: float DEFAULT: 0.1
layer_norm_epsilon	The epsilon value for layer normalization. Defaults to 1e-06. TYPE: float DEFAULT: 1e-06
initializer_factor	The factor for initializer. Defaults to 1.0. TYPE: float DEFAULT: 1.0
feed_forward_proj	The type of feed forward projection. Defaults to 'relu'. TYPE: str DEFAULT: 'relu'
is_encoder_decoder	Indicates if the model is an encoder-decoder. Defaults to True. TYPE: bool DEFAULT: True
use_cache	Indicates if cache is used. Defaults to True. TYPE: bool DEFAULT: True
pad_token_id	The token id for padding. Defaults to 0. TYPE: int DEFAULT: 0
eos_token_id	The token id for end of sequence. Defaults to 1. TYPE: int DEFAULT: 1
classifier_dropout	The dropout rate for the classifier. Defaults to 0.0. TYPE: float DEFAULT: 0.0

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the feed_forward_proj is not a valid activation function of the dense layer.

Source code in mindnlp/transformers/models/t5/configuration_t5.py

def __init__(
    self,
    vocab_size=32128,
    d_model=512,
    d_kv=64,
    d_ff=2048,
    num_layers=6,
    num_decoder_layers=None,
    num_heads=8,
    relative_attention_num_buckets=32,
    relative_attention_max_distance=128,
    dropout_rate=0.1,
    layer_norm_epsilon=1e-6,
    initializer_factor=1.0,
    feed_forward_proj="relu",
    is_encoder_decoder=True,
    use_cache=True,
    pad_token_id=0,
    eos_token_id=1,
    classifier_dropout=0.0,
    **kwargs,
):
    """
    Initializes a new instance of the T5Config class.

    Args:
        vocab_size (int, optional): The size of the vocabulary. Defaults to 32128.
        d_model (int, optional): The dimension of the model. Defaults to 512.
        d_kv (int, optional): The dimension of the key and value. Defaults to 64.
        d_ff (int, optional): The dimension of the feed forward layer. Defaults to 2048.
        num_layers (int, optional): The number of layers. Defaults to 6.
        num_decoder_layers (int, optional): The number of decoder layers. Defaults to the value of num_layers
            if not provided.
        num_heads (int, optional): The number of attention heads. Defaults to 8.
        relative_attention_num_buckets (int, optional): The number of buckets for relative attention. Defaults to 32.
        relative_attention_max_distance (int, optional): The maximum distance for relative attention. Defaults to 128.
        dropout_rate (float, optional): The dropout rate. Defaults to 0.1.
        layer_norm_epsilon (float, optional): The epsilon value for layer normalization. Defaults to 1e-06.
        initializer_factor (float, optional): The factor for initializer. Defaults to 1.0.
        feed_forward_proj (str, optional): The type of feed forward projection. Defaults to 'relu'.
        is_encoder_decoder (bool, optional): Indicates if the model is an encoder-decoder. Defaults to True.
        use_cache (bool, optional): Indicates if cache is used. Defaults to True.
        pad_token_id (int, optional): The token id for padding. Defaults to 0.
        eos_token_id (int, optional): The token id for end of sequence. Defaults to 1.
        classifier_dropout (float, optional): The dropout rate for the classifier. Defaults to 0.0.

    Returns:
        None.

    Raises:
        ValueError: If the feed_forward_proj is not a valid activation function of the dense layer.
    """
    self.vocab_size = vocab_size
    self.d_model = d_model
    self.d_kv = d_kv
    self.d_ff = d_ff
    self.num_layers = num_layers
    self.num_decoder_layers = (
        num_decoder_layers if num_decoder_layers is not None else self.num_layers
    )  # default = symmetry
    self.num_heads = num_heads
    self.relative_attention_num_buckets = relative_attention_num_buckets
    self.relative_attention_max_distance = relative_attention_max_distance
    self.dropout_rate = dropout_rate
    self.classifier_dropout = classifier_dropout
    self.layer_norm_epsilon = layer_norm_epsilon
    self.initializer_factor = initializer_factor
    self.feed_forward_proj = feed_forward_proj
    self.use_cache = use_cache

    act_info = self.feed_forward_proj.split("-")
    self.dense_act_fn = act_info[-1]
    self.is_gated_act = act_info[0] == "gated"

    if len(act_info) > 1 and act_info[0] != "gated" or len(act_info) > 2:
        raise ValueError(
            f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer. "
            "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
            "'gated-gelu' or 'relu'"
        )

    # for backwards compatibility
    if feed_forward_proj == "gated-gelu":
        self.dense_act_fn = "gelu_new"

    super().__init__(
        pad_token_id=pad_token_id,
        eos_token_id=eos_token_id,
        is_encoder_decoder=is_encoder_decoder,
        **kwargs,
    )

mindnlp.transformers.models.t5.tokenization_t5

Tokenization class for model T5.

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer

Bases: PreTrainedTokenizer

Construct a T5 tokenizer. Based on SentencePiece.

This tokenizer inherits from [PreTrainedTokenizer] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	SentencePiece file (generally has a .spm extension) that contains the vocabulary necessary to instantiate a tokenizer. TYPE: `str`
eos_token	The end of sequence token. When building a sequence using special tokens, this is not the token that is used for the end of sequence. The token used is the sep_token. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<unk>"` DEFAULT: '<unk>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"<pad>"` DEFAULT: '<pad>'
extra_ids	Add a number of extra ids added to the vocabulary for use as sentinels. These tokens are accessible as "" where "{%d}" is a number between 0 and extra_ids-1. These tokens can be retrieved by calling get_sentinel_tokens method and token ids can be by calling get_sentinel_token_ids method additional_special_tokens (List[str], optional): Additional special tokens used by the tokenizer. TYPE: `int`, *optional*, defaults to 100 DEFAULT: 100
sp_model_kwargs	Will be passed to the SentencePieceProcessor.__init__() method. The Python wrapper for SentencePiece can be used, among other things, to set: enable_sampling: Enable subword regularization. nbest_size: Sampling parameters for unigram. Invalid for BPE-Dropout. nbest_size = {0,1}: No sampling is performed. nbest_size > 1: samples from the nbest_size results. nbest_size < 0: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. alpha: Smoothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. TYPE: `dict`, *optional* DEFAULT: None
legacy	Whether or not the legacy behaviour of the tokenizer should be used. Legacy is before the merge of #24622 and #25224 which includes fixes to properly handle tokens that appear after special tokens. A simple example: legacy=True: >>> from transformers import T5Tokenizer ... >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=True) >>> tokenizer.encode("Hello <extra_id_0>.") [8774, 32099, 3, 5, 1] legacy=False: >>> from transformers import T5Tokenizer ... >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=False) >>> tokenizer.encode("Hello <extra_id_0>.") # the extra space `[3]` is no longer here [8774, 32099, 5, 1] Checkout the pull request for more details. TYPE: `bool`, *optional* DEFAULT: None

ATTRIBUTE	DESCRIPTION
sp_model	The SentencePiece processor that is used for every conversion (string, tokens and IDs). TYPE: `SentencePieceProcessor`

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

class T5Tokenizer(PreTrainedTokenizer):
    """
    Construct a T5 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        extra_ids (`int`, *optional*, defaults to 100):
           Add a number of extra ids added to the vocabulary for use as sentinels. These tokens are
            accessible as "<extra_id_{%d}>" where "{%d}" is a number between 0 and extra_ids-1. These tokens can be
            retrieved by calling get_sentinel_tokens method and token ids can be by calling get_sentinel_token_ids
            method
         additional_special_tokens (`List[str]`, *optional*):
            Additional special tokens used by the tokenizer.
        sp_model_kwargs (`dict`, *optional*):
            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
            to set:

            - `enable_sampling`: Enable subword regularization.
            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.

                - `nbest_size = {0,1}`: No sampling is performed.
                - `nbest_size > 1`: samples from the nbest_size results.
                - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
                using forward-filtering-and-backward-sampling algorithm.
            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
            BPE-dropout.
        legacy (`bool`, *optional*):
            Whether or not the `legacy` behaviour of the tokenizer should be used. Legacy is before the merge of #24622
            and #25224 which includes fixes to properly handle tokens that appear after special tokens. A simple
            example:

            - `legacy=True`:
            ```python
            >>> from transformers import T5Tokenizer
            ...
            >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=True)
            >>> tokenizer.encode("Hello <extra_id_0>.")
            [8774, 32099, 3, 5, 1]
            ```

            - `legacy=False`:
            ```python
            >>> from transformers import T5Tokenizer
            ...
            >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=False)
            >>> tokenizer.encode("Hello <extra_id_0>.")  # the extra space `[3]` is no longer here
            [8774, 32099, 5, 1]
            ```
            Checkout the [pull request](https://github.com/huggingface/transformers/pull/24565) for more details.

    Attributes:
        sp_model (`SentencePieceProcessor`):
            The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        eos_token="</s>",
        unk_token="<unk>",
        pad_token="<pad>",
        extra_ids=100,
        additional_special_tokens=None,
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        legacy=None,
        **kwargs,
    ) -> None:
        """
        Initializes a T5Tokenizer instance.

        Args:
            self (object): The T5Tokenizer instance itself.
            vocab_file (str): The file path to the vocabulary file.
            eos_token (str, optional): The end-of-sequence token. Defaults to '</s>'.
            unk_token (str, optional): The unknown token. Defaults to '<unk>'.
            pad_token (str, optional): The padding token. Defaults to '<pad>'.
            extra_ids (int): The number of extra tokens to be added to the vocabulary.
            additional_special_tokens (List[str], optional): Additional special tokens to be added to the vocabulary.
                Defaults to None.
            sp_model_kwargs (Dict[str, Any], optional): Additional keyword arguments for the SentencePieceProcessor.
                Defaults to None.
            legacy (bool, optional): Flag to indicate whether to use the default legacy behavior. Defaults to None.

        Returns:
            None.

        Raises:
            ValueError: If both extra_ids and additional_special_tokens are provided and additional_special_tokens
                do not include the extra_ids tokens.
            Warning: If using the default legacy behavior, a warning is issued to notify the user about the behavior
                and provide guidance on changing it.
        """
        pad_token = AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token
        unk_token = AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token
        eos_token = AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token

        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

        self.vocab_file = vocab_file
        self._extra_ids = extra_ids

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(vocab_file)

        if additional_special_tokens is not None:
            extra_tokens = [x for x in additional_special_tokens if "<extra_id_" in str(x)]
            if len(extra_tokens) < 1:
                additional_special_tokens += [f"<extra_id_{i}>" for i in range(extra_ids)]
            elif extra_ids > 0 and extra_ids != len(extra_tokens):
                raise ValueError(
                    f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
                    " tokens"
                )
        else:
            extra_tokens = [f"<extra_id_{i}>" for i in range(extra_ids)]
            additional_special_tokens = extra_tokens

        # for legacy purpose, we keep this. Will be removed and tests updated. (when `added_tokens_decoder` is not passed as kwargs)
        self._added_tokens_decoder = {}
        for i in range(len(extra_tokens)):
            self._added_tokens_decoder[len(self.sp_model) - 1 + extra_ids - i] = AddedToken(
                f"<extra_id_{i}>", single_word=True, lstrip=True, rstrip=True, special=True
            )

        if legacy is None:
            logger.warning_once(
                f"You are using the default legacy behaviour of the {self.__class__}. This is"
                " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you."
                " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it"
                " means, and thouroughly read the reason why this was added as explained in"
                " https://github.com/huggingface/transformers/pull/24565"
            )
            legacy = True

        self.legacy = legacy
        self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
        self.vocab_file = vocab_file
        self._extra_ids = extra_ids

        super().__init__(
            eos_token=eos_token,
            unk_token=unk_token,
            pad_token=pad_token,
            extra_ids=extra_ids,
            additional_special_tokens=additional_special_tokens,
            sp_model_kwargs=self.sp_model_kwargs,
            legacy=legacy,
            **kwargs,
        )

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_spm_processor
    def get_spm_processor(self, from_slow=False):
        """
        This method is responsible for retrieving the SentencePieceProcessor for the T5Tokenizer class.

        Args:
            self (T5Tokenizer): The instance of the T5Tokenizer class.
            from_slow (bool): A flag indicating whether to use the slow SentencePiece model. Defaults to False.
                If set to True, the method will load the tokenizer from the slow model.

        Returns:
            None: This method does not return any value, rather it modifies the internal state of the T5Tokenizer
                instance by loading the SentencePieceProcessor.

        Raises:
            IOError: If there is an issue with reading the vocab_file or if the file is not found.
            ImportError: If there is an issue with importing the protobuf module.
            AttributeError: If an attribute error occurs during the method execution.
            RuntimeError: If there is a runtime error while loading the SentencePieceProcessor.
        """
        tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        if self.legacy or from_slow:  # no dependency on protobuf
            tokenizer.Load(self.vocab_file)
            return tokenizer

        with open(self.vocab_file, "rb") as f:
            sp_model = f.read()
            model_pb2 = import_protobuf(f"The new behaviour of {self.__class__.__name__} (with `self.legacy = False`)")
            model = model_pb2.ModelProto.FromString(sp_model)
            normalizer_spec = model_pb2.NormalizerSpec()
            normalizer_spec.add_dummy_prefix = False
            model.normalizer_spec.MergeFrom(normalizer_spec)
            sp_model = model.SerializeToString()
            tokenizer.LoadFromSerializedProto(sp_model)
        return tokenizer

    @staticmethod
    def _eventually_correct_t5_max_length(pretrained_model_name_or_path, max_model_length, init_max_model_length):
        """
        This method is a static method in the `T5Tokenizer` class and is named `_eventually_correct_t5_max_length`.

        Args:
            pretrained_model_name_or_path (str): The name or path of the pretrained model.
            max_model_length (int): The maximum model length.
            init_max_model_length (int): The initial maximum model length.

        Returns:
            None.

        Raises:
            FutureWarning: If the tokenizer was incorrectly instantiated with a model max length that will be
                corrected in Transformers v5. This warning is to ensure backward compatibility when  padding/encoding
                with `truncation` set to True. It is recommended not to rely on automatic truncation to the deprecated
                max length. To encode/pad to sequences longer than the deprecated max length, either instantiate the
                tokenizer with `model_max_length` or pass `max_length` when encoding/padding.
        """
        if pretrained_model_name_or_path in T5Tokenizer.max_model_input_sizes:
            deprecated_max_model_length = T5Tokenizer.max_model_input_sizes[pretrained_model_name_or_path]
            if init_max_model_length is not None and init_max_model_length != max_model_length:
                return init_max_model_length
            if init_max_model_length is None:
                warnings.warn(
                    "This tokenizer was incorrectly instantiated with a model max length of"
                    f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
                    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
                    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
                    f" {pretrained_model_name_or_path} automatically truncating your input to"
                    f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
                    f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
                    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
                    " instantiate this tokenizer with `model_max_length` set to your preferred value.",
                    FutureWarning,
                )

        return max_model_length

    @property
    def vocab_size(self):
        """
        Method to get the vocabulary size of the T5Tokenizer instance.

        Args:
            self (T5Tokenizer): The instance of the T5Tokenizer class.
                This parameter is required to access the tokenizer's properties and methods.

        Returns:
            int: The method returns the vocabulary size of the T5Tokenizer instance as an integer value.

        Raises:
            None.
        """
        return self.sp_model.get_piece_size()

    def get_vocab(self):
        """
        Retrieves the vocabulary of the T5Tokenizer.

        Args:
            self (T5Tokenizer): The instance of the T5Tokenizer class.

        Returns:
            dict: A dictionary containing the vocabulary of the tokenizer. The keys are the tokens in the vocabulary,
                and the values are the corresponding token IDs. The vocabulary includes both the original vocabulary
                of the T5Tokenizer and any additional tokens that have been added.

        Raises:
            None.
        """
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        # normal case: some special tokens
        if token_ids_1 is None:
            return ([0] * len(token_ids_0)) + [1]
        return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]

    def get_sentinel_tokens(self):
        """
        This method, get_sentinel_tokens, belongs to the class T5Tokenizer and retrieves sentinel tokens from the additional_special_tokens list.

        Args:
            self:
                An instance of the T5Tokenizer class.

                - Type: T5Tokenizer object.
                - Purpose: Represents the current instance of the T5Tokenizer class.
                - Restrictions: None.

        Returns:
            None: this method modifies the internal state of the object.

        Raises:
            None.
        """
        return list(
            set(filter(lambda x: bool(re.search(r"<extra_id_\d+>", x)) is not None, self.additional_special_tokens))
        )

    def get_sentinel_token_ids(self):
        """
        Method to retrieve the token IDs for sentinel tokens in the T5Tokenizer class.

        Args:
            self (T5Tokenizer): An instance of the T5Tokenizer class.
                Represents the tokenizer object that the method is called on.
                It is used to access the necessary methods and attributes within the tokenizer.

        Returns:
            None:
                The method does not return a value but directly returns the list of token IDs for sentinel tokens.

        Raises:
            None:
                This method does not raise any exceptions.
        """
        return [self.convert_tokens_to_ids(token) for token in self.get_sentinel_tokens()]

    def _add_eos_if_not_present(self, token_ids: List[int]) -> List[int]:
        """Do not add eos again if user already added it."""
        if len(token_ids) > 0 and token_ids[-1] == self.eos_token_id:
            warnings.warn(
                f"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
                " eos tokens being added."
            )
            return token_ids
        return token_ids + [self.eos_token_id]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
        use of token type ids, therefore a list of zeros is returned.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of zeros.
        """
        eos = [self.eos_token_id]

        if token_ids_1 is None:
            return len(token_ids_0 + eos) * [0]
        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A sequence has the following format:

        - single sequence: `X </s>`
        - pair of sequences: `A </s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        token_ids_0 = self._add_eos_if_not_present(token_ids_0)
        if token_ids_1 is None:
            return token_ids_0
        token_ids_1 = self._add_eos_if_not_present(token_ids_1)
        return token_ids_0 + token_ids_1

    def __getstate__(self):
        """
        This method __getstate__ is defined in the class T5Tokenizer and is used to retrieve the state of the tokenizer.

        Args:
            self (T5Tokenizer): The instance of the T5Tokenizer class.

        Returns:
            dict: A dictionary containing the state of the tokenizer with the 'sp_model' key set to None.

        Raises:
            This method does not raise any exceptions.
        """
        state = self.__dict__.copy()
        state["sp_model"] = None
        return state

    def __setstate__(self, d):
        """
        Method '__setstate__' in the class 'T5Tokenizer'.

        Args:
            self (T5Tokenizer): The instance of the T5Tokenizer class.
                This parameter represents the current instance of the T5Tokenizer class.
            d (dict): A dictionary containing the state information to be set.
                The dictionary 'd' holds the state information that needs to be set for the T5Tokenizer instance.

        Returns:
            None.

        Raises:
            None.
        """
        self.__dict__ = d

        # for backward compatibility
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(self.vocab_file)

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.tokenize
    def tokenize(self, text: "TextInput", **kwargs) -> List[str]:
        """
        Converts a string to a list of tokens. If `self.legacy` is set to `False`, a prefix token is added unless the
        first token is special.
        """
        if self.legacy or len(text) == 0:
            return super().tokenize(text, **kwargs)

        tokens = super().tokenize(SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " "), **kwargs)

        if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:
            tokens = tokens[1:]
        return tokens

    @property
    def unk_token_length(self):
        """
        This method returns the length of the encoded form of the unknown token in the T5Tokenizer.

        Args:
            self: T5Tokenizer
                The instance of the T5Tokenizer class.

        Returns:
            int:
                The length of the encoded form of the unknown token.

        Raises:
            None.
        """
        return len(self.sp_model.encode(str(self.unk_token)))

    def _tokenize(self, text, **kwargs):
        """
        Returns a tokenized string.

        We de-activated the `add_dummy_prefix` option, thus the sentencepiece internals will always strip any
        SPIECE_UNDERLINE. For example: `self.sp_model.encode(f"{SPIECE_UNDERLINE}Hey", out_type = str)` will give
        `['H', 'e', 'y']` instead of `['▁He', 'y']`. Thus we always encode `f"{unk_token}text"` and strip the
        `unk_token`. Here is an example with `unk_token = "<unk>"` and `unk_token_length = 4`.
        `self.tokenizer.sp_model.encode("<unk> Hey", out_type = str)[4:]`.
        """
        tokens = self.sp_model.encode(text, out_type=str)
        if self.legacy or not text.startswith((SPIECE_UNDERLINE, " ")):
            return tokens

        # 1. Encode string + prefix ex: "<unk> Hey"
        tokens = self.sp_model.encode(self.unk_token + text, out_type=str)
        # 2. Remove self.unk_token from ['<','unk','>', '▁Hey']
        return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.sp_model.piece_to_id(token)

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        token = self.sp_model.IdToPiece(index)
        return token

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        current_sub_tokens = []
        # since we manually add the prefix space, we have to remove it
        tokens[0] = tokens[0].lstrip(SPIECE_UNDERLINE)
        out_string = ""
        prev_is_special = False
        for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                if not prev_is_special:
                    out_string += " "
                out_string += self.sp_model.decode(current_sub_tokens) + token
                prev_is_special = True
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
                prev_is_special = False
        out_string += self.sp_model.decode(current_sub_tokens)
        return out_string.strip()

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        '''
        Save the vocabulary to the specified directory with an optional filename prefix.

        Args:
            self (T5Tokenizer): The instance of the T5Tokenizer class.
            save_directory (str): The directory where the vocabulary will be saved. It should be a valid directory path.
            filename_prefix (Optional[str]): An optional prefix to be added to the vocabulary filename. Default is None.

        Returns:
            Tuple[str]: A tuple containing the path to the saved vocabulary file.

        Raises:
            OSError: If the save_directory is not a valid directory path.
            FileNotFoundError: If the self.vocab_file does not exist.
        '''
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.unk_token_length property

This method returns the length of the encoded form of the unknown token in the T5Tokenizer.

PARAMETER	DESCRIPTION
self	T5Tokenizer The instance of the T5Tokenizer class.

RETURNS	DESCRIPTION
int	The length of the encoded form of the unknown token.

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.vocab_size property

Method to get the vocabulary size of the T5Tokenizer instance.

PARAMETER	DESCRIPTION
self	The instance of the T5Tokenizer class. This parameter is required to access the tokenizer's properties and methods. TYPE: T5Tokenizer

RETURNS	DESCRIPTION
int	The method returns the vocabulary size of the T5Tokenizer instance as an integer value.

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.__getstate__()

This method getstate is defined in the class T5Tokenizer and is used to retrieve the state of the tokenizer.

PARAMETER	DESCRIPTION
self	The instance of the T5Tokenizer class. TYPE: T5Tokenizer

RETURNS	DESCRIPTION
dict	A dictionary containing the state of the tokenizer with the 'sp_model' key set to None.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def __getstate__(self):
    """
    This method __getstate__ is defined in the class T5Tokenizer and is used to retrieve the state of the tokenizer.

    Args:
        self (T5Tokenizer): The instance of the T5Tokenizer class.

    Returns:
        dict: A dictionary containing the state of the tokenizer with the 'sp_model' key set to None.

    Raises:
        This method does not raise any exceptions.
    """
    state = self.__dict__.copy()
    state["sp_model"] = None
    return state

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.__init__(vocab_file, eos_token='</s>', unk_token='<unk>', pad_token='<pad>', extra_ids=100, additional_special_tokens=None, sp_model_kwargs=None, legacy=None, **kwargs)

Initializes a T5Tokenizer instance.

PARAMETER	DESCRIPTION
self	The T5Tokenizer instance itself. TYPE: object
vocab_file	The file path to the vocabulary file. TYPE: str
eos_token	The end-of-sequence token. Defaults to ''. TYPE: str DEFAULT: '</s>'
unk_token	The unknown token. Defaults to ''. TYPE: str DEFAULT: '<unk>'
pad_token	The padding token. Defaults to ''. TYPE: str DEFAULT: '<pad>'
extra_ids	The number of extra tokens to be added to the vocabulary. TYPE: int DEFAULT: 100
additional_special_tokens	Additional special tokens to be added to the vocabulary. Defaults to None. TYPE: List[str] DEFAULT: None
sp_model_kwargs	Additional keyword arguments for the SentencePieceProcessor. Defaults to None. TYPE: Dict[str, Any] DEFAULT: None
legacy	Flag to indicate whether to use the default legacy behavior. Defaults to None. TYPE: bool DEFAULT: None

RETURNS	DESCRIPTION
None	None.

RAISES	DESCRIPTION
ValueError	If both extra_ids and additional_special_tokens are provided and additional_special_tokens do not include the extra_ids tokens.
Warning	If using the default legacy behavior, a warning is issued to notify the user about the behavior and provide guidance on changing it.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def __init__(
    self,
    vocab_file,
    eos_token="</s>",
    unk_token="<unk>",
    pad_token="<pad>",
    extra_ids=100,
    additional_special_tokens=None,
    sp_model_kwargs: Optional[Dict[str, Any]] = None,
    legacy=None,
    **kwargs,
) -> None:
    """
    Initializes a T5Tokenizer instance.

    Args:
        self (object): The T5Tokenizer instance itself.
        vocab_file (str): The file path to the vocabulary file.
        eos_token (str, optional): The end-of-sequence token. Defaults to '</s>'.
        unk_token (str, optional): The unknown token. Defaults to '<unk>'.
        pad_token (str, optional): The padding token. Defaults to '<pad>'.
        extra_ids (int): The number of extra tokens to be added to the vocabulary.
        additional_special_tokens (List[str], optional): Additional special tokens to be added to the vocabulary.
            Defaults to None.
        sp_model_kwargs (Dict[str, Any], optional): Additional keyword arguments for the SentencePieceProcessor.
            Defaults to None.
        legacy (bool, optional): Flag to indicate whether to use the default legacy behavior. Defaults to None.

    Returns:
        None.

    Raises:
        ValueError: If both extra_ids and additional_special_tokens are provided and additional_special_tokens
            do not include the extra_ids tokens.
        Warning: If using the default legacy behavior, a warning is issued to notify the user about the behavior
            and provide guidance on changing it.
    """
    pad_token = AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token
    unk_token = AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token
    eos_token = AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token

    self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

    self.vocab_file = vocab_file
    self._extra_ids = extra_ids

    self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
    self.sp_model.Load(vocab_file)

    if additional_special_tokens is not None:
        extra_tokens = [x for x in additional_special_tokens if "<extra_id_" in str(x)]
        if len(extra_tokens) < 1:
            additional_special_tokens += [f"<extra_id_{i}>" for i in range(extra_ids)]
        elif extra_ids > 0 and extra_ids != len(extra_tokens):
            raise ValueError(
                f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
                " tokens"
            )
    else:
        extra_tokens = [f"<extra_id_{i}>" for i in range(extra_ids)]
        additional_special_tokens = extra_tokens

    # for legacy purpose, we keep this. Will be removed and tests updated. (when `added_tokens_decoder` is not passed as kwargs)
    self._added_tokens_decoder = {}
    for i in range(len(extra_tokens)):
        self._added_tokens_decoder[len(self.sp_model) - 1 + extra_ids - i] = AddedToken(
            f"<extra_id_{i}>", single_word=True, lstrip=True, rstrip=True, special=True
        )

    if legacy is None:
        logger.warning_once(
            f"You are using the default legacy behaviour of the {self.__class__}. This is"
            " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you."
            " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it"
            " means, and thouroughly read the reason why this was added as explained in"
            " https://github.com/huggingface/transformers/pull/24565"
        )
        legacy = True

    self.legacy = legacy
    self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
    self.vocab_file = vocab_file
    self._extra_ids = extra_ids

    super().__init__(
        eos_token=eos_token,
        unk_token=unk_token,
        pad_token=pad_token,
        extra_ids=extra_ids,
        additional_special_tokens=additional_special_tokens,
        sp_model_kwargs=self.sp_model_kwargs,
        legacy=legacy,
        **kwargs,
    )

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.__setstate__(d)

Method 'setstate' in the class 'T5Tokenizer'.

PARAMETER	DESCRIPTION
self	The instance of the T5Tokenizer class. This parameter represents the current instance of the T5Tokenizer class. TYPE: T5Tokenizer
d	A dictionary containing the state information to be set. The dictionary 'd' holds the state information that needs to be set for the T5Tokenizer instance. TYPE: dict

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def __setstate__(self, d):
    """
    Method '__setstate__' in the class 'T5Tokenizer'.

    Args:
        self (T5Tokenizer): The instance of the T5Tokenizer class.
            This parameter represents the current instance of the T5Tokenizer class.
        d (dict): A dictionary containing the state information to be set.
            The dictionary 'd' holds the state information that needs to be set for the T5Tokenizer instance.

    Returns:
        None.

    Raises:
        None.
    """
    self.__dict__ = d

    # for backward compatibility
    if not hasattr(self, "sp_model_kwargs"):
        self.sp_model_kwargs = {}

    self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
    self.sp_model.Load(self.vocab_file)

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A sequence has the following format:

• single sequence: X </s>
• pair of sequences: A </s> B </s>

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. A sequence has the following format:

    - single sequence: `X </s>`
    - pair of sequences: `A </s> B </s>`

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    token_ids_0 = self._add_eos_if_not_present(token_ids_0)
    if token_ids_1 is None:
        return token_ids_0
    token_ids_1 = self._add_eos_if_not_present(token_ids_1)
    return token_ids_0 + token_ids_1

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (string) in a single string.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (string) in a single string."""
    current_sub_tokens = []
    # since we manually add the prefix space, we have to remove it
    tokens[0] = tokens[0].lstrip(SPIECE_UNDERLINE)
    out_string = ""
    prev_is_special = False
    for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
            if not prev_is_special:
                out_string += " "
            out_string += self.sp_model.decode(current_sub_tokens) + token
            prev_is_special = True
            current_sub_tokens = []
        else:
            current_sub_tokens.append(token)
            prev_is_special = False
    out_string += self.sp_model.decode(current_sub_tokens)
    return out_string.strip()

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make use of token type ids, therefore a list of zeros is returned.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of zeros.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
    use of token type ids, therefore a list of zeros is returned.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of zeros.
    """
    eos = [self.eos_token_id]

    if token_ids_1 is None:
        return len(token_ids_0 + eos) * [0]
    return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.get_sentinel_token_ids()

Method to retrieve the token IDs for sentinel tokens in the T5Tokenizer class.

PARAMETER	DESCRIPTION
self	An instance of the T5Tokenizer class. Represents the tokenizer object that the method is called on. It is used to access the necessary methods and attributes within the tokenizer. TYPE: T5Tokenizer

RETURNS	DESCRIPTION
None	The method does not return a value but directly returns the list of token IDs for sentinel tokens.

RAISES	DESCRIPTION
None	This method does not raise any exceptions.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def get_sentinel_token_ids(self):
    """
    Method to retrieve the token IDs for sentinel tokens in the T5Tokenizer class.

    Args:
        self (T5Tokenizer): An instance of the T5Tokenizer class.
            Represents the tokenizer object that the method is called on.
            It is used to access the necessary methods and attributes within the tokenizer.

    Returns:
        None:
            The method does not return a value but directly returns the list of token IDs for sentinel tokens.

    Raises:
        None:
            This method does not raise any exceptions.
    """
    return [self.convert_tokens_to_ids(token) for token in self.get_sentinel_tokens()]

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.get_sentinel_tokens()

This method, get_sentinel_tokens, belongs to the class T5Tokenizer and retrieves sentinel tokens from the additional_special_tokens list.

PARAMETER	DESCRIPTION
self	An instance of the T5Tokenizer class. Type: T5Tokenizer object. Purpose: Represents the current instance of the T5Tokenizer class. Restrictions: None.

RETURNS	DESCRIPTION
None	this method modifies the internal state of the object.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def get_sentinel_tokens(self):
    """
    This method, get_sentinel_tokens, belongs to the class T5Tokenizer and retrieves sentinel tokens from the additional_special_tokens list.

    Args:
        self:
            An instance of the T5Tokenizer class.

            - Type: T5Tokenizer object.
            - Purpose: Represents the current instance of the T5Tokenizer class.
            - Restrictions: None.

    Returns:
        None: this method modifies the internal state of the object.

    Raises:
        None.
    """
    return list(
        set(filter(lambda x: bool(re.search(r"<extra_id_\d+>", x)) is not None, self.additional_special_tokens))
    )

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    # normal case: some special tokens
    if token_ids_1 is None:
        return ([0] * len(token_ids_0)) + [1]
    return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.get_spm_processor(from_slow=False)

This method is responsible for retrieving the SentencePieceProcessor for the T5Tokenizer class.

PARAMETER	DESCRIPTION
self	The instance of the T5Tokenizer class. TYPE: T5Tokenizer
from_slow	A flag indicating whether to use the slow SentencePiece model. Defaults to False. If set to True, the method will load the tokenizer from the slow model. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
None	This method does not return any value, rather it modifies the internal state of the T5Tokenizer instance by loading the SentencePieceProcessor.

RAISES	DESCRIPTION
IOError	If there is an issue with reading the vocab_file or if the file is not found.
ImportError	If there is an issue with importing the protobuf module.
AttributeError	If an attribute error occurs during the method execution.
RuntimeError	If there is a runtime error while loading the SentencePieceProcessor.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def get_spm_processor(self, from_slow=False):
    """
    This method is responsible for retrieving the SentencePieceProcessor for the T5Tokenizer class.

    Args:
        self (T5Tokenizer): The instance of the T5Tokenizer class.
        from_slow (bool): A flag indicating whether to use the slow SentencePiece model. Defaults to False.
            If set to True, the method will load the tokenizer from the slow model.

    Returns:
        None: This method does not return any value, rather it modifies the internal state of the T5Tokenizer
            instance by loading the SentencePieceProcessor.

    Raises:
        IOError: If there is an issue with reading the vocab_file or if the file is not found.
        ImportError: If there is an issue with importing the protobuf module.
        AttributeError: If an attribute error occurs during the method execution.
        RuntimeError: If there is a runtime error while loading the SentencePieceProcessor.
    """
    tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
    if self.legacy or from_slow:  # no dependency on protobuf
        tokenizer.Load(self.vocab_file)
        return tokenizer

    with open(self.vocab_file, "rb") as f:
        sp_model = f.read()
        model_pb2 = import_protobuf(f"The new behaviour of {self.__class__.__name__} (with `self.legacy = False`)")
        model = model_pb2.ModelProto.FromString(sp_model)
        normalizer_spec = model_pb2.NormalizerSpec()
        normalizer_spec.add_dummy_prefix = False
        model.normalizer_spec.MergeFrom(normalizer_spec)
        sp_model = model.SerializeToString()
        tokenizer.LoadFromSerializedProto(sp_model)
    return tokenizer

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.get_vocab()

Retrieves the vocabulary of the T5Tokenizer.

PARAMETER	DESCRIPTION
self	The instance of the T5Tokenizer class. TYPE: T5Tokenizer

RETURNS	DESCRIPTION
dict	A dictionary containing the vocabulary of the tokenizer. The keys are the tokens in the vocabulary, and the values are the corresponding token IDs. The vocabulary includes both the original vocabulary of the T5Tokenizer and any additional tokens that have been added.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def get_vocab(self):
    """
    Retrieves the vocabulary of the T5Tokenizer.

    Args:
        self (T5Tokenizer): The instance of the T5Tokenizer class.

    Returns:
        dict: A dictionary containing the vocabulary of the tokenizer. The keys are the tokens in the vocabulary,
            and the values are the corresponding token IDs. The vocabulary includes both the original vocabulary
            of the T5Tokenizer and any additional tokens that have been added.

    Raises:
        None.
    """
    vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
    vocab.update(self.added_tokens_encoder)
    return vocab

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary to the specified directory with an optional filename prefix.

PARAMETER	DESCRIPTION
self	The instance of the T5Tokenizer class. TYPE: T5Tokenizer
save_directory	The directory where the vocabulary will be saved. It should be a valid directory path. TYPE: str
filename_prefix	An optional prefix to be added to the vocabulary filename. Default is None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the path to the saved vocabulary file.

RAISES	DESCRIPTION
OSError	If the save_directory is not a valid directory path.
FileNotFoundError	If the self.vocab_file does not exist.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    '''
    Save the vocabulary to the specified directory with an optional filename prefix.

    Args:
        self (T5Tokenizer): The instance of the T5Tokenizer class.
        save_directory (str): The directory where the vocabulary will be saved. It should be a valid directory path.
        filename_prefix (Optional[str]): An optional prefix to be added to the vocabulary filename. Default is None.

    Returns:
        Tuple[str]: A tuple containing the path to the saved vocabulary file.

    Raises:
        OSError: If the save_directory is not a valid directory path.
        FileNotFoundError: If the self.vocab_file does not exist.
    '''
    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory")
        return
    out_vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )

    if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file, out_vocab_file)
    elif not os.path.isfile(self.vocab_file):
        with open(out_vocab_file, "wb") as fi:
            content_spiece_model = self.sp_model.serialized_model_proto()
            fi.write(content_spiece_model)

    return (out_vocab_file,)

mindnlp.transformers.models.t5.tokenization_t5.T5Tokenizer.tokenize(text, **kwargs)

Converts a string to a list of tokens. If self.legacy is set to False, a prefix token is added unless the first token is special.

Source code in mindnlp/transformers/models/t5/tokenization_t5.py

def tokenize(self, text: "TextInput", **kwargs) -> List[str]:
    """
    Converts a string to a list of tokens. If `self.legacy` is set to `False`, a prefix token is added unless the
    first token is special.
    """
    if self.legacy or len(text) == 0:
        return super().tokenize(text, **kwargs)

    tokens = super().tokenize(SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " "), **kwargs)

    if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:
        tokens = tokens[1:]
    return tokens

mindnlp.transformers.models.t5.tokenization_t5_fast

Tokenization class for model T5.

mindnlp.transformers.models.t5.tokenization_t5_fast.T5TokenizerFast

Bases: PreTrainedTokenizerFast

Construct a "fast" T5 tokenizer (backed by HuggingFace's tokenizers library). Based on Unigram.

This tokenizer inherits from [PreTrainedTokenizerFast] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	SentencePiece file (generally has a .spm extension) that contains the vocabulary necessary to instantiate a tokenizer. TYPE: `str` DEFAULT: None
eos_token	The end of sequence token. When building a sequence using special tokens, this is not the token that is used for the end of sequence. The token used is the sep_token. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<unk>"` DEFAULT: '<unk>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"<pad>"` DEFAULT: '<pad>'
extra_ids	Add a number of extra ids added to the vocabulary for use as sentinels. These tokens are accessible as "" where "{%d}" is a number between 0 and extra_ids-1. These tokens can be retrieved by calling get_sentinel_tokens method and token ids can be by calling get_sentinel_token_ids method TYPE: `int`, *optional*, defaults to 100 DEFAULT: 100
additional_special_tokens	Additional special tokens used by the tokenizer. TYPE: `List[str]`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/t5/tokenization_t5_fast.py

class T5TokenizerFast(PreTrainedTokenizerFast):
    """
    Construct a "fast" T5 tokenizer (backed by HuggingFace's *tokenizers* library). Based on
    [Unigram](https://hf-mirror.com/docs/tokenizers/python/latest/components.html?highlight=unigram#models).

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        extra_ids (`int`, *optional*, defaults to 100):
            Add a number of extra ids added to the vocabulary for use as sentinels. These tokens are accessible as
            "<extra_id_{%d}>" where "{%d}" is a number between 0 and extra_ids-1. These tokens can be retrieved by
            calling get_sentinel_tokens method and token ids can be by calling get_sentinel_token_ids method
        additional_special_tokens (`List[str]`, *optional*):
            Additional special tokens used by the tokenizer.
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = T5Tokenizer

    prefix_tokens: List[int] = []

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        eos_token="</s>",
        unk_token="<unk>",
        pad_token="<pad>",
        extra_ids=100,
        additional_special_tokens=None,
        **kwargs,
    ):
        """
        Initializes a new instance of the T5TokenizerFast class.

        Args:
            self (T5TokenizerFast): The instance of the T5TokenizerFast class that the method is called on.
            vocab_file (str, optional): The path to the vocabulary file. Default is None.
            tokenizer_file (str, optional): The path to the tokenizer file. Default is None.
            eos_token (str, optional): The end-of-sentence token. Default is '</s>'.
            unk_token (str, optional): The unknown token. Default is '<unk>'.
            pad_token (str, optional): The padding token. Default is '<pad>'.
            extra_ids (int, optional): The number of extra tokens to be added. Default is 100.
            additional_special_tokens (list, optional):
                Additional special tokens to be added. Default is None.

                - If provided, it must include the extra_ids tokens.
                - If not provided, extra_ids number of '<extra_id_i>' tokens will be added automatically.
                - If provided and no '<extra_id_i>' tokens are found, extra_ids number of '<extra_id_i>' tokens
                will be added automatically.
            **kwargs (dict): Additional keyword arguments.

        Returns:
            None.

        Raises:
            ValueError: If both extra_ids and additional_special_tokens are provided, but additional_special_tokens
                does not include the extra_ids tokens.
        """
        # Add extra_ids to the special token list
        if additional_special_tokens is not None:
            extra_tokens = [x for x in additional_special_tokens if "<extra_id_" in str(x)]
            if len(extra_tokens) < 1:
                additional_special_tokens += [f"<extra_id_{i}>" for i in range(extra_ids)]
            elif extra_ids > 0 and extra_ids != len(extra_tokens):
                raise ValueError(
                    f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
                    " tokens"
                )
        else:
            extra_tokens = [f"<extra_id_{i}>" for i in range(extra_ids)]
            additional_special_tokens = extra_tokens

        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            eos_token=eos_token,
            unk_token=unk_token,
            pad_token=pad_token,
            extra_ids=extra_ids,
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )

        self.vocab_file = vocab_file
        self._extra_ids = extra_ids

    @property
    def can_save_slow_tokenizer(self) -> bool:
        """
        This method checks if the slow tokenizer can be saved.

        Args:
            self (T5TokenizerFast): The instance of the T5TokenizerFast class.
                It is used to access the vocab_file attribute which is required for checking
                if the slow tokenizer can be saved.

        Returns:
            bool: Returns a boolean value indicating whether the slow tokenizer can be saved.
                True if the vocab_file exists, otherwise False.

        Raises:
            None
        """
        return os.path.isfile(self.vocab_file) if self.vocab_file else False

    @staticmethod
    def _eventually_correct_t5_max_length(pretrained_model_name_or_path, max_model_length, init_max_model_length):
        """
        This method updates the maximum model length for the T5 tokenizer. It checks if the provided
        `pretrained_model_name_or_path` is valid and compares the `init_max_model_length` with the
        `max_model_length` to determine the final value.

        Args:
            pretrained_model_name_or_path (str): The name or path of the pretrained model.
            max_model_length (int): The maximum length for the model.
            init_max_model_length (int or None): The initial maximum model length.

        Returns:
            None.

        Raises:
            FutureWarning: If the tokenizer was incorrectly instantiated with a deprecated maximum model length,
            a warning is raised. This is to maintain backwards compatibility and inform the user about possible issues
            when padding or encoding with `truncation` set to True.

        Note:
            - If `pretrained_model_name_or_path` is in the list of `T5TokenizerFast.max_model_input_sizes`,
            the deprecated maximum model length will be retrieved.
            - If `init_max_model_length` is provided and different from `max_model_length`,
            it will be returned as the final value.
            - If `init_max_model_length` is None, a FutureWarning will be raised to inform about the deprecated behavior
            and recommend explicit specification of `max_length` or `model_max_length` when encoding
            or padding sequences longer than the deprecated maximum model length.
        """
        if pretrained_model_name_or_path in T5TokenizerFast.max_model_input_sizes:
            deprecated_max_model_length = T5TokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
            if init_max_model_length is not None and init_max_model_length != max_model_length:
                return init_max_model_length
            if init_max_model_length is None:
                warnings.warn(
                    "This tokenizer was incorrectly instantiated with a model max length of"
                    f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
                    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
                    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
                    f" {pretrained_model_name_or_path} automatically truncating your input to"
                    f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
                    f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
                    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
                    " instantiate this tokenizer with `model_max_length` set to your preferred value.",
                    FutureWarning,
                )

        return max_model_length

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Saves the vocabulary for a slow tokenizer.

        Args:
            self (T5TokenizerFast): An instance of the T5TokenizerFast class.
            save_directory (str): The directory where the vocabulary will be saved.
            filename_prefix (Optional[str], optional): A prefix to be added to the filename. Defaults to None.

        Returns:
            Tuple[str]: A tuple containing the path to the saved vocabulary file.

        Raises:
            ValueError: If the fast tokenizer does not have the necessary information to save the vocabulary for
                a slow tokenizer.
            FileNotFoundError: If the save_directory does not exist.

        Note:
            The method assumes that the fast tokenizer has the necessary information to save the vocabulary for
            a slow tokenizer.

        Example:
            ```python
            >>> tokenizer = T5TokenizerFast()
            >>> tokenizer.save_vocabulary('/path/to/save', 'vocab')
            ```
        """
        if not self.can_save_slow_tokenizer:
            raise ValueError(
                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                "tokenizer."
            )

        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
            logger.info(f"Copy vocab file to {out_vocab_file}")

        return (out_vocab_file,)

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A sequence has the following format:

        - single sequence: `X </s>`
        - pair of sequences: `A </s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        token_ids_0 = token_ids_0 + [self.eos_token_id]
        if token_ids_1 is None:
            return self.prefix_tokens + token_ids_0
        token_ids_1 = token_ids_1 + [self.eos_token_id]
        return self.prefix_tokens + token_ids_0 + token_ids_1

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
        use of token type ids, therefore a list of zeros is returned.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of zeros.
        """
        eos = [self.eos_token_id]

        if token_ids_1 is None:
            return len(token_ids_0 + eos) * [0]
        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

    def get_sentinel_tokens(self):
        """
        This method retrieves the sentinel tokens from the T5TokenizerFast instance.

        Args:
            self (T5TokenizerFast): The T5TokenizerFast instance.

        Returns:
            list: A list of sentinel tokens filtered from the additional_special_tokens attribute of the
                T5TokenizerFast instance.

        Raises:
            None.
        """
        return list(
            set(filter(lambda x: bool(re.search(r"<extra_id_\d+>", x)) is not None, self.additional_special_tokens))
        )

    def get_sentinel_token_ids(self):
        """
        This method 'get_sentinel_token_ids' in the class 'T5TokenizerFast' retrieves the token IDs corresponding
        to the sentinel tokens.

        Args:
            self (T5TokenizerFast): The instance of the T5TokenizerFast class.
                Represents the tokenizer object which provides the necessary methods for tokenization.

        Returns:
            list of int: A list containing the token IDs of the sentinel tokens obtained by converting each sentinel
                token using the 'convert_tokens_to_ids' method.

        Raises:
            None.
        """
        return [self.convert_tokens_to_ids(token) for token in self.get_sentinel_tokens()]

mindnlp.transformers.models.t5.tokenization_t5_fast.T5TokenizerFast.can_save_slow_tokenizer: bool property

This method checks if the slow tokenizer can be saved.

PARAMETER	DESCRIPTION
self	The instance of the T5TokenizerFast class. It is used to access the vocab_file attribute which is required for checking if the slow tokenizer can be saved. TYPE: T5TokenizerFast

RETURNS	DESCRIPTION
bool	Returns a boolean value indicating whether the slow tokenizer can be saved. True if the vocab_file exists, otherwise False. TYPE: bool

mindnlp.transformers.models.t5.tokenization_t5_fast.T5TokenizerFast.__init__(vocab_file=None, tokenizer_file=None, eos_token='</s>', unk_token='<unk>', pad_token='<pad>', extra_ids=100, additional_special_tokens=None, **kwargs)

Initializes a new instance of the T5TokenizerFast class.

PARAMETER	DESCRIPTION
self	The instance of the T5TokenizerFast class that the method is called on. TYPE: T5TokenizerFast
vocab_file	The path to the vocabulary file. Default is None. TYPE: str DEFAULT: None
tokenizer_file	The path to the tokenizer file. Default is None. TYPE: str DEFAULT: None
eos_token	The end-of-sentence token. Default is ''. TYPE: str DEFAULT: '</s>'
unk_token	The unknown token. Default is ''. TYPE: str DEFAULT: '<unk>'
pad_token	The padding token. Default is ''. TYPE: str DEFAULT: '<pad>'
extra_ids	The number of extra tokens to be added. Default is 100. TYPE: int DEFAULT: 100
additional_special_tokens	Additional special tokens to be added. Default is None. If provided, it must include the extra_ids tokens. If not provided, extra_ids number of '' tokens will be added automatically. If provided and no '' tokens are found, extra_ids number of '' tokens will be added automatically. TYPE: list DEFAULT: None
**kwargs	Additional keyword arguments. TYPE: dict DEFAULT: {}

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If both extra_ids and additional_special_tokens are provided, but additional_special_tokens does not include the extra_ids tokens.

Source code in mindnlp/transformers/models/t5/tokenization_t5_fast.py

def __init__(
    self,
    vocab_file=None,
    tokenizer_file=None,
    eos_token="</s>",
    unk_token="<unk>",
    pad_token="<pad>",
    extra_ids=100,
    additional_special_tokens=None,
    **kwargs,
):
    """
    Initializes a new instance of the T5TokenizerFast class.

    Args:
        self (T5TokenizerFast): The instance of the T5TokenizerFast class that the method is called on.
        vocab_file (str, optional): The path to the vocabulary file. Default is None.
        tokenizer_file (str, optional): The path to the tokenizer file. Default is None.
        eos_token (str, optional): The end-of-sentence token. Default is '</s>'.
        unk_token (str, optional): The unknown token. Default is '<unk>'.
        pad_token (str, optional): The padding token. Default is '<pad>'.
        extra_ids (int, optional): The number of extra tokens to be added. Default is 100.
        additional_special_tokens (list, optional):
            Additional special tokens to be added. Default is None.

            - If provided, it must include the extra_ids tokens.
            - If not provided, extra_ids number of '<extra_id_i>' tokens will be added automatically.
            - If provided and no '<extra_id_i>' tokens are found, extra_ids number of '<extra_id_i>' tokens
            will be added automatically.
        **kwargs (dict): Additional keyword arguments.

    Returns:
        None.

    Raises:
        ValueError: If both extra_ids and additional_special_tokens are provided, but additional_special_tokens
            does not include the extra_ids tokens.
    """
    # Add extra_ids to the special token list
    if additional_special_tokens is not None:
        extra_tokens = [x for x in additional_special_tokens if "<extra_id_" in str(x)]
        if len(extra_tokens) < 1:
            additional_special_tokens += [f"<extra_id_{i}>" for i in range(extra_ids)]
        elif extra_ids > 0 and extra_ids != len(extra_tokens):
            raise ValueError(
                f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
                " tokens"
            )
    else:
        extra_tokens = [f"<extra_id_{i}>" for i in range(extra_ids)]
        additional_special_tokens = extra_tokens

    super().__init__(
        vocab_file,
        tokenizer_file=tokenizer_file,
        eos_token=eos_token,
        unk_token=unk_token,
        pad_token=pad_token,
        extra_ids=extra_ids,
        additional_special_tokens=additional_special_tokens,
        **kwargs,
    )

    self.vocab_file = vocab_file
    self._extra_ids = extra_ids

mindnlp.transformers.models.t5.tokenization_t5_fast.T5TokenizerFast.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A sequence has the following format:

• single sequence: X </s>
• pair of sequences: A </s> B </s>

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp/transformers/models/t5/tokenization_t5_fast.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. A sequence has the following format:

    - single sequence: `X </s>`
    - pair of sequences: `A </s> B </s>`

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    token_ids_0 = token_ids_0 + [self.eos_token_id]
    if token_ids_1 is None:
        return self.prefix_tokens + token_ids_0
    token_ids_1 = token_ids_1 + [self.eos_token_id]
    return self.prefix_tokens + token_ids_0 + token_ids_1

mindnlp.transformers.models.t5.tokenization_t5_fast.T5TokenizerFast.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make use of token type ids, therefore a list of zeros is returned.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of zeros.

Source code in mindnlp/transformers/models/t5/tokenization_t5_fast.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
    use of token type ids, therefore a list of zeros is returned.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of zeros.
    """
    eos = [self.eos_token_id]

    if token_ids_1 is None:
        return len(token_ids_0 + eos) * [0]
    return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

mindnlp.transformers.models.t5.tokenization_t5_fast.T5TokenizerFast.get_sentinel_token_ids()

This method 'get_sentinel_token_ids' in the class 'T5TokenizerFast' retrieves the token IDs corresponding to the sentinel tokens.

PARAMETER	DESCRIPTION
self	The instance of the T5TokenizerFast class. Represents the tokenizer object which provides the necessary methods for tokenization. TYPE: T5TokenizerFast

RETURNS	DESCRIPTION
	list of int: A list containing the token IDs of the sentinel tokens obtained by converting each sentinel token using the 'convert_tokens_to_ids' method.

Source code in mindnlp/transformers/models/t5/tokenization_t5_fast.py

def get_sentinel_token_ids(self):
    """
    This method 'get_sentinel_token_ids' in the class 'T5TokenizerFast' retrieves the token IDs corresponding
    to the sentinel tokens.

    Args:
        self (T5TokenizerFast): The instance of the T5TokenizerFast class.
            Represents the tokenizer object which provides the necessary methods for tokenization.

    Returns:
        list of int: A list containing the token IDs of the sentinel tokens obtained by converting each sentinel
            token using the 'convert_tokens_to_ids' method.

    Raises:
        None.
    """
    return [self.convert_tokens_to_ids(token) for token in self.get_sentinel_tokens()]

mindnlp.transformers.models.t5.tokenization_t5_fast.T5TokenizerFast.get_sentinel_tokens()

This method retrieves the sentinel tokens from the T5TokenizerFast instance.

PARAMETER	DESCRIPTION
self	The T5TokenizerFast instance. TYPE: T5TokenizerFast

RETURNS	DESCRIPTION
list	A list of sentinel tokens filtered from the additional_special_tokens attribute of the T5TokenizerFast instance.

Source code in mindnlp/transformers/models/t5/tokenization_t5_fast.py

def get_sentinel_tokens(self):
    """
    This method retrieves the sentinel tokens from the T5TokenizerFast instance.

    Args:
        self (T5TokenizerFast): The T5TokenizerFast instance.

    Returns:
        list: A list of sentinel tokens filtered from the additional_special_tokens attribute of the
            T5TokenizerFast instance.

    Raises:
        None.
    """
    return list(
        set(filter(lambda x: bool(re.search(r"<extra_id_\d+>", x)) is not None, self.additional_special_tokens))
    )

mindnlp.transformers.models.t5.tokenization_t5_fast.T5TokenizerFast.save_vocabulary(save_directory, filename_prefix=None)

Saves the vocabulary for a slow tokenizer.

PARAMETER	DESCRIPTION
self	An instance of the T5TokenizerFast class. TYPE: T5TokenizerFast
save_directory	The directory where the vocabulary will be saved. TYPE: str
filename_prefix	A prefix to be added to the filename. Defaults to None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the path to the saved vocabulary file.

RAISES	DESCRIPTION
ValueError	If the fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer.
FileNotFoundError	If the save_directory does not exist.

Note

The method assumes that the fast tokenizer has the necessary information to save the vocabulary for a slow tokenizer.

Example

>>> tokenizer = T5TokenizerFast()
>>> tokenizer.save_vocabulary('/path/to/save', 'vocab')

Source code in mindnlp/transformers/models/t5/tokenization_t5_fast.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Saves the vocabulary for a slow tokenizer.

    Args:
        self (T5TokenizerFast): An instance of the T5TokenizerFast class.
        save_directory (str): The directory where the vocabulary will be saved.
        filename_prefix (Optional[str], optional): A prefix to be added to the filename. Defaults to None.

    Returns:
        Tuple[str]: A tuple containing the path to the saved vocabulary file.

    Raises:
        ValueError: If the fast tokenizer does not have the necessary information to save the vocabulary for
            a slow tokenizer.
        FileNotFoundError: If the save_directory does not exist.

    Note:
        The method assumes that the fast tokenizer has the necessary information to save the vocabulary for
        a slow tokenizer.

    Example:
        ```python
        >>> tokenizer = T5TokenizerFast()
        >>> tokenizer.save_vocabulary('/path/to/save', 'vocab')
        ```
    """
    if not self.can_save_slow_tokenizer:
        raise ValueError(
            "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
            "tokenizer."
        )

    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory")
        return
    out_vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )

    if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
        copyfile(self.vocab_file, out_vocab_file)
        logger.info(f"Copy vocab file to {out_vocab_file}")

    return (out_vocab_file,)

mindnlp.transformers.models.t5.chatyuan_tokenizer

Tokenization classes for ChatYuan.

mindnlp.transformers.models.t5.chatyuan_tokenizer.ChatYuanTokenizer

Bases: PreTrainedTokenizer

Tokenizer for ChatYuan

Source code in mindnlp/transformers/models/t5/chatyuan_tokenizer.py

class ChatYuanTokenizer(PreTrainedTokenizer):
    """Tokenizer for ChatYuan"""
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_map = PRETRAINED_VOCAB_MAP
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(self, vocab_file, **kwargs):
        """
        __init__

        Initializes a new instance of the ChatYuanTokenizer class.

        Args:
            vocab_file (str): The file path to the vocabulary file used for tokenization.
            **kwargs: Additional keyword arguments.
                return_token (bool, optional): A flag indicating whether to return the token. Defaults to False.

        Returns:
            None: This method does not return any value.

        Raises:
            None.
        """
        super().__init__()

        return_token = kwargs.pop('return_token', False)
        self.return_token = return_token

        self.vocab_file = vocab_file
        self._tokenizer = self.get_spm_processor()

    def get_spm_processor(self):
        """Get SentencePieceProcessor Tokenizer."""
        tokenizer = spm.SentencePieceProcessor()
        tokenizer.Load(self.vocab_file)
        return tokenizer

    def execute_py(self, text_input):
        """Execute method."""
        return self.tokenize(text_input)

    def _execute_py(self, text_input):
        """Execute method."""
        return self._tokenize(text_input)

    def tokenize(self, text_input) -> List[str]:
        """
        This method tokenizes the input text using the ChatYuanTokenizer.

        Args:
            self (ChatYuanTokenizer): An instance of the ChatYuanTokenizer class.
            text_input (str): The input text to be tokenized.

        Returns:
            List[str]: A list of strings representing the tokens extracted from the input text.

        Raises:
            This method does not explicitly raise any exceptions.
        """
        return self._execute_py(text_input)

    def _tokenize(self, text_input):
        """
        Returns a tokenized string.
        """
        text_input = self._convert_to_unicode(text_input)

        tokens = self._tokenizer.encode(text_input, out_type=str)
        if self.return_token:
            return tokens
        # return ids
        return np.array(self.convert_tokens_to_ids(tokens))

    def _convert_to_unicode(self, text_input):
        """Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
        if isinstance(text_input, str):
            return text_input
        if isinstance(text_input, bytes):
            return text_input.decode("utf-8", "ignore")
        if isinstance(text_input, np.ndarray):
            if text_input.dtype.type is np.bytes_:
                text_input = np.char.decode(text_input, "utf-8")
            return str(text_input)
        raise ValueError(f"Unsupported string type: {type(text_input)}, {text_input.dtype}")

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self._tokenizer.piece_to_id(token)

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        token = self._tokenizer.IdToPiece(index)
        return token

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        # since we manually add the prefix space, we have to remove it when decoding
        if tokens[0].startswith(SPIECE_UNDERLINE):
            tokens[0] = tokens[0][1:]

        current_sub_tokens = []
        out_string = ""
        prev_is_special = False
        for i, token in enumerate(tokens):
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                if not prev_is_special and i != 0 and self.legacy:
                    out_string += " "
                out_string += self._tokenizer.decode(current_sub_tokens) + token
                prev_is_special = True
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
                prev_is_special = False
        out_string += self._tokenizer.decode(current_sub_tokens)
        return out_string

mindnlp.transformers.models.t5.chatyuan_tokenizer.ChatYuanTokenizer.__init__(vocab_file, **kwargs)

init

Initializes a new instance of the ChatYuanTokenizer class.

PARAMETER	DESCRIPTION
vocab_file	The file path to the vocabulary file used for tokenization. TYPE: str
**kwargs	Additional keyword arguments. return_token (bool, optional): A flag indicating whether to return the token. Defaults to False. DEFAULT: {}

RETURNS	DESCRIPTION
None	This method does not return any value.

Source code in mindnlp/transformers/models/t5/chatyuan_tokenizer.py

def __init__(self, vocab_file, **kwargs):
    """
    __init__

    Initializes a new instance of the ChatYuanTokenizer class.

    Args:
        vocab_file (str): The file path to the vocabulary file used for tokenization.
        **kwargs: Additional keyword arguments.
            return_token (bool, optional): A flag indicating whether to return the token. Defaults to False.

    Returns:
        None: This method does not return any value.

    Raises:
        None.
    """
    super().__init__()

    return_token = kwargs.pop('return_token', False)
    self.return_token = return_token

    self.vocab_file = vocab_file
    self._tokenizer = self.get_spm_processor()

mindnlp.transformers.models.t5.chatyuan_tokenizer.ChatYuanTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (string) in a single string.

Source code in mindnlp/transformers/models/t5/chatyuan_tokenizer.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (string) in a single string."""
    # since we manually add the prefix space, we have to remove it when decoding
    if tokens[0].startswith(SPIECE_UNDERLINE):
        tokens[0] = tokens[0][1:]

    current_sub_tokens = []
    out_string = ""
    prev_is_special = False
    for i, token in enumerate(tokens):
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
            if not prev_is_special and i != 0 and self.legacy:
                out_string += " "
            out_string += self._tokenizer.decode(current_sub_tokens) + token
            prev_is_special = True
            current_sub_tokens = []
        else:
            current_sub_tokens.append(token)
            prev_is_special = False
    out_string += self._tokenizer.decode(current_sub_tokens)
    return out_string

mindnlp.transformers.models.t5.chatyuan_tokenizer.ChatYuanTokenizer.execute_py(text_input)

Execute method.

Source code in mindnlp/transformers/models/t5/chatyuan_tokenizer.py

def execute_py(self, text_input):
    """Execute method."""
    return self.tokenize(text_input)

mindnlp.transformers.models.t5.chatyuan_tokenizer.ChatYuanTokenizer.get_spm_processor()

Get SentencePieceProcessor Tokenizer.

Source code in mindnlp/transformers/models/t5/chatyuan_tokenizer.py

def get_spm_processor(self):
    """Get SentencePieceProcessor Tokenizer."""
    tokenizer = spm.SentencePieceProcessor()
    tokenizer.Load(self.vocab_file)
    return tokenizer

mindnlp.transformers.models.t5.chatyuan_tokenizer.ChatYuanTokenizer.tokenize(text_input)

This method tokenizes the input text using the ChatYuanTokenizer.

PARAMETER	DESCRIPTION
self	An instance of the ChatYuanTokenizer class. TYPE: ChatYuanTokenizer
text_input	The input text to be tokenized. TYPE: str

RETURNS	DESCRIPTION
List[str]	List[str]: A list of strings representing the tokens extracted from the input text.

Source code in mindnlp/transformers/models/t5/chatyuan_tokenizer.py

def tokenize(self, text_input) -> List[str]:
    """
    This method tokenizes the input text using the ChatYuanTokenizer.

    Args:
        self (ChatYuanTokenizer): An instance of the ChatYuanTokenizer class.
        text_input (str): The input text to be tokenized.

    Returns:
        List[str]: A list of strings representing the tokens extracted from the input text.

    Raises:
        This method does not explicitly raise any exceptions.
    """
    return self._execute_py(text_input)