ai-agents-model-adaptation-using-mt5encodermodel-or-t5encodermodel-with-huggi-e26b303b

THEN Create a custom module that wraps the encoder model (e.g., MT5EncoderModel) with a sequence classification head. In the forward method, accept 'labels', compute logits, calculate loss using cross-entropy, and return the loss and logits. This mirrors how BertForSequenceClassification is implemented.

class MT5ForSequenceClassification(nn.Module):
    def __init__(self, model_name, num_labels):
        super().__init__()
        self.encoder = MT5EncoderModel.from_pretrained(model_name)
        self.dropout = nn.Dropout(0.1)
        self.classifier = nn.Linear(self.encoder.config.hidden_size, num_labels)

    def forward(self, input_ids, attention_mask=None, labels=None):
        outputs = self.encoder(input_ids, attention_mask=attention_mask)
        pooled = outputs.last_hidden_state[:, 0, :]  # take <s> token
        pooled = self.dropout(pooled)
        logits = self.classifier(pooled)
        loss = None
        if labels is not None:
            loss_fn = nn.CrossEntropyLoss()
            loss = loss_fn(logits, labels)
        return (loss, logits) if loss is not None else logits