Amr Keleg AMR-KELEG

Unigram Weighting

Precision: 0.84776 +- 0.00871

Recall: 0.83987 +- 0.00888

	import numpy as np
	from transformers import AutoTokenizer, AutoModelForSequenceClassification

	model_name = "AMR-KELEG/Sentence-ALDi"
	tokenizer = AutoTokenizer.from_pretrained(model_name)
	model = AutoModelForSequenceClassification.from_pretrained(model_name)

	def compute_score(sentence):
	"""Returns a normalized divergence 'distance' score from MSA in [0, 1]"""
	# Warning -- inputs longer than 512 subtokens are truncated

	import re
	import torch
	import pandas as pd
	from tqdm import tqdm
	from transformers import AutoModelForSequenceClassification, AutoTokenizer

	DIALECTS = [
	"Algeria",
	"Bahrain",
	"Egypt",