# --- Correcteurs : chacun renvoie une fonction texte -> bool ---
def _nombres(texte):
return [float(x.replace(",", "")) for x in re.findall(r"-?\d[\d,]*(?:\.\d+)?", texte)]
def dernier_nombre(attendu, tol=1e-6):
"""Le DERNIER nombre de la reponse ('9 x 10 x 11 = 990' -> 990) doit valoir `attendu`."""
return lambda t: bool(_nombres(t)) and abs(_nombres(t)[-1] - attendu) <= tol
def premier_candidat(attendu, candidats):
"""Parmi des candidats connus, celui qui apparait EN PREMIER doit etre `attendu`."""
def f(t):
tl = t.lower()
pos = {c: tl.find(c) for c in candidats if tl.find(c) >= 0}
return bool(pos) and min(pos, key=pos.get) == attendu
return f
def contient(*mots):
return lambda t: any(m.lower() in t.lower() for m in mots)
def oui_non(attendu):
def f(t):
m = re.search(r"\b(yes|no)\b", t.lower())
return bool(m) and m.group(1) == attendu
return f
def nb_mots_exact(n):
return lambda t: len(re.findall(r"[A-Za-z']+", t.strip().split("\n")[0])) == n
def derivee_3x2(t):
s = t.lower().replace(" ", "").replace("*", "").replace("²", "^2")
return "3x^2" in s
def proba_1_36(t):
s = t.replace(" ", "")
return ("1/36" in s or re.search(r"frac\{1\}\{36\}", s) is not None
or any(abs(x - 1 / 36) < 1e-3 for x in _nombres(t)))
ITEMS = []
def item(domaine, ident, prompt, correcteur, max_tokens=60, poids=1.0, **extra):
ITEMS.append({"domaine": domaine, "id": ident, "prompt": prompt, "correcteur": correcteur,
"max_tokens": max_tokens, "poids": poids, **extra})
# K - connaissances generales (dont une question de sens commun)
item("K", "K-or", "What is the chemical symbol for gold? Answer with the symbol only.",
lambda t: re.search(r"\bAu\b", t) is not None)
item("K", "K-berlin", "In which year did the Berlin Wall fall? Answer with the year only.", dernier_nombre(1989))
item("K", "K-planete", "What is the largest planet in the Solar System? Answer with one word.", contient("jupiter"))
item("K", "K-hugo", "Who wrote the novel 'Les Miserables'? Answer with the author's name only.", contient("hugo"))
item("K", "K-co2", "Which gas do plants absorb from the air for photosynthesis? Answer briefly.",
contient("carbon dioxide", "co2", "co₂"))
item("K", "K-glace", "If you leave a glass of water in a freezer overnight, what has the water turned into "
"by morning? Answer with one word.", contient("ice", "frozen", "solid"))
# RW - comprehension (dont une paire de type Winograd) et ecriture sous contrainte
item("RW", "RW-vote", "Passage: 'The committee postponed the vote because two members were absent; it will "
"reconvene on Thursday.' Question: why was the vote postponed? Answer in at most eight words.", contient("absent"))
item("RW", "RW-nage", "Passage: 'Unlike her brother, Mara never learned to swim, although she grew up by the sea.' "
"Question: can Mara swim? Answer yes or no.", oui_non("no"))
item("RW", "RW-trophee-grand", "The trophy doesn't fit in the brown suitcase because it is too big. "
"What is too big? Answer with one word.", premier_candidat("trophy", ["trophy", "suitcase"]))
item("RW", "RW-trophee-petit", "The trophy doesn't fit in the brown suitcase because it is too small. "
"What is too small? Answer with one word.", premier_candidat("suitcase", ["trophy", "suitcase"]))
item("RW", "RW-7mots", "Write one sentence about rain containing exactly seven words. Output only the sentence.",
nb_mots_exact(7))
# M - mathematiques, de l'arithmetique au calcul. Budget large : le domaine M mesure le calcul,
# pas la concision ; un modele qui pose son raisonnement doit pouvoir atteindre sa valeur finale.
BUDGET_M = 400
item("M", "M-produit", "What is 17 x 24? Answer with the number only.", dernier_nombre(408), max_tokens=BUDGET_M)
item("M", "M-equation", "Solve for x: 3x + 7 = 25. Answer with the value of x only.", dernier_nombre(6),
max_tokens=BUDGET_M)
item("M", "M-de", "A fair six-sided die is rolled twice. What is the probability that both rolls show a 6? "
"Answer as a fraction.", proba_1_36, max_tokens=BUDGET_M)
item("M", "M-derivee", "What is the derivative of x^3 with respect to x? Answer with the expression only.", derivee_3x2,
max_tokens=BUDGET_M)
item("M", "M-cercle", "What is the area of a circle of radius 3? Give the numerical value rounded to two decimals.",
dernier_nombre(28.27, tol=0.006), max_tokens=BUDGET_M)
item("M", "M-puissance", "What is 2 to the power of 10? Answer with the number only.", dernier_nombre(1024),
max_tokens=BUDGET_M)
# R - raisonnement sur le vif : deduction (R13 p. 9), theorie de l'esprit (R13 p. 9), induction, syllogisme
item("R", "R-david", "David knows Mr. Zhang's friend Jack, and Jack knows David's friend Ms. Lin. Everyone of "
"them who knows Jack has a master's degree, and everyone of them who knows Ms. Lin is from Shanghai. "
"Who is from Shanghai and has a master's degree? Answer with the name only.",
premier_candidat("david", ["david", "jack", "zhang", "lin"]))
item("R", "R-pringles", "The can of Pringles has moldy chips in it. Mary picks up the can in the supermarket and "
"walks to the cashier. Is Mary likely to be aware that the can of Pringles has moldy chips in it? "
"Answer yes or no.", oui_non("no"))
item("R", "R-suite", "What is the next number in the sequence 2, 6, 12, 20, 30, ...? Answer with the number only.",
dernier_nombre(42))
item("R", "R-tailles", "Alice is taller than Bob. Carol is shorter than Bob. Dave is taller than Alice. "
"Who is the second tallest? Answer with the name only.",
premier_candidat("alice", ["alice", "bob", "carol", "dave"]))
item("R", "R-syllogisme", "All bloops are razzies. Some razzies are lazzies. Does it necessarily follow that "
"some bloops are lazzies? Answer yes or no.", oui_non("no"))
# S - vitesse (R13 p. 12) : correct ET rapide ; le seuil est une convention de ce notebook
SEUIL_VITESSE_S = 2.0
item("S", "S-produit", "Compute 9 x 10 x 11. Answer with the number only.", dernier_nombre(990), max_tokens=16)
item("S", "S-compare", "Quickly choose the larger number: 5.11 or 5.9. Answer with the number only.",
premier_candidat("5.9", ["5.11", "5.9"]), max_tokens=16)
item("S", "S-max", "Find the largest number in '48291, 93652, 12844, 59277'. Answer with the number only.",
premier_candidat("93652", ["48291", "93652", "12844", "59277"]), max_tokens=16)
item("S", "S-hello", "After reading this, immediately say 'hello'.", contient("hello"), max_tokens=16)
from collections import Counter
print("Items par domaine :", dict(Counter(i["domaine"] for i in ITEMS)))
print("Correcteurs essayes sur des reponses fictives, avant tout appel de modele :")
c_produit = [i for i in ITEMS if i["id"] == "S-produit"][0]["correcteur"]
c_compare = [i for i in ITEMS if i["id"] == "S-compare"][0]["correcteur"]
for rep in ["990", "9 x 10 x 11 = 990", "900"]:
print(f" S-produit {rep!r:28s} -> {c_produit(rep)}")
for rep in ["5.9", "5.9 is larger than 5.11", "5.11 is larger than 5.9"]:
print(f" S-compare {rep!r:28s} -> {c_compare(rep)}")
