From 4f9459e55844c690ab8a1b778abcdab7c08ab7a4 Mon Sep 17 00:00:00 2001 From: laurent Date: Sun, 22 Feb 2026 16:47:56 +0100 Subject: [PATCH] fix: implement robust model mapping using slugs and normalized names --- README.md | 104 +++++++++++++++++++------------------- update_models.py | 128 +++++++++++++++++++++++------------------------ 2 files changed, 116 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index e20e648..6625a7d 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,76 @@ # Table des Modèles Mammouth.ai -*Mise à jour automatique via Artificial Analysis & Mammouth Public API* +*Généré automatiquement à partir des benchmarks d'Artificial Analysis et des tarifs Mammouth.ai.* -Dernière mise à jour : 2026-02-22 16:45:41 +Dernière mise à jour : 2026-02-22 16:47:56 ## Coding -| Modèle | Prix (In / Out / 1M) | Intelligence Index | Vitesse (TPS) | +| Modèle | Prix (In / Out / 1M) | Score (Intelligence) | Vitesse (TPS) | | :--- | :--- | :--- | :--- | -| grok-code-fast-1 | $0.20 / $1.50 | N/A | N/A | +| grok-code-fast-1 | $0.20 / $1.50 | **23.7** | 314.1 | | qwen3-coder | $0.22 / $0.95 | N/A | N/A | | codestral-2508 | $0.30 / $0.90 | N/A | N/A | -| qwen3-coder-flash | $0.50 / $2.00 | N/A | N/A | | qwen3-coder-plus | $1.80 / $9.00 | N/A | N/A | +| qwen3-coder-flash | $0.50 / $2.00 | N/A | N/A | ## Agents -| Modèle | Prix (In / Out / 1M) | Intelligence Index | Vitesse (TPS) | +| Modèle | Prix (In / Out / 1M) | Score (Intelligence) | Vitesse (TPS) | | :--- | :--- | :--- | :--- | +| sonar-pro | $3.00 / $15.00 | **15.2** | 129.2 | | sonar-deep-research | $2.00 / $8.00 | N/A | N/A | -| sonar-pro | $3.00 / $15.00 | N/A | N/A | ## General -| Modèle | Prix (In / Out / 1M) | Intelligence Index | Vitesse (TPS) | +| Modèle | Prix (In / Out / 1M) | Score (Intelligence) | Vitesse (TPS) | | :--- | :--- | :--- | :--- | -| text-embedding-3-small | $0.02 / $0.00 | N/A | N/A | -| gpt-5-nano | $0.05 / $0.40 | N/A | N/A | -| llama-4-scout | $0.08 / $0.50 | N/A | N/A | -| gpt-4.1-nano | $0.10 / $0.40 | N/A | N/A | -| mistral-small-3.2-24b-instruct | $0.10 / $0.30 | N/A | N/A | +| gemini-3-pro-preview | $2.00 / $12.00 | **48.4** | 138.8 | +| kimi-k2.5 | $0.60 / $3.00 | **46.7** | 44.8 | +| claude-opus-4-6 | $5.00 / $25.00 | **46.4** | 66.8 | +| claude-opus-4-5 | $5.00 / $25.00 | **43.0** | 65.1 | +| grok-4-0709 | $3.00 / $15.00 | **41.4** | 39.4 | +| gpt-5-mini | $0.25 / $2.00 | **41.0** | 75.3 | +| kimi-k2-thinking | $0.55 / $2.50 | **40.7** | 86.9 | +| gemini-3-flash-preview | $0.50 / $3.00 | **35.1** | 177.9 | +| gemini-2.5-pro | $2.50 / $15.00 | **34.5** | 158.9 | +| o4-mini | $1.10 / $4.40 | **33.0** | 133.8 | +| claude-4-sonnet-20250522 | $3.00 / $15.00 | **33.0** | 72.8 | +| deepseek-v3.2 | $0.27 / $0.42 | **32.1** | 49.1 | +| claude-3-7-sonnet-20250219 | $3.00 / $15.00 | **30.8** | N/A | +| deepseek-v3.1-terminus | $0.27 / $1.00 | **28.4** | N/A | +| deepseek-v3.1 | $0.27 / $1.00 | **28.0** | N/A | +| deepseek-r1-0528 | $0.50 / $2.18 | **27.0** | N/A | +| gpt-5-nano | $0.05 / $0.40 | **26.7** | 130.9 | +| kimi-k2-instruct | $0.50 / $2.50 | **26.2** | 40.8 | +| gpt-4.1 | $2.00 / $8.00 | **25.6** | 103.9 | +| grok-3 | $3.00 / $15.00 | **25.0** | 67.7 | +| grok-4-1-fast | $0.20 / $0.50 | **23.5** | 119.7 | +| mistral-large-3 | $0.50 / $1.50 | **22.7** | 55.9 | +| gpt-4.1-mini | $0.40 / $1.60 | **22.4** | 77.4 | +| mistral-medium-3.1 | $0.40 / $2.00 | **21.1** | 86.8 | +| gemini-2.5-flash | $0.30 / $2.50 | **20.5** | 235.9 | +| mistral-medium-3 | $0.40 / $2.00 | **18.7** | 90.2 | +| claude-3-5-haiku-20241022 | $0.80 / $4.00 | **18.7** | 46.4 | +| llama-4-maverick | $0.15 / $0.60 | **18.3** | 126.7 | +| gpt-4o | $2.50 / $10.00 | **17.3** | 168.6 | +| deepseek-v3-0324 | $0.25 / $1.00 | **16.4** | N/A | +| claude-3-5-sonnet-20241022 | $3.00 / $15.00 | **15.9** | N/A | +| llama-4-scout | $0.08 / $0.50 | **13.5** | 158.6 | +| gpt-4.1-nano | $0.10 / $0.40 | **12.9** | 141.9 | +| mistral-large-2411 | $2.00 / $6.00 | **9.9** | N/A | | text-embedding-3-large | $0.13 / $0.00 | N/A | N/A | -| llama-4-maverick | $0.15 / $0.60 | N/A | N/A | -| grok-4-1-fast | $0.20 / $0.50 | N/A | N/A | -| gpt-5-mini | $0.25 / $2.00 | N/A | N/A | -| deepseek-v3-0324 | $0.25 / $1.00 | N/A | N/A | -| deepseek-v3.2 | $0.27 / $0.42 | N/A | N/A | -| deepseek-v3.1 | $0.27 / $1.00 | N/A | N/A | -| deepseek-v3.1-terminus | $0.27 / $1.00 | N/A | N/A | -| deepseek-v3.2-exp | $0.27 / $0.41 | N/A | N/A | -| gemini-2.5-flash-image | $0.30 / $2.50 | N/A | N/A | -| grok-3-mini | $0.30 / $0.50 | N/A | N/A | -| gemini-2.5-flash | $0.30 / $2.50 | N/A | N/A | -| grok-4-fast-non-reasoning | $0.40 / $1.00 | N/A | N/A | -| mistral-medium-3 | $0.40 / $2.00 | N/A | N/A | -| gpt-4.1-mini | $0.40 / $1.60 | N/A | N/A | -| mistral-medium-3.1 | $0.40 / $2.00 | N/A | N/A | -| gemini-3-flash-preview | $0.50 / $3.00 | N/A | N/A | -| kimi-k2-instruct | $0.50 / $2.50 | N/A | N/A | -| mistral-large-3 | $0.50 / $1.50 | N/A | N/A | -| deepseek-r1-0528 | $0.50 / $2.18 | N/A | N/A | -| kimi-k2-thinking | $0.55 / $2.50 | N/A | N/A | -| kimi-k2.5 | $0.60 / $3.00 | N/A | N/A | -| claude-3-5-haiku-20241022 | $0.80 / $4.00 | N/A | N/A | -| claude-haiku-4-5 | $1.00 / $5.00 | N/A | N/A | -| o4-mini | $1.10 / $4.40 | N/A | N/A | | gpt-5-chat | $1.25 / $10.00 | N/A | N/A | -| gpt-5.1-chat | $1.25 / $10.00 | N/A | N/A | -| gpt-5.2-chat | $1.75 / $14.00 | N/A | N/A | -| mistral-large-2411 | $2.00 / $6.00 | N/A | N/A | -| gpt-4.1 | $2.00 / $8.00 | N/A | N/A | -| gemini-3-pro-image-preview | $2.00 / $12.00 | N/A | N/A | -| gemini-3-pro-preview | $2.00 / $12.00 | N/A | N/A | -| gpt-4o | $2.50 / $10.00 | N/A | N/A | -| gemini-2.5-pro | $2.50 / $15.00 | N/A | N/A | -| claude-3-7-sonnet-20250219 | $3.00 / $15.00 | N/A | N/A | -| claude-4-sonnet-20250522 | $3.00 / $15.00 | N/A | N/A | +| grok-4-fast-non-reasoning | $0.40 / $1.00 | N/A | N/A | | claude-sonnet-4-5 | $3.00 / $15.00 | N/A | N/A | -| claude-3-5-sonnet-20241022 | $3.00 / $15.00 | N/A | N/A | -| grok-3 | $3.00 / $15.00 | N/A | N/A | -| grok-4-0709 | $3.00 / $15.00 | N/A | N/A | -| claude-opus-4-6 | $5.00 / $25.00 | N/A | N/A | -| claude-opus-4-5 | $5.00 / $25.00 | N/A | N/A | +| gpt-5.1-chat | $1.25 / $10.00 | N/A | N/A | +| claude-haiku-4-5 | $1.00 / $5.00 | N/A | N/A | +| gemini-2.5-flash-image | $0.30 / $2.50 | N/A | N/A | | claude-opus-4-1-20250805 | $15.00 / $75.00 | N/A | N/A | +| deepseek-v3.2-exp | $0.27 / $0.41 | N/A | N/A | +| gpt-5.2-chat | $1.75 / $14.00 | N/A | N/A | +| grok-3-mini | $0.30 / $0.50 | N/A | N/A | +| mistral-small-3.2-24b-instruct | $0.10 / $0.30 | N/A | N/A | +| gemini-3-pro-image-preview | $2.00 / $12.00 | N/A | N/A | +| text-embedding-3-small | $0.02 / $0.00 | N/A | N/A | diff --git a/update_models.py b/update_models.py index 78c69a5..fe3d51e 100644 --- a/update_models.py +++ b/update_models.py @@ -2,17 +2,19 @@ import os import requests import json import time +import re from dotenv import load_dotenv +# Charger .env.global load_dotenv("../.env.global") - AIANALASYS_APIKEY = os.getenv("AIANALASYS_APIKEY") def get_mammouth_models(): - # URL correcte fournie par l'utilisateur url = "https://api.mammouth.ai/public/models" try: - response = requests.get(url, verify=False) # verify=False au cas où il y a des soucis de certifs + # Désactiver les warnings InsecureRequest car verify=False est utilisé + requests.packages.urllib3.disable_warnings() + response = requests.get(url, verify=False) response.raise_for_status() return response.json().get('data', []) except Exception as e: @@ -27,99 +29,97 @@ def get_aa_data(): response.raise_for_status() return response.json().get('data', []) except Exception as e: - print(f"Error fetching Artificial Analysis data: {e}") + print(f"Error fetching AA data: {e}") return [] +def clean_id(model_id): + # Nettoyage agressif pour favoriser le mapping + id_clean = re.sub(r'-\d{4,8}', '', model_id.lower()) + id_clean = id_clean.replace('-latest', '').replace('-preview', '').replace('-instruct', '') + return id_clean.strip() + def generate_markdown(models_data): categories = {} for m in models_data: cat = m.get('category', 'General') - if cat not in categories: - categories[cat] = [] + if cat not in categories: categories[cat] = [] categories[cat].append(m) - + md = "# Table des Modèles Mammouth.ai\n\n" - md += "*Mise à jour automatique via Artificial Analysis & Mammouth Public API*\n\n" + md += "*Généré automatiquement à partir des benchmarks d'Artificial Analysis et des tarifs Mammouth.ai.*\n\n" md += f"Dernière mise à jour : {time.strftime('%Y-%m-%d %H:%M:%S')}\n\n" - + order = ['Coding', 'Agents', 'General'] sorted_cats = sorted(categories.keys(), key=lambda x: order.index(x) if x in order else 99) for cat in sorted_cats: - models = categories[cat] - # Tri : Score (desc), puis Prix (asc) - models.sort(key=lambda x: (x.get('score') or 0, -(x.get('price_in') or 999)), reverse=True) - md += f"## {cat}\n\n" - md += "| Modèle | Prix (In / Out / 1M) | Intelligence Index | Vitesse (TPS) |\n" + md += "| Modèle | Prix (In / Out / 1M) | Score (Intelligence) | Vitesse (TPS) |\n" md += "| :--- | :--- | :--- | :--- |\n" + models = categories[cat] + # Tri : Score (desc), puis Nom + models.sort(key=lambda x: (x.get('score') or 0), reverse=True) for m in models: - p_in = f"${m['price_in']:.2f}" - p_out = f"${m['price_out']:.2f}" - score = f"**{m['score']:.1f}**" if m['score'] else "N/A" - speed = f"{m['speed']:.1f}" if m['speed'] else "N/A" - md += f"| {m['name']} | {p_in} / {p_out} | {score} | {speed} |\n" + score_str = f"**{m['score']:.1f}**" if m['score'] else "N/A" + speed_str = f"{m['speed']:.1f}" if m['speed'] else "N/A" + md += f"| {m['name']} | ${m['price_in']:.2f} / ${m['price_out']:.2f} | {score_str} | {speed_str} |\n" md += "\n" return md def main(): - print("Fetching Mammouth public models...") - mammouth_models = get_mammouth_models() - print(f"Found {len(mammouth_models)} models from Mammouth.") + print("Fetching data from Mammouth and Artificial Analysis...") + m_models = get_mammouth_models() + aa_data = get_aa_data() - print("Fetching Artificial Analysis data...") - aa_raw = get_aa_data() - - # Mapping AA + # Mapping table (slug -> data) aa_map = {} - for aa_m in aa_raw: - m_id = aa_m.get('model_id', '').lower() - m_name = aa_m.get('model_name', '').lower() - if m_id: aa_map[m_id] = aa_m - if m_name: aa_map[m_name] = aa_m + for aa_m in aa_data: + slug = aa_m.get('slug', '').lower() + name = aa_m.get('name', '').lower() + if slug: aa_map[slug] = aa_m + if name: aa_map[name] = aa_m - enriched_models = [] - for m in mammouth_models: + enriched = [] + for m in m_models: m_id = m.get('id', '') info = m.get('model_info', {}) - if not m_id: continue - # Mapping intelligent - m_id_low = m_id.lower() - aa_info = aa_map.get(m_id_low) + m_id_clean = clean_id(m_id) + short_id = m_id_clean.split('/')[-1] - # Si pas de match exact, on cherche une correspondance partielle + # Match mapping + aa_info = aa_map.get(m_id_clean) or aa_map.get(short_id) + + # Recherche floue (ex: claude-3-5-sonnet -> claude-3.5-sonnet) if not aa_info: - for key in aa_map: - if key in m_id_low or m_id_low in key: - # On vérifie que ce n'est pas un faux positif (ex: gpt-4 vs gpt-4-turbo) - if abs(len(key) - len(m_id_low)) < 5: - aa_info = aa_map[key] - break + normalized_m_id = m_id_clean.replace('-', '').replace('.', '') + for key, val in aa_map.items(): + if key.replace('-', '').replace('.', '') == normalized_m_id: + aa_info = val + break + + price_in = float(info.get('input_cost_per_token', 0)) * 1000000 + price_out = float(info.get('output_cost_per_token', 0)) * 1000000 + + category = "General" + if any(x in m_id_clean for x in ['coding', 'code', 'starcoder', 'codestral', 'coder']): + category = "Coding" + elif any(x in m_id_clean for x in ['agent', 'hermes', 'tool', 'function', 'sonar']): + category = "Agents" - # Extraction des prix - try: - price_in = float(info.get('input_cost_per_token', 0)) * 1000000 - price_out = float(info.get('output_cost_per_token', 0)) * 1000000 - except: - price_in, price_out = 0, 0 - score = None speed = None if aa_info: evals = aa_info.get('evaluations', {}) - score = evals.get('artificial_analysis_intelligence_index') - speed = aa_info.get('median_output_tokens_per_second') + # On prend le score coding si c'est la catégorie, sinon intelligence index + score = evals.get('artificial_analysis_coding_index') if category == "Coding" else None + if not score: + score = evals.get('artificial_analysis_intelligence_index') - # Catégorisation - category = "General" - if any(x in m_id_low for x in ['coding', 'code', 'starcoder', 'coder', 'codestral']): - category = "Coding" - elif any(x in m_id_low for x in ['agent', 'hermes', 'tool', 'function', 'sonar']): - category = "Agents" + speed = aa_info.get('median_output_tokens_per_second') - enriched_models.append({ + enriched.append({ 'name': m_id, 'price_in': price_in, 'price_out': price_out, @@ -128,13 +128,13 @@ def main(): 'category': category }) - # Filtrer les modèles (on garde tout ce qui a un prix ou un score) - final_list = [m for m in enriched_models if m['price_in'] > 0 or m['score'] is not None] + # On ne garde que les modèles avec prix > 0 + final = [x for x in enriched if x['price_in'] > 0] with open("README.md", "w", encoding="utf-8") as f: - f.write(generate_markdown(final_list)) + f.write(generate_markdown(final)) - print(f"Done! README.md updated with {len(final_list)} models.") + print(f"Success! {len(final)} models processed.") if __name__ == "__main__": main()