diff --git a/app/classifier.py b/app/classifier.py index 52cf302..4ca61bb 100644 --- a/app/classifier.py +++ b/app/classifier.py @@ -214,3 +214,67 @@ def confirm_and_apply_rule(regle_id: int, db_path: Optional[str] = None) -> Opti reclassify_all_non_manual(db_path=db_path) return rule + + +def classify_batch(mails: List[Dict[str, Any]], db_path: Optional[str] = None) -> List[Dict[str, Any]]: + """Classe un lot de mails en une seule connexion : règles lues une fois, classements existants lus en une + requête, écriture seulement quand un classement change (nouveau mail ou règle modifiée), le tout en une + transaction. Un classement manuel n'est jamais écrasé. Même résultat que classify_mail, mail par mail.""" + from app.db import get_connection + from app.config import get_tunis_now_iso + + mails = [m for m in mails if m.get("message_id")] + if not mails: + return [] + rules = list_regles(statut="active", db_path=db_path) + conn = get_connection(db_path) + try: + ids = [m["message_id"] for m in mails] + existants: Dict[str, Dict[str, Any]] = {} + for i in range(0, len(ids), 400): + lot = ids[i:i + 400] + req = ("SELECT message_id, categorie, source, horodatage, traite, expediteur, objet, date_mail, etiquettes " + "FROM classements WHERE message_id IN (%s)" % ",".join("?" * len(lot))) + for r in conn.execute(req, lot): + d = dict(r) + d["traite"] = bool(d["traite"]) + existants[d["message_id"]] = d + + maintenant = get_tunis_now_iso() + sortie: List[Dict[str, Any]] = [] + ecritures = [] + for m in mails: + mid = m["message_id"] + ex = existants.get(mid) + if ex and ex.get("source") == "manuel": + sortie.append(ex) + continue + categorie = match_mail_to_rules(m, rules) + tags = m.get("etiquettes") or m.get("tags") or [] + tags_json = json.dumps(tags) if isinstance(tags, list) else str(tags) + exp = (m.get("expediteur") or m.get("from") or "") or (ex["expediteur"] if ex else "") + obj = (m.get("objet") or m.get("subject") or "") or (ex["objet"] if ex else "") + date_mail = (m.get("date") or "") or (ex["date_mail"] if ex else "") + etiq = tags_json if tags_json != "[]" else ((ex.get("etiquettes") if ex else None) or "[]") + if ex and (ex["categorie"], ex["expediteur"], ex["objet"], ex["date_mail"], ex.get("etiquettes") or "[]") == (categorie, exp, obj, date_mail, etiq): + sortie.append(ex) # inchangé : aucune écriture + continue + traite = ex["traite"] if ex else False + ecritures.append((mid, categorie, maintenant, 1 if traite else 0, exp, obj, date_mail, etiq)) + sortie.append({"message_id": mid, "categorie": categorie, "source": "regle", "horodatage": maintenant, + "traite": traite, "expediteur": exp, "objet": obj, "date_mail": date_mail, "etiquettes": etiq}) + if ecritures: + with conn: + conn.executemany( + """ + INSERT INTO classements (message_id, categorie, source, horodatage, traite, expediteur, objet, date_mail, etiquettes) + VALUES (?, ?, 'regle', ?, ?, ?, ?, ?, ?) + ON CONFLICT(message_id) DO UPDATE SET + categorie = excluded.categorie, source = 'regle', horodatage = excluded.horodatage, + expediteur = excluded.expediteur, objet = excluded.objet, date_mail = excluded.date_mail, + etiquettes = CASE WHEN excluded.etiquettes != '[]' THEN excluded.etiquettes ELSE classements.etiquettes END + WHERE classements.source != 'manuel' + """, ecritures) + return sortie + finally: + conn.close() diff --git a/app/main.py b/app/main.py index 74ee67a..8c7c8dd 100644 --- a/app/main.py +++ b/app/main.py @@ -6,7 +6,7 @@ from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field -from app.config import get_tunis_now_iso +from app.config import get_tunis_now_iso, ENV from app.db import ( init_db, list_categories, @@ -24,6 +24,7 @@ from app.db import ( ) from app.classifier import ( classify_mail, + classify_batch, manual_classify_and_propose, confirm_and_apply_rule, reclassify_today_mails @@ -70,6 +71,8 @@ async def lifespan(app: FastAPI): reclassify_today_mails(mirror_client=mirror_client) except Exception: pass + if ENV != "test": + mirror_client.demarrer_rafraichissement() yield app = FastAPI( @@ -149,9 +152,12 @@ def get_mails_du_jour(date: Optional[str] = None, n: int = 50): "miroir_erreur": str(e) } + classes = {c["message_id"]: c for c in classify_batch(mails)} classified_results = [] for m in mails: - c = classify_mail(m) + c = classes.get(m.get("message_id")) + if c is None: + continue classified_results.append({ "message_id": m["message_id"], "date": m.get("date"), @@ -178,13 +184,15 @@ def get_mails_periode(jours: int = 7): mails = mirror_client.get_mails_since(debut, max_items=min(150 * jours, 800)) except Exception as e: return {"debut": debut, "fin": fin.isoformat(), "count": 0, "mails": [], "miroir_erreur": str(e)} + classes = {c["message_id"]: c for c in classify_batch(mails)} resultats = [] for m in mails: - if not m.get("message_id"): + c = classes.get(m.get("message_id")) + if c is None: continue resultats.append({ "message_id": m["message_id"], "date": m.get("date"), "expediteur": m.get("expediteur"), - "objet": m.get("objet"), "etiquettes": m.get("etiquettes", []), "classement": classify_mail(m), + "objet": m.get("objet"), "etiquettes": m.get("etiquettes", []), "classement": c, }) return {"debut": debut, "fin": fin.isoformat(), "count": len(resultats), "mails": resultats} diff --git a/app/mirror_client.py b/app/mirror_client.py index 85e5e06..edcf55e 100644 --- a/app/mirror_client.py +++ b/app/mirror_client.py @@ -1,4 +1,5 @@ import json +import threading import time from collections import deque from typing import Optional, Dict, Any, List @@ -8,6 +9,9 @@ import urllib.error from app.config import MIRROR_URL, read_mirror_token class MirrorClient: + # Le miroir renvoie au plus 100 mails par lecture : toutes les vues partagent donc une seule clé de cache. + MIRROR_MAX_LATEST = 100 + def __init__(self, base_url: Optional[str] = None, token: Optional[str] = None, cache_ttl_seconds: int = 60): self.base_url = (base_url or MIRROR_URL).rstrip("/") self._token = token or read_mirror_token() @@ -16,6 +20,8 @@ class MirrorClient: self._cache: Dict[str, tuple[float, Any]] = {} self._call_times: deque = deque() self._request_counter = 0 + self._lock = threading.Lock() + self._refresh_started = False def _check_rate_limit(self) -> None: """Applique strictement la limite de 60 requêtes par minute.""" @@ -70,12 +76,21 @@ class MirrorClient: except Exception as e: raise ConnectionError(f"Échec initialisation session MCP miroir: {e}") - def call_tool(self, tool_name: str, arguments: Optional[Dict[str, Any]] = None, use_cache: bool = True) -> Any: + def call_tool(self, tool_name: str, arguments: Optional[Dict[str, Any]] = None, use_cache: bool = True, force: bool = False) -> Any: + """Appel d'un outil du miroir : cache d'abord (sauf force), réseau sérialisé par un verrou + (la tâche de fond et les requêtes ne partagent jamais la session en même temps).""" arguments = arguments or {} cache_key = f"{tool_name}:{json.dumps(arguments, sort_keys=True)}" + if use_cache and not force and cache_key in self._cache: + ts, val = self._cache[cache_key] + if time.time() - ts < self.cache_ttl: + return val + with self._lock: + return self._appel_reseau(tool_name, arguments, use_cache, cache_key, force) + def _appel_reseau(self, tool_name: str, arguments: Dict[str, Any], use_cache: bool, cache_key: str, force: bool) -> Any: now = time.time() - if use_cache and cache_key in self._cache: + if use_cache and not force and cache_key in self._cache: ts, val = self._cache[cache_key] if now - ts < self.cache_ttl: return val @@ -161,7 +176,7 @@ class MirrorClient: """Récupère les derniers emails (mirror_mail_latest).""" res = self.call_tool("mirror_mail_latest", { "folder": folder, - "n": n, + "n": min(n, self.MIRROR_MAX_LATEST), "inclure_exclus": inclure_exclus }) if isinstance(res, dict) and "donnees" in res: @@ -233,3 +248,24 @@ class MirrorClient: "apercu": m.get("snippet") or m.get("apercu") or "" }) return out + + def rafraichir_derniers(self) -> None: + """Relit les derniers mails en ignorant le cache (même clé que les vues).""" + self.call_tool("mirror_mail_latest", {"folder": "*", "n": self.MIRROR_MAX_LATEST, "inclure_exclus": False}, force=True) + + def demarrer_rafraichissement(self, intervalle: int = 45) -> None: + """Préchauffe le cache puis le rafraîchit en tâche de fond (1 lecture toutes les 45 s, loin des 60 par minute), + pour que l'ouverture d'une vue n'attende jamais le miroir. Idempotent.""" + if self._refresh_started: + return + self._refresh_started = True + + def boucle() -> None: + while True: + try: + self.rafraichir_derniers() + except Exception: + pass # le cache expire seul et la requête suivante relira le miroir + time.sleep(intervalle) + + threading.Thread(target=boucle, name="miroir-rafraichissement", daemon=True).start() diff --git a/app/static/app.js b/app/static/app.js index 58cb63b..a342cc5 100644 --- a/app/static/app.js +++ b/app/static/app.js @@ -29,7 +29,7 @@ var COULEUR_NOM = { 'Tunis': 1, 'Demande FB': 2, 'CI': 3, 'Câbles': 4, 'Sans objet': 5, 'À classer': 0 }; var SYSTEME = 'À classer'; - var S = { vue: 'tri', jour: auj(), filtre: null, cats: [], mails: [], etat: 'chargement', banniere: null, notif: null, regles: [], reglesEtat: 'ok', miroirErreur: false, majHeure: '', groupe: { jours: 7, cat: null, etq: null, mails: [], etat: 'chargement' } }; + var S = { chargeA: 0, chargeJour: '', vue: 'tri', jour: auj(), filtre: null, cats: [], mails: [], etat: 'chargement', banniere: null, notif: null, regles: [], reglesEtat: 'ok', miroirErreur: false, majHeure: '', groupe: { jours: 7, cat: null, etq: null, mails: [], etat: 'chargement' } }; /* ---------- Utilitaires ---------- */ function auj() { return new Intl.DateTimeFormat('en-CA', { timeZone: TZ }).format(new Date()); } @@ -120,7 +120,7 @@ var r = rs[1]; S.miroirErreur = !!r.miroir_erreur; S.mails = (r.mails || []).map(normaliser).sort(function (a, b) { return String(b.date).localeCompare(String(a.date)); }); - S.etat = 'ok'; S.majHeure = heureTunis(new Date().toISOString()); + S.etat = 'ok'; S.majHeure = heureTunis(new Date().toISOString()); S.chargeA = Date.now(); S.chargeJour = jour; rendre(); }).catch(function (e) { if (e.message === 'session') return; if (jour !== S.jour) return; S.etat = 'erreur'; rendre(); }); } @@ -149,7 +149,9 @@ } function allerA(vue) { S.vue = vue; S.banniere = null; - if (vue === 'tri' || vue === 'categories') chargerMails(); + if (vue === 'tri' || vue === 'categories') { + if (S.etat === 'ok' && S.chargeJour === S.jour && Date.now() - S.chargeA < 30000) rendre(); else chargerMails(); + } else if (vue === 'groupes') chargerGroupe(); else if (vue === 'regles') chargerRegles(); else rendre(); diff --git a/app/static/index.html b/app/static/index.html index 74dc55b..fbd40e2 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -10,7 +10,7 @@ - +
@@ -20,6 +20,6 @@
- + diff --git a/tests/test_perf_065.py b/tests/test_perf_065.py new file mode 100644 index 0000000..b9f2550 --- /dev/null +++ b/tests/test_perf_065.py @@ -0,0 +1,66 @@ +import time +from pathlib import Path + +from app.classifier import classify_batch, classify_mail +from app.db import save_classement_manuel, get_classement, add_regle +from app.mirror_client import MirrorClient + +STATIC = Path(__file__).resolve().parent.parent / "app" / "static" + + +def _mail(i, objet="Bonjour", tags=None, exp="a@b.tn"): + return {"message_id": "m%d" % i, "date": "2026-11-17T09:%02d:00+01:00" % i, "expediteur": exp, + "objet": objet, "etiquettes": tags or []} + + +def test_lot_equivalent_au_classement_mail_par_mail(temp_db): + mails = [_mail(1, tags=["tunis"]), _mail(2, objet=""), _mail(3), _mail(4, tags=["sans_objet"])] + lot = classify_batch(mails, db_path=temp_db) + seuls = [classify_mail(m, db_path=temp_db) for m in mails] + assert [(c["message_id"], c["categorie"], c["source"]) for c in lot] == [(c["message_id"], c["categorie"], c["source"]) for c in seuls] + + +def test_lot_sans_ecriture_si_inchange(temp_db): + mails = [_mail(i) for i in range(1, 6)] + premier = classify_batch(mails, db_path=temp_db) + time.sleep(1.1) + second = classify_batch(mails, db_path=temp_db) + assert [c["horodatage"] for c in premier] == [c["horodatage"] for c in second] + + +def test_lot_ne_touche_pas_au_classement_manuel(temp_db): + classify_batch([_mail(1, tags=["tunis"])], db_path=temp_db) + save_classement_manuel("m1", "CI", expediteur="a@b.tn", objet="Bonjour", date_mail="2026-11-17T09:01:00+01:00", db_path=temp_db) + res = classify_batch([_mail(1, tags=["tunis"])], db_path=temp_db) + assert res[0]["categorie"] == "CI" and res[0]["source"] == "manuel" + assert get_classement("m1", temp_db)["categorie"] == "CI" + + +def test_lot_conserve_traite_quand_la_regle_change(temp_db): + classify_batch([_mail(1)], db_path=temp_db) + from app.db import get_connection + with get_connection(temp_db) as conn: + conn.execute("UPDATE classements SET traite = 1 WHERE message_id = 'm1'") + add_regle(categorie="CI", expediteur="a@b.tn", mot_cle=None, statut="active", db_path=temp_db) + res = classify_batch([_mail(1)], db_path=temp_db) + assert res[0]["categorie"] == "CI" and res[0]["traite"] is True + + +def test_lot_ignore_un_mail_sans_identifiant(temp_db): + assert classify_batch([{"expediteur": "x"}], db_path=temp_db) == [] + + +def test_lecture_miroir_une_seule_cle_et_rafraichissement_force(monkeypatch): + appels = [] + c = MirrorClient(base_url="http://x/mcp", token="t") + monkeypatch.setattr(c, "call_tool", lambda nom, args=None, use_cache=True, force=False: appels.append((nom, dict(args or {}), force)) or []) + c.get_latest_mails(folder="*", n=200) + c.get_latest_mails(folder="*", n=400) + c.rafraichir_derniers() + assert appels[0][1] == appels[1][1] == appels[2][1] + assert appels[0][1]["n"] == 100 and appels[2][2] is True and appels[0][2] is False + + +def test_onglet_tri_rouvert_sans_rechargement_sous_30_secondes(): + js = (STATIC / "app.js").read_text(encoding="utf-8") + assert "S.chargeJour === S.jour && Date.now() - S.chargeA < 30000" in js