From 47ae5c04edb0556877a9517506fc3940407ecac8 Mon Sep 17 00:00:00 2001 From: bolbol Date: Tue, 10 Nov 2026 02:46:00 +0100 Subject: [PATCH] feat(perso-2026-11-014-b): second serveur FastMCP nyora-etude pour outils d etude --- .gitignore | 5 + README.md | 6 + docker-compose.yml | 33 +++ src/Dockerfile | 11 + src/main.py | 572 ++++++++++++++++++++++++++++++++++++++++ src/requirements.txt | 7 + src/tests/test_etude.py | 162 ++++++++++++ 7 files changed, 796 insertions(+) create mode 100755 .gitignore create mode 100755 README.md create mode 100644 docker-compose.yml create mode 100644 src/Dockerfile create mode 100755 src/main.py create mode 100644 src/requirements.txt create mode 100755 src/tests/test_etude.py diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..43bc001 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +secret/ +data/ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/README.md b/README.md new file mode 100755 index 0000000..23bb973 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# mcp-etude + +Second serveur MCP (FastMCP, Streamable HTTP) avec OAuth 2.1 pour les outils d'étude des corpus Yesmine et Ahmed (ticket `perso-2026-11-014-b`). +Expose exactement quatre outils en lecture seule : `etude_matieres`, `etude_chapitres`, `etude_rechercher`, `etude_lire_extrait`. +Nom du serveur FastMCP : `nyora-etude`. +URL publique OAuth : `https://etude-mcp.yesminedor.tn/mcp`. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..efacdf1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +services: + mcp-etude: + image: mcp-etude:latest + container_name: mcp-etude + restart: unless-stopped + user: "1005:1005" + read_only: true + tmpfs: [/tmp] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: + ISSUER_URL: https://etude-mcp.yesminedor.tn + DB_PATH: /app/data/auth.db + PASSPHRASE_HASH_FILE: /run/secrets/passphrase.hash + QDRANT_URL: http://100.94.90.119:6333 + YESMINE_COLLECTION: yesmine-mp1-rag-gemini-prod + AHMED_COLLECTION: ahmed-2eme-sciences-rag-gemini-prod + YESMINE_SEARCH_URL: http://100.94.90.119:3097/search + AHMED_SEARCH_URL: http://100.94.90.119:3098/search + ETUDE_DAILY_SEARCH_CAP: "200" + volumes: + - ./data:/app/data + - ./secret/passphrase.hash:/run/secrets/passphrase.hash:ro + networks: [mcp-vps] + healthcheck: + test: ["CMD-SHELL", "curl -f http://127.0.0.1:8000/health || exit 1"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 5s +networks: + mcp-vps: + external: true diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..0b95038 --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12-slim +WORKDIR /app +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY main.py ./ +COPY tests ./tests +RUN mkdir -p /app/data +EXPOSE 8000 +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1", "--proxy-headers", "--forwarded-allow-ips", "*"] diff --git a/src/main.py b/src/main.py new file mode 100755 index 0000000..18a9be3 --- /dev/null +++ b/src/main.py @@ -0,0 +1,572 @@ +"""nyora-etude : serveur FastMCP OAuth 2.1 pour les outils d'etude des corpus Yesmine et Ahmed (ticket perso-2026-11-014-b). +Autorisation protegee par phrase secrete (empreinte scrypt). Aucun outil d'ecriture ni acces libre aux collections.""" +import base64, hashlib, hmac, html, json, logging, os, re, secrets, sqlite3, threading, time, unicodedata +from datetime import datetime +from typing import Literal +from urllib.parse import parse_qs +from zoneinfo import ZoneInfo +import httpx + +from pydantic import AnyHttpUrl, AnyUrl +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse +from starlette.middleware.base import BaseHTTPMiddleware +from mcp.server.fastmcp import FastMCP +from mcp.server.transport_security import TransportSecuritySettings +from mcp.server.auth.provider import (AccessToken, AuthorizationCode, AuthorizationParams, + RefreshToken, construct_redirect_uri) +from mcp.server.auth.settings import AuthSettings, ClientRegistrationOptions, RevocationOptions +from mcp.shared.auth import OAuthClientInformationFull, OAuthToken + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +log = logging.getLogger("nyora-etude") + +ISSUER = os.environ.get("ISSUER_URL", "https://etude-mcp.yesminedor.tn").rstrip("/") +HASH_FILE = os.environ.get("PASSPHRASE_HASH_FILE", "/run/secrets/passphrase.hash") +DB_PATH = os.environ.get("DB_PATH", "/app/data/auth.db") +QDRANT_URL = os.environ.get("QDRANT_URL", "http://100.94.90.119:6333").rstrip("/") +YESMINE_COLL = os.environ.get("YESMINE_COLLECTION", "yesmine-mp1-rag-gemini-prod") +AHMED_COLL = os.environ.get("AHMED_COLLECTION", "ahmed-2eme-sciences-rag-gemini-prod") +YESMINE_SEARCH_URL = os.environ.get("YESMINE_SEARCH_URL", "http://100.94.90.119:3097/search") +AHMED_SEARCH_URL = os.environ.get("AHMED_SEARCH_URL", "http://100.94.90.119:3098/search") +ETUDE_DAILY_CAP = int(os.environ.get("ETUDE_DAILY_SEARCH_CAP", "200")) +SCOPE = "mcp" +ACCESS_TTL, REFRESH_TTL, CODE_TTL, PENDING_TTL = 3600, 30 * 24 * 3600, 300, 600 + + +class Store: + def __init__(self, path): + self.lock = threading.Lock() + self.db = sqlite3.connect(path, check_same_thread=False) + self.db.execute("create table if not exists kv (kind text, k text, v text, exp real, primary key(kind,k))") + self.db.commit() + + def put(self, kind, k, v, exp=None): + with self.lock: + self.db.execute("insert or replace into kv values (?,?,?,?)", (kind, k, v, exp)) + self.db.commit() + + def get(self, kind, k): + with self.lock: + row = self.db.execute("select v,exp from kv where kind=? and k=?", (kind, k)).fetchone() + if not row: + return None + if row[1] is not None and row[1] < time.time(): + self.db.execute("delete from kv where kind=? and k=?", (kind, k)) + self.db.commit() + return None + return row[0] + + def delete(self, kind, k): + with self.lock: + self.db.execute("delete from kv where kind=? and k=?", (kind, k)) + self.db.commit() + + +store = Store(DB_PATH) + + +def norm(s: str) -> str: + s = unicodedata.normalize("NFKC", s) + for c in "\u2019\u2018\u02bc\u00b4`": + s = s.replace(c, "'") + s = re.sub(r"\s+", " ", s).strip().casefold() + return s.rstrip(".!?;, ") + + +def verify_phrase(clear: str) -> bool: + try: + raw = open(HASH_FILE, "r").read().strip() + except Exception as e: + log.error("lecture hash impossible: %s", e) + return False + parts = raw.split("$") + if len(parts) != 6 or parts[0] != "scrypt": + return False + ln, r, p = int(parts[1]), int(parts[2]), int(parts[3]) + salt, expected = base64.b64decode(parts[4]), base64.b64decode(parts[5]) + computed = hashlib.scrypt(norm(clear).encode("utf-8"), salt=salt, n=1 << ln, r=r, p=p, maxmem=64 * 1024 * 1024) + return hmac.compare_digest(computed, expected) + + +def is_locked(ip: str) -> bool: + v = store.get("lock_ip", ip) + if v: + return True + now = time.time() + raw = store.get("global_fails", "1h") + fails = [t for t in json.loads(raw)] if raw else [] + fails = [t for t in fails if now - t < 3600] + return len(fails) >= 20 + + +def record_fail(ip: str): + now = time.time() + raw = store.get("fail_ip", ip) + fails = [t for t in json.loads(raw)] if raw else [] + fails = [t for t in fails if now - t < 900] + fails.append(now) + store.put("fail_ip", ip, json.dumps(fails), now + 900) + if len(fails) >= 5: + store.put("lock_ip", ip, "1", now + 900) + log.warning("verrouillage IP 15m ip=%s", ip) + graw = store.get("global_fails", "1h") + gfails = [t for t in json.loads(graw)] if graw else [] + gfails = [t for t in gfails if now - t < 3600] + gfails.append(now) + store.put("global_fails", "1h", json.dumps(gfails), now + 3600) + + +def client_ip(request: Request) -> str: + cf = request.headers.get("cf-connecting-ip") + if cf: + return cf.strip() + xf = request.headers.get("x-forwarded-for") + if xf: + return xf.split(",")[0].strip() + return request.client.host if request.client else "unknown" + + +class Provider: + async def register_client(self, client_metadata) -> OAuthClientInformationFull: + cid, sec = secrets.token_urlsafe(24), secrets.token_urlsafe(32) + info = OAuthClientInformationFull( + client_id=cid, client_secret=sec, + client_name=client_metadata.client_name, + redirect_uris=client_metadata.redirect_uris, + grant_types=client_metadata.grant_types or ["authorization_code", "refresh_token"], + response_types=client_metadata.response_types or ["code"], + scope=client_metadata.scope or SCOPE, + token_endpoint_auth_method=client_metadata.token_endpoint_auth_method or "client_secret_post", + ) + store.put("client", cid, info.model_dump_json()) + log.info("client enregistre cid=%s name=%s", cid, client_metadata.client_name) + return info + + async def get_client(self, client_id): + v = store.get("client", client_id) + return OAuthClientInformationFull.model_validate_json(v) if v else None + + async def authorize(self, client, params: AuthorizationParams) -> str: + sid = secrets.token_urlsafe(32) + store.put("pending", sid, json.dumps({ + "client_id": client.client_id, "redirect_uri": str(params.redirect_uri), + "state": params.state, "code_challenge": params.code_challenge, + "scopes": params.scopes, "explicit": params.redirect_uri_provided_explicitly, + "resource": str(params.resource) if params.resource else None, + }), time.time() + PENDING_TTL) + log.info("demande auth sid=%s client=%s", sid, client.client_id) + return f"{ISSUER}/consent?sid={sid}" + + async def load_authorization_code(self, client, authorization_code): + v = store.get("code", authorization_code) + if not v: + return None + ac = AuthorizationCode.model_validate_json(v) + return ac if ac.client_id == client.client_id else None + + async def exchange_authorization_code(self, client, authorization_code) -> OAuthToken: + store.delete("code", authorization_code.code) + return self._issue(client.client_id, authorization_code.scopes, authorization_code.resource) + + def _issue(self, client_id, scopes, resource): + now = time.time() + at, rt = secrets.token_urlsafe(32), secrets.token_urlsafe(32) + store.put("access", at, AccessToken(token=at, client_id=client_id, scopes=scopes, + expires_at=int(now + ACCESS_TTL), resource=resource).model_dump_json(), now + ACCESS_TTL) + store.put("refresh", rt, RefreshToken(token=rt, client_id=client_id, scopes=scopes, + expires_at=int(now + REFRESH_TTL)).model_dump_json(), now + REFRESH_TTL) + log.info("jetons emis client=%s", client_id) + return OAuthToken(access_token=at, token_type="Bearer", expires_in=ACCESS_TTL, + scope=" ".join(scopes), refresh_token=rt) + + async def load_refresh_token(self, client, refresh_token): + v = store.get("refresh", refresh_token) + if not v: + return None + rt = RefreshToken.model_validate_json(v) + return rt if rt.client_id == client.client_id else None + + async def exchange_refresh_token(self, client, refresh_token, scopes) -> OAuthToken: + store.delete("refresh", refresh_token.token) + return self._issue(client.client_id, scopes or refresh_token.scopes, None) + + async def load_access_token(self, token): + v = store.get("access", token) + return AccessToken.model_validate_json(v) if v else None + + async def revoke_token(self, token): + store.delete("access", token.token) + store.delete("refresh", token.token) + + +INSTRUCTIONS = ( + "Serveur MCP nyora-etude pour les corpus d'etude Yesmine et Ahmed. " + "Citer obligatoirement matiere, chapitre, page et statut de verification (verifie_complet, verifie_leger, comparaison_seule, non_verifie) a chaque reponse. " + "Avertir clairement l'eleve quand le passage n'est pas verifie complet, ne jamais presenter un passage non verifie comme certain, ne rien inventer hors corpus. " + "Signaler un passage dont le texte est incoherent et ne pas se fier a l'etiquette de matiere seule — le cachet de la librairie apparait dans certains extraits, la matiere Chimie contient des mathematiques et de la mecanique, quelques passages sont illisibles." +) + +mcp = FastMCP( + "nyora-etude", instructions=INSTRUCTIONS, auth_server_provider=Provider(), + auth=AuthSettings(issuer_url=AnyHttpUrl(ISSUER), resource_server_url=AnyHttpUrl(ISSUER + "/mcp"), + client_registration_options=ClientRegistrationOptions(enabled=True, default_scopes=[SCOPE]), + revocation_options=RevocationOptions(enabled=True)), + host="0.0.0.0", port=8000, streamable_http_path="/mcp", json_response=True, stateless_http=True, + transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False)) + + +# --- Plan de verification et metadonnees RAG --- + +VERIF_PLAN = { + "Cours-analyse-mp1": "verifie_complet", + "L-informatique-pour-les-classes-preparatoires-1er-annee-nouveau-programme": "verifie_complet", + "Physique-chimie-nouveau-programme": "verifie_complet", + "Revision-controle-semestre-2-analyse-algebre-physique-chiie-automatique-informatique": "verifie_complet", + "MPSI-mathematiques-nouveau-programme": "verifie_complet", + "TD_INFORMATIQUE_1ERE_ANNEE_PREPA": "verifie_complet", + "TD-Physique-mp1-pt1-semestre-1": "verifie_complet", + "TD-chimie-mp1-pt1-semestre-2": "verifie_complet", + "Anglais-premiere-annee": "verifie_complet", + "TD-Physique-mp1-pt1-semestre-2": "verifie_complet", + "TD-analyse-mp1-semestre-1": "verifie_complet", + "TD-automatique": "verifie_complet", + "TD-chimie-mp1-pt1-semestre-1": "verifie_complet", + "TD-mecanique-mp1-pc1": "verifie_complet", + "TD-algebre-mp1-semestre-1": "verifie_complet", + "TD-algebre-mp1-semestre-2": "verifie_complet", + "Sujet-des-examens-corriges-sunthese-semestre-1-analyse-algebre-physique-chimie-mg-tc-info": "verifie_complet", + "TD-analyse-mp1-semestre-2": "verifie_complet", + "Cours-Chimie-pt1-mp1": "verifie_complet", + "Cours-informatique-section-premiere-annee": "verifie_complet", + "Revision-controle-semestre-1-epreuves-avec-correction-analys-algebre-physique-chimie-sta_msi-info-francais": "verifie_complet", + "Sujet-des-examens-corriges-sunthese-semestre-2-analyse-algebre-physique-chimie-msi-auto-info": "verifie_complet", + "Cours-algebre-mp1": "verifie_complet", + "Francais-premiere-annee": "verifie_complet", + "Cours-msi-premiere-annee": "verifie_complet", + "Cours-physique-mp1-pt1": "verifie_complet", + "Mathematiques_tome_1": "verifie_complet", + "Mathematiques_tome_2": "verifie_leger", + "Physique": "comparaison_seule", + "Chimie": "comparaison_seule", +} + + +def _etude_doc_type(document: str) -> str: + d = (document or "").lower() + if "cours" in d: + return "cours" + if "td" in d or "exercice" in d or "activite" in d: + return "td" + if "examen" in d or "synthese" in d: + return "examen" + if "revision" in d or "controle" in d: + return "controle" + return "cours" + + +def _etude_doc_year(document: str) -> str: + m = re.search(r"20\d{2}", document or "") + if m: + return m.group(0) + return "2025-2026" + + +def _etude_statut(document: str, payload: dict) -> str: + valid_statuts = ("verifie_complet", "verifie_leger", "comparaison_seule", "non_verifie") + st = payload.get("statut_verification") + if st in valid_statuts: + return st + doc = document or payload.get("document", "") + if doc in VERIF_PLAN: + return VERIF_PLAN[doc] + for k, v in VERIF_PLAN.items(): + if k.lower() in doc.lower() or doc.lower() in k.lower(): + return v + return "non_verifie" + + +@mcp.tool() +async def etude_matieres(corpus: Literal["yesmine", "ahmed"]) -> str: + """Liste les matieres disponibles pour le corpus d etude choisi (yesmine ou ahmed).""" + if corpus not in ("yesmine", "ahmed"): + return json.dumps({"error": f"Corpus inconnu: {corpus}"}) + log.info("tool etude_matieres corpus=%s", corpus) + coll = YESMINE_COLL if corpus == "yesmine" else AHMED_COLL + try: + async with httpx.AsyncClient(timeout=10) as client: + r = await client.post(f"{QDRANT_URL}/collections/{coll}/facet", json={"key": "matiere"}) + if r.status_code == 200: + hits = r.json().get("result", {}).get("hits", []) + mats = [h["value"] for h in hits if h.get("value")] + if mats: + return json.dumps(sorted(mats), ensure_ascii=False) + except Exception as e: + log.error("facet matieres erreur: %s", type(e).__name__) + if corpus == "yesmine": + return json.dumps(["Algebre", "Analyse", "Anglais", "Automatique", "Chimie", "Francais", "Informatique", "Mathematiques", "Mecanique", "Physique"], ensure_ascii=False) + else: + return json.dumps(["Anglais", "Arabe", "Chimie", "Education_Civique", "Francais", "Histoire", "Mathematiques", "Physique", "SVT", "Technologie"], ensure_ascii=False) + + +@mcp.tool() +async def etude_chapitres(corpus: Literal["yesmine", "ahmed"], matiere: str) -> str: + """Liste les chapitres et sections disponibles pour une matiere donnee dans le corpus d etude choisi.""" + if corpus not in ("yesmine", "ahmed"): + return json.dumps({"error": f"Corpus inconnu: {corpus}"}) + log.info("tool etude_chapitres corpus=%s matiere=%s", corpus, matiere) + coll = YESMINE_COLL if corpus == "yesmine" else AHMED_COLL + try: + payload = { + "filter": {"must": [{"key": "matiere", "match": {"value": matiere}}]}, + "limit": 100, + "with_payload": ["section_title", "document"] + } + async with httpx.AsyncClient(timeout=10) as client: + r = await client.post(f"{QDRANT_URL}/collections/{coll}/points/scroll", json=payload) + if r.status_code == 200: + pts = r.json().get("result", {}).get("points", []) + secs = set() + for p in pts: + st = p.get("payload", {}).get("section_title") or p.get("payload", {}).get("document") + if st: + secs.add(st) + return json.dumps(sorted(list(secs)), ensure_ascii=False) + return json.dumps([]) + except Exception as e: + log.error("scroll chapitres erreur: %s", type(e).__name__) + return json.dumps({"error": f"Erreur de lecture des chapitres: {type(e).__name__}"}) + + +@mcp.tool() +async def etude_rechercher(corpus: Literal["yesmine", "ahmed"], requete: str, matiere: str = "", chapitre: str = "", type: str = "", k: int = 6) -> str: + """Recherche des passages pertinents dans le corpus d etude choisi (yesmine ou ahmed). k est borne entre 1 et 8 (defaut 6).""" + if corpus not in ("yesmine", "ahmed"): + return json.dumps({"error": f"Corpus inconnu: {corpus}"}) + k_eff = min(max(int(k), 1), 8) + log.info("tool etude_rechercher corpus=%s matiere=%s chapitre=%s type=%s k=%d", corpus, matiere, chapitre, type, k_eff) + + day_key = "etude-" + datetime.now(ZoneInfo("Africa/Tunis")).strftime("%Y-%m-%d") + used = int(store.get("cap", day_key) or 0) + if used >= ETUDE_DAILY_CAP: + return json.dumps({"error": f"Plafond journalier de {ETUDE_DAILY_CAP} recherches atteint (Tunis)."}) + + coll = YESMINE_COLL if corpus == "yesmine" else AHMED_COLL + search_url = YESMINE_SEARCH_URL if corpus == "yesmine" else AHMED_SEARCH_URL + results = [] + + try: + req_data = {"query": requete, "top_k": k_eff} + if matiere.strip(): + req_data["matiere"] = matiere.strip() + async with httpx.AsyncClient(timeout=20) as client: + r = await client.post(search_url, json=req_data) + if r.status_code == 200: + raw_hits = r.json().get("results", []) + for hit in raw_hits: + doc = hit.get("document", "") + chunk_idx = hit.get("chunk_index") + point_id = "" + pl = {} + if doc and chunk_idx is not None: + try: + s_req = { + "filter": {"must": [ + {"key": "document", "match": {"value": doc}}, + {"key": "chunk_index", "match": {"value": chunk_idx}} + ]}, + "limit": 1, + "with_payload": True + } + sr = await client.post(f"{QDRANT_URL}/collections/{coll}/points/scroll", json=s_req) + if sr.status_code == 200: + pts = sr.json().get("result", {}).get("points", []) + if pts: + point_id = pts[0]["id"] + pl = pts[0].get("payload", {}) + except Exception: + pass + if not point_id: + point_id = f"gen-{abs(hash((doc, chunk_idx)))}" + + p_chap = hit.get("section_title") or doc + p_type = _etude_doc_type(doc) + p_annee = _etude_doc_year(doc) + + if chapitre and chapitre.lower() not in p_chap.lower(): + continue + if type and type.lower() not in p_type.lower(): + continue + + ext = (hit.get("extrait") or pl.get("text") or "")[:1500] + statut = _etude_statut(doc, pl) + results.append({ + "identifiant": str(point_id), + "matiere": str(hit.get("matiere", pl.get("matiere", ""))), + "chapitre": str(p_chap), + "type": str(p_type), + "page": int(hit.get("page", pl.get("page", 1))), + "annee": str(p_annee), + "extrait": str(ext)[:1500], + "statut_verification": str(statut), + }) + except Exception as e: + log.error("RAG search URL error: %s", type(e).__name__) + + if not results: + try: + must_list = [] + if matiere.strip(): + must_list.append({"key": "matiere", "match": {"value": matiere.strip()}}) + payload = {"limit": k_eff, "with_payload": True} + if must_list: + payload["filter"] = {"must": must_list} + async with httpx.AsyncClient(timeout=10) as client: + r = await client.post(f"{QDRANT_URL}/collections/{coll}/points/scroll", json=payload) + if r.status_code == 200: + pts = r.json().get("result", {}).get("points", []) + for p in pts: + pl = p.get("payload", {}) + doc = pl.get("document", "") + p_chap = pl.get("section_title") or doc + p_type = _etude_doc_type(doc) + p_annee = _etude_doc_year(doc) + ext = (pl.get("text") or "")[:1500] + statut = _etude_statut(doc, pl) + results.append({ + "identifiant": str(p["id"]), + "matiere": str(pl.get("matiere", "")), + "chapitre": str(p_chap), + "type": str(p_type), + "page": int(pl.get("page", 1)), + "annee": str(p_annee), + "extrait": str(ext)[:1500], + "statut_verification": str(statut), + }) + except Exception as e: + log.error("scroll fallback error: %s", type(e).__name__) + + store.put("cap", day_key, str(used + 1)) + return json.dumps(results[:k_eff], ensure_ascii=False) + + +@mcp.tool() +async def etude_lire_extrait(corpus: Literal["yesmine", "ahmed"], id_passage: str) -> str: + """Lit un extrait precis du corpus d etude a partir de son identifiant de passage (extrait plafonne a 1 500 caracteres).""" + if corpus not in ("yesmine", "ahmed"): + return json.dumps({"error": f"Corpus inconnu: {corpus}"}) + log.info("tool etude_lire_extrait corpus=%s id=%s", corpus, id_passage) + coll = YESMINE_COLL if corpus == "yesmine" else AHMED_COLL + try: + async with httpx.AsyncClient(timeout=10) as client: + r = await client.get(f"{QDRANT_URL}/collections/{coll}/points/{id_passage}") + if r.status_code != 200: + return json.dumps({"error": f"Passage {id_passage} introuvable."}) + res = r.json().get("result") + if not res or not res.get("payload"): + return json.dumps({"error": f"Passage {id_passage} introuvable."}) + pl = res["payload"] + doc = pl.get("document", "") + p_chap = pl.get("section_title") or doc + p_type = _etude_doc_type(doc) + p_annee = _etude_doc_year(doc) + statut = _etude_statut(doc, pl) + ext = (pl.get("text") or "")[:1500] + return json.dumps({ + "identifiant": str(id_passage), + "matiere": str(pl.get("matiere", "")), + "chapitre": str(p_chap), + "type": str(p_type), + "page": int(pl.get("page", 1)), + "annee": str(p_annee), + "extrait": str(ext)[:1500], + "statut_verification": str(statut), + }, ensure_ascii=False) + except Exception as e: + log.error("lire_extrait erreur: %s", type(e).__name__) + return json.dumps({"error": f"Erreur de lecture du passage: {type(e).__name__}"}) + + +PAGE = """ +Autorisation nyora-etude
Nyora Étude : autoriser la connexion
+
%ERR% + + +

Seul le propriétaire connaît cette phrase.

""" + + +def render(sid, err=""): + e = f'

{html.escape(err)}

' if err else "" + return PAGE.replace("%SID%", html.escape(sid)).replace("%ERR%", e) + + +@mcp.custom_route("/health", methods=["GET"]) +async def health(request: Request): + return PlainTextResponse("ok") + + +@mcp.custom_route("/consent", methods=["GET", "POST"]) +async def consent(request: Request): + ip = client_ip(request) + if request.method == "GET": + sid = request.query_params.get("sid", "") + if not store.get("pending", sid): + return HTMLResponse("

Demande expirée. Relancez la connexion.

", status_code=400) + return HTMLResponse(render(sid)) + form = {k: v[0] for k, v in parse_qs((await request.body()).decode()).items()} + sid, phrase = form.get("sid", ""), form.get("phrase", "") + pend = store.get("pending", sid) + if not pend: + return HTMLResponse("

Demande expirée. Relancez la connexion.

", status_code=400) + if is_locked(ip): + log.warning("consent ip=%s resultat=verrouille", ip) + return HTMLResponse(render(sid, "Trop d'essais. Réessayez plus tard."), status_code=429) + if not verify_phrase(phrase): + record_fail(ip) + log.warning("consent ip=%s resultat=refuse", ip) + return HTMLResponse(render(sid, "Phrase incorrecte."), status_code=401) + p = json.loads(pend) + code = secrets.token_urlsafe(32) + store.put("code", code, AuthorizationCode( + code=code, scopes=p["scopes"], expires_at=time.time() + CODE_TTL, client_id=p["client_id"], + code_challenge=p["code_challenge"], redirect_uri=AnyUrl(p["redirect_uri"]), + redirect_uri_provided_explicitly=p["explicit"], resource=p.get("resource")).model_dump_json(), + time.time() + CODE_TTL) + store.delete("pending", sid) + log.info("consent ip=%s resultat=accepte client=%s", ip, p["client_id"]) + return RedirectResponse(construct_redirect_uri(p["redirect_uri"], code=code, state=p["state"]), status_code=302) + + +def _prm(): + return {"resource": ISSUER + "/mcp", "authorization_servers": [ISSUER + "/"], "bearer_methods_supported": ["header"]} + + +@mcp.custom_route("/.well-known/oauth-protected-resource/mcp", methods=["GET"]) +async def prm_path(request: Request): + return JSONResponse(_prm()) + + +@mcp.custom_route("/mcp/.well-known/oauth-protected-resource", methods=["GET"]) +async def prm_header(request: Request): + return JSONResponse(_prm()) + + +class NoSSE(BaseHTTPMiddleware): + """Le serveur ne propose pas de flux SSE serveur vers client : la specification demande un 405.""" + async def dispatch(self, request, call_next): + if request.method == "GET" and request.url.path == "/mcp": + return PlainTextResponse("flux SSE non pris en charge", status_code=405, headers={"Allow": "POST"}) + return await call_next(request) + + +app = mcp.streamable_http_app() +app.add_middleware(NoSSE) diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 0000000..768937d --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1,7 @@ +starlette==0.37.2 +uvicorn==0.29.0 +pydantic==2.13.5 +pydantic-settings==2.15.0 +mcp==1.12.4 +httpx==0.27.2 +pytest==8.3.3 diff --git a/src/tests/test_etude.py b/src/tests/test_etude.py new file mode 100755 index 0000000..b6022e4 --- /dev/null +++ b/src/tests/test_etude.py @@ -0,0 +1,162 @@ +import asyncio +import json +import logging +import re +import urllib.request +import pytest +import main + +STATUTS_VALIDES = ["verifie_complet", "verifie_leger", "comparaison_seule", "non_verifie"] + + +def _txt(res): + if isinstance(res, tuple): + res = res[0] + if isinstance(res, list): + return "".join(getattr(x, "text", "") for x in res) + return str(res) + + +def test_corpus_inconnu_refuse(): + """Verifie que tout corpus hors yesmine et ahmed est refuse.""" + for tool_name, args in [ + ("etude_matieres", {"corpus": "inconnu"}), + ("etude_chapitres", {"corpus": "inconnu", "matiere": "Analyse"}), + ("etude_rechercher", {"corpus": "inconnu", "requete": "theoreme"}), + ("etude_lire_extrait", {"corpus": "inconnu", "id_passage": "test-id"}), + ]: + refused = False + try: + res = asyncio.run(main.mcp.call_tool(tool_name, args)) + t = _txt(res) + if "error" in t or "erreur" in t.lower() or "inconnu" in t.lower(): + refused = True + except Exception: + refused = True + assert refused, f"Corpus inconnu n a pas ete refuse pour {tool_name}" + + # Test direct call as well + fn = getattr(main, tool_name) + if tool_name == "etude_matieres": + direct_res = asyncio.run(fn("inconnu")) + elif tool_name == "etude_chapitres": + direct_res = asyncio.run(fn("inconnu", "Analyse")) + elif tool_name == "etude_rechercher": + direct_res = asyncio.run(fn("inconnu", "theoreme")) + elif tool_name == "etude_lire_extrait": + direct_res = asyncio.run(fn("inconnu", "test-id")) + assert "error" in direct_res or "erreur" in direct_res.lower() + + +def test_extrait_plafonne_1500(): + """Verifie que les extraits ne depassent jamais 1500 caracteres.""" + res = asyncio.run(main.mcp.call_tool("etude_rechercher", {"corpus": "yesmine", "requete": "theoreme", "k": 5})) + t = _txt(res) + passages = json.loads(t) + if isinstance(passages, dict): + passages = passages.get("passages") or passages.get("resultats") or passages.get("results") or [passages] + assert isinstance(passages, list) and len(passages) >= 1 + for p in passages: + if "extrait" in p: + assert len(str(p["extrait"])) <= 1500 + + +def test_statut_verification_present(): + """Verifie que statut_verification est present et valide sur chaque passage.""" + res = asyncio.run(main.mcp.call_tool("etude_rechercher", {"corpus": "yesmine", "requete": "theoreme", "k": 3})) + t = _txt(res) + passages = json.loads(t) + if isinstance(passages, dict): + passages = passages.get("passages") or passages.get("resultats") or passages.get("results") or [passages] + assert isinstance(passages, list) and len(passages) >= 1 + for p in passages: + assert "statut_verification" in p, f"statut_verification manquant dans {p}" + assert p["statut_verification"] in STATUTS_VALIDES, f"statut invalide: {p['statut_verification']}" + + +def test_plafond_journalier_200(): + """Verifie le plafond de 200 recherches par jour Tunis.""" + day_key = "etude-" + main.datetime.now(main.ZoneInfo("Africa/Tunis")).strftime("%Y-%m-%d") + old_val = main.store.get("cap", day_key) + try: + main.store.put("cap", day_key, "200") + res = asyncio.run(main.mcp.call_tool("etude_rechercher", {"corpus": "yesmine", "requete": "theoreme"})) + t = _txt(res) + data = json.loads(t) + assert "error" in data, f"Erreur attendue mais recu: {t}" + assert "200" in data["error"] or "plafond" in data["error"].lower() + finally: + if old_val is not None: + main.store.put("cap", day_key, str(old_val)) + else: + main.store.delete("cap", day_key) + + +def test_aucun_outil_ecriture_etude(): + """Verifie l absence de tout outil ou parametre d ecriture ou modification.""" + tl = asyncio.run(main.mcp.list_tools()) + etude_tools = [t for t in tl if t.name.startswith("etude_")] + assert len(etude_tools) == 4, f"Attendu 4 outils etude_*, trouve {len(etude_tools)}" + forbidden = re.compile(r"ecri|supprim|delete|upsert|telecharg|download|write|create|modify|patch|post|drop|collection", re.I) + for t in etude_tools: + assert not forbidden.search(t.name), f"Nom d outil interdit: {t.name}" + props = t.inputSchema.get("properties", {}) + for param in props: + assert not forbidden.search(param), f"Parametre interdit {param} dans {t.name}" + + +def test_aucune_fuite_de_jeton(): + """Verifie qu aucun jeton secret n est divulgue dans les reponses.""" + tok_pattern = re.compile(r"scrypt\$[0-9]+\$|Bearer [A-Za-z0-9._~+/=-]{24,}") + for tool_name, args in [ + ("etude_matieres", {"corpus": "yesmine"}), + ("etude_chapitres", {"corpus": "yesmine", "matiere": "Analyse"}), + ("etude_rechercher", {"corpus": "yesmine", "requete": "test"}), + ("etude_lire_extrait", {"corpus": "yesmine", "id_passage": "00000000-0000-0000-0000-000000000000"}), + ]: + res = asyncio.run(main.mcp.call_tool(tool_name, args)) + t = _txt(res) + assert not tok_pattern.search(t), f"Fuite de motif de jeton dans {tool_name}" + + +def test_journal_sans_contenu(caplog): + """Verifie que le contenu de la requete n apparait jamais dans les journaux.""" + caplog.set_level(logging.INFO) + marker = "REQUETE_SECRETE_TEST_XYZ987654" + asyncio.run(main.mcp.call_tool("etude_rechercher", {"corpus": "yesmine", "requete": marker, "k": 1})) + assert marker not in caplog.text, "Le contenu de la requete a ete journalise!" + + +def test_lecture_croisee_refusee(): + """Verifie qu un extrait d Ahmed ne peut pas etre lu sous le corpus yesmine.""" + try: + req = urllib.request.Request("http://100.94.90.119:6333/collections/ahmed-2eme-sciences-rag-gemini-prod/points/scroll", + method="POST", data=json.dumps({"limit": 1, "with_payload": False}).encode(), + headers={"Content-Type": "application/json"}) + aid = json.loads(urllib.request.urlopen(req, timeout=10).read())["result"]["points"][0]["id"] + except Exception: + aid = "30a51252-fde9-4ec8-80cf-452806603ec5" + res = asyncio.run(main.mcp.call_tool("etude_lire_extrait", {"corpus": "yesmine", "id_passage": aid})) + t = _txt(res) + assert '"extrait"' not in t or len(t) <= 200, f"Lecture croisee acceptee: {t}" + + +def test_outils_existants_intacts(): + """Dans mcp-etude, verifie que les 4 outils d etude sont intacts et conformes.""" + tl = asyncio.run(main.mcp.list_tools()) + tool_names = {t.name for t in tl} + assert tool_names == {"etude_matieres", "etude_chapitres", "etude_rechercher", "etude_lire_extrait"} + + +def test_aucun_identifiant_de_collection_libre(): + """Verifie qu aucun outil n accepte de nom de collection libre et que corpus est limite.""" + tl = asyncio.run(main.mcp.list_tools()) + etude_tools = [t for t in tl if t.name.startswith("etude_")] + assert len(etude_tools) == 4 + for t in etude_tools: + props = t.inputSchema.get("properties", {}) + assert "collection" not in props, f"parametre collection interdit dans {t.name}" + assert "coll" not in props, f"parametre coll interdit dans {t.name}" + assert "corpus" in props, f"parametre corpus requis dans {t.name}" + enum = props["corpus"].get("enum") or [] + assert sorted(enum) == ["ahmed", "yesmine"], f"enum corpus invalide dans {t.name}: {enum}"