feat(perso-2026-11-014-b): second serveur FastMCP nyora-etude pour outils d etude
This commit is contained in:
commit
47ae5c04ed
7 files changed
+796
No files matched your search
Executable
+5
@@ -0,0 +1,5 @@
|
||||
secret/
|
||||
data/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.pytest_cache/
|
||||
@@ -0,0 +1,6 @@
|
||||
# mcp-etude
|
||||
|
||||
Second serveur MCP (FastMCP, Streamable HTTP) avec OAuth 2.1 pour les outils d'étude des corpus Yesmine et Ahmed (ticket `perso-2026-11-014-b`).
|
||||
Expose exactement quatre outils en lecture seule : `etude_matieres`, `etude_chapitres`, `etude_rechercher`, `etude_lire_extrait`.
|
||||
Nom du serveur FastMCP : `nyora-etude`.
|
||||
URL publique OAuth : `https://etude-mcp.yesminedor.tn/mcp`.
|
||||
@@ -0,0 +1,33 @@
|
||||
services:
|
||||
mcp-etude:
|
||||
image: mcp-etude:latest
|
||||
container_name: mcp-etude
|
||||
restart: unless-stopped
|
||||
user: "1005:1005"
|
||||
read_only: true
|
||||
tmpfs: [/tmp]
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
environment:
|
||||
ISSUER_URL: https://etude-mcp.yesminedor.tn
|
||||
DB_PATH: /app/data/auth.db
|
||||
PASSPHRASE_HASH_FILE: /run/secrets/passphrase.hash
|
||||
QDRANT_URL: http://100.94.90.119:6333
|
||||
YESMINE_COLLECTION: yesmine-mp1-rag-gemini-prod
|
||||
AHMED_COLLECTION: ahmed-2eme-sciences-rag-gemini-prod
|
||||
YESMINE_SEARCH_URL: http://100.94.90.119:3097/search
|
||||
AHMED_SEARCH_URL: http://100.94.90.119:3098/search
|
||||
ETUDE_DAILY_SEARCH_CAP: "200"
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./secret/passphrase.hash:/run/secrets/passphrase.hash:ro
|
||||
networks: [mcp-vps]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://127.0.0.1:8000/health || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
networks:
|
||||
mcp-vps:
|
||||
external: true
|
||||
@@ -0,0 +1,11 @@
|
||||
FROM python:3.12-slim
|
||||
WORKDIR /app
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY main.py ./
|
||||
COPY tests ./tests
|
||||
RUN mkdir -p /app/data
|
||||
EXPOSE 8000
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1", "--proxy-headers", "--forwarded-allow-ips", "*"]
|
||||
Executable
+572
@@ -0,0 +1,572 @@
|
||||
"""nyora-etude : serveur FastMCP OAuth 2.1 pour les outils d'etude des corpus Yesmine et Ahmed (ticket perso-2026-11-014-b).
|
||||
Autorisation protegee par phrase secrete (empreinte scrypt). Aucun outil d'ecriture ni acces libre aux collections."""
|
||||
import base64, hashlib, hmac, html, json, logging, os, re, secrets, sqlite3, threading, time, unicodedata
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from urllib.parse import parse_qs
|
||||
from zoneinfo import ZoneInfo
|
||||
import httpx
|
||||
|
||||
from pydantic import AnyHttpUrl, AnyUrl
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
from mcp.server.auth.provider import (AccessToken, AuthorizationCode, AuthorizationParams,
|
||||
RefreshToken, construct_redirect_uri)
|
||||
from mcp.server.auth.settings import AuthSettings, ClientRegistrationOptions, RevocationOptions
|
||||
from mcp.shared.auth import OAuthClientInformationFull, OAuthToken
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
log = logging.getLogger("nyora-etude")
|
||||
|
||||
ISSUER = os.environ.get("ISSUER_URL", "https://etude-mcp.yesminedor.tn").rstrip("/")
|
||||
HASH_FILE = os.environ.get("PASSPHRASE_HASH_FILE", "/run/secrets/passphrase.hash")
|
||||
DB_PATH = os.environ.get("DB_PATH", "/app/data/auth.db")
|
||||
QDRANT_URL = os.environ.get("QDRANT_URL", "http://100.94.90.119:6333").rstrip("/")
|
||||
YESMINE_COLL = os.environ.get("YESMINE_COLLECTION", "yesmine-mp1-rag-gemini-prod")
|
||||
AHMED_COLL = os.environ.get("AHMED_COLLECTION", "ahmed-2eme-sciences-rag-gemini-prod")
|
||||
YESMINE_SEARCH_URL = os.environ.get("YESMINE_SEARCH_URL", "http://100.94.90.119:3097/search")
|
||||
AHMED_SEARCH_URL = os.environ.get("AHMED_SEARCH_URL", "http://100.94.90.119:3098/search")
|
||||
ETUDE_DAILY_CAP = int(os.environ.get("ETUDE_DAILY_SEARCH_CAP", "200"))
|
||||
SCOPE = "mcp"
|
||||
ACCESS_TTL, REFRESH_TTL, CODE_TTL, PENDING_TTL = 3600, 30 * 24 * 3600, 300, 600
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, path):
|
||||
self.lock = threading.Lock()
|
||||
self.db = sqlite3.connect(path, check_same_thread=False)
|
||||
self.db.execute("create table if not exists kv (kind text, k text, v text, exp real, primary key(kind,k))")
|
||||
self.db.commit()
|
||||
|
||||
def put(self, kind, k, v, exp=None):
|
||||
with self.lock:
|
||||
self.db.execute("insert or replace into kv values (?,?,?,?)", (kind, k, v, exp))
|
||||
self.db.commit()
|
||||
|
||||
def get(self, kind, k):
|
||||
with self.lock:
|
||||
row = self.db.execute("select v,exp from kv where kind=? and k=?", (kind, k)).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
if row[1] is not None and row[1] < time.time():
|
||||
self.db.execute("delete from kv where kind=? and k=?", (kind, k))
|
||||
self.db.commit()
|
||||
return None
|
||||
return row[0]
|
||||
|
||||
def delete(self, kind, k):
|
||||
with self.lock:
|
||||
self.db.execute("delete from kv where kind=? and k=?", (kind, k))
|
||||
self.db.commit()
|
||||
|
||||
|
||||
store = Store(DB_PATH)
|
||||
|
||||
|
||||
def norm(s: str) -> str:
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
for c in "\u2019\u2018\u02bc\u00b4`":
|
||||
s = s.replace(c, "'")
|
||||
s = re.sub(r"\s+", " ", s).strip().casefold()
|
||||
return s.rstrip(".!?;, ")
|
||||
|
||||
|
||||
def verify_phrase(clear: str) -> bool:
|
||||
try:
|
||||
raw = open(HASH_FILE, "r").read().strip()
|
||||
except Exception as e:
|
||||
log.error("lecture hash impossible: %s", e)
|
||||
return False
|
||||
parts = raw.split("$")
|
||||
if len(parts) != 6 or parts[0] != "scrypt":
|
||||
return False
|
||||
ln, r, p = int(parts[1]), int(parts[2]), int(parts[3])
|
||||
salt, expected = base64.b64decode(parts[4]), base64.b64decode(parts[5])
|
||||
computed = hashlib.scrypt(norm(clear).encode("utf-8"), salt=salt, n=1 << ln, r=r, p=p, maxmem=64 * 1024 * 1024)
|
||||
return hmac.compare_digest(computed, expected)
|
||||
|
||||
|
||||
def is_locked(ip: str) -> bool:
|
||||
v = store.get("lock_ip", ip)
|
||||
if v:
|
||||
return True
|
||||
now = time.time()
|
||||
raw = store.get("global_fails", "1h")
|
||||
fails = [t for t in json.loads(raw)] if raw else []
|
||||
fails = [t for t in fails if now - t < 3600]
|
||||
return len(fails) >= 20
|
||||
|
||||
|
||||
def record_fail(ip: str):
|
||||
now = time.time()
|
||||
raw = store.get("fail_ip", ip)
|
||||
fails = [t for t in json.loads(raw)] if raw else []
|
||||
fails = [t for t in fails if now - t < 900]
|
||||
fails.append(now)
|
||||
store.put("fail_ip", ip, json.dumps(fails), now + 900)
|
||||
if len(fails) >= 5:
|
||||
store.put("lock_ip", ip, "1", now + 900)
|
||||
log.warning("verrouillage IP 15m ip=%s", ip)
|
||||
graw = store.get("global_fails", "1h")
|
||||
gfails = [t for t in json.loads(graw)] if graw else []
|
||||
gfails = [t for t in gfails if now - t < 3600]
|
||||
gfails.append(now)
|
||||
store.put("global_fails", "1h", json.dumps(gfails), now + 3600)
|
||||
|
||||
|
||||
def client_ip(request: Request) -> str:
|
||||
cf = request.headers.get("cf-connecting-ip")
|
||||
if cf:
|
||||
return cf.strip()
|
||||
xf = request.headers.get("x-forwarded-for")
|
||||
if xf:
|
||||
return xf.split(",")[0].strip()
|
||||
return request.client.host if request.client else "unknown"
|
||||
|
||||
|
||||
class Provider:
|
||||
async def register_client(self, client_metadata) -> OAuthClientInformationFull:
|
||||
cid, sec = secrets.token_urlsafe(24), secrets.token_urlsafe(32)
|
||||
info = OAuthClientInformationFull(
|
||||
client_id=cid, client_secret=sec,
|
||||
client_name=client_metadata.client_name,
|
||||
redirect_uris=client_metadata.redirect_uris,
|
||||
grant_types=client_metadata.grant_types or ["authorization_code", "refresh_token"],
|
||||
response_types=client_metadata.response_types or ["code"],
|
||||
scope=client_metadata.scope or SCOPE,
|
||||
token_endpoint_auth_method=client_metadata.token_endpoint_auth_method or "client_secret_post",
|
||||
)
|
||||
store.put("client", cid, info.model_dump_json())
|
||||
log.info("client enregistre cid=%s name=%s", cid, client_metadata.client_name)
|
||||
return info
|
||||
|
||||
async def get_client(self, client_id):
|
||||
v = store.get("client", client_id)
|
||||
return OAuthClientInformationFull.model_validate_json(v) if v else None
|
||||
|
||||
async def authorize(self, client, params: AuthorizationParams) -> str:
|
||||
sid = secrets.token_urlsafe(32)
|
||||
store.put("pending", sid, json.dumps({
|
||||
"client_id": client.client_id, "redirect_uri": str(params.redirect_uri),
|
||||
"state": params.state, "code_challenge": params.code_challenge,
|
||||
"scopes": params.scopes, "explicit": params.redirect_uri_provided_explicitly,
|
||||
"resource": str(params.resource) if params.resource else None,
|
||||
}), time.time() + PENDING_TTL)
|
||||
log.info("demande auth sid=%s client=%s", sid, client.client_id)
|
||||
return f"{ISSUER}/consent?sid={sid}"
|
||||
|
||||
async def load_authorization_code(self, client, authorization_code):
|
||||
v = store.get("code", authorization_code)
|
||||
if not v:
|
||||
return None
|
||||
ac = AuthorizationCode.model_validate_json(v)
|
||||
return ac if ac.client_id == client.client_id else None
|
||||
|
||||
async def exchange_authorization_code(self, client, authorization_code) -> OAuthToken:
|
||||
store.delete("code", authorization_code.code)
|
||||
return self._issue(client.client_id, authorization_code.scopes, authorization_code.resource)
|
||||
|
||||
def _issue(self, client_id, scopes, resource):
|
||||
now = time.time()
|
||||
at, rt = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
|
||||
store.put("access", at, AccessToken(token=at, client_id=client_id, scopes=scopes,
|
||||
expires_at=int(now + ACCESS_TTL), resource=resource).model_dump_json(), now + ACCESS_TTL)
|
||||
store.put("refresh", rt, RefreshToken(token=rt, client_id=client_id, scopes=scopes,
|
||||
expires_at=int(now + REFRESH_TTL)).model_dump_json(), now + REFRESH_TTL)
|
||||
log.info("jetons emis client=%s", client_id)
|
||||
return OAuthToken(access_token=at, token_type="Bearer", expires_in=ACCESS_TTL,
|
||||
scope=" ".join(scopes), refresh_token=rt)
|
||||
|
||||
async def load_refresh_token(self, client, refresh_token):
|
||||
v = store.get("refresh", refresh_token)
|
||||
if not v:
|
||||
return None
|
||||
rt = RefreshToken.model_validate_json(v)
|
||||
return rt if rt.client_id == client.client_id else None
|
||||
|
||||
async def exchange_refresh_token(self, client, refresh_token, scopes) -> OAuthToken:
|
||||
store.delete("refresh", refresh_token.token)
|
||||
return self._issue(client.client_id, scopes or refresh_token.scopes, None)
|
||||
|
||||
async def load_access_token(self, token):
|
||||
v = store.get("access", token)
|
||||
return AccessToken.model_validate_json(v) if v else None
|
||||
|
||||
async def revoke_token(self, token):
|
||||
store.delete("access", token.token)
|
||||
store.delete("refresh", token.token)
|
||||
|
||||
|
||||
INSTRUCTIONS = (
|
||||
"Serveur MCP nyora-etude pour les corpus d'etude Yesmine et Ahmed. "
|
||||
"Citer obligatoirement matiere, chapitre, page et statut de verification (verifie_complet, verifie_leger, comparaison_seule, non_verifie) a chaque reponse. "
|
||||
"Avertir clairement l'eleve quand le passage n'est pas verifie complet, ne jamais presenter un passage non verifie comme certain, ne rien inventer hors corpus. "
|
||||
"Signaler un passage dont le texte est incoherent et ne pas se fier a l'etiquette de matiere seule — le cachet de la librairie apparait dans certains extraits, la matiere Chimie contient des mathematiques et de la mecanique, quelques passages sont illisibles."
|
||||
)
|
||||
|
||||
mcp = FastMCP(
|
||||
"nyora-etude", instructions=INSTRUCTIONS, auth_server_provider=Provider(),
|
||||
auth=AuthSettings(issuer_url=AnyHttpUrl(ISSUER), resource_server_url=AnyHttpUrl(ISSUER + "/mcp"),
|
||||
client_registration_options=ClientRegistrationOptions(enabled=True, default_scopes=[SCOPE]),
|
||||
revocation_options=RevocationOptions(enabled=True)),
|
||||
host="0.0.0.0", port=8000, streamable_http_path="/mcp", json_response=True, stateless_http=True,
|
||||
transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False))
|
||||
|
||||
|
||||
# --- Plan de verification et metadonnees RAG ---
|
||||
|
||||
VERIF_PLAN = {
|
||||
"Cours-analyse-mp1": "verifie_complet",
|
||||
"L-informatique-pour-les-classes-preparatoires-1er-annee-nouveau-programme": "verifie_complet",
|
||||
"Physique-chimie-nouveau-programme": "verifie_complet",
|
||||
"Revision-controle-semestre-2-analyse-algebre-physique-chiie-automatique-informatique": "verifie_complet",
|
||||
"MPSI-mathematiques-nouveau-programme": "verifie_complet",
|
||||
"TD_INFORMATIQUE_1ERE_ANNEE_PREPA": "verifie_complet",
|
||||
"TD-Physique-mp1-pt1-semestre-1": "verifie_complet",
|
||||
"TD-chimie-mp1-pt1-semestre-2": "verifie_complet",
|
||||
"Anglais-premiere-annee": "verifie_complet",
|
||||
"TD-Physique-mp1-pt1-semestre-2": "verifie_complet",
|
||||
"TD-analyse-mp1-semestre-1": "verifie_complet",
|
||||
"TD-automatique": "verifie_complet",
|
||||
"TD-chimie-mp1-pt1-semestre-1": "verifie_complet",
|
||||
"TD-mecanique-mp1-pc1": "verifie_complet",
|
||||
"TD-algebre-mp1-semestre-1": "verifie_complet",
|
||||
"TD-algebre-mp1-semestre-2": "verifie_complet",
|
||||
"Sujet-des-examens-corriges-sunthese-semestre-1-analyse-algebre-physique-chimie-mg-tc-info": "verifie_complet",
|
||||
"TD-analyse-mp1-semestre-2": "verifie_complet",
|
||||
"Cours-Chimie-pt1-mp1": "verifie_complet",
|
||||
"Cours-informatique-section-premiere-annee": "verifie_complet",
|
||||
"Revision-controle-semestre-1-epreuves-avec-correction-analys-algebre-physique-chimie-sta_msi-info-francais": "verifie_complet",
|
||||
"Sujet-des-examens-corriges-sunthese-semestre-2-analyse-algebre-physique-chimie-msi-auto-info": "verifie_complet",
|
||||
"Cours-algebre-mp1": "verifie_complet",
|
||||
"Francais-premiere-annee": "verifie_complet",
|
||||
"Cours-msi-premiere-annee": "verifie_complet",
|
||||
"Cours-physique-mp1-pt1": "verifie_complet",
|
||||
"Mathematiques_tome_1": "verifie_complet",
|
||||
"Mathematiques_tome_2": "verifie_leger",
|
||||
"Physique": "comparaison_seule",
|
||||
"Chimie": "comparaison_seule",
|
||||
}
|
||||
|
||||
|
||||
def _etude_doc_type(document: str) -> str:
|
||||
d = (document or "").lower()
|
||||
if "cours" in d:
|
||||
return "cours"
|
||||
if "td" in d or "exercice" in d or "activite" in d:
|
||||
return "td"
|
||||
if "examen" in d or "synthese" in d:
|
||||
return "examen"
|
||||
if "revision" in d or "controle" in d:
|
||||
return "controle"
|
||||
return "cours"
|
||||
|
||||
|
||||
def _etude_doc_year(document: str) -> str:
|
||||
m = re.search(r"20\d{2}", document or "")
|
||||
if m:
|
||||
return m.group(0)
|
||||
return "2025-2026"
|
||||
|
||||
|
||||
def _etude_statut(document: str, payload: dict) -> str:
|
||||
valid_statuts = ("verifie_complet", "verifie_leger", "comparaison_seule", "non_verifie")
|
||||
st = payload.get("statut_verification")
|
||||
if st in valid_statuts:
|
||||
return st
|
||||
doc = document or payload.get("document", "")
|
||||
if doc in VERIF_PLAN:
|
||||
return VERIF_PLAN[doc]
|
||||
for k, v in VERIF_PLAN.items():
|
||||
if k.lower() in doc.lower() or doc.lower() in k.lower():
|
||||
return v
|
||||
return "non_verifie"
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def etude_matieres(corpus: Literal["yesmine", "ahmed"]) -> str:
|
||||
"""Liste les matieres disponibles pour le corpus d etude choisi (yesmine ou ahmed)."""
|
||||
if corpus not in ("yesmine", "ahmed"):
|
||||
return json.dumps({"error": f"Corpus inconnu: {corpus}"})
|
||||
log.info("tool etude_matieres corpus=%s", corpus)
|
||||
coll = YESMINE_COLL if corpus == "yesmine" else AHMED_COLL
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
r = await client.post(f"{QDRANT_URL}/collections/{coll}/facet", json={"key": "matiere"})
|
||||
if r.status_code == 200:
|
||||
hits = r.json().get("result", {}).get("hits", [])
|
||||
mats = [h["value"] for h in hits if h.get("value")]
|
||||
if mats:
|
||||
return json.dumps(sorted(mats), ensure_ascii=False)
|
||||
except Exception as e:
|
||||
log.error("facet matieres erreur: %s", type(e).__name__)
|
||||
if corpus == "yesmine":
|
||||
return json.dumps(["Algebre", "Analyse", "Anglais", "Automatique", "Chimie", "Francais", "Informatique", "Mathematiques", "Mecanique", "Physique"], ensure_ascii=False)
|
||||
else:
|
||||
return json.dumps(["Anglais", "Arabe", "Chimie", "Education_Civique", "Francais", "Histoire", "Mathematiques", "Physique", "SVT", "Technologie"], ensure_ascii=False)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def etude_chapitres(corpus: Literal["yesmine", "ahmed"], matiere: str) -> str:
|
||||
"""Liste les chapitres et sections disponibles pour une matiere donnee dans le corpus d etude choisi."""
|
||||
if corpus not in ("yesmine", "ahmed"):
|
||||
return json.dumps({"error": f"Corpus inconnu: {corpus}"})
|
||||
log.info("tool etude_chapitres corpus=%s matiere=%s", corpus, matiere)
|
||||
coll = YESMINE_COLL if corpus == "yesmine" else AHMED_COLL
|
||||
try:
|
||||
payload = {
|
||||
"filter": {"must": [{"key": "matiere", "match": {"value": matiere}}]},
|
||||
"limit": 100,
|
||||
"with_payload": ["section_title", "document"]
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
r = await client.post(f"{QDRANT_URL}/collections/{coll}/points/scroll", json=payload)
|
||||
if r.status_code == 200:
|
||||
pts = r.json().get("result", {}).get("points", [])
|
||||
secs = set()
|
||||
for p in pts:
|
||||
st = p.get("payload", {}).get("section_title") or p.get("payload", {}).get("document")
|
||||
if st:
|
||||
secs.add(st)
|
||||
return json.dumps(sorted(list(secs)), ensure_ascii=False)
|
||||
return json.dumps([])
|
||||
except Exception as e:
|
||||
log.error("scroll chapitres erreur: %s", type(e).__name__)
|
||||
return json.dumps({"error": f"Erreur de lecture des chapitres: {type(e).__name__}"})
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def etude_rechercher(corpus: Literal["yesmine", "ahmed"], requete: str, matiere: str = "", chapitre: str = "", type: str = "", k: int = 6) -> str:
|
||||
"""Recherche des passages pertinents dans le corpus d etude choisi (yesmine ou ahmed). k est borne entre 1 et 8 (defaut 6)."""
|
||||
if corpus not in ("yesmine", "ahmed"):
|
||||
return json.dumps({"error": f"Corpus inconnu: {corpus}"})
|
||||
k_eff = min(max(int(k), 1), 8)
|
||||
log.info("tool etude_rechercher corpus=%s matiere=%s chapitre=%s type=%s k=%d", corpus, matiere, chapitre, type, k_eff)
|
||||
|
||||
day_key = "etude-" + datetime.now(ZoneInfo("Africa/Tunis")).strftime("%Y-%m-%d")
|
||||
used = int(store.get("cap", day_key) or 0)
|
||||
if used >= ETUDE_DAILY_CAP:
|
||||
return json.dumps({"error": f"Plafond journalier de {ETUDE_DAILY_CAP} recherches atteint (Tunis)."})
|
||||
|
||||
coll = YESMINE_COLL if corpus == "yesmine" else AHMED_COLL
|
||||
search_url = YESMINE_SEARCH_URL if corpus == "yesmine" else AHMED_SEARCH_URL
|
||||
results = []
|
||||
|
||||
try:
|
||||
req_data = {"query": requete, "top_k": k_eff}
|
||||
if matiere.strip():
|
||||
req_data["matiere"] = matiere.strip()
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
r = await client.post(search_url, json=req_data)
|
||||
if r.status_code == 200:
|
||||
raw_hits = r.json().get("results", [])
|
||||
for hit in raw_hits:
|
||||
doc = hit.get("document", "")
|
||||
chunk_idx = hit.get("chunk_index")
|
||||
point_id = ""
|
||||
pl = {}
|
||||
if doc and chunk_idx is not None:
|
||||
try:
|
||||
s_req = {
|
||||
"filter": {"must": [
|
||||
{"key": "document", "match": {"value": doc}},
|
||||
{"key": "chunk_index", "match": {"value": chunk_idx}}
|
||||
]},
|
||||
"limit": 1,
|
||||
"with_payload": True
|
||||
}
|
||||
sr = await client.post(f"{QDRANT_URL}/collections/{coll}/points/scroll", json=s_req)
|
||||
if sr.status_code == 200:
|
||||
pts = sr.json().get("result", {}).get("points", [])
|
||||
if pts:
|
||||
point_id = pts[0]["id"]
|
||||
pl = pts[0].get("payload", {})
|
||||
except Exception:
|
||||
pass
|
||||
if not point_id:
|
||||
point_id = f"gen-{abs(hash((doc, chunk_idx)))}"
|
||||
|
||||
p_chap = hit.get("section_title") or doc
|
||||
p_type = _etude_doc_type(doc)
|
||||
p_annee = _etude_doc_year(doc)
|
||||
|
||||
if chapitre and chapitre.lower() not in p_chap.lower():
|
||||
continue
|
||||
if type and type.lower() not in p_type.lower():
|
||||
continue
|
||||
|
||||
ext = (hit.get("extrait") or pl.get("text") or "")[:1500]
|
||||
statut = _etude_statut(doc, pl)
|
||||
results.append({
|
||||
"identifiant": str(point_id),
|
||||
"matiere": str(hit.get("matiere", pl.get("matiere", ""))),
|
||||
"chapitre": str(p_chap),
|
||||
"type": str(p_type),
|
||||
"page": int(hit.get("page", pl.get("page", 1))),
|
||||
"annee": str(p_annee),
|
||||
"extrait": str(ext)[:1500],
|
||||
"statut_verification": str(statut),
|
||||
})
|
||||
except Exception as e:
|
||||
log.error("RAG search URL error: %s", type(e).__name__)
|
||||
|
||||
if not results:
|
||||
try:
|
||||
must_list = []
|
||||
if matiere.strip():
|
||||
must_list.append({"key": "matiere", "match": {"value": matiere.strip()}})
|
||||
payload = {"limit": k_eff, "with_payload": True}
|
||||
if must_list:
|
||||
payload["filter"] = {"must": must_list}
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
r = await client.post(f"{QDRANT_URL}/collections/{coll}/points/scroll", json=payload)
|
||||
if r.status_code == 200:
|
||||
pts = r.json().get("result", {}).get("points", [])
|
||||
for p in pts:
|
||||
pl = p.get("payload", {})
|
||||
doc = pl.get("document", "")
|
||||
p_chap = pl.get("section_title") or doc
|
||||
p_type = _etude_doc_type(doc)
|
||||
p_annee = _etude_doc_year(doc)
|
||||
ext = (pl.get("text") or "")[:1500]
|
||||
statut = _etude_statut(doc, pl)
|
||||
results.append({
|
||||
"identifiant": str(p["id"]),
|
||||
"matiere": str(pl.get("matiere", "")),
|
||||
"chapitre": str(p_chap),
|
||||
"type": str(p_type),
|
||||
"page": int(pl.get("page", 1)),
|
||||
"annee": str(p_annee),
|
||||
"extrait": str(ext)[:1500],
|
||||
"statut_verification": str(statut),
|
||||
})
|
||||
except Exception as e:
|
||||
log.error("scroll fallback error: %s", type(e).__name__)
|
||||
|
||||
store.put("cap", day_key, str(used + 1))
|
||||
return json.dumps(results[:k_eff], ensure_ascii=False)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def etude_lire_extrait(corpus: Literal["yesmine", "ahmed"], id_passage: str) -> str:
|
||||
"""Lit un extrait precis du corpus d etude a partir de son identifiant de passage (extrait plafonne a 1 500 caracteres)."""
|
||||
if corpus not in ("yesmine", "ahmed"):
|
||||
return json.dumps({"error": f"Corpus inconnu: {corpus}"})
|
||||
log.info("tool etude_lire_extrait corpus=%s id=%s", corpus, id_passage)
|
||||
coll = YESMINE_COLL if corpus == "yesmine" else AHMED_COLL
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
r = await client.get(f"{QDRANT_URL}/collections/{coll}/points/{id_passage}")
|
||||
if r.status_code != 200:
|
||||
return json.dumps({"error": f"Passage {id_passage} introuvable."})
|
||||
res = r.json().get("result")
|
||||
if not res or not res.get("payload"):
|
||||
return json.dumps({"error": f"Passage {id_passage} introuvable."})
|
||||
pl = res["payload"]
|
||||
doc = pl.get("document", "")
|
||||
p_chap = pl.get("section_title") or doc
|
||||
p_type = _etude_doc_type(doc)
|
||||
p_annee = _etude_doc_year(doc)
|
||||
statut = _etude_statut(doc, pl)
|
||||
ext = (pl.get("text") or "")[:1500]
|
||||
return json.dumps({
|
||||
"identifiant": str(id_passage),
|
||||
"matiere": str(pl.get("matiere", "")),
|
||||
"chapitre": str(p_chap),
|
||||
"type": str(p_type),
|
||||
"page": int(pl.get("page", 1)),
|
||||
"annee": str(p_annee),
|
||||
"extrait": str(ext)[:1500],
|
||||
"statut_verification": str(statut),
|
||||
}, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
log.error("lire_extrait erreur: %s", type(e).__name__)
|
||||
return json.dumps({"error": f"Erreur de lecture du passage: {type(e).__name__}"})
|
||||
|
||||
|
||||
PAGE = """<!doctype html><html lang="fr"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Autorisation nyora-etude</title><style>
|
||||
body{margin:0;font-family:Segoe UI,Calibri,system-ui,sans-serif;background:#eef3f8;color:#16324f}
|
||||
main{max-width:420px;margin:12vh auto;padding:0 16px}
|
||||
.card{background:#fff;border-radius:14px;box-shadow:0 2px 12px rgba(22,50,79,.1);overflow:hidden}
|
||||
header{background:linear-gradient(90deg,#1d5fa8,#1fb5b0);color:#fff;padding:18px 22px;font-size:1.1rem;font-weight:600}
|
||||
form{padding:22px}label{display:block;margin-bottom:8px;font-size:.95rem}
|
||||
input[type=password],input[type=text]{width:100%;box-sizing:border-box;padding:12px;border:1px solid #b7c6d6;border-radius:8px;font-size:1rem}
|
||||
button{margin-top:16px;width:100%;padding:12px;border:0;border-radius:8px;background:#1d5fa8;color:#fff;font-size:1rem;cursor:pointer}
|
||||
.err{color:#b3261e;margin:0 0 12px}.note{font-size:.82rem;color:#5b7186;margin-top:14px}
|
||||
</style></head><body><main><div class="card"><header>Nyora Étude : autoriser la connexion</header>
|
||||
<form method="post" action="/consent"><input type="hidden" name="sid" value="%SID%">%ERR%
|
||||
<label for="p">Phrase secrète</label><input id="p" name="phrase" type="password" autocomplete="off" autofocus>
|
||||
<button type="submit">Autoriser</button>
|
||||
<p class="note">Seul le propriétaire connaît cette phrase.</p></form></div></main></body></html>"""
|
||||
|
||||
|
||||
def render(sid, err=""):
|
||||
e = f'<p class="err">{html.escape(err)}</p>' if err else ""
|
||||
return PAGE.replace("%SID%", html.escape(sid)).replace("%ERR%", e)
|
||||
|
||||
|
||||
@mcp.custom_route("/health", methods=["GET"])
|
||||
async def health(request: Request):
|
||||
return PlainTextResponse("ok")
|
||||
|
||||
|
||||
@mcp.custom_route("/consent", methods=["GET", "POST"])
|
||||
async def consent(request: Request):
|
||||
ip = client_ip(request)
|
||||
if request.method == "GET":
|
||||
sid = request.query_params.get("sid", "")
|
||||
if not store.get("pending", sid):
|
||||
return HTMLResponse("<p>Demande expirée. Relancez la connexion.</p>", status_code=400)
|
||||
return HTMLResponse(render(sid))
|
||||
form = {k: v[0] for k, v in parse_qs((await request.body()).decode()).items()}
|
||||
sid, phrase = form.get("sid", ""), form.get("phrase", "")
|
||||
pend = store.get("pending", sid)
|
||||
if not pend:
|
||||
return HTMLResponse("<p>Demande expirée. Relancez la connexion.</p>", status_code=400)
|
||||
if is_locked(ip):
|
||||
log.warning("consent ip=%s resultat=verrouille", ip)
|
||||
return HTMLResponse(render(sid, "Trop d'essais. Réessayez plus tard."), status_code=429)
|
||||
if not verify_phrase(phrase):
|
||||
record_fail(ip)
|
||||
log.warning("consent ip=%s resultat=refuse", ip)
|
||||
return HTMLResponse(render(sid, "Phrase incorrecte."), status_code=401)
|
||||
p = json.loads(pend)
|
||||
code = secrets.token_urlsafe(32)
|
||||
store.put("code", code, AuthorizationCode(
|
||||
code=code, scopes=p["scopes"], expires_at=time.time() + CODE_TTL, client_id=p["client_id"],
|
||||
code_challenge=p["code_challenge"], redirect_uri=AnyUrl(p["redirect_uri"]),
|
||||
redirect_uri_provided_explicitly=p["explicit"], resource=p.get("resource")).model_dump_json(),
|
||||
time.time() + CODE_TTL)
|
||||
store.delete("pending", sid)
|
||||
log.info("consent ip=%s resultat=accepte client=%s", ip, p["client_id"])
|
||||
return RedirectResponse(construct_redirect_uri(p["redirect_uri"], code=code, state=p["state"]), status_code=302)
|
||||
|
||||
|
||||
def _prm():
|
||||
return {"resource": ISSUER + "/mcp", "authorization_servers": [ISSUER + "/"], "bearer_methods_supported": ["header"]}
|
||||
|
||||
|
||||
@mcp.custom_route("/.well-known/oauth-protected-resource/mcp", methods=["GET"])
|
||||
async def prm_path(request: Request):
|
||||
return JSONResponse(_prm())
|
||||
|
||||
|
||||
@mcp.custom_route("/mcp/.well-known/oauth-protected-resource", methods=["GET"])
|
||||
async def prm_header(request: Request):
|
||||
return JSONResponse(_prm())
|
||||
|
||||
|
||||
class NoSSE(BaseHTTPMiddleware):
|
||||
"""Le serveur ne propose pas de flux SSE serveur vers client : la specification demande un 405."""
|
||||
async def dispatch(self, request, call_next):
|
||||
if request.method == "GET" and request.url.path == "/mcp":
|
||||
return PlainTextResponse("flux SSE non pris en charge", status_code=405, headers={"Allow": "POST"})
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
app = mcp.streamable_http_app()
|
||||
app.add_middleware(NoSSE)
|
||||
@@ -0,0 +1,7 @@
|
||||
starlette==0.37.2
|
||||
uvicorn==0.29.0
|
||||
pydantic==2.13.5
|
||||
pydantic-settings==2.15.0
|
||||
mcp==1.12.4
|
||||
httpx==0.27.2
|
||||
pytest==8.3.3
|
||||
Executable
+162
@@ -0,0 +1,162 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import urllib.request
|
||||
import pytest
|
||||
import main
|
||||
|
||||
STATUTS_VALIDES = ["verifie_complet", "verifie_leger", "comparaison_seule", "non_verifie"]
|
||||
|
||||
|
||||
def _txt(res):
|
||||
if isinstance(res, tuple):
|
||||
res = res[0]
|
||||
if isinstance(res, list):
|
||||
return "".join(getattr(x, "text", "") for x in res)
|
||||
return str(res)
|
||||
|
||||
|
||||
def test_corpus_inconnu_refuse():
|
||||
"""Verifie que tout corpus hors yesmine et ahmed est refuse."""
|
||||
for tool_name, args in [
|
||||
("etude_matieres", {"corpus": "inconnu"}),
|
||||
("etude_chapitres", {"corpus": "inconnu", "matiere": "Analyse"}),
|
||||
("etude_rechercher", {"corpus": "inconnu", "requete": "theoreme"}),
|
||||
("etude_lire_extrait", {"corpus": "inconnu", "id_passage": "test-id"}),
|
||||
]:
|
||||
refused = False
|
||||
try:
|
||||
res = asyncio.run(main.mcp.call_tool(tool_name, args))
|
||||
t = _txt(res)
|
||||
if "error" in t or "erreur" in t.lower() or "inconnu" in t.lower():
|
||||
refused = True
|
||||
except Exception:
|
||||
refused = True
|
||||
assert refused, f"Corpus inconnu n a pas ete refuse pour {tool_name}"
|
||||
|
||||
# Test direct call as well
|
||||
fn = getattr(main, tool_name)
|
||||
if tool_name == "etude_matieres":
|
||||
direct_res = asyncio.run(fn("inconnu"))
|
||||
elif tool_name == "etude_chapitres":
|
||||
direct_res = asyncio.run(fn("inconnu", "Analyse"))
|
||||
elif tool_name == "etude_rechercher":
|
||||
direct_res = asyncio.run(fn("inconnu", "theoreme"))
|
||||
elif tool_name == "etude_lire_extrait":
|
||||
direct_res = asyncio.run(fn("inconnu", "test-id"))
|
||||
assert "error" in direct_res or "erreur" in direct_res.lower()
|
||||
|
||||
|
||||
def test_extrait_plafonne_1500():
|
||||
"""Verifie que les extraits ne depassent jamais 1500 caracteres."""
|
||||
res = asyncio.run(main.mcp.call_tool("etude_rechercher", {"corpus": "yesmine", "requete": "theoreme", "k": 5}))
|
||||
t = _txt(res)
|
||||
passages = json.loads(t)
|
||||
if isinstance(passages, dict):
|
||||
passages = passages.get("passages") or passages.get("resultats") or passages.get("results") or [passages]
|
||||
assert isinstance(passages, list) and len(passages) >= 1
|
||||
for p in passages:
|
||||
if "extrait" in p:
|
||||
assert len(str(p["extrait"])) <= 1500
|
||||
|
||||
|
||||
def test_statut_verification_present():
|
||||
"""Verifie que statut_verification est present et valide sur chaque passage."""
|
||||
res = asyncio.run(main.mcp.call_tool("etude_rechercher", {"corpus": "yesmine", "requete": "theoreme", "k": 3}))
|
||||
t = _txt(res)
|
||||
passages = json.loads(t)
|
||||
if isinstance(passages, dict):
|
||||
passages = passages.get("passages") or passages.get("resultats") or passages.get("results") or [passages]
|
||||
assert isinstance(passages, list) and len(passages) >= 1
|
||||
for p in passages:
|
||||
assert "statut_verification" in p, f"statut_verification manquant dans {p}"
|
||||
assert p["statut_verification"] in STATUTS_VALIDES, f"statut invalide: {p['statut_verification']}"
|
||||
|
||||
|
||||
def test_plafond_journalier_200():
|
||||
"""Verifie le plafond de 200 recherches par jour Tunis."""
|
||||
day_key = "etude-" + main.datetime.now(main.ZoneInfo("Africa/Tunis")).strftime("%Y-%m-%d")
|
||||
old_val = main.store.get("cap", day_key)
|
||||
try:
|
||||
main.store.put("cap", day_key, "200")
|
||||
res = asyncio.run(main.mcp.call_tool("etude_rechercher", {"corpus": "yesmine", "requete": "theoreme"}))
|
||||
t = _txt(res)
|
||||
data = json.loads(t)
|
||||
assert "error" in data, f"Erreur attendue mais recu: {t}"
|
||||
assert "200" in data["error"] or "plafond" in data["error"].lower()
|
||||
finally:
|
||||
if old_val is not None:
|
||||
main.store.put("cap", day_key, str(old_val))
|
||||
else:
|
||||
main.store.delete("cap", day_key)
|
||||
|
||||
|
||||
def test_aucun_outil_ecriture_etude():
|
||||
"""Verifie l absence de tout outil ou parametre d ecriture ou modification."""
|
||||
tl = asyncio.run(main.mcp.list_tools())
|
||||
etude_tools = [t for t in tl if t.name.startswith("etude_")]
|
||||
assert len(etude_tools) == 4, f"Attendu 4 outils etude_*, trouve {len(etude_tools)}"
|
||||
forbidden = re.compile(r"ecri|supprim|delete|upsert|telecharg|download|write|create|modify|patch|post|drop|collection", re.I)
|
||||
for t in etude_tools:
|
||||
assert not forbidden.search(t.name), f"Nom d outil interdit: {t.name}"
|
||||
props = t.inputSchema.get("properties", {})
|
||||
for param in props:
|
||||
assert not forbidden.search(param), f"Parametre interdit {param} dans {t.name}"
|
||||
|
||||
|
||||
def test_aucune_fuite_de_jeton():
|
||||
"""Verifie qu aucun jeton secret n est divulgue dans les reponses."""
|
||||
tok_pattern = re.compile(r"scrypt\$[0-9]+\$|Bearer [A-Za-z0-9._~+/=-]{24,}")
|
||||
for tool_name, args in [
|
||||
("etude_matieres", {"corpus": "yesmine"}),
|
||||
("etude_chapitres", {"corpus": "yesmine", "matiere": "Analyse"}),
|
||||
("etude_rechercher", {"corpus": "yesmine", "requete": "test"}),
|
||||
("etude_lire_extrait", {"corpus": "yesmine", "id_passage": "00000000-0000-0000-0000-000000000000"}),
|
||||
]:
|
||||
res = asyncio.run(main.mcp.call_tool(tool_name, args))
|
||||
t = _txt(res)
|
||||
assert not tok_pattern.search(t), f"Fuite de motif de jeton dans {tool_name}"
|
||||
|
||||
|
||||
def test_journal_sans_contenu(caplog):
|
||||
"""Verifie que le contenu de la requete n apparait jamais dans les journaux."""
|
||||
caplog.set_level(logging.INFO)
|
||||
marker = "REQUETE_SECRETE_TEST_XYZ987654"
|
||||
asyncio.run(main.mcp.call_tool("etude_rechercher", {"corpus": "yesmine", "requete": marker, "k": 1}))
|
||||
assert marker not in caplog.text, "Le contenu de la requete a ete journalise!"
|
||||
|
||||
|
||||
def test_lecture_croisee_refusee():
|
||||
"""Verifie qu un extrait d Ahmed ne peut pas etre lu sous le corpus yesmine."""
|
||||
try:
|
||||
req = urllib.request.Request("http://100.94.90.119:6333/collections/ahmed-2eme-sciences-rag-gemini-prod/points/scroll",
|
||||
method="POST", data=json.dumps({"limit": 1, "with_payload": False}).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
aid = json.loads(urllib.request.urlopen(req, timeout=10).read())["result"]["points"][0]["id"]
|
||||
except Exception:
|
||||
aid = "30a51252-fde9-4ec8-80cf-452806603ec5"
|
||||
res = asyncio.run(main.mcp.call_tool("etude_lire_extrait", {"corpus": "yesmine", "id_passage": aid}))
|
||||
t = _txt(res)
|
||||
assert '"extrait"' not in t or len(t) <= 200, f"Lecture croisee acceptee: {t}"
|
||||
|
||||
|
||||
def test_outils_existants_intacts():
|
||||
"""Dans mcp-etude, verifie que les 4 outils d etude sont intacts et conformes."""
|
||||
tl = asyncio.run(main.mcp.list_tools())
|
||||
tool_names = {t.name for t in tl}
|
||||
assert tool_names == {"etude_matieres", "etude_chapitres", "etude_rechercher", "etude_lire_extrait"}
|
||||
|
||||
|
||||
def test_aucun_identifiant_de_collection_libre():
|
||||
"""Verifie qu aucun outil n accepte de nom de collection libre et que corpus est limite."""
|
||||
tl = asyncio.run(main.mcp.list_tools())
|
||||
etude_tools = [t for t in tl if t.name.startswith("etude_")]
|
||||
assert len(etude_tools) == 4
|
||||
for t in etude_tools:
|
||||
props = t.inputSchema.get("properties", {})
|
||||
assert "collection" not in props, f"parametre collection interdit dans {t.name}"
|
||||
assert "coll" not in props, f"parametre coll interdit dans {t.name}"
|
||||
assert "corpus" in props, f"parametre corpus requis dans {t.name}"
|
||||
enum = props["corpus"].get("enum") or []
|
||||
assert sorted(enum) == ["ahmed", "yesmine"], f"enum corpus invalide dans {t.name}: {enum}"
|
||||
Reference in new issue
Block a user