4) Indexation & vecteurs — RAG

FAISS (PoC rapide)

# faiss_build.py
import faiss, numpy as np

DIM = 1024
xb = np.array(embeddings, dtype="float32")      # déjà L2-normés
# Flat L2
index = faiss.IndexFlatL2(DIM)                  # ou IndexFlatIP si non-normalisés
index.add(xb)

# IVF + HNSW (optionnel)
quantizer = faiss.IndexHNSWFlat(DIM, 32)
ivf = faiss.IndexIVFFlat(quantizer, DIM, 200)   # lists=200
ivf.train(xb)
ivf.add(xb)
# recherche
D, I = ivf.search(query_vecs, k=24)

pgvector (production SQL-first)

-- schema.sql (PostgreSQL)
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE rag_chunks(
  id TEXT PRIMARY KEY,
  text TEXT,
  meta JSONB,
  embedding VECTOR(1024)
);
-- IVFFlat
CREATE INDEX rag_chunks_ivf ON rag_chunks USING ivfflat (embedding vector_l2_ops) WITH (lists=200);
-- HNSW (si dispo)
-- CREATE INDEX rag_chunks_hnsw ON rag_chunks USING hnsw (embedding vector_l2_ops) WITH (m=16, ef_construction=64);

-- requête: kNN + filtres
-- :qvec = embedding de la requête (VECTOR(1024))
SELECT id, text, meta
FROM rag_chunks
WHERE (meta->>'lang') = 'fr'
  AND (meta->>'bu') = 'IT'
  AND (meta->>'type') IN ('documentation','policy')
  AND (meta->>'last_modified')::date >= (now()::date - interval '90 days')
ORDER BY embedding <-> :qvec   -- L2 ; si vecteurs L2-normés => approx. cosine
LIMIT 24;

OpenSearch / Elasticsearch (dense vectors)

// OpenSearch (KNN plugin)
PUT /rag_chunks
{
  "settings": { "index": { "knn": true } },
  "mappings": {
    "properties": {
      "text": { "type": "text" },
      "lang": { "type": "keyword" },
      "bu":   { "type": "keyword" },
      "type": { "type": "keyword" },
      "last_modified": { "type": "date" },
      "embedding": {
        "type": "knn_vector",
        "dimension": 1024,
        "method": {"name":"hnsw","space_type":"cosinesimil","engine":"nmslib"}
      }
    }
  }
}

// recherche (kNN + bool filters)
POST /rag_chunks/_search
{
  "size": 24,
  "query": {
    "bool": {
      "filter": [
        {"term": {"lang":"fr"}},
        {"term": {"bu":"IT"}},
        {"terms": {"type":["documentation","policy"]}},
        {"range":{"last_modified":{"gte":"now-90d"}}}
      ],
      "must": {
        "knn": {"embedding": {"vector": [..qvec..], "k": 24}}
      }
    }
  }
}

Weaviate / Milvus (volumétrie élevée)

# Weaviate (schéma)
{
  "class": "RagChunk",
  "vectorizer": "none",
  "vectorIndexConfig": {"distance":"cosine","efConstruction":64,"maxConnections":16}
}

# Milvus (pymilvus)
from pymilvus import Collection, FieldSchema, CollectionSchema, DataType, connections, utility
dim = 1024
fields = [
  FieldSchema("id", DataType.VARCHAR, is_primary=True, max_length=128),
  FieldSchema("embedding", DataType.FLOAT_VECTOR, dim=dim),
  FieldSchema("lang", DataType.VARCHAR, max_length=8),
  FieldSchema("bu", DataType.VARCHAR, max_length=32),
  FieldSchema("type", DataType.VARCHAR, max_length=32),
  FieldSchema("last_modified", DataType.INT64)
]
schema = CollectionSchema(fields, "rag chunks")
col = Collection("rag_chunks", schema)
col.create_index("embedding", {"index_type":"HNSW","metric_type":"COSINE","params":{"M":16,"efConstruction":64}})
# recherche
col.load()
res = col.search([qvec], "embedding", {"metric_type":"COSINE","params":{"ef":80}}, limit=24,
                 expr="lang=='fr' and bu=='IT' and type in ['documentation','policy'] and last_modified >= now()-90*86400")

Schéma de métadonnées (rappel)

• lang : fr/en…
• bu : IT/HR/Finance…
• type : documentation/policy/faq…
• last_modified : ISO datetime
• + source_id, version, hash

Requête hybride (pgvector + fulltext)

-- Exemple combiné (dense + BM25) avec filtres
WITH dense AS (
  SELECT id, text, meta, 1 - (embedding <-> :qvec) AS s_d
  FROM rag_chunks
  WHERE (meta->>'lang')='fr' AND (meta->>'bu')='IT'
    AND (meta->>'type') IN ('documentation','policy')
    AND (meta->>'last_modified')::date >= (now()::date - interval '90 days')
  ORDER BY embedding <-> :qvec
  LIMIT 60
),
sparse AS (
  SELECT id, text, meta, ts_rank_cd(to_tsvector('french', text), plainto_tsquery('french', :q)) AS s_s
  FROM rag_chunks
  WHERE to_tsvector('french', text) @@ plainto_tsquery('french', :q)
  LIMIT 60
)
SELECT * FROM (
  SELECT *, s_d*0.6 AS score FROM dense
  UNION
  SELECT *, s_s*0.6 AS score FROM sparse
) u
ORDER BY score DESC
LIMIT 24;

Livrables attendus

• Index créé (config documentée : modèle, dim, metric, ann, params).
• Scripts de build/rebuild (full & incrémental) + readme.
• Tests de rappel (recall@k) sur golden set.

Contrat YAML de livraison

{
  "embedding":{"model":"bge-m3","dim":1024,"normalize_l2":true},
  "index":{"store":"pgvector","metric":"cosine","ann":"ivfflat","params":{"lists":200}},
  "filters":{"lang":["fr","en"],"bu":["IT","HR"],"freshness_days":90,"type":["documentation","policy"]},
  "artifacts":["schema.sql","build_index.py","rebuild_index.py","recall_report.json"]
}

Requête FAISS

# search_faiss.py
D, I = index.search(np.array([qvec]).astype("float32"), k=24)
results = [corpus[idx] for idx in I[0]]

Requête pgvector (SQL)

SELECT id, text, meta
FROM rag_chunks
WHERE (meta->>'lang')='fr' AND (meta->>'bu')='IT'
  AND (meta->>'type') IN ('documentation','policy')
  AND (meta->>'last_modified')::date >= (now()::date - interval '90 days')
ORDER BY embedding <-> :qvec
LIMIT 24;

Requête OpenSearch (KNN + filtres)

POST /rag_chunks/_search
{
  "size": 24,
  "query": {
    "bool": {
      "filter": [
        {"term":{"lang":"fr"}},{"term":{"bu":"IT"}},
        {"terms":{"type":["documentation","policy"]}},
        {"range":{"last_modified":{"gte":"now-90d"}}}
      ],
      "must": {"knn":{"embedding":{"vector":[..qvec..],"k":24}}}
    }
  }
}

Règles de dimensionnement

• Stockage ≈ nb_chunks × dim × 4 bytes (float32) + overhead index.
• IVFFlat : lists ≈ sqrt(nb_vectors) (ajustez par mesures).
• HNSW : M 12–24 ; ef_search 50–150 (qualité vs latence).
• Batch d’upsert 128–512 pour saturer le CPU/IO.

Contrôles d’intégrité

• Vérifier dim constant (assert sur longueur des vecteurs).
• Vérifier L2-norm==1 si similarité cosinus.
• Compter trous/suppressions → vacuum périodique.

# checks.py
assert all(abs(np.linalg.norm(v)-1) < 1e-3 for v in vecs)

Protocole recall@k

• Golden set : (question → ids de passages pertinents).
• Évaluer pour k ∈ {5,10,20} : recall@k = % de questions où ≥1 passage pertinent est dans top-k.
• Comparer variantes : modèle, dim, ann, params, filtres.

Script (pseudo)

# recall.py
def recall_at_k(queries, gold, k=10):
    ok = 0
    for q in queries:
        qvec = embed_texts([q.text])[0]
        top = search(qvec, k=k)  # retourne [ids]
        if set(top) & set(gold[q.id]): ok += 1
    return ok/len(queries)

report = {
  "k5":  recall_at_k(queries, gold, 5),
  "k10": recall_at_k(queries, gold, 10),
  "k20": recall_at_k(queries, gold, 20),
}
save_json(report, "recall_report.json")