LlamaIndex : Le Framework RAG Optimisé pour la Recherche et l'Indexation Intelligente

Documents (vos données)
    ↓
Data Connectors (loaders)
    ↓
Indexes (structures de données)
    ↓
Query Engines (recherche)
    ↓
Response Synthesizers (génération)
    ↓
Réponse finale

from llama_index import (
    SimpleDirectoryReader,
    download_loader
)

# Charger des fichiers locaux
documents = SimpleDirectoryReader('./docs').load_data()

# Charger depuis des sources externes
GoogleDocsReader = download_loader("GoogleDocsReader")
loader = GoogleDocsReader()
documents = loader.load_data(document_ids=["doc_id"])

# Plus de 100 loaders disponibles via LlamaHub

from llama_index import VectorStoreIndex, ServiceContext
from llama_index.llms import OpenAI

# Configuration du LLM
llm = OpenAI(model="gpt-4", temperature=0)
service_context = ServiceContext.from_defaults(llm=llm)

# Création de l'index vectoriel
index = VectorStoreIndex.from_documents(
    documents,
    service_context=service_context
)

from llama_index import TreeIndex

# Index en arbre pour résumés hiérarchiques
tree_index = TreeIndex.from_documents(documents)

from llama_index import KeywordTableIndex

# Index par mots-clés
keyword_index = KeywordTableIndex.from_documents(documents)

from llama_index import SummaryIndex

# Index liste pour parcours séquentiel
list_index = SummaryIndex.from_documents(documents)

from llama_index import KnowledgeGraphIndex

# Index graphe pour relations
graph_index = KnowledgeGraphIndex.from_documents(documents)

# Query engine simple
query_engine = index.as_query_engine()

response = query_engine.query("Qu'est-ce que le fine-tuning ?")
print(response)

# Mode compact (défaut)
query_engine = index.as_query_engine(
    response_mode="compact"  # Combine chunks intelligemment
)

# Mode tree_summarize (hiérarchique)
query_engine = index.as_query_engine(
    response_mode="tree_summarize"  # Bottom-up summarization
)

# Mode simple_summarize
query_engine = index.as_query_engine(
    response_mode="simple_summarize"  # Résumé de tous les chunks
)

# Mode no_text (juste les sources)
query_engine = index.as_query_engine(
    response_mode="no_text"  # Retourne seulement les chunks pertinents
)

from llama_index.response_synthesizers import get_response_synthesizer

response_synthesizer = get_response_synthesizer(
    response_mode="compact",
    structured_answer_filtering=True  # Filtre les réponses mal formées
)

query_engine = index.as_query_engine(
    response_synthesizer=response_synthesizer
)

pip install llama-index llama-index-llms-openai

from llama_index import VectorStoreIndex, SimpleDirectoryReader
from llama_index.llms import OpenAI
import os

# 1. Configuration
os.environ["OPENAI_API_KEY"] = "sk-..."

# 2. Charger les documents
documents = SimpleDirectoryReader("./data").load_data()
print(f"Chargé {len(documents)} documents")

# 3. Créer l'index (automatique : chunking + embeddings + vectorstore)
index = VectorStoreIndex.from_documents(documents)

# 4. Query engine
query_engine = index.as_query_engine()

# 5. Poser des questions
response = query_engine.query("Résume le document principal")
print(response)

# Avec sources
response = query_engine.query("Qu'est-ce que LoRA ?")
print(f"Réponse: {response}")
print(f"\nSources: {response.source_nodes}")

# Sauvegarder
index.storage_context.persist(persist_dir="./storage")

# Recharger
from llama_index import StorageContext, load_index_from_storage

storage_context = StorageContext.from_defaults(persist_dir="./storage")
index = load_index_from_storage(storage_context)

from llama_index.node_parser import SimpleNodeParser

# Chunking manuel
node_parser = SimpleNodeParser.from_defaults(
    chunk_size=512,
    chunk_overlap=50
)

nodes = node_parser.get_nodes_from_documents(documents)
index = VectorStoreIndex(nodes)

from llama_index.embeddings import HuggingFaceEmbedding

# Modèle d'embedding open source
embed_model = HuggingFaceEmbedding(
    model_name="sentence-transformers/all-MiniLM-L6-v2"
)

from llama_index import ServiceContext

service_context = ServiceContext.from_defaults(
    embed_model=embed_model
)

index = VectorStoreIndex.from_documents(
    documents,
    service_context=service_context
)

import chromadb
from llama_index.vector_stores import ChromaVectorStore

# Chroma
chroma_client = chromadb.PersistentClient(path="./chroma_db")
chroma_collection = chroma_client.get_or_create_collection("my_collection")

vector_store = ChromaVectorStore(chroma_collection=chroma_collection)

from llama_index import StorageContext

storage_context = StorageContext.from_defaults(vector_store=vector_store)

index = VectorStoreIndex.from_documents(
    documents,
    storage_context=storage_context
)

from llama_index.vector_stores.types import ExactMatchFilter, MetadataFilters

# Ajouter des métadonnées aux documents
for doc in documents:
    doc.metadata = {
        "category": "technical",
        "year": 2025,
        "author": "Sophie"
    }

index = VectorStoreIndex.from_documents(documents)

# Query avec filtres
filters = MetadataFilters(
    filters=[
        ExactMatchFilter(key="category", value="technical"),
        ExactMatchFilter(key="year", value=2025)
    ]
)

query_engine = index.as_query_engine(
    filters=filters,
    similarity_top_k=5
)

response = query_engine.query("Question technique")

from llama_index import VectorStoreIndex, KeywordTableIndex
from llama_index.indices.composability import ComposableGraph

# Index vectoriel
vector_index = VectorStoreIndex.from_documents(documents)

# Index mots-clés
keyword_index = KeywordTableIndex.from_documents(documents)

# Composition
graph = ComposableGraph.from_indices(
    VectorStoreIndex,
    [vector_index, keyword_index],
    index_summaries=["Recherche sémantique", "Recherche par mots-clés"]
)

query_engine = graph.as_query_engine()
response = query_engine.query("Question complexe")

from llama_index.tools import QueryEngineTool
from llama_index.query_engine import SubQuestionQueryEngine

# Créer un index par document
doc1_index = VectorStoreIndex.from_documents([doc1])
doc2_index = VectorStoreIndex.from_documents([doc2])

# Outils de query
query_tools = [
    QueryEngineTool(
        query_engine=doc1_index.as_query_engine(),
        metadata={"name": "Document 1", "description": "Documentation technique"}
    ),
    QueryEngineTool(
        query_engine=doc2_index.as_query_engine(),
        metadata={"name": "Document 2", "description": "Guide utilisateur"}
    )
]

# Query engine qui décompose automatiquement en sous-questions
query_engine = SubQuestionQueryEngine.from_defaults(
    query_engine_tools=query_tools
)

# Question complexe nécessitant plusieurs documents
response = query_engine.query(
    "Compare les approches techniques décrites dans les deux documents"
)

from llama_index.chat_engine import CondensePlusContextChatEngine

# Chat engine avec historique
chat_engine = index.as_chat_engine(
    chat_mode="condense_plus_context",
    verbose=True
)

# Conversation
response1 = chat_engine.chat("Parle-moi de LoRA")
print(response1)

response2 = chat_engine.chat("Quels sont ses avantages ?")  # Contexte conservé
print(response2)

# Réinitialiser l'historique
chat_engine.reset()

from llama_index import VectorStoreIndex, SimpleDirectoryReader
from llama_index.llms import OpenAI

# Charger toute la doc entreprise
documents = SimpleDirectoryReader(
    "./company_docs",
    recursive=True,
    required_exts=[".pdf", ".docx", ".txt"]
).load_data()

# Ajouter métadonnées
for doc in documents:
    # Extraire département du path
    if "hr" in doc.metadata["file_path"]:
        doc.metadata["department"] = "HR"
    elif "tech" in doc.metadata["file_path"]:
        doc.metadata["department"] = "Tech"

# Index persistant
index = VectorStoreIndex.from_documents(documents)
index.storage_context.persist(persist_dir="./company_index")

# Query avec département
query_engine = index.as_query_engine(
    similarity_top_k=5,
    filters=MetadataFilters(
        filters=[ExactMatchFilter(key="department", value="HR")]
    )
)

response = query_engine.query("Quelle est la politique de congés ?")

from llama_index import VectorStoreIndex
from llama_index.response_synthesizers import ResponseMode

# Index de base de connaissances
knowledge_base = VectorStoreIndex.from_documents(articles)

# Query engine optimisé pour la recherche
search_engine = knowledge_base.as_query_engine(
    response_mode="no_text",  # Juste les passages pertinents
    similarity_top_k=10
)

# Recherche
query = "techniques de fine-tuning économiques"
results = search_engine.query(query)

# Afficher les résultats
for i, node in enumerate(results.source_nodes):
    print(f"\n--- Résultat {i+1} ---")
    print(f"Score: {node.score:.3f}")
    print(f"Source: {node.metadata.get('file_name')}")
    print(f"Extrait: {node.text[:200]}...")

from llama_index import VectorStoreIndex
from llama_index.tools import QueryEngineTool
from llama_index.query_engine import SubQuestionQueryEngine

# Index par concurrent
competitor_docs = {
    "OpenAI": load_documents("./openai_docs"),
    "Anthropic": load_documents("./anthropic_docs"),
    "Google": load_documents("./google_docs")
}

tools = []
for name, docs in competitor_docs.items():
    index = VectorStoreIndex.from_documents(docs)
    tool = QueryEngineTool(
        query_engine=index.as_query_engine(),
        metadata={
            "name": name,
            "description": f"Documentation de {name}"
        }
    )
    tools.append(tool)

# Query engine comparatif
comparative_engine = SubQuestionQueryEngine.from_defaults(
    query_engine_tools=tools,
    verbose=True
)

# Question comparative
response = comparative_engine.query("""
Compare les modèles GPT-4, Claude Sonnet 4.5 et Gemini sur :
1. Capacités de raisonnement
2. Support multimodal
3. Limites de contexte
4. Prix
""")

print(response)

from llama_index import TreeIndex

# Charger rapport long (100+ pages)
documents = SimpleDirectoryReader("./annual_report.pdf").load_data()

# Tree index pour résumé hiérarchique
tree_index = TreeIndex.from_documents(documents)

# Query engine avec tree_summarize
query_engine = tree_index.as_query_engine(
    response_mode="tree_summarize"
)

# Résumé exécutif
executive_summary = query_engine.query("""
Crée un résumé exécutif en 5 points clés couvrant :
- Performance financière
- Défis principaux
- Réalisations majeures
- Perspectives futures
""")

print(executive_summary)

from llama_index.node_parser import SentenceSplitter

# Chunking par phrases (meilleur pour la cohérence)
splitter = SentenceSplitter(
    chunk_size=1024,
    chunk_overlap=20,
    paragraph_separator="\n\n"
)

nodes = splitter.get_nodes_from_documents(documents)
index = VectorStoreIndex(nodes)

from llama_index import ServiceContext
from llama_index.embeddings import OpenAIEmbedding

# Cache d'embeddings
embed_model = OpenAIEmbedding(embed_batch_size=100)

service_context = ServiceContext.from_defaults(
    embed_model=embed_model
)

# Les embeddings sont automatiquement cachés lors de la persistance
index = VectorStoreIndex.from_documents(
    documents,
    service_context=service_context
)

index.storage_context.persist()  # Cache sauvegardé

from llama_index import VectorStoreIndex

# Indexation par batches (économique)
all_documents = []  # Milliers de documents

batch_size = 100
for i in range(0, len(all_documents), batch_size):
    batch = all_documents[i:i+batch_size]

    if i == 0:
        index = VectorStoreIndex.from_documents(batch)
    else:
        # Ajouter au index existant
        for doc in batch:
            index.insert(doc)

    print(f"Indexé {i+len(batch)}/{len(all_documents)} documents")

# Sauvegarder
index.storage_context.persist()

from llama_index.callbacks import CallbackManager, LlamaDebugHandler

# Callback de debug
llama_debug = LlamaDebugHandler(print_trace_on_end=True)
callback_manager = CallbackManager([llama_debug])

service_context = ServiceContext.from_defaults(
    callback_manager=callback_manager
)

index = VectorStoreIndex.from_documents(
    documents,
    service_context=service_context
)

query_engine = index.as_query_engine()

# Les métriques s'affichent automatiquement
response = query_engine.query("Question")

# Accéder aux métriques
print(f"Événements: {llama_debug.get_events()}")

from llama_index.evaluation import ResponseEvaluator

# Évaluateur automatique
evaluator = ResponseEvaluator(service_context=service_context)

# Tester une réponse
response = query_engine.query("Question test")

eval_result = evaluator.evaluate_response(
    query="Question test",
    response=response
)

print(f"Score: {eval_result.score}")
print(f"Feedback: {eval_result.feedback}")

from llama_index import VectorStoreIndex
from llama_index.langchain_helpers.agents import IndexToolConfig, LlamaIndexTool
from langchain.agents import initialize_agent
from langchain_openai import ChatOpenAI

# Index LlamaIndex
index = VectorStoreIndex.from_documents(documents)

# Convertir en outil LangChain
tool_config = IndexToolConfig(
    query_engine=index.as_query_engine(),
    name="DocumentSearch",
    description="Recherche dans la documentation technique"
)

llama_tool = LlamaIndexTool.from_tool_config(tool_config)

# Utiliser dans un agent LangChain
llm = ChatOpenAI(model="gpt-4")
agent = initialize_agent(
    [llama_tool],
    llm,
    agent="zero-shot-react-description"
)

# L'agent LangChain peut utiliser l'index LlamaIndex
result = agent.run("Cherche des infos sur le fine-tuning")