[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"pack-detail-ml-engineer-rag-eval-zh":3,"seo:pack:ml-engineer-rag-eval:zh":101},{"code":4,"message":5,"data":6},200,"操作成功",{"pack":7},{"slug":8,"icon":9,"tone":10,"status":11,"status_label":12,"title":13,"description":14,"items":15,"install_cmd":100},"ml-engineer-rag-eval","🧠","#8B5CF6","new","本周新建","ML 工程师的 RAG + 评测栈","十件生产级资产，给真正要把 RAG 上线的 ML 工程师：分块、embedding 服务、向量库（pgvector + Qdrant）、检索框架、reranker、评测、漂移监控、tracing。没评测 = 没进步。",[16,28,38,46,53,61,69,76,84,93],{"id":17,"uuid":18,"slug":19,"title":20,"description":21,"author_name":22,"view_count":23,"vote_count":24,"lang_type":25,"type":26,"type_label":27},439,"c2ba9909-f624-414f-8aeb-fbd95c50766e","unstructured-document-etl-llm-pipelines-c2ba9909","Unstructured — Document ETL for LLM Pipelines","Extract clean data from PDFs, DOCX, HTML, images, and emails for RAG and LLM ingestion. 14K+ GitHub stars.","MCP Hub",212,0,"en","mcp","MCP",{"id":29,"uuid":30,"slug":31,"title":32,"description":33,"author_name":34,"view_count":35,"vote_count":24,"lang_type":25,"type":36,"type_label":37},2513,"19c58bfa-45e0-11f1-9bc6-00163e2b0d79","text-embeddings-inference-high-performance-embedding-server-19c58bfa","Text Embeddings Inference — High-Performance Embedding Server by Hugging Face","A blazing-fast inference server for text embedding and reranking models. TEI serves any Sentence Transformers or cross-encoder model with optimized Rust and CUDA kernels, token-based dynamic batching, and an OpenAI-compatible API.","Hugging Face",131,"skill","Skill",{"id":39,"uuid":40,"slug":41,"title":42,"description":43,"author_name":44,"view_count":45,"vote_count":24,"lang_type":25,"type":36,"type_label":37},286,"596096ff-e0fb-41bd-a964-03817dafce9d","sentence-transformers-state-art-embeddings-596096ff","Sentence Transformers — State-of-the-Art Embeddings","Sentence Transformers computes text embeddings for semantic search, similarity, and reranking. 18.5K+ GitHub stars. 15,000+ pre-trained models, dense\u002Fsparse\u002Freranker, multi-lingual. Apache 2.0.","Script Depot",123,{"id":47,"uuid":48,"slug":49,"title":50,"description":51,"author_name":44,"view_count":52,"vote_count":24,"lang_type":25,"type":36,"type_label":37},1459,"121fb0d5-3920-11f1-9bc6-00163e2b0d79","pgvector-vector-similarity-search-inside-postgresql-121fb0d5","pgvector — Vector Similarity Search Inside PostgreSQL","A PostgreSQL extension that adds a native `vector` type, HNSW and IVFFlat indexes, and distance operators so semantic search, RAG and recommendation workloads can reuse the same database as the rest of the app.",164,{"id":54,"uuid":55,"slug":56,"title":57,"description":58,"author_name":59,"view_count":60,"vote_count":24,"lang_type":25,"type":36,"type_label":37},215,"1566710d-f5ed-46da-af8c-757475a10420","qdrant-high-performance-vector-database-1566710d","Qdrant — High-Performance Vector Database","Vector database and search engine for AI applications. Handles billion-scale similarity search with filtering, sparse vectors, and multi-tenancy. Rust-powered. 30K+ stars.","AI Open Source",187,{"id":62,"uuid":63,"slug":64,"title":65,"description":66,"author_name":67,"view_count":68,"vote_count":24,"lang_type":25,"type":36,"type_label":37},407,"2126f372-519e-45bd-8817-69d70e061bb0","haystack-production-rag-agent-framework-2126f372","Haystack — Production RAG & Agent Framework","Build composable AI pipelines for RAG, agents, and search. Model-agnostic, production-ready, by deepset. 18K+ stars.","Skill Factory",147,{"id":70,"uuid":71,"slug":72,"title":73,"description":74,"author_name":44,"view_count":75,"vote_count":24,"lang_type":25,"type":36,"type_label":37},157,"1bd234e2-5c10-459f-91f4-00675625103b","llamaindex-data-framework-llm-applications-1bd234e2","LlamaIndex — Data Framework for LLM Applications","Connect your data to large language models. The leading framework for RAG, document indexing, knowledge graphs, and structured data extraction.",176,{"id":77,"uuid":78,"slug":79,"title":80,"description":81,"author_name":82,"view_count":83,"vote_count":24,"lang_type":25,"type":36,"type_label":37},2824,"bf323939-d2b6-4426-aa9f-9325666e7eaa","cohere-rerank-boost-rag-accuracy-with-rerank-3","Cohere Rerank — Boost RAG Accuracy with Rerank-3","Cohere Rerank scores candidates against a query using a cross-encoder. Drop into any RAG to boost top-1 hit rate by 30-50% over vector search alone.","Cohere",92,{"id":85,"uuid":86,"slug":87,"title":88,"description":89,"author_name":90,"view_count":91,"vote_count":24,"lang_type":92,"type":36,"type_label":37},4260,"ea696ee5-0736-48e3-a789-f5a026223bd0","embedding-drift-monitoring-retrieval-regression-runbook-ea696ee5","Embedding Drift Monitoring — Retrieval Regression Runbook","Embedding drift monitoring runbook for RAG and agent search. Uses golden queries, recall@K, rank delta, and rollback gates.","henuwangkai",40,"",{"id":94,"uuid":95,"slug":96,"title":97,"description":98,"author_name":44,"view_count":99,"vote_count":24,"lang_type":25,"type":36,"type_label":37},3576,"41cdac3f-4ea4-11f1-9bc6-00163e2b0d79","arize-phoenix-open-source-ai-observability-evaluation-41cdac3f","Arize Phoenix — Open Source AI Observability and Evaluation","Arize Phoenix is an open-source platform for monitoring, evaluating, and debugging AI applications, providing tracing, experiment tracking, and automated evaluation for LLM and ML pipelines.",55,"tokrepo install pack\u002Fml-engineer-rag-eval",{"pageType":102,"pageKey":8,"locale":103,"title":104,"metaDescription":105,"h1":106,"tldr":107,"bodyMarkdown":108,"faq":109,"schema":125,"internalLinks":130,"citations":143,"wordCount":156,"generatedAt":157},"pack","zh","ML 工程师的 RAG + 评测栈 — 10 件真正能上线的生产资产","Unstructured \u002F Text Embeddings Inference \u002F Sentence Transformers \u002F pgvector \u002F Qdrant \u002F Haystack \u002F LlamaIndex \u002F Cohere Rerank \u002F Embedding Drift Monitoring \u002F Arize Phoenix — ML 工程师真正用来把 RAG 推上线的栈。从分块到 tracing，按推荐安装顺序排列。","ML 工程师的 RAG + 评测栈 — 真正上线 LLM 应用要用的资产","十件经过生产验证的资产，按 ML 工程师实际搭建顺序排列：分块和入库、embedding 服务、embedding 模型、向量库（pgvector 或 Qdrant）、检索框架、reranker、评测、漂移监控、tracing。一句惨痛教训：没评测就没进步。","## 这个 pack 包含什么\n\n这是 demo RAG 跑通了、老板很兴奋、现在你必须把它放到真用户面前还不能让它幻觉到惹官司时，你会搭的那一套。每个资产都是**生产级**、**活跃维护**，代表 ML 工程师必须亲自负责的某一层 —— 不是用胶水代码把困难抹平的库。\n\n这个 pack **刻意区别于** TokRepo 上已有的 `rag-pipelines` pack。那一套是端到端 RAG 框架横评（Quivr \u002F RAGFlow \u002F GraphRAG \u002F Kotaemon \u002F Verba —— 想要现成 RAG app 时用）。这一套是**底下的基础设施层**：当任何现成框架都给不了你团队需要的延迟、控制力或评测严谨度时，你自己拼起来的组件。\n\n所有这些资产串起来回答同一个问题，也是大多数 ML 团队在 RAG 上线第 3 周才痛苦意识到的：**demo 指标无意义，唯一重要的数字是 faithfulness + answer-relevance，在真实评测集上每次改动前后都跑一遍。** 这里一半的资产存在的意义就是让这个 loop 跑得快。\n\n## 推荐安装顺序（分块 → embedding → 向量库 → 检索 → 评测 → tracing）\n\n1. **Unstructured** —— 文档 ETL。从这里开始，因为 garbage in 仍然等于 garbage out。Unstructured 处理带表格的 PDF、扫描表单、HTML、.docx、.pptx、.eml，返回干净的 chunk 加上元素级元数据（`Title` \u002F `NarrativeText` \u002F `Table`），下游的过滤和 rerank 都靠这个信号。\n2. **Text Embeddings Inference**（Hugging Face）—— 你的 embedding 服务。自托管、低延迟、批处理，BGE \u002F E5 \u002F GTE \u002F Jina \u002F Nomic 都开箱支持。一张 GPU 跑一个服务，下游所有微服务都 POST 过去。别在 12 个微服务里各自调 OpenAI 的 embedding API。\n3. **Sentence Transformers** —— 大多数值得跑的 embedding 背后的模型库。你会用它做离线批量 embedding、训练自己领域微调的模型、以及在**你自己的语料**上 benchmark BGE-large vs E5 vs nomic-embed（唯一有意义的 benchmark）。\n4. **pgvector** —— 向量库方案 A。如果你已经在跑 Postgres，5000 万向量以内最便宜的正确答案就是 pgvector + HNSW。一个数据库、一套备份、事务性写入、能 join 你已有的元数据表。在 pgvector 真正撑不住之前别加独立向量库。\n5. **Qdrant** —— 向量库方案 B。pgvector 顶不住之后（1 亿+ 向量上的过滤查询、低延迟混合搜索、动态 schema），Qdrant 是开源升级路径。Rust 内核、索引期 payload 过滤、分片集群、MIT 协议。\n6. **Haystack** —— 生产级 RAG 与 agent 框架。Pipeline graph 抽象，每个组件都可替换，原生 async。当 LangChain 让你觉得在跟它打架、你想要可追踪可测试的显式 DAG 时，就拿它。\n7. **LlamaIndex** —— LLM 应用的数据框架。强在**入库**和**检索**侧：150+ 数据加载器、可组合的 query engine（router → sub-question → response synth）、`LlamaParse` 处理硬 PDF。配 Haystack 用，或单独用都行。\n8. **Cohere Rerank** —— 你这辈子能用最便宜的方式拿到的 +10–20 分检索质量提升。双编码器检索 top-50，cross-encoder 重排到 top-5。几乎每个一开始没有 reranker 的生产 RAG 团队，都在一个季度内加上了。\n9. **Embedding Drift Monitoring** —— 检索回归运维手册。当同一个 query 两个月后返回不同的文档，因为你的 embedding 模型悄悄被重新量化了，或者文档分布漂移了，你需要一个漂移看板。这就是抓这件事的生产运维手册。\n10. **Arize Phoenix** —— 开源 AI 可观测性 + 评测。OpenInference 兼容，给每个 LLM 调用和检索步骤打 trace；评测框架在每次 commit 时跑 LLM-as-judge 对你的测试集打分。Tracing + eval 这个闭环不可妥协，Phoenix 是用一套工具同时做到这两件事的开源方案。\n\n## 它们怎么协同（生产级 RAG 流水线）\n\n```\n┌─────────────────────────────────────────────────────────────┐\n│  入库（INGESTION）                                          │\n│   Unstructured  ──►  chunks + 元素元数据                    │\n│        │                                                    │\n│        ▼                                                    │\n│  EMBEDDING                                                  │\n│   Text Embeddings Inference（服务）                         │\n│        ▲    （模型 = Sentence Transformers \u002F BGE）          │\n│        │                                                    │\n│        ▼                                                    │\n│  向量库                                                     │\n│   pgvector（≤50M） 或  Qdrant（>50M、混合搜索、过滤）       │\n└─────────────────────────────────────────────────────────────┘\n                          │\n                          ▼\n┌─────────────────────────────────────────────────────────────┐\n│  查询时                                                     │\n│   LlamaIndex \u002F Haystack  ──►  retrieve top-50               │\n│        │                                                    │\n│        ▼                                                    │\n│   Cohere Rerank  ──►  cross-encoder 重排到 top-5            │\n│        │                                                    │\n│        ▼                                                    │\n│   LLM（你的生成器） ──► 答案                                │\n└─────────────────────────────────────────────────────────────┘\n                          │\n                          ▼\n┌─────────────────────────────────────────────────────────────┐\n│  可观测性                                                   │\n│   Arize Phoenix tracing  ◄── 每一个 span（检索、rerank）    │\n│   Phoenix evals（LLM-as-judge）  ◄── 每个 PR 跑评测集       │\n│   Embedding Drift Monitoring   ◄── 每晚 cron，触发告警      │\n└─────────────────────────────────────────────────────────────┘\n```\n\n这个切分是故意的：入库是批处理（Unstructured → TEI → 向量库），查询时是热路径（向量库 → LlamaIndex\u002FHaystack → Rerank → LLM），可观测性裹在最外面。没有第三个框，你根本看不出上周改的 prompt 是帮了你还是悄悄把 faithfulness 拉低了 8%。\n\n## 你会遇到的取舍\n\n- **pgvector vs Pinecone vs Qdrant** —— Postgres 上的 pgvector 在运维简洁性上赢（一个 DB、一套备份、能 join 已有元数据），到 1000–5000 万向量配 HNSW 都是真的够用。Pinecone 赢在「我不想跑基础设施」和弹性扩展，但成本叠加很快、锁定也重。Qdrant 赢在大规模上需要带过滤的混合搜索而你想自托管。默认 pgvector。过滤延迟扛不住时切 Qdrant。运维人手是硬约束才动 Pinecone。\n- **OpenAI embedding vs 开源（BGE \u002F E5 \u002F Nomic）** —— OpenAI `text-embedding-3-large` 通用英语很强、上手简单。开源 embedding 走 Text Embeddings Inference 在量大时便宜 ~10 倍、能离线、能在自己领域上微调。决策通常归结到一个问题：你有没有一个足够好的评测集能 A\u002FB 它们？有的话开源经常赢。没的话先用 OpenAI，把评测集先建起来。\n- **Haystack vs LlamaIndex vs LangChain** —— Haystack：显式 pipeline graph，更好测，稍微 verbose 一些。LlamaIndex：入库 + 检索组合更强，全 agent loop 的抽象稍弱。LangChain：表面积最大、原型最快，多数生产团队最终都重构掉它。比较成熟的栈最后往往是 **LlamaIndex 做入库 + 检索** + **Haystack 或纯 Python 做编排**。\n- **Reranker 延迟** —— Cohere Rerank 加 100–250ms。几乎总是值得。如果真的扛不住，就在自家 GPU 上跑一个小的开源 reranker（BGE-reranker-base）。\n\n## 常见踩坑\n\n- **分块太激进** —— 512-token chunk + 50-token overlap 是默认值，通常是错的。技术文档 Q&A 上，更大的语义 chunk（1000–1500 token，用 Unstructured 元素元数据按标题边界切）一致表现更好。在你自己的评测集上测，别猜。\n- **没评测集 = 没进步** —— 最常见的死法。没有 50–200 个手工标注的 query \u002F 期望 context 对，每次改动都是凭感觉。第一周就把评测集建起来。每次有真实用户反馈坏答案，就更新一个。这是任何 RAG 项目里单点 ROI 最高的工程投入。\n- **换 embedding 模型不重建索引** —— 把 `text-embedding-ada-002` 悄悄换成 `text-embedding-3-small`，你的旧向量瞬间变成垃圾。给 embedding 打版本号。换模型就重建索引。\n- **单一检索策略** —— 纯 dense 检索会漏掉词汇型 query（产品 SKU、错误码、版本号）。加一个 BM25 \u002F 关键词检索器并行跑然后合并。LlamaIndex 和 Haystack 两行代码就支持。\n- **没 reranker** —— 双编码器快但有损。在 top-50 候选上跑 cross-encoder 重排是整个 RAG 栈上最稳的质量提升。「太花延迟」这个理由在 faithfulness 才是瓶颈的时候是假经济。\n- **Tracing 当事后补救** —— 等你出了质量问题再装 Phoenix，比第一天装难 5 倍。在第一个老板 demo 之前就把它装上。",[110,113,116,119,122],{"q":111,"a":112},"这个 pack 跟 TokRepo 已有的 `rag-pipelines` pack 有什么区别？","rag-pipelines 是框架横评 —— Quivr \u002F RAGFlow \u002F GraphRAG \u002F Kotaemon \u002F Verba —— 想直接部署一个现成 RAG app 时用的。这个 pack 是底下的基础设施层：分块（Unstructured）、自己跑的 embedding 服务（Text Embeddings Inference）、自己运维的向量库（pgvector \u002F Qdrant）、reranker、漂移监控、可观测性层。受众不同、workflow ID 零重叠。两个搭配用：从 rag-pipelines 选一个框架，等你要把它推过 demo 阶段时再回这里挑组件。",{"q":114,"a":115},"我真的需要同时装 pgvector 和 Qdrant 吗？","不需要，选一个。pack 里两个都列是因为答案真的取决于规模和已有基础设施。如果你已经在跑 Postgres、向量数少于 5000 万，pgvector + HNSW 是正确答案，加第二个 DB 是无谓的复杂度。如果你需要在几亿向量上做低延迟的过滤混合搜索，Qdrant 的运维成本是值的。先 pgvector，只有真实负载上的 benchmark 显示它跟不上时再切。",{"q":117,"a":118},"为什么这个 pack 里没有 LangChain？","LangChain 做原型没问题，多数团队第一个 RAG 仓库里都有它。这个 pack 反映的是成熟 ML 团队真正留在生产里的那些资产 —— Haystack 的显式 pipeline 和 LlamaIndex 的检索组合，在项目超过半年之后，可测试性和可维护性一致更胜出。能跑通原型就用什么。已经用 LangChain 上线且工作良好就别动；2026 年新起项目，这里的资产更经得住时间。",{"q":120,"a":121},"Reranker 真的值这点延迟吗？","值，几乎总是。双编码器向量检索 20ms 返回 top-50 候选，但有损，因为它比较的是预计算的独立表示。Cross-encoder reranker 对每个 query\u002Fdoc 对联合重打分，常规能把 NDCG@5 拉高 10–20 分。Cohere Rerank 加 ~100–250ms，是整个栈上最便宜的质量跳跃。如果 100ms 真的不能接受，就在 CPU 上跑一个小的开源 reranker（BGE-reranker-base），50ms 内搞定 —— 但一定要上 reranker。",{"q":123,"a":124},"最小可用的评测集是多少？","50 个 query 手写理想 context 和可接受答案就足以有意义地度量 faithfulness、answer-relevance、context-precision。在调任何东西之前的第一周就把它建起来。然后接 Ragas \u002F Phoenix Evals \u002F DeepEval 之类的工具自动化打分循环。每个真实用户投诉都变成一条新的评测 case。三个月之后你会有 200–500 个 case，每个 PR 都在 CI 里跑一遍 —— 这才是真正驱动 RAG 质量的 loop。",{"@context":126,"@type":127,"name":13,"description":128,"numberOfItems":129,"inLanguage":103},"https:\u002F\u002Fschema.org","ItemList","十件生产级开源资产，给真正要把 RAG 流水线推上线的 ML 工程师：分块、embedding 服务、向量库、检索框架、reranker、漂移监控、评测、tracing。",10,[131,135,139],{"url":132,"anchor":133,"reason":134},"\u002Fzh\u002Ftopics\u002Frag-pipelines","RAG 流水线 pack — 现成框架","配套 pack：可以直接部署的端到端 RAG app，本 pack 是它下面的基础设施层",{"url":136,"anchor":137,"reason":138},"\u002Fzh\u002Ftopics\u002Fllm-observability","LLM 可观测性 pack","比 Phoenix 更广的可观测性覆盖 —— Langfuse \u002F AgentOps \u002F LangSmith 给生产 LLM app 用",{"url":140,"anchor":141,"reason":142},"\u002Fzh\u002Ftopics\u002Fvector-db-showdown","向量数据库横评 pack","当本 pack 的 pgvector vs Qdrant 二选一不够时，深入对比 Chroma \u002F Weaviate \u002F Pinecone \u002F Milvus",[144,148,152],{"claim":145,"source_name":146,"source_url":147},"Cross-encoder 重排相比仅用双编码器检索能显著提升 NDCG","Cohere Rerank 文档","https:\u002F\u002Fcohere.com\u002Frerank",{"claim":149,"source_name":150,"source_url":151},"pgvector 在 Postgres 内置支持 HNSW 索引做快速近似最近邻搜索","pgvector GitHub","https:\u002F\u002Fgithub.com\u002Fpgvector\u002Fpgvector",{"claim":153,"source_name":154,"source_url":155},"Hugging Face Text Embeddings Inference 是高性能 embedding 服务，支持 BGE \u002F E5 \u002F GTE \u002F Jina \u002F Nomic 模型","Text Embeddings Inference 仓库","https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Ftext-embeddings-inference",1050,"2026-05-22T00:00:00Z"]