1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

version: "3.1"

volumes:
  data:
  config:
  plugin:

networks:
  es:

services:
  elasticsearch:
    image: elasticsearch
    container_name: elasticsearch  # 容器名保持为 elasticsearch
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - es
    environment:
      - discovery.type=single-node
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.enabled=true
      - xpack.security.autoconfiguration.enabled=true
      # - xpack.security.enrollment.enabled=true
      # 移除 NODE_OPTIONS（OpenSSL 警告不影响功能，无需强制关闭）
      # 注意：ES 环境变量中没有 ELASTICSEARCH_USERNAME/PASSWORD，需手动创建该用户
    volumes:
      - /usr/volume/ElasticSearch/data:/usr/share/elasticsearch/data
      - /usr/volume/ElasticSearch/config:/usr/share/elasticsearch/config
      - /usr/volume/ElasticSearch/plugins:/usr/share/elasticsearch/plugins
    user: "1000:1000"

  kibana:
    image: kibana
    container_name: kibana
    ports:
      - 5601:5601
    networks:
      - es
    environment:
      - SERVER_HOST=0.0.0.0  # 仅保留此环境变量，避免与配置文件冲突
      # 移除 ELASTICSEARCH_HOSTS 环境变量（由 kibana.yml 统一配置）
      # 彻底删除服务账户令牌相关配置（避免混淆）
    volumes:
      - /usr/volume/ElasticSearch/kibana.yml:/usr/share/kibana/config/kibana.yml
    depends_on:
      - elasticsearch  # 依赖正确的 ES 服务名

# ------------------------------------ kibana.yml 文件内容--------------------------------------------------------------
server.host: "0.0.0.0"  # 明确允许外部访问，替换 "0" 避免歧义
server.shutdownTimeout: "5s"
# 与 ES 容器名保持一致（elasticsearch）
elasticsearch.hosts: ["http://elasticsearch:9200"]
monitoring.ui.container.elasticsearch.enabled: true
# 仅保留用户名密码认证（与 ES 配置匹配）
elasticsearch.username: "kibana_user"
elasticsearch.password: "xxxx"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <!--   spring-ai-alibaba     -->
        <dependency>
            <groupId>com.alibaba.cloud.ai</groupId>
            <artifactId>spring-ai-alibaba-starter</artifactId>
            <version>1.0.0-M5.1</version>
        </dependency>

        <!-- Elasticsearch Java Client -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
        </dependency>
        <!-- Jackson for JSON processing -->
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.15.2</version>
        </dependency>
        <!-- 如果你需要处理向量计算 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-math3</artifactId>
            <version>3.6.1</version>
        </dependency>

        <!--   poi     -->
        <!-- Apache POI for Excel processing -->
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi</artifactId>
            <version>5.2.4</version>
        </dependency>
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-ooxml</artifactId>
            <version>5.2.4</version>
        </dependency>
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-scratchpad</artifactId>
            <version>5.2.4</version>
        </dependency>

        <!--   pdfbox     -->
        <!-- Apache PDFBox：解析PDF文件 -->
        <dependency>
            <groupId>org.apache.pdfbox</groupId>
            <artifactId>pdfbox</artifactId>
            <version>2.0.32</version> <!-- 使用最新稳定版 -->
        </dependency>
        <!-- 可选：处理PDF中的字体问题（避免中文乱码） -->
        <dependency>
            <groupId>org.apache.pdfbox</groupId>
            <artifactId>fontbox</artifactId>
            <version>2.0.32</version>
        </dependency>

    </dependencies>

    <!-- 拉取spring ai指定仓库   -->
    <repositories>
        <repository>
            <id>spring-milestones</id>
            <name>Spring Milestones</name>
            <url>https://repo.spring.io/milestone</url>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </repository>
    </repositories>

1
2
3
4
5
6
7
8

spring.application.name=xl-ai-rag
server.port=9006
spring.ai.dashscope.api-key=sk-c3*****************7d1da1b

# elasticsearch 
spring.elasticsearch.uris= http://192.168.0.1:9200
spring.elasticsearch.username= elastic
spring.elasticsearch.password= xxxx

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

@Component
@AllArgsConstructor
@Slf4j
public class VectorIndexManager {

    private final ElasticsearchClient client;
    private static final String INDEX_NAME = "vector-documents";
    private static final int INDEX_DIMS = 1536;

    /**
     * 创建向量索引
     * 服务启动时自动检查并创建索引（仅执行一次）
     */
    @PostConstruct
    public void createVectorIndex() throws Exception {
        // 检查索引是否存在
        boolean exists = client.indices().exists(e -> e.index(INDEX_NAME)).value();

        if (!exists) {
            CreateIndexResponse response = client.indices().create(c -> c
                    .index(INDEX_NAME)
                    .mappings(m -> m
                            .properties("id", p -> p.keyword(k -> k))
                            .properties("title", p -> p.text(t -> t
                                    .analyzer("standard")
                                    .fields("keyword", f -> f.keyword(k -> k))
                            ))
                            .properties("content", p -> p.text(t -> t
                                    .analyzer("standard")
                            ))
                            .properties("vector", p -> p.denseVector(d -> d
                                    .dims(INDEX_DIMS)  //  根据你的向量维度调整
                                    .similarity("cosine")  // 支持 cosine, l2_norm, dot_product
                            ))
                    )
                    .settings(s -> s
                            .numberOfShards("1")
                            .numberOfReplicas("0")
                    )
            );

            log.info("索引创建成功: " + response.index());
        } else {
            log.info("索引已存在");
        }
    }

    /**
     * 删除索引
     */
    public void deleteIndex() throws Exception {
        DeleteIndexResponse response = client.indices().delete(d -> d.index(INDEX_NAME));
        log.info("索引删除成功: " + response.acknowledged());
    }

    /**
     *     更新文档（部分字段，如content或vector）
      */
    public void updateDocument(String id, Map<String, Object> fields) throws IOException {
        Map<String, Object> doc = new HashMap<>();
        doc.put("doc", fields);
        client.update(u -> u.index(INDEX_NAME).id(id).doc(doc), VectorDocument.class);
    }

}

1
2
3
4
5
6
7
8
9

@Data
public class VectorDocument {

    private String id;
    private String title;
    private String content;
    private float[] vector;

}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

@Service
@AllArgsConstructor
@Slf4j
public class VectorService {
    private final ElasticsearchClient client;
    private static final String INDEX_NAME = "vector-documents";


    /**
     * 插入向量文档
     */
    public String insertDocument(VectorDocument document) throws Exception {
        IndexResponse response = client.index(i -> i
                .index(INDEX_NAME)
                .id(document.getId())
                .document(document)
        );

        log.info("文档插入成功: " + response.id());
        return response.id();
    }

    /**
     * 批量插入向量文档
     */
    public boolean bulkInsertDocuments(List<VectorDocument> documents) throws Exception {
        BulkRequest.Builder br = new BulkRequest.Builder();

        for (VectorDocument doc : documents) {
            br.operations(op -> op
                    .index(idx -> idx
                            .index(INDEX_NAME)
                            .id(doc.getId())
                            .document(doc)
                    )
            );
        }

        BulkResponse response = client.bulk(br.build());

        if (response.errors()) {
            log.error("批量插入存在错误");
            response.items().forEach(item -> {
                if (item.error() != null) {
                    log.error("错误: " + item.error().reason());
                }
            });
            return false;
        } else {
            log.info("批量插入成功，处理了 " + documents.size() + " 个文档");
            return true ;
        }
    }

    /**
     * 向量搜索
     */
    public List<VectorDocument> vectorSearch(List<Float> queryVector, Long k1) throws Exception {
        SearchResponse<VectorDocument> response = client.search(s -> s
                        .index(INDEX_NAME)
                        .knn(k -> k
                                .field("vector")
                                .queryVector(queryVector)
                                .k(k1)
                                .numCandidates(100L)
                        )
                        .source(src -> src.filter(f -> f
                                .includes("id", "title", "content")
                        )),
                VectorDocument.class
        );

        List<VectorDocument> results = new ArrayList<>();
        for (Hit<VectorDocument> hit : response.hits().hits()) {
            VectorDocument doc = hit.source();
            if (doc != null) {
                doc.setId(hit.id());
                results.add(doc);
            }
        }

        return results;
    }

    /**
     * 混合搜索（向量 + 文本）
     */
    public List<VectorDocument> hybridSearch(List<Float> queryVector, String queryText, Long k1) throws Exception {
        SearchResponse<VectorDocument> response = client.search(s -> s
                        .index(INDEX_NAME)
                        .knn(k -> k
                                .field("vector")
                                .queryVector(queryVector)
                                .k(k1)
                                .numCandidates(100L)
                        )
                        .query(q -> q
                                .multiMatch(m -> m
                                        .fields("title", "content")
                                        .query(queryText)
                                )
                        )
                        .source(src -> src.filter(f -> f
                                .includes("id", "title", "content")
                        )),
                VectorDocument.class
        );

        List<VectorDocument> results = new ArrayList<>();
        for (Hit<VectorDocument> hit : response.hits().hits()) {
            VectorDocument doc = hit.source();
            if (doc != null) {
                doc.setId(hit.id());
                results.add(doc);
            }
        }

        return results;
    }

    /**
     * 根据ID获取文档
     */
    public VectorDocument getDocumentById(String id) throws Exception {
        GetResponse<VectorDocument> response = client.get(g -> g
                        .index(INDEX_NAME)
                        .id(id),
                VectorDocument.class
        );

        if (response.found()) {
            VectorDocument doc = response.source();
            doc.setId(response.id());
            return doc;
        }
        return null;
    }

}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

@Service
@AllArgsConstructor
@Slf4j
public class FileUploadService {

    private final VectorService vectorService;
    private final EmbeddingModel embeddingModel;
    private final DocumentParser documentParser;

    /**
     * 处理用户上传的文件，生成向量并插入ES
     * @param file 用户上传的文件（MultipartFile）
     * @param userId 上传用户ID（用于多用户隔离）
     * @return 插入成功的文档数量
     */
    public int handleUploadedFile(MultipartFile file, String userId) {
        try {
            // 步骤1：解析文件（根据后缀名选择解析器，如PDF用PDFBox，DOCX用POI）
            String fileName = file.getOriginalFilename();
            List<Document> fileContents = documentParser.parse(file.getInputStream(), fileName);

            // 步骤2：文本分割（避免单段文本过长，影响向量质量和检索精度）1200token/段，重叠350
            TokenTextSplitter splitter =  new TokenTextSplitter(1200,
                    350, 5,
                    100, true);
            List<Document> splitDocs = splitter.split(fileContents);

            // 步骤3：生成向量文档列表（给每个分割后的文本生成向量）
            List<VectorDocument> vectorDocuments = splitDocs.stream()
                    .map(text -> {
                        VectorDocument doc = new VectorDocument();
                        doc.setId(UUID.randomUUID().toString());
                        doc.setTitle(fileName);
                        doc.setContent(text.getText());
                        // 步骤4：生成向量（核心！必须在插入前赋值）
                        float[] embed = embeddingModel.embed(text);
                        log.info("嵌入模型生成的向量维度：{}", embed.length);
                        doc.setVector(embed);
                        return doc;
                    })
                    .collect(Collectors.toList());

            // 步骤5：批量插入向量文档（调用你现有的 bulkInsertDocuments 方法）
            boolean isSuccess = vectorService.bulkInsertDocuments(vectorDocuments);
            if (isSuccess){
                log.info("用户 {} 上传文件 {} 处理完成，插入 {} 个向量文档", userId, fileName, vectorDocuments.size());
            }
            return vectorDocuments.size();

        } catch (Exception e) {
            log.error("处理用户 {} 上传文件失败", userId, e);
            throw new RuntimeException("文件导入知识库失败", e);
        }
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

@Component
public class DocumentParser {

    /**
     *  根据文件后缀名选择解析逻辑
     * @param inputStream
     * @param fileName
     * @return
     * @throws Exception
     */
    public List<Document> parse(InputStream inputStream, String fileName) throws Exception {
        if (fileName.endsWith(".txt")) {
            return parseTxt(inputStream,fileName);
        } else if (fileName.endsWith(".pdf")) {
            return parsePdf(inputStream,fileName);
        } else if (fileName.endsWith(".docx")) {
            return parseDocx(inputStream,fileName);
        } else {
            throw new UnsupportedOperationException("不支持的文件格式：" + fileName);
        }
    }

    /**
     * 解析TXT
     * @param inputStream
     * @return
     * @throws IOException
     */
    private List<Document> parseTxt(InputStream inputStream, String fileName) throws IOException {
        List<String> lines = new ArrayList<>();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
            String line;
            while ((line = reader.readLine()) != null) {
                if (!line.trim().isEmpty()) {
                    lines.add(line);
                }
            }
        }
        // 封装为 Spring AI Document（content 为全文，metadata 可添加文件名等信息）
        Document doc = new Document(String.join("\n", lines));
        doc.getMetadata().put("fileName", fileName); // 添加元数据
        return List.of(doc);
    }

    /**
     * 解析PDF（依赖 PDFBox 库）
     * @param inputStream
     * @return
     * @throws IOException
     */
    private List<Document> parsePdf(InputStream inputStream, String fileName) throws Exception {
        try (PDDocument document = PDDocument.load(inputStream)) {
            PDFTextStripper stripper = new PDFTextStripper();
            String content = stripper.getText(document);
            Document doc = new Document(content);
            doc.getMetadata().put("fileName", fileName);
            return List.of(doc);
        }
    }

    /**
     * 解析DOCX（依赖 POI 库）
     * @param inputStream
     * @return
     * @throws IOException
     * @throws XmlException
     */
    private List<Document> parseDocx(InputStream inputStream, String fileName) throws Exception {

        try (XWPFDocument document = new XWPFDocument(inputStream)) {
            XWPFWordExtractor extractor = new XWPFWordExtractor(document);
            String content = extractor.getText();
            Document doc = new Document(content);
            doc.getMetadata().put("fileName", fileName);
            return List.of(doc);
        }

    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

private final FileUploadService fileUploadService;
/**
     * 用户上传文件到知识库
     * @param file 上传的文件
     * @param userId 用户ID（从登录态获取，此处简化为参数）
     * @return 处理结果
     */
    @PostMapping("/upload")
    public ResponseEntity<Map<String, Object>> uploadFile(
            @RequestParam("file") MultipartFile file,
            @RequestParam("userId") String userId) {

        if (file.isEmpty()) {
            return ResponseEntity.badRequest().body(Collections.singletonMap("msg", "文件不能为空"));
        }

        int count = fileUploadService.handleUploadedFile(file, userId);
        Map<String, Object> result = new HashMap<>();
        result.put("msg", "文件导入成功");
        // 插入的向量文档数量
        result.put("documentCount", count);
        return ResponseEntity.ok(result);
    }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

@Service
@AllArgsConstructor
@Slf4j
public class RAGService {

    private final VectorService vectorService;

    /**
     *  文本嵌入模型
     */
    private final EmbeddingModel embeddingModel;

    /**
     * 大模型客户端
     */
    private final ChatModel chatModel;

    /**
     * RAG核心方法：基于知识库回答用户问题
     * @param userQuestion 用户问题
     * @param topK 检索最相关的前K个文档
     * @return 结合知识库的回答
     */
    public String answerWithKnowledge(String userQuestion, Long topK) {
        try {
            // 步骤1：将用户问题转换为向量
            float[] queryVector = embeddingModel.embed(userQuestion);
            ArrayList queryVectorList = new ArrayList<>();
            for (float f : queryVector) {
                queryVectorList.add(f);
            }

            // 步骤2：调用VectorService检索相关文档
             List<VectorDocument> relevantDocs = vectorService.hybridSearch(queryVectorList, userQuestion, topK);

            // 步骤3：将检索到的文档拼接为上下文
            String context = buildContext(relevantDocs);

            // 步骤4：构建提示词（Prompt）
            String promptStr = buildPrompt(userQuestion, context);
            Prompt prompt = new Prompt(promptStr);
            List<Generation> results = chatModel.call(prompt).getResults();
            // 步骤5：调用大模型生成回答
            return results.stream().map(x -> x.getOutput().getContent()).collect(Collectors.joining());
        } catch (Exception e) {
            log.error("RAG回答生成失败", e);
            return "抱歉，处理您的问题时出错了，请稍后再试。";
        }
    }

    /**
     * 拼接检索到的文档为上下文文本
     */
    private String buildContext(List<VectorDocument> docs) {
        if (docs.isEmpty()) {
            return "没有找到相关信息。";
        }
        StringBuilder contextBuilder = new StringBuilder();
        contextBuilder.append("以下是相关参考信息：\n");
        for (int i = 0; i < docs.size(); i++) {
            VectorDocument doc = docs.get(i);
            contextBuilder.append(String.format("【参考%d】标题：%s\n内容：%s\n\n",
                    i + 1, doc.getTitle(), doc.getContent()));
        }
        return contextBuilder.toString();
    }

    /**
     * 构建大模型系统提示词（Prompt Engineering）
     */
    private String buildPrompt(String userQuestion, String context) {
        return String.format("请基于以下参考信息，回答用户的问题。如果参考信息中没有相关内容，直接说明“没有找到相关信息”，不要编造答案。\n" +
                        "参考信息：\n%s\n" +
                        "用户问题：%s\n" +
                        "回答：",
                context, userQuestion);
    }

}

1
2
3
4
5

private final RAGService ragService;
 	@GetMapping("/know")
    public String know(@RequestParam("userQuestion")String userQuestion, @RequestParam("topK")Long topK){
        return ragService.answerWithKnowledge(userQuestion, topK);
    }