项目五：多模态内容理解与检索（电商平台跨模态商品搜索系统）
·
项目原型
🖼️ 多模态智能检索系统
==================================================
[检索输入区]
请选择检索方式：
🔘 文本检索 🔘 图像检索 🔘 视频检索
文本查询: [红色连衣裙 夏季新款 ]
[检索结果] 找到 1,247 个相关商品
排序方式: ▢ 相关性 ▢ 价格 ▢ 销量 ▢ 上新时间
[结果展示]
┌─────────────────────────────────────────────────────┐
│ 📸 商品图片1 📸 商品图片2 📸 商品图片3 │
│ 红色蕾丝连衣裙 红色波点连衣裙 红色雪纺连衣裙 │
│ 💰 299元 💰 359元 💰 199元 │
│ ⭐ 4.8 (245) ⭐ 4.7 (189) ⭐ 4.9 (567) │
│ 🔍 相似度: 0.92 🔍 相似度: 0.89 🔍 相似度: 0.87 │
└─────────────────────────────────────────────────────┘
[相似推荐]
基于您的搜索，我们还推荐：
• 红色半身裙 (相似度: 0.85)
• 红色衬衫 (相似度: 0.82)
• 红色外套 (相似度: 0.79)
[筛选面板]
▢ 仅显示图片商品 ▢ 仅显示视频商品
💰 价格区间: [0 - 1000] ▢ 0-100 ▢ 100-300 ▢ 300-500 ▢ 500+
⭐ 评分: ▢ 4.5以上 ▢ 4.0以上 ▢ 3.5以上
🚚 配送: ▢ 次日达 ▢ 包邮 ▢ 货到付款
[操作选项]
1. 🛒 加入购物车 2. 💖 收藏商品 3. 📤 分享结果
4. 🔄 重新检索 5. ⚙️ 高级搜索 6. ❓ 帮助
请输入选择 [1-6]:
配置文件
# config/multimodal_config.yaml
clip_model:
model_name: "openai/clip-vit-large-patch14"
image_size: 224
batch_size: 32
retrieval:
top_k: 10
similarity_threshold: 0.3
modality_boost: 1.1 # 同模态检索分数提升
index:
type: "HNSW"
space: "l2"
m: 32 # HNSW参数
ef_construction: 200
video_processing:
frame_rate: 1 # 每秒帧数
max_frames: 100
frame_size: [224, 224]
preprocessing:
image_extensions: [".jpg", ".jpeg", ".png", ".bmp", ".webp"]
video_extensions: [".mp4", ".avi", ".mov", ".mkv"]
max_file_size: 100MB
api:
port: 8002
max_concurrent_requests: 100
timeout: 60
核心代码
import torch
import torch.nn as nn
from transformers import CLIPModel,CLIPProcessor,AutoTokenizer,AutoModel
import faiss
import numpy as np
from PIL import Image
import cv2
import json
from typing import List,Dict,Tuple
import os
class MultiModalRetrievalSystem:
"""多模态检索系统
支持图像、文本、视频的跨模态检索
"""
def __init__(self,config:Dict):
self.config = config
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
#初始化CLIP模型-图文多模态
self.clip_model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14")
self.clip_processor = CLIPProcessor.from_pretrained("sentence-transformers/all-mpnet-base-v2")
self.clip_model.to(self.device)
self.clip_model.eval()
# 初始化文本嵌入模型 (备用)
self.text_model = AutoModel.from_pretrained("sentence-transformers/all-mpnet-base-v2")
self.text_tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-mpnet-base-v2")
self.text_model.to(self.device)
self.text_model.eval()
#初始化向量数据库
self.vector_dim = 512 #CLIP特征维度
self.index = faiss.IndexHNSWFlat() #HNSW图索引
#元数据存储
self.metadata = []
self.id_to_metadata = {}
#视频处理配置
self.video_frame_rate = 1 #每秒抽取帧数
def encode_image(self,image_path:str)->np.ndarray:
"""编码图像为向量"""
try:
image = Image.open(image_path).convert('RGB')
inputs = self.clip_processor(images=image, return_tensors="pt", padding=True)
inputs = {k: v.to(self.device) for k, v in inputs.items()}
with torch.no_grad():
image_features = self.clip_model.get_image_features(**inputs)
# L2归一化
image_features = image_features / image_features.norm(dim=-1, keepdim=True)
return image_features.cpu().numpy().astype('float32')
except Exception as e:
print(f"图像编码失败: {e}")
return np.zeros((1,self.vector_dim),dtype='float32')
def encode_text(self,text:str)->np.ndarray:
"""编码文本为向量"""
try:
inputs = clip_processor(text=text,return_tensors="pt",padding=True)
inputs = {k:v.to(self.device) for k,v in inputs.items()}
with torch.no_grad():
text_features = self.clip_model.get_text_features(**inputs)
text_features = text_features / text_features.norm(dim=-1,keepdim=True)
return text_features.cpu().numpy().astype('float32')
except Exception as e:
print(f"CLIP文本编码失败: {e}, 使用备用编码器")
# 备用文本编码
return self._encode_text_fallback(text)
def _encode_text_fallback(self, text: str) -> np.ndarray:
"""备用文本编码方法"""
inputs = self.text_tokenizer(
text,
return_tensors="pt",
max_length=512,
truncation=True,
padding=True
)
inputs = {k:v.to(self.divice) for k,v in inputs.items()}
with torch.no_trad():
outputs = self.text_model(**inputs)
#使用平均池化
embeddings = outputs.last_hidden_state.mean(dim=1)
embeddings = embeddings / embeddings.normal(dim=-1,keepdim=True)
return embeddings.cpu().numpy().astype('float32')
def encode_video(self,video_path:str)->List[np.ndarray]:
"""编码视频的多帧向量"""
frame_vectors = []
try:
#提取视频关键帧
frames = self._extract_video_frames(video_path)
for frame in frames:
#临时保存帧图像
temp_path = f"/tmp/frame_{hash(str(frame.tobytes()))}.jpg"
cv2.imwrite(temp_path,frame)
#编码帧图像
frame_vector = self.encode_image(temp_path)
frame_vectors.append(frame_vector[0])
#清理临时文件
os.remove(temp_path)
return frame_vectors
except Exception as e:
print(f"视频编码失败: {e}")
return []
def _extract_video_frames(self,video_path:str)->List:
"""提取视频关键帧"""
frames = []
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
return frames
#获取视频信息
fps = cap.get(cv2.CAP_PROP_FPS)
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = total_frames /fps if fps > 0 else 0
#计算采样间隔
if duration > 0:
frame_interval = max(1,int(fps / self.video_frame_rate))
else:
frame_interval = 30 #默认间隔
frame_count = 0
while True:
ret,frame = cap.read()
if not ret:
break
#按间隔采样帧
if frame_count % frame_interval == 0:
#调整帧大小
frame = cv2.resize(frame,(224,224))
frames.append(frame)
frame_count += 1
#限制最大处理帧数
if len(frames) >= 100 #最多处理100帧
break
cap.release()
return frames
def add_to_index(self,items:List[Dict]):
"""添加项目到检索索引"""
vectors = []
new_metadata = []
for item in items:
if item['type'] == 'image':
vector = self.encode_image(item['path'])
elif item['type'] == 'text':
vector = self.encode_Text(item['content'])
elif item['type'] == 'video':
frame_vectors = self.encode_video(item[item['path'])
#使用平均向量表示视频
if frame_vectors:
vector = np.mean(frame_vectors,axis=0,keepdims=True)
else:
continue
else:
continue
vectors.append(vector[0])
metadata_entry = {
'id': len(self.metadata) + len(new_metadata),
'type': item['type'],
'path': item.get('path', ''),
'content': item.get('content', ''),
'category': item.get('category', ''),
'tags': item.get('tags', []),
'timestamp': item.get('timestamp', '')
}
new_metadata.append(metadata_entry)
if vectors:
#转换为numpy数组
vectors_array = np.array(vectors).astype('float32')
#添加到FAISS索引
if self.index.ntotal == 0:
self.index.add(vectors_array)
else:
#增量添加
self.index.add(vectors_array)
#更新元数据
start_d = len(self.metadata)
for i,metadata in enumerate(new_metadata):
metadata['id'] = start_id + i
self.metadata.append(metadata)
self.id_to_metadata[metadata['id']] = metadata
def search(self,query:str,top_k:int=10,modality:str=None)->List[Dict]:
"""多模态检索"""
if os.path.exists(query):
query_vector = self.encode_image(query)
query_type = 'image'
else:
query_vector = self.encode_text(query)
query_type = 'text'
#执行检索
query_vector = query_vector.astype('float32')
scores,indices = self.index.search(query_vector,top_k * 2) #多取一些用于过滤
results = []
seen_ids = set()
for score,idx in zip(scores[0],indices[0]):
if idx < len(self.metadata) and idx not in seen_ids:
metadata = self.metadata[idx]
#模态过滤
if modality and metadata['type'] != modality:
continue
#计算调整后的分数（基于模态匹配）
adjusted_score = self._adjust_score(score,query_type,metadata['type'])
results.append({
'metadata':metadata,
'score':float(adjusted)
'similarity':float(score)
})
seen_ids.add(idx)
if len(results) >= top_k:
break
#按分数排序
results.sort(key=lambda x:x['score'],reverse=True)
return results
def _adjust_score(self,original_score:float,query_type:str,result_type:str)->float:
"""根据查询和结果模态调整分数"""
#同模态检索分数更高
if query_type == result_type:
return original_score * 1.1
else:
return priginal_score
def build_index_from_directory(self,data_dir:str):
"""从目录构建索引（批量处理）"""
items = []
#处理图像文件
image_extensions = {'.jpg','.jpeg','.png','.bmp','.webp'}
for root,dirs,files in os.walk(data_dir):
for file in files:
file_ext = os.path.splitext(file)[1].lower()
if file_ext in image_extensions:
item = {
'type': 'image',
'path': os.path.join(root, file),
'category': os.path.basename(root),
'tags': self._extract_tags_from_path(os.path.join(root, file))
}
items.append(item)
#分批处理避免内存溢出
batch_size = 100
for i in range(0,len(items),batch_size):
batch = items[i:i + batch_size]
print(f"处理批次 {i//batch_size + 1}/{(len(items)-1)//batch_size + 1}")
self.add_to_index(batch)
def _extract_tags_from_path(self,path:str)->List[str]:
"""从文件路径提取标签"""
tags = []
dir_name = os.path.basename(os.path.dirname(path))
file_name = os.path.splitext(os.path.basename(path))[0]
if dir_name and dir_name != '.':
tags.append(dir_name)
#从文件名提取关键词
import re
words = re.findall((r'[a-zA-Z]+', file_name)
tags.extend(words)
return tags
def save_index(self,index_path:str,metadata_path:str):
"""保存索引和元数据"""
#保存FAISS索引
faiss.write_index(self.index,index_path)
#保存元数据
with open(metadata_path,'w',encoding='utf-8')as f:
json.dump(self.metadata,f,ensure_ascii=False,indent=2)
def load_index(self,index_path:str,metadata_path:str):
"""加载索引和元数据"""
#加载FAISS索引
self.index = faiss.read_index(index_path)
#加载元数据
with open(metadata_path,'r',encoding='utf-8')as f:
self.metadata = json.load(f)
#重建ID映射
self.id_to_metadata = {item['id']:item for item in self.metadata}
class MultiModalSearchAPI:
"""多模态检索API服务"""
def __init__(self,retrieval_system:MultiModalRetrievalSystem):
self.retrieval_system = retrieval_system
def handle_search_request(self,request_Data:Dict)->Dict:
"""处理搜索请求"""
query = request_data.get('query', '')
top_k = request_data.get('top_k', 10)
modality = request_data.get('modality')
filters = request_data.get('filters', {})
try:
# 执行检索
results = self.retrieval_system.search(query, top_k, modality)
# 应用后过滤
filtered_results = self._apply_filters(results, filters)
return {
'status': 'success',
'query': query,
'results': filtered_results,
'total_count': len(filtered_results)
}
except Exception as e:
return {
'status': 'error',
'message': str(e),
'results': []
}
def _apply_filters(self,results:List[Dict],filters:Dict)->List[Dict]:
"""应用后过滤器"""
if not filters:
return results
filtered_results = []
for result in results:
metadata = result['metadata']
# 类别过滤
if 'category' in filters and metadata.get('category') not in filters['category']:
continue
# 标签过滤
if 'tags' in filters:
required_tags = set(filters['tags'])
item_tags = set(metadata.get('tags', []))
if not required_tags.issubset(item_tags):
continue
# 类型过滤
if 'type' in filters and metadata.get('type') not in filters['type']:
continue
filtered_results.append(result)
return filtered_results
===========================================
Java架构
多模态检索在制造业质量追溯的应用
业务场景：汽车零部件制造商通过图片、视频快速检索相似质量问题和解决方案
┌─ 制造业质量知识库 ───────────────────────────────────────┐
│ 问题描述: "变速箱壳体铸造气孔缺陷" | 分类: 铸造工艺 │
├─────────────────────────────────────────────────────────┤
│ 【多模态检索】 │
│ 上传图片: █████████████████ [缺陷部位标记] │
│ │
│ 检索条件: │
│ □ 图片特征匹配 □ 工艺参数 □ 材料批次 □ 设备信息 │
│ │
│ 【相似案例匹配】 │
│ 匹配度92% ┌─────────────────────────────────────────┐ │
│ │ 案例ID: QC202400123 │ │
│ │ 时间: 2024-01-15 生产线: 铸造3线 │ │
│ │ 缺陷类型: 气孔 位置: 壳体结合面 │ │
│ │ 根本原因: 模具温度过高 │ │
│ │ 解决方案: 调整模具冷却参数至65±5℃ │ │
│ └─────────────────────────────────────────┘ │
│ │
│ 匹配度85% ┌─────────────────────────────────────────┐ │
│ │ 相关视频: 气孔缺陷形成过程演示 │ │
│ │ 时长: 2:15 录制: 2024-02-20 │ │
│ │ 分析: 熔炼脱气不充分导致 │ │
│ └─────────────────────────────────────────┘ │
│ │
│ 【关联知识】 │
│ 📄 铸造工艺规范_v3.1 📊 历史质量数据统计 │
│ 🔧 模具维护记录 🎥 标准操作视频 │
│ │
│ [生成质量报告] [通知工艺工程师] [添加到知识库] │
└─────────────────────────────────────────────────────────┘
项目结构
manufacturing-quality-system/
├── src/main/java/com/company/quality/
│ ├── controller/ # REST API
│ ├── service/ # 业务逻辑
│ ├── repository/ # 数据访问
│ ├── entity/ # JPA实体
│ ├── dto/ # 数据传输对象
│ ├── config/ # 配置类
│ ├── feature/ # 特征提取模块
│ ├── milvus/ # 向量数据库操作
│ └── job/ # 定时任务
├── src/test/java/ # 测试代码
└── src/main/resources/
├── application.yml
└── model/ # AI模型文件
配置文件
# application.yml
spring:
datasource:
url: jdbc:mysql://localhost:3306/quality_management?useSSL=false&serverTimezone=Asia/Shanghai
username: ${DB_USERNAME:root}
password: ${DB_PASSWORD:password}
hikari:
maximum-pool-size: 20
minimum-idle: 5
jpa:
hibernate:
ddl-auto: update
show-sql: true
properties:
hibernate:
format_sql: true
dialect: org.hibernate.dialect.MySQL8Dialect
data:
elasticsearch:
cluster-nodes: localhost:9200
cluster-name: elasticsearch
redis:
host: localhost
port: 6379
password: ${REDIS_PASSWORD:}
timeout: 3000ms
lettuce:
pool:
max-active: 20
max-idle: 10
min-idle: 5
rabbitmq:
host: localhost
port: 5672
username: ${RABBITMQ_USER:guest}
password: ${RABBITMQ_PASSWORD:guest}
servlet:
multipart:
max-file-size: 50MB
max-request-size: 50MB
milvus:
host: localhost
port: 19530
collection:
name: defect_cases
minio:
endpoint: http://localhost:9000
access-key: ${MINIO_ACCESS_KEY:minioadmin}
secret-key: ${MINIO_SECRET_KEY:minioadmin}
bucket:
name: quality-images
logging:
level:
com.company.quality: DEBUG
org.hibernate.SQL: DEBUG
file:
name: logs/quality-system.log
pattern:
file: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n"
server:
port: 8080
servlet:
context-path: /quality-api
核心实体类
// 缺陷案例实体
@Entity
@Table(name = "quality_defect_case")
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DefectCase {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column(nullable = false, unique = true)
private String caseId; // QC202400123
@Column(nullable = false)
private String defectType; // 气孔、裂纹等
@Column(nullable = false)
private String productLine; // 铸造3线
@Column(nullable = false)
private String component; // 变速箱壳体
@Enumerated(EnumType.STRING)
private DefectSeverity severity; // LOW, MEDIUM, HIGH, CRITICAL
@Column(columnDefinition = "TEXT")
private String rootCause; // 根本原因分析
@Column(columnDefinition = "TEXT")
private String solution; // 解决方案
private LocalDateTime occurrenceTime;
private LocalDateTime resolvedTime;
@OneToMany(cascade = CascadeType.ALL, mappedBy = "defectCase")
private List<DefectImage> defectImages;
@OneToMany(cascade = CascadeType.ALL, mappedBy = "defectCase")
private List<ProcessParameter> processParameters;
@CreationTimestamp
private LocalDateTime createTime;
@UpdateTimestamp
private LocalDateTime updateTime;
}
// 缺陷图片实体
@Entity
@Table(name = "quality_defect_image")
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DefectImage {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column(nullable = false)
private String imageName;
@Column(nullable = false)
private String filePath; // MinIO存储路径
@Column(nullable = false)
private String fileType;
private Long fileSize;
@Column(columnDefinition = "JSON")
private String imageMetadata; // 图片元数据JSON
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "case_id")
private DefectCase defectCase;
@Column(unique = true)
private String vectorId; // Milvus中的向量ID
}
// 工艺参数实体
@Entity
@Table(name = "quality_process_parameter")
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ProcessParameter {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column(nullable = false)
private String parameterName; // 模具温度、熔炼时间等
private Double parameterValue;
private String unit;
private String standardRange; // 标准范围
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "case_id")
private DefectCase defectCase;
}
REST API控制器
@RestController
@RequestMapping("/api/v1/quality/defects")
@Validated
@Slf4j
public class DefectCaseController {
private final QualityDefectService qualityDefectService;
public DefectCaseController(QualityDefectService qualityDefectService) {
this.qualityDefectService = qualityDefectService;
}
/**
* 创建缺陷案例
*/
@PostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity<ApiResponse<DefectCaseDTO>> createDefectCase(
@Valid @ModelAttribute DefectCaseCreateRequest request) {
log.info("收到创建缺陷案例请求: {}", request.getCaseId());
DefectCaseDTO result = qualityDefectService.createDefectCase(request);
return ResponseEntity.ok(ApiResponse.success("缺陷案例创建成功", result));
}
/**
* 多模态相似案例搜索
*/
@PostMapping(value = "/search/similar", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity<ApiResponse<SimilarCaseSearchResult>> searchSimilarCases(
@Valid @ModelAttribute SimilarCaseSearchRequest request) {
log.info("收到相似案例搜索请求");
SimilarCaseSearchResult result = qualityDefectService.searchSimilarCases(request);
return ResponseEntity.ok(ApiResponse.success("搜索完成", result));
}
/**
* 生成质量分析报告
*/
@GetMapping("/{caseId}/report")
public ResponseEntity<ApiResponse<QualityAnalysisReport>> generateAnalysisReport(
@PathVariable String caseId) {
log.info("收到生成质量报告请求，案例ID: {}", caseId);
QualityAnalysisReport report = qualityDefectService.generateAnalysisReport(caseId);
return ResponseEntity.ok(ApiResponse.success("报告生成成功", report));
}
/**
* 获取案例详情
*/
@GetMapping("/{caseId}")
public ResponseEntity<ApiResponse<DefectCaseDTO>> getDefectCase(
@PathVariable String caseId) {
DefectCaseDTO result = qualityDefectService.getDefectCaseByCaseId(caseId);
return ResponseEntity.ok(ApiResponse.success("查询成功", result));
}
}
核心业务服务
@Service
@Slf4j
@Transactional
public class QualityDefectService {
private final DefectCaseRepository defectCaseRepository;
private final FeatureExtractionService featureExtractionService;
private final MilvusService milvusService;
private final FileStorageService fileStorageService;
private final ElasticsearchService elasticsearchService;
public QualityDefectService(DefectCaseRepository defectCaseRepository,
FeatureExtractionService featureExtractionService,
MilvusService milvusService,
FileStorageService fileStorageService,
ElasticsearchService elasticsearchService) {
this.defectCaseRepository = defectCaseRepository;
this.featureExtractionService = featureExtractionService;
this.milvusService = milvusService;
this.fileStorageService = fileStorageService;
this.elasticsearchService = elasticsearchService;
}
/**
* 创建缺陷案例（完整业务流程）
*/
public DefectCaseDTO createDefectCase(DefectCaseCreateRequest request) {
log.info("开始创建缺陷案例，产品线: {}, 缺陷类型: {}",
request.getProductLine(), request.getDefectType());
//1.保存基础案例信息
DefectCase defectCase = buildDefectCase(request);
defectCase = defectCaseRepository.save(defectCase);
//2.处理并保存图片
List<DefectImage> savedImages = processAndSaveImages(request.getImages(),defectCase);
defectCase.setDefectImages(savedImages);
//3.保存工艺参数
List<ProcessParameter> parameters = buildProcessParameters(request.getProcessParameters(), defectCase);
defectCase.setProcessParameters(parameters);
// 4. 特征提取和向量化（异步处理）
processCaseVectorsAsync(defectCase, savedImages, parameters);.
// 5. 索引到Elasticsearch
elasticsearchService.indexDefectCase(defectCase);
log.info("缺陷案例创建完成，案例ID: {}", defectCase.getCaseId());
return DefectCaseMapper.INSTANCE.toDTO(defectCase);
}
/**
* 多模态相似案例搜索
*/
public SimilarCaseSearchResult searchSimilarCases(SimilarCaseSearchRequest request) {
log.info("开始相似案例搜索，搜索条件: {}", request);
SimilarCaseSearchResult result = new SimilarCaseSearchResult();
//1.提取查询图片特征
float[] imageVector = null;
if(request.getQueryImage()!=null && !request.getQueryImage().isEmpty()){
imageVector = featureExtractionService.extractImageFeatures(request.getQueryImage());
}
//2.提取工艺参数特征
float[] processVector = null;
if(request.getProcessParameters()!=null && !request.getProcessParamters().isEmpty()){
processVector = featureExtractionService.extractProcessFeatures(request.getProcessParameters());
}
//3.向量相似性搜索
SearchCondition searchCondition = SearchCondition.builder()
.imageSearch(imageVector != null)
.processSearch(processVector != null)
.build();
List<SearchResult> vectorResults = milvusService.multimodalearch(imageVector, processVector,
request.getTopK(), searchCondition);
//4.获取完整案例信息
List<> caseIds = vectorResults.stream()
.map(SearchResult::getCaseId)
.collect(Collectors.toList());
List<> similarCases = defectCaseRepository.findByCaseIdIn(caseIds);
//5.构建返回结果
result.setTotal(similarCases.size());
result.setSimilarCases(buildSimilarCaseDTOs(similarCases, vectorResults));
result.setSearchCondition(searchCondition);
log.info("相似案例搜索完成，找到 {} 个匹配案例", similarCases.size());
return result;
}
/**
生成质量分析报告
*/
/**
* 生成质量分析报告
*/
public QualityAnalysisReport generateAnalysisReport(String caseId) {
DefectCase defectCase = defectCaseRepository.findByCaseId(caseId)
.orElseThrow(() -> new ResourceNotFoundException("缺陷案例不存在: " + caseId));
// 1. 搜索高度相似案例
SimilarCaseSearchRequest searchRequest = buildSearchRequestFromCase(defectCase);
SimilarCaseSearchResult similarCases = searchSimilarCases(searchRequest);
// 2. 统计分析
StatisticalAnalysis stats = performStatisticalAnalysis(defectCase, similarCases);
// 3. 生成改进建议
List<ImprovementSuggestion> suggestions = generateImprovementSuggestions(defectCase, similarCases);
// 4. 构建报告
return QualityAnalysisReport.builder()
.reportId(generateReportId())
.caseId(caseId)
.generatedTime(LocalDateTime.now())
.defectCase(DefectCaseMapper.INSTANCE.toDTO(defectCase))
.similarCases(similarCases.getSimilarCases())
.statisticalAnalysis(stats)
.improvementSuggestions(suggestions)
.riskAssessment(assessRiskLevel(defectCase, stats))
.build();
}
private void processCaseVectorsAsync(DefectCase defectCase, List<DefectImage> images,
List<ProcessParameter> parameters) {
CompletableFuture.runAsync(() -> {
try {
// 提取主图片特征
DefectImage mainImage = images.get(0);
byte[] imageData = fileStorageService.downloadFile(mainImage.getFilePath());
// 这里需要将byte[]转换为MultipartFile，实际项目中需要适当处理
float[] imageVector = featureExtractionService.extractImageFeatures(
createMultipartFile(imageData, mainImage.getImageName(), mainImage.getFileType()));
// 提取工艺参数特征
List<ProcessParameterDTO> parameterDTOs = parameters.stream()
.map(ProcessParameterMapper.INSTANCE::toDTO)
.collect(Collectors.toList());
float[] processVector = featureExtractionService.extractProcessFeatures(parameterDTOs);
// 存储到向量数据库
milvusService.insertCaseVectors(defectCase, imageVector, processVector);
// 更新图片记录向量ID
mainImage.setVectorId(defectCase.getCaseId() + "_image");
defectCaseRepository.save(defectCase);
log.info("案例向量化完成，案例ID: {}", defectCase.getCaseId());
} catch (Exception e) {
log.error("案例向量化处理失败，案例ID: {}", defectCase.getCaseId(), e);
}
});
}
private StatisticalAnalysis performStatisticalAnalysis(DefectCase currentCase,
SimilarCaseSearchResult similarCases) {
// 实现统计分析逻辑
return StatisticalAnalysis.builder()
.similarCaseCount(similarCases.getTotal())
.commonRootCauses(extractCommonRootCauses(similarCases))
.effectiveSolutions(extractEffectiveSolutions(similarCases))
.recurrenceRate(calculateRecurrenceRate(currentCase, similarCases))
.build();
}
}
向量数据库服务
@Service
@Slf4j
public class MilvusService {
@Value("${milvus.host:localhost}")
private String milvusHost;
@Value("${milvus.port:19530}")
private int milvusPort;
private MilvusClient client;
private final String COLLECTION_NAME = "defect_cases";
@PostConstruct
public void init() {
ConnectParam connectParam = ConnectParam.newBuilder()
.withHost(milvusHost)
.withPort(milvusPort)
.build();
this.client = new MilvusServiceClient(connectParam);
createCollectionIfNotExists();
}
/**
* 创建向量集合
*/
private void createCollectionIfNotExists() {
try {
// 检查集合是否存在
if (client.hasCollection(HasCollectionParam.newBuilder()
.withCollectionName(COLLECTION_NAME)
.build())) {
return;
}
// 定义字段
FieldType fieldType1 = FieldType.newBuilder()
.withName("case_id")
.withDataType(DataType.VarChar)
.withMaxLength(64)
.withPrimaryKey(true)
.build();
FieldType fieldType2 = FieldType.newBuilder()
.withName("image_vector")
.withDataType(DataType.FloatVector)
.withDimension(2048)
.build();
FieldType fieldType3 = FieldType.newBuilder()
.withName("process_vector")
.withDataType(DataType.FloatVector)
.withDimension(50)
.build();
// 创建集合
CreateCollectionParam createParam = CreateCollectionParam.newBuilder()
.withCollectionName(COLLECTION_NAME)
.withFieldTypes(Arrays.asList(fieldType1, fieldType2, fieldType3))
.build();
client.createCollection(createParam);
// 创建索引
CreateIndexParam indexParam = CreateIndexParam.newBuilder()
.withCollectionName(COLLECTION_NAME)
.withFieldName("image_vector")
.withIndexType(IndexType.IVF_FLAT)
.withMetricType(MetricType.L2)
.withExtraParam("{\"nlist\":1024}")
.build();
client.createIndex(indexParam);
} catch (Exception e) {
log.error("创建Milvus集合失败: {}", e.getMessage(), e);
}
}
/**
* 插入案例向量
*/
public void insertCaseVectors(DefectCase defectCase, float[] imageVector, float[] processVector) {
try {
List<InsertParam.Field> fields = new ArrayList<>();
fields.add(new InsertParam.Field("case_id",
Collections.singletonList(defectCase.getCaseId())));
fields.add(new InsertParam.Field("image_vector",
Collections.singletonList(imageVector)));
fields.add(new InsertParam.Field("process_vector",
Collections.singletonList(processVector)));
InsertParam insertParam = InsertParam.newBuilder()
.withCollectionName(COLLECTION_NAME)
.withFields(fields)
.build();
client.insert(insertParam);
// 刷新数据
client.flush(FlushParam.newBuilder()
.addCollectionName(COLLECTION_NAME)
.build());
} catch (Exception e) {
log.error("插入向量数据失败: {}", e.getMessage(), e);
throw new BusinessException("向量数据存储失败");
}
}
/**
* 多模态相似性搜索
*/
public List<SearchResult> multimodalSearch(float[] imageVector, float[] processVector,
int topK, SearchCondition condition) {
try {
// 构建搜索参数
String vectorField = condition.isImageSearch() ? "image_vector" : "process_vector";
float[] searchVector = condition.isImageSearch() ? imageVector : processVector;
List<String> outputFields = Arrays.asList("case_id");
List<List<Float>> searchVectors = Collections.singletonList(
Arrays.stream(searchVector).boxed().collect(Collectors.toList()));
SearchParam searchParam = SearchParam.newBuilder()
.withCollectionName(COLLECTION_NAME)
.withVectorFieldName(vectorField)
.withVectors(searchVectors)
.withTopK(topK)
.withMetricType(MetricType.L2)
.withParams("{\"nprobe\":10}")
.withOutFields(outputFields)
.build();
SearchResults searchResults = client.search(searchParam);
return parseSearchResults(searchResults);
} catch (Exception e) {
log.error("向量搜索失败: {}", e.getMessage(), e);
throw new BusinessException("相似案例搜索失败");
}
}
private List<SearchResult> parseSearchResults(SearchResults searchResults) {
List<SearchResult> results = new ArrayList<>();
for (int i = 0; i < searchResults.getResults().get(0).getFields().get("case_id").size(); i++) {
String caseId = (String) searchResults.getResults().get(0).getFields().get("case_id").get(i);
float score = searchResults.getResults().get(0).getScores().get(i);
results.add(SearchResult.builder()
.caseId(caseId)
.similarityScore(1 - score) // 转换为相似度分数
.build());
}
return results;
}
}
特征提取服务
@Service
@Slf4j
public class FeatureExtractionService {
private final Session session;
private final Graph graph;
private final SavedModelBundle model;
public FeatureExtractionService() {
// 加载预训练的ResNet50模型进行特征提取
this.model = SavedModelBundle.load("src/main/resources/model/resnet50", "serve");
this.graph = model.graph();
this.session = model.session();
}
/**
* 提取图片特征向量
*/
public float[] extractImageFeatures(MultipartFile imageFile) {
try {
// 1. 图片预处理
Mat image = preprocessImage(imageFile);
// 2. 转换为Tensor
Tensor<?> imageTensor = convertMatToTensor(image);
// 3. 特征提取
try (Tensor<?> result = session.runner()
.feed("input_1", imageTensor)
.fetch("global_average_pooling2d")
.run()
.get(0)) {
// 4. 获取特征向量
float[][][][] output = new float[1][1][1][2048];
result.copyTo(output);
return output[0][0][0];
}
} catch (Exception e) {
log.error("特征提取失败: {}", e.getMessage(), e);
throw new BusinessException("图片特征提取失败");
}
}
/**
* 图片预处理
*/
private Mat preprocessImage(MultipartFile imageFile) throws IOException {
byte[] bytes = imageFile.getBytes();
Mat image = Imgcodecs.imdecode(new MatOfByte(bytes), Imgcodecs.IMREAD_COLOR);
// 调整尺寸为224x224
Mat resizedImage = new Mat();
Imgproc.resize(image, resizedImage, new Size(224, 224));
// 归一化
resizedImage.convertTo(resizedImage, CvType.CV_32F, 1.0 / 255);
// 均值减法 (ImageNet数据集均值)
Core.subtract(resizedImage, new Scalar(0.485, 0.456, 0.406), resizedImage);
Core.divide(resizedImage, new Scalar(0.229, 0.224, 0.225), resizedImage);
return resizedImage;
}
/**
* 转换Mat到Tensor
*/
private Tensor<?> convertMatToTensor(Mat image) {
int height = image.rows();
int width = image.cols();
int channels = image.channels();
float[][][][] floatArray = new float[1][height][width][channels];
for (int h = 0; h < height; h++) {
for (int w = 0; w < width; w++) {
float[] pixel = new float[channels];
image.get(h, w, pixel);
floatArray[0][h][w] = pixel;
}
}
return Tensor.create(floatArray, Float.class);
}
/**
* 提取工艺参数特征
*/
public float[] extractProcessFeatures(List<ProcessParameterDTO> parameters) {
// 将工艺参数转换为特征向量
float[] features = new float[parameters.size() * 2];
for (int i = 0; i < parameters.size(); i++) {
ProcessParameterDTO param = parameters.get(i);
features[i * 2] = param.getParameterValue().floatValue();
features[i * 2 + 1] = normalizeParameter(param);
}
return features;
}
private float normalizeParameter(ProcessParameterDTO param) {
// 参数归一化逻辑
return (float) (param.getParameterValue() / 100.0);
}
}
电商企业物流数字化转型必备！快递鸟 API 接口，72 小时快速完成物流系统集成。全流程实战1V1指导，营造开放的API技术生态圈。
更多推荐
7
0- 0
已为社区贡献5条内容
所有评论(0)