feat: 添加 OpenAI 兼容 API 支持和独立 Vision 服务

- 添加 OpenAI AI SDK provider 支持 (@ai-sdk/openai)
- 支持 OpenAI 兼容服务的 baseUrl 配置(如阿里云百炼)
- 添加独立的 Vision 配置(visionProvider/visionApiKey/visionBaseUrl/visionModel)
- 实现图片引用语法 @path/to/image.png,支持带空格的路径
- 当主模型不支持 vision 时,自动调用配置的 Vision 服务分析图片
- 添加图片处理工具函数和单元测试
This commit is contained in:
2025-12-11 17:49:16 +08:00
parent a476a4240c
commit 32fdb244f0
11 changed files with 1096 additions and 42 deletions
+90 -10
View File
@@ -1,5 +1,6 @@
import { createAnthropic } from '@ai-sdk/anthropic';
import { createDeepSeek } from '@ai-sdk/deepseek';
import { createOpenAI } from '@ai-sdk/openai';
import {
generateText,
streamText,
@@ -8,7 +9,7 @@ import {
type Tool as AITool,
type LanguageModel,
} from 'ai';
import type { Tool, ToolResult, Message, AgentConfig, ProviderType } from '../types/index.js';
import type { Tool, ToolResult, Message, AgentConfig, ProviderType, UserInput, ContentBlock } from '../types/index.js';
import { buildZodSchema } from '../types/index.js';
import { ToolRegistry } from '../tools/registry.js';
import { SessionManager } from '../session/index.js';
@@ -19,17 +20,27 @@ import {
} from '../context/index.js';
import type { AgentInfo } from '../agent/types.js';
// Provider 配置
interface ProviderOptions {
apiKey: string;
baseUrl?: string;
}
// Provider 工厂函数类型
type ProviderFactory = (apiKey: string) => (model: string) => LanguageModel;
type ProviderFactory = (options: ProviderOptions) => (model: string) => LanguageModel;
// Provider 注册表
const providers: Record<ProviderType, ProviderFactory> = {
anthropic: (apiKey) => {
const client = createAnthropic({ apiKey });
anthropic: ({ apiKey, baseUrl }) => {
const client = createAnthropic({ apiKey, baseURL: baseUrl });
return (model) => client(model);
},
deepseek: (apiKey) => {
const client = createDeepSeek({ apiKey });
deepseek: ({ apiKey, baseUrl }) => {
const client = createDeepSeek({ apiKey, baseURL: baseUrl });
return (model) => client(model);
},
openai: ({ apiKey, baseUrl }) => {
const client = createOpenAI({ apiKey, baseURL: baseUrl });
return (model) => client(model);
},
};
@@ -68,7 +79,7 @@ export class Agent {
if (!providerFactory) {
throw new Error(`不支持的 provider: ${config.provider}`);
}
this.getModel = providerFactory(config.apiKey);
this.getModel = providerFactory({ apiKey: config.apiKey, baseUrl: config.baseUrl });
// 初始化压缩管理器
this.compressionManager = new CompressionManager(compressionConfig);
@@ -221,13 +232,49 @@ export class Agent {
/**
* 发送消息并处理响应(流式)
* @param userMessage 用户消息文本或包含图片的 UserInput
* @param onStream 流式输出回调
*/
async chat(userMessage: string, onStream?: (text: string) => void): Promise<string> {
async chat(userMessage: string | UserInput, onStream?: (text: string) => void): Promise<string> {
// 构建消息内容
let messageContent: string | ContentBlock[];
if (typeof userMessage === 'string') {
// 纯文本消息
messageContent = userMessage;
} else {
// 带图片的消息
const blocks: ContentBlock[] = [];
// 添加图片
if (userMessage.images && userMessage.images.length > 0) {
for (const img of userMessage.images) {
blocks.push({
type: 'image',
image: img.data,
mimeType: img.mimeType,
});
}
}
// 添加文本
if (userMessage.text) {
blocks.push({
type: 'text',
text: userMessage.text,
});
}
messageContent = blocks.length === 1 && blocks[0].type === 'text'
? blocks[0].text
: blocks;
}
// 添加用户消息到历史
this.conversationHistory.push({
role: 'user',
content: userMessage,
});
content: messageContent,
} as ModelMessage);
const vercelTools = this.getVercelTools();
let fullResponse = '';
@@ -436,4 +483,37 @@ export class Agent {
getAgentModeName(): string {
return this.currentAgentMode?.name ?? 'default';
}
/**
* 检查当前模型是否支持 vision(图片理解)
*/
supportsVision(): boolean {
const model = this.config.model.toLowerCase();
// Anthropic Claude 模型支持 vision
if (this.config.provider === 'anthropic') {
// Claude 3 及以上版本支持 vision
return model.includes('claude-3') || model.includes('claude-4');
}
// OpenAI GPT-4 系列支持 vision
if (this.config.provider === 'openai') {
// GPT-4o, GPT-4 Turbo, GPT-4 Vision 等支持
return model.includes('gpt-4');
}
// DeepSeek 目前不支持 vision
if (this.config.provider === 'deepseek') {
return false;
}
return false;
}
/**
* 获取当前配置
*/
getConfig(): AgentConfig {
return { ...this.config };
}
}