feat: 添加 OpenAI 兼容 API 支持和独立 Vision 服务
- 添加 OpenAI AI SDK provider 支持 (@ai-sdk/openai) - 支持 OpenAI 兼容服务的 baseUrl 配置(如阿里云百炼) - 添加独立的 Vision 配置(visionProvider/visionApiKey/visionBaseUrl/visionModel) - 实现图片引用语法 @path/to/image.png,支持带空格的路径 - 当主模型不支持 vision 时,自动调用配置的 Vision 服务分析图片 - 添加图片处理工具函数和单元测试
This commit is contained in:
+90
-10
@@ -1,5 +1,6 @@
|
||||
import { createAnthropic } from '@ai-sdk/anthropic';
|
||||
import { createDeepSeek } from '@ai-sdk/deepseek';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import {
|
||||
generateText,
|
||||
streamText,
|
||||
@@ -8,7 +9,7 @@ import {
|
||||
type Tool as AITool,
|
||||
type LanguageModel,
|
||||
} from 'ai';
|
||||
import type { Tool, ToolResult, Message, AgentConfig, ProviderType } from '../types/index.js';
|
||||
import type { Tool, ToolResult, Message, AgentConfig, ProviderType, UserInput, ContentBlock } from '../types/index.js';
|
||||
import { buildZodSchema } from '../types/index.js';
|
||||
import { ToolRegistry } from '../tools/registry.js';
|
||||
import { SessionManager } from '../session/index.js';
|
||||
@@ -19,17 +20,27 @@ import {
|
||||
} from '../context/index.js';
|
||||
import type { AgentInfo } from '../agent/types.js';
|
||||
|
||||
// Provider 配置
|
||||
interface ProviderOptions {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
// Provider 工厂函数类型
|
||||
type ProviderFactory = (apiKey: string) => (model: string) => LanguageModel;
|
||||
type ProviderFactory = (options: ProviderOptions) => (model: string) => LanguageModel;
|
||||
|
||||
// Provider 注册表
|
||||
const providers: Record<ProviderType, ProviderFactory> = {
|
||||
anthropic: (apiKey) => {
|
||||
const client = createAnthropic({ apiKey });
|
||||
anthropic: ({ apiKey, baseUrl }) => {
|
||||
const client = createAnthropic({ apiKey, baseURL: baseUrl });
|
||||
return (model) => client(model);
|
||||
},
|
||||
deepseek: (apiKey) => {
|
||||
const client = createDeepSeek({ apiKey });
|
||||
deepseek: ({ apiKey, baseUrl }) => {
|
||||
const client = createDeepSeek({ apiKey, baseURL: baseUrl });
|
||||
return (model) => client(model);
|
||||
},
|
||||
openai: ({ apiKey, baseUrl }) => {
|
||||
const client = createOpenAI({ apiKey, baseURL: baseUrl });
|
||||
return (model) => client(model);
|
||||
},
|
||||
};
|
||||
@@ -68,7 +79,7 @@ export class Agent {
|
||||
if (!providerFactory) {
|
||||
throw new Error(`不支持的 provider: ${config.provider}`);
|
||||
}
|
||||
this.getModel = providerFactory(config.apiKey);
|
||||
this.getModel = providerFactory({ apiKey: config.apiKey, baseUrl: config.baseUrl });
|
||||
|
||||
// 初始化压缩管理器
|
||||
this.compressionManager = new CompressionManager(compressionConfig);
|
||||
@@ -221,13 +232,49 @@ export class Agent {
|
||||
|
||||
/**
|
||||
* 发送消息并处理响应(流式)
|
||||
* @param userMessage 用户消息文本或包含图片的 UserInput
|
||||
* @param onStream 流式输出回调
|
||||
*/
|
||||
async chat(userMessage: string, onStream?: (text: string) => void): Promise<string> {
|
||||
async chat(userMessage: string | UserInput, onStream?: (text: string) => void): Promise<string> {
|
||||
// 构建消息内容
|
||||
let messageContent: string | ContentBlock[];
|
||||
|
||||
if (typeof userMessage === 'string') {
|
||||
// 纯文本消息
|
||||
messageContent = userMessage;
|
||||
} else {
|
||||
// 带图片的消息
|
||||
const blocks: ContentBlock[] = [];
|
||||
|
||||
// 添加图片
|
||||
if (userMessage.images && userMessage.images.length > 0) {
|
||||
for (const img of userMessage.images) {
|
||||
blocks.push({
|
||||
type: 'image',
|
||||
image: img.data,
|
||||
mimeType: img.mimeType,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 添加文本
|
||||
if (userMessage.text) {
|
||||
blocks.push({
|
||||
type: 'text',
|
||||
text: userMessage.text,
|
||||
});
|
||||
}
|
||||
|
||||
messageContent = blocks.length === 1 && blocks[0].type === 'text'
|
||||
? blocks[0].text
|
||||
: blocks;
|
||||
}
|
||||
|
||||
// 添加用户消息到历史
|
||||
this.conversationHistory.push({
|
||||
role: 'user',
|
||||
content: userMessage,
|
||||
});
|
||||
content: messageContent,
|
||||
} as ModelMessage);
|
||||
|
||||
const vercelTools = this.getVercelTools();
|
||||
let fullResponse = '';
|
||||
@@ -436,4 +483,37 @@ export class Agent {
|
||||
getAgentModeName(): string {
|
||||
return this.currentAgentMode?.name ?? 'default';
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查当前模型是否支持 vision(图片理解)
|
||||
*/
|
||||
supportsVision(): boolean {
|
||||
const model = this.config.model.toLowerCase();
|
||||
|
||||
// Anthropic Claude 模型支持 vision
|
||||
if (this.config.provider === 'anthropic') {
|
||||
// Claude 3 及以上版本支持 vision
|
||||
return model.includes('claude-3') || model.includes('claude-4');
|
||||
}
|
||||
|
||||
// OpenAI GPT-4 系列支持 vision
|
||||
if (this.config.provider === 'openai') {
|
||||
// GPT-4o, GPT-4 Turbo, GPT-4 Vision 等支持
|
||||
return model.includes('gpt-4');
|
||||
}
|
||||
|
||||
// DeepSeek 目前不支持 vision
|
||||
if (this.config.provider === 'deepseek') {
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前配置
|
||||
*/
|
||||
getConfig(): AgentConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user