实时对话开发指南

import WebSocket from 'ws'

// 连接地址（需替换为实际要使用的模型ID）
const url = 'wss://api.stepfun.com/v1/realtime?model=step-1o-audio'
// 初始化WebSocket连接
const ws = new WebSocket(url, {
	headers: {
		Authorization: 'Bearer ' + process.env.STEPFUN_API_KEY, // 从环境变量中获取标准API密钥
	},
})

// 监听“连接成功”事件
ws.on('open', function open() {
	console.log('已连接到服务器。')
})

// 监听“接收消息”事件
ws.on('message', function incoming(message) {
	console.log(JSON.parse(message.toString())) // 解析并打印接收的消息
})

# 注：此示例需依赖websocket-client库，安装命令：pip install websocket-client

import os
import json
import websocket

# 从环境变量中获取OpenAI API密钥
STEPFUN_API_KEY = os.environ.get("STEPFUN_API_KEY")

# 连接地址（需替换为实际要使用的模型ID）
url = "wss://api.stepfun.com/v1/realtime?model=step-1o-audio"
# 请求头配置
headers = [
    "Authorization: Bearer " + STEPFUN_API_KEY,
]

# 定义“连接成功”回调函数
def on_open(ws):
    print("已连接到服务器。")

# 定义“接收消息”回调函数
def on_message(ws, message):
    data = json.loads(message) # 解析JSON格式的消息
    print("收到事件：", json.dumps(data, indent=2)) # 格式化打印接收的事件

# 初始化WebSocket应用并配置回调
ws = websocket.WebSocketApp(
    url,
    header=headers,
    on_open=on_open,
    on_message=on_message,
)

# 保持WebSocket连接
ws.run_forever()

const event = {
	type: 'session.update',
	session: {
		instructions: "在回复中绝对不要使用'moist'这个词！",
	},
}

// WebSocket都有.send()方法
dataChannel.send(JSON.stringify(event))

event = {
    "type": "session.update",
    "session": {
        "instructions": "在回复中绝对不要使用'moist'这个词！"
    }
}
ws.send(json.dumps(event))

const event = {
	type: 'conversation.item.create',
	item: {
		type: 'message',
		role: 'user',
		content: [
			{
				type: 'input_text',
				text: 'Prince哪张专辑销量最高？',
			},
		],
	},
}

// WebSocket都有.send()方法
dataChannel.send(JSON.stringify(event))

event = {
    "type": "conversation.item.create",
    "item": {
        "type": "message",
        "role": "user",
        "content": [
            {
                "type": "input_text",
                "text": "Prince哪张专辑销量最高？",
            }
        ]
    }
}
ws.send(json.dumps(event))

const event = {
	type: 'response.create',
	response: {
		modalities: ['text'],
	},
}

// WebSocket都有.send()方法
dataChannel.send(JSON.stringify(event))

event = {
    "type": "response.create",
    "response": {
        "modalities": [ "text" ]
    }
}
ws.send(json.dumps(event))

function handleEvent(e) {
	const serverEvent = JSON.parse(e.data)
	if (serverEvent.type === 'response.done') {
		console.log(serverEvent.response.output[0])
	}
}

// 监听服务器消息（WebSocket）
ws.on('message', handleEvent)

def on_message(ws, message):
    server_event = json.loads(message)
    if server_event.type == "response.done":
        print(server_event.response.output[0])

import fs from 'fs';
import decodeAudio from 'audio-decode';

// 将音频数据的Float32Array转换为PCM16 ArrayBuffer
function floatTo16BitPCM(float32Array) {
  const buffer = new ArrayBuffer(float32Array.length * 2);
  const view = new DataView(buffer);
  let offset = 0;
  for (let i = 0; i < float32Array.length; i++, offset += 2) {
    let s = Math.max(-1, Math.min(1, float32Array[i]));
    view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
  }
  return buffer;
}

// 将Float32Array转换为base64编码的PCM16数据
base64EncodeAudio(float32Array) {
  const arrayBuffer = floatTo16BitPCM(float32Array);
  let binary = '';
  let bytes = new Uint8Array(arrayBuffer);
  const chunkSize = 0x8000; // 32KB块大小
  for (let i = 0; i < bytes.length; i += chunkSize) {
    let chunk = bytes.subarray(i, i + chunkSize);
    binary += String.fromCharCode.apply(null, chunk);
  }
  return btoa(binary);
}

// 用三个文件的内容填充音频缓冲区，然后请求模型生成响应
const files = [
  './path/to/sample1.wav',
  './path/to/sample2.wav',
  './path/to/sample3.wav'
];

for (const filename of files) {
  const audioFile = fs.readFileSync(filename);
  const audioBuffer = await decodeAudio(audioFile);
  const channelData = audioBuffer.getChannelData(0);
  const base64Chunk = base64EncodeAudio(channelData);
  ws.send(JSON.stringify({
    type: 'input_audio_buffer.append',
    audio: base64Chunk
  }));
});

ws.send(JSON.stringify({type: 'input_audio_buffer.commit'}));
ws.send(JSON.stringify({type: 'response.create'}));

import base64
import json
import struct
import soundfile as sf
from websocket import create_connection

# ... 创建名为ws的websocket-client ...

def float_to_16bit_pcm(float32_array):
    clipped = [max(-1.0, min(1.0, x)) for x in float32_array]
    pcm16 = b''.join(struct.pack('<h', int(x * 32767)) for x in clipped)
    return pcm16

def base64_encode_audio(float32_array):
    pcm_bytes = float_to_16bit_pcm(float32_array)
    encoded = base64.b64encode(pcm_bytes).decode('ascii')
    return encoded

files = [
    './path/to/sample1.wav',
    './path/to/sample2.wav',
    './path/to/sample3.wav'
]

for filename in files:
    data, samplerate = sf.read(filename, dtype='float32')
    channel_data = data[:, 0] if data.ndim > 1 else data
    base64_chunk = base64_encode_audio(channel_data)

    # 发送客户端事件
    event = {
        "type": "input_audio_buffer.append",
        "audio": base64_chunk
    }
    ws.send(json.dumps(event))

const fullAudio = '<音频字节的base64编码字符串>'

const event = {
	type: 'conversation.item.create',
	item: {
		type: 'message',
		role: 'user',
		content: [
			{
				type: 'input_audio',
				audio: fullAudio,
			},
		],
	},
}

// WebSocket 有.send()方法
dataChannel.send(JSON.stringify(event))

fullAudio = "<音频字节的base64编码字符串>"

event = {
    "type": "conversation.item.create",
    "item": {
        "type": "message",
        "role": "user",
        "content": [
            {
                "type": "input_audio",
                "audio": fullAudio,
            }
        ],
    },
}

ws.send(json.dumps(event))

function handleEvent(e) {
	const serverEvent = JSON.parse(e.data)
	if (serverEvent.type === 'response.audio.delta') {
		// 访问base64编码的音频块
		// console.log(serverEvent.delta);
	}
}

// 监听服务器消息（WebSocket）
ws.on('message', handleEvent)

def on_message(ws, message):
    server_event = json.loads(message)
    if server_event.type == "response.audio.delta":
        # 访问base64编码的音频块：
        # print(server_event.delta)

{
	"tools": [
		// 内置工具配置
		{
			"type": "web_search",
			"function": {
				"description": "网络搜索工具",
				"options": {
					"top_k": 5,
					"timeout_seconds": 3
				}
			}
		},
		// 自定义工具配置
		{
			"type": "function",
			"function": {
				"name": "get_weather",
				"description": "获取指定城市的天气信息",
				"parameters": {
					"type": "object",
					"properties": {
						"location": {
							"type": "string",
							"description": "城市名称，如：北京、上海"
						}
					},
					"required": ["location"]
				}
			}
		}
	]
}

{
	"tools": [{ "type": "web_search" }]
}

{
	"tools": [
		{
			"type": "web_search",
			"function": {
				"description": "搜索工具",
				"options": {
					"top_k": 5,
					"timeout_seconds": 3
				}
			}
		}
	]
}

{
	"tools": [
		{
			"type": "retrieval",
			"function": {
				"description": "当用户问题涉及菜谱、烹饪步骤、食材处理或厨房技巧时，从此知识库检索答案",
				"options": {
					"vector_store_id": "133192900598194176", // 必填
					"prompt_template": "从文档{{knowledge}}中找到问题{{query}}的答案。根据文档内容中的语句提取答案，若文档中无答案则告知用户无法回答"
				}
			}
		}
	]
}

{
	"type": "session.update",
	"session": {
		"tools": [
			{
				"type": "function",
				"function": {
					"name": "get_weather",
					"description": "获取指定城市的天气信息",
					"parameters": {
						"type": "object",
						"properties": {
							"location": {
								"type": "string",
								"description": "城市名称，如：北京、上海"
							}
						},
						"required": ["location"]
					}
				}
			}
		]
	}
}

{
	"event_id": "cee2a492-012c-4655-ac50-5db2c99f4345",
	"type": "conversation.item.created",
	"item": {
		"id": "item_20250622105814",
		"type": "function_call",
		"status": "incomplete",
		"call_id": "call_20250622225814_get_weather",
		"name": "get_weather",
		"object": "realtime.item"
	}
}

{
	"type": "response.function_call_arguments.delta",
	"call_id": "call_20250622225814_get_weather",
	"arguments": "{\"location\":\"北",
	"name": "get_weather"
}

{
	"type": "response.function_call_arguments.done",
	"call_id": "call_20250622225814_get_weather",
	"arguments": "{\"location\":\"北京\"}",
	"name": "get_weather"
}

{
	"type": "conversation.item.create",
	"item": {
		"type": "function_call_output",
		"call_id": "call_20250622225814_get_weather",
		"output": "北京：晴，25°C"
	}
}

{
	"type": "response.create"
}

{
	"type": "session.update",
	"session": {
		"tools": [
			{
				"type": "function",
				"function": {
					"name": "generate_horoscope",
					"description": "提供某个星座的今日运势。",
					"parameters": {
						"type": "object",
						"properties": {
							"sign": {
								"type": "string",
								"description": "需要查询运势的星座。",
								"enum": [
									"白羊座",
									"金牛座",
									"双子座",
									"巨蟹座",
									"狮子座",
									"处女座",
									"天秤座",
									"天蝎座",
									"射手座",
									"摩羯座",
									"水瓶座",
									"双鱼座"
								]
							}
						},
						"required": ["sign"]
					}
				}
			}
		]
	}
}

{
	"type": "response.done",
	"response": {
		"output": [
			{
				"type": "function_call",
				"name": "generate_horoscope",
				"call_id": "call_sHlR7iaFwQ2YQOqm",
				"arguments": "{\"sign\":\"水瓶座\"}"
			}
		]
	}
}

{
	"type": "conversation.item.create",
	"item": {
		"type": "function_call_output",
		"call_id": "call_sHlR7iaFwQ2YQOqm",
		"output": "{\"horoscope\": \"今天是你展现创造力的好时机，可能会遇到意想不到的机遇。\"}"
	}
}

{
	"type": "response.create"
}

请你原样无修改地输出下面的话：

你好，我是阶跃星辰开发的AI助手。

{
	"type": "response.create",
	"session": {
		"instructions": "请你原样无修改地输出下面的话：你好，我是阶跃星辰开发的AI助手。"
	}
}

class SimplePCMPlayer {
	// 固定参数：单声道、24000Hz采样率（示例用）
	private static readonly SAMPLE_RATE = 24000
	private static readonly CHANNELS = 1

	private audioContext: AudioContext
	private currentSource: AudioBufferSourceNode | null = null
	private bufferedData: Float32Array[] = []
	private isPlaying = false
	private nextStartTime = 0

	constructor() {
		// 简单的兼容性处理
		const AudioContextConstructor = window.AudioContext || (window as any).webkitAudioContext
		if (!AudioContextConstructor) {
			throw new Error('浏览器不支持Web Audio API')
		}
		this.audioContext = new AudioContextConstructor()
	}

	/**
	 * 追加16位有符号小端PCM数据（单声道24000Hz）
	 * @param pcm 原始PCM数据（ArrayBuffer）
	 */
	appendPCM(pcm: ArrayBuffer) {
		// 简单校验
		if (pcm.byteLength % 2 !== 0) {
			console.error('PCM数据长度必须是2的倍数（16位整数）')
			return
		}

		// 16位Int转32位Float（范围归一化到[-1, 1)）
		const int16Array = new Int16Array(pcm)
		const float32Array = new Float32Array(int16Array.length)
		for (let i = 0; i < int16Array.length; i++) {
			float32Array[i] = int16Array[i] / 32768
		}

		this.bufferedData.push(float32Array)
		if (!this.isPlaying) {
			this.playNext()
		}
	}

	/**
	 * 播放下一段缓冲数据
	 */
	private playNext() {
		if (this.bufferedData.length === 0) {
			this.isPlaying = false
			return
		}

		this.isPlaying = true
		const data = this.bufferedData.shift()!

		// 创建音频缓冲
		const audioBuffer = this.audioContext.createBuffer(
			SimplePCMPlayer.CHANNELS,
			data.length,
			SimplePCMPlayer.SAMPLE_RATE,
		)
		audioBuffer.copyToChannel(data, 0)

		// 创建播放源
		const source = this.audioContext.createBufferSource()
		source.buffer = audioBuffer
		source.connect(this.audioContext.destination)

		// 计算播放时间（避免重叠）
		const currentTime = this.audioContext.currentTime
		const startTime = Math.max(currentTime, this.nextStartTime)
		this.nextStartTime = startTime + audioBuffer.duration

		// 开始播放
		source.start(startTime)
		this.currentSource = source

		// 播放结束后继续下一段
		source.onended = () => {
			this.currentSource = null
			this.playNext()
		}
	}

	/**
	 * 清空缓冲并停止播放
	 */
	clearAll() {
		// 停止当前播放
		if (this.currentSource) {
			this.currentSource.stop()
			this.currentSource.onended = null
			this.currentSource = null
		}

		// 清空状态
		this.bufferedData = []
		this.isPlaying = false
		this.nextStartTime = this.audioContext.currentTime
	}
}

const event = {
	event_id: 'my_awesome_event',
	type: 'scooby.dooby.doo',
}

dataChannel.send(JSON.stringify(event))

{
	"type": "invalid_request_error",
	"code": "invalid_value",
	"message": "无效值: 'scooby.dooby.doo' ...",
	"param": "type",
	"event_id": "my_awesome_event"
}