Video Input

import zeroruntime
from zeroruntime import Agent, Pipeline
from zeroruntime.plugins import OpenAIRealtime
from zeroruntime.inference import AICousticsDenoise

AGENT_ID = "vision-agent"

pipeline = Pipeline(
    llm=OpenAIRealtime(
        model="gpt-4o-realtime-preview", config={"voice": "alloy"}
    ),
    denoise=AICousticsDenoise(model_id="quail-vf-2.2-l-16khz"),
)

class VisionAgent(Agent):
    def __init__(self):
        super().__init__(
            agent_id=AGENT_ID,
            instructions="You can see the participant's video. Describe what you observe.",
            pipeline=pipeline,
        )

if __name__ == "__main__":
    # Pass the class itself (not an instance): serve() builds a fresh VisionAgent +
    # pipeline per call, which is required for correct per-call state under concurrent calls.
    zeroruntime.serve(
        VisionAgent,
        on_ready=lambda: zeroruntime.invoke(
            AGENT_ID, room=zeroruntime.Room(playground=True, vision=True)
        ),
    )

import * as zeroruntime from '@zeroruntime/js-sdk';
import { Agent, Pipeline } from '@zeroruntime/js-sdk';
import { OpenAIRealtime } from '@zeroruntime/js-sdk/plugins';
import { AICousticsDenoise } from '@zeroruntime/js-sdk/inference';

const AGENT_ID = 'vision-agent';

const pipeline = Pipeline({
  llm: OpenAIRealtime({
    model: 'gpt-4o-realtime-preview', config: {voice: 'alloy'},
  }),
  denoise: AICousticsDenoise({ model_id: 'quail-vf-2.2-l-16khz' }),
});

class VisionAgent extends Agent {
  constructor() {
    super({
      agent_id: AGENT_ID,
      instructions: "You can see the participant's video. Describe what you observe.",
      pipeline,
    });
  }
}

// Pass the class itself (not an instance): serve() builds a fresh VisionAgent +
// pipeline per call, which is required for correct per-call state under concurrent calls.
zeroruntime.serve(VisionAgent, {
  on_ready: () => zeroruntime.invoke(AGENT_ID, { room: zeroruntime.Room({ playground: true, vision: true }) }),
});