Image Input

# Show the model the two most recent frames and ask it to describe them
await session.reply(
    "Describe what you see in this frame in one sentence.",
    frames=2,
)

// Show the model the two most recent frames and ask it to describe them
await session.reply('Describe what you see in this frame in one sentence.', {
  frames: 2,
});

from zeroruntime import Agent, PubSubSubscribeConfig, Room

class VisionAgent(Agent):
    async def on_enter(self) -> None:
        await self.session.subscribe_to_pubsub(
            PubSubSubscribeConfig(topic="CHAT", cb=self.on_chat)
        )

    async def on_chat(self, frame: dict, backlog: bool) -> None:
        if backlog or frame.get("message") != "capture_frames":
            return
        await self.session.reply(
            "Analyze this frame and describe what you see.", frames=2
        )

# Turn the video track on; the topic is subscribed once the session exists.
zeroruntime.invoke(AGENT_ID, room=Room(vision=True))

import * as zeroruntime from '@zeroruntime/js-sdk';
import { Agent, PubSubSubscribeConfig, Room } from '@zeroruntime/js-sdk';

class VisionAgent extends Agent {
  async on_enter(): Promise<void> {
    await this.session!.subscribe_to_pubsub(
      PubSubSubscribeConfig({ topic: 'CHAT', cb: this.on_chat.bind(this) }),
    );
  }

  async on_chat(frame: Record<string, any>, backlog: boolean): Promise<void> {
    if (backlog || frame.message !== 'capture_frames') {
      return;
    }
    await this.session!.reply('Analyze this frame and describe what you see.', { frames: 2 });
  }
}

// Turn the video track on; the topic is subscribed once the session exists.
zeroruntime.invoke(AGENT_ID, { room: Room({ vision: true }) });