Video-Connector für Node.js

import { VonageVideoClient } from '@vonage/video-connector';

const client = new VonageVideoClient();

try {
  const session = await client.connect(applicationId, sessionId, token, {
    // Fires only for errors after the connection is established
    onError: (session, description, code) => {
      console.error(`Session error [${code}]: ${description}`);
    },
  });
  console.log(`Connected to session ${session.sessionId}`);
} catch (error) {
  // Failure to connect surfaces here, not in onError
  console.error('Failed to connect:', error);
}

interface Session {
  sessionId: string;  // Unique identifier for the session
}

interface Connection {
  connectionId: string;  // Unique identifier for the connection
  creationTime: number;  // Unix timestamp in milliseconds
  data: string;          // Connection data (encoded in the token)
}

interface Stream {
  streamId: string;       // Unique identifier for the stream
  connection: Connection; // The connection that published this stream
}

interface Publisher {
  stream: Stream;  // The underlying stream for this publisher
}

interface Subscriber {
  stream: Stream;  // The stream being subscribed to
}

interface AudioData {
  data: Buffer;                        // Raw PCM 16-bit LE samples
  sampleRate: SampleRate;              // 8000 | 12000 | 16000 | 24000 | 32000 | 44100 | 48000
  numberOfChannels: NumberOfChannels;  // 1 (mono) or 2 (stereo)
  numberOfFrames: number;              // One frame = one sample per channel
}

interface VideoFrame {
  data: Buffer;               // Raw pixel data
  width: number;              // Frame width in pixels
  height: number;             // Frame height in pixels
  format?: VideoFrameFormat;  // 'YUV420P' | 'RGB24' | 'ARGB32', defaults to 'YUV420P'
}

interface VideoResolution {
  width: number;   // Width in pixels. Must be a positive integer
  height: number;  // Height in pixels. Must be a positive integer
}

interface CaptionsData {
  text: string;     // The caption text content
  isFinal: boolean; // true for final captions, false for interim/partial captions
}

interface BufferStats {
  durationMs: number;  // Duration of queued media in milliseconds
}

interface MediaBufferStats {
  audio: BufferStats | null;  // null if no audio publisher is active
  video: BufferStats | null;  // null if no video publisher is active
}

type SampleRate = 8000 | 12000 | 16000 | 24000 | 32000 | 44100 | 48000;
type NumberOfChannels = 1 | 2;
type LogLevel = 'ERROR' | 'WARN' | 'INFO' | 'DEBUG' | 'TRACE';
type VideoFrameFormat = 'YUV420P' | 'RGB24' | 'ARGB32';

import type { SessionSettings } from '@vonage/video-connector';

const sessionSettings: SessionSettings = {
  enableMigration: false,   // Enable automatic session migration. Defaults to false
  av: avSettings,           // Optional audio/video configuration
  logging: { level: 'INFO' } // Optional logging configuration
};

import type { SessionAVSettings } from '@vonage/video-connector';

const avSettings: SessionAVSettings = {
  audioPublisher: { sampleRate: 48000, channels: 2 },
  audioSubscribersMix: { sampleRate: 48000, channels: 1 },
  videoPublisher: { width: 1280, height: 720, fps: 30, format: 'YUV420P' }
};

import type { SessionAudioSettings } from '@vonage/video-connector';

const audioSettings: SessionAudioSettings = {
  sampleRate: 48000,  // Optional, defaults to 48000
  channels: 1         // Optional, 1 (mono) or 2 (stereo), defaults to 2
};

import type { SessionVideoPublisherSettings } from '@vonage/video-connector';

const videoSettings: SessionVideoPublisherSettings = {
  width: 1280,        // Optional, defaults to 640
  height: 720,        // Optional, defaults to 480
  fps: 30,            // Optional, 1-30, defaults to 30
  format: 'YUV420P'   // Optional, defaults to 'YUV420P'
};

import type { LoggingSettings } from '@vonage/video-connector';

const loggingSettings: LoggingSettings = {
  level: 'INFO'  // Required: 'ERROR' | 'WARN' | 'INFO' | 'DEBUG' | 'TRACE'
};

import type { PublisherSettings } from '@vonage/video-connector';

const publisherSettings: PublisherSettings = {
  name: 'My Application',  // Required: name for your published stream
  hasAudio: true,          // Required: whether to publish audio
  hasVideo: true,          // Required: whether to publish video
  enableCaptions: false,   // Optional: enable live captions for this stream. Defaults to false
  audioSettings: {         // Optional
    enableStereoMode: true,
    enableOpusDtx: false
  }
};

import type { PublisherAudioSettings } from '@vonage/video-connector';

const audioSettings: PublisherAudioSettings = {
  enableStereoMode: true,  // Optional: publish in stereo. Defaults to true
  enableOpusDtx: false     // Optional: enable discontinuous transmission. Defaults to false
};

import type { SubscriberSettings } from '@vonage/video-connector';

const subscriberSettings: SubscriberSettings = {
  subscribeToAudio: true,     // Optional, defaults to true
  subscribeToVideo: true,     // Optional, defaults to true
  subscribeToCaptions: false, // Optional, defaults to false
  videoSettings: {
    preferredResolution: { width: 640, height: 480 },
    preferredFrameRate: 15
  }
};

import type { SubscriberVideoSettings } from '@vonage/video-connector';

const videoSettings: SubscriberVideoSettings = {
  preferredResolution: { width: 640, height: 480 },  // Optional
  preferredFrameRate: 15                             // Optional, 1-30
};

import { VonageVideoClient, type SessionSettings } from '@vonage/video-connector';

const client = new VonageVideoClient();

const sessionSettings: SessionSettings = {
  enableMigration: false,
  av: {
    audioSubscribersMix: { sampleRate: 48000, channels: 1 }
  },
  logging: { level: 'INFO' }
};

const session = await client.connect(
  'your_application_id',
  'your_session_id',
  'your_token',
  { sessionSettings }
);

console.log(`Connected to session: ${session.sessionId}`);

const session = await client.connect(
  'your_application_id',
  'your_session_id',
  'your_token',
  {
    sessionSettings,
    onError: onSessionError,
    onDisconnected: onSessionDisconnected,
    onConnectionCreated: onConnectionCreated,
    onConnectionDropped: onConnectionDropped,
    onStreamReceived: onStreamReceived,
    onStreamDropped: onStreamDropped,
    onAudioData: onAudioData,
    onReadyForAudio: onReadyForAudio,
    onMediaBufferDrained: onMediaBufferDrained
  }
);

await client.disconnect();

if (client.isConnected()) {
  console.log('Still connected');
}

import type { SessionAVSettings } from '@vonage/video-connector';

const avSettings: SessionAVSettings = {
  // Format of the audio you supply via addAudio()
  audioPublisher: {
    sampleRate: 48000,  // Valid: 8000, 12000, 16000, 24000, 32000, 44100, 48000
    channels: 2         // 1 for mono, 2 for stereo
  },
  // Format of the mixed audio delivered to your session onAudioData callback
  audioSubscribersMix: {
    sampleRate: 48000,
    channels: 1
  },
  // Geometry and format of the video you supply via addVideo()
  videoPublisher: {
    width: 1280,
    height: 720,
    fps: 30,
    format: 'YUV420P'  // Valid: YUV420P, RGB24, ARGB32
  }
};

await client.connect(applicationId, sessionId, token, {
  sessionSettings: { av: avSettings }
});

await client.connect(applicationId, sessionId, token, {
  sessionSettings: {
    logging: { level: 'DEBUG' }  // Valid: ERROR, WARN, INFO, DEBUG, TRACE
  }
});

await client.connect(applicationId, sessionId, token, {
  sessionSettings: {
    enableMigration: true,
    av: avSettings,
    logging: { level: 'INFO' }
  }
});

import type { PublisherSettings } from '@vonage/video-connector';

// Audio and video publisher
const publisherSettings: PublisherSettings = {
  name: 'AI Assistant Bot',
  hasAudio: true,
  hasVideo: true,
  enableCaptions: true,
  audioSettings: {
    enableStereoMode: true,
    enableOpusDtx: false
  }
};

// Or audio-only publisher
const audioOnlySettings: PublisherSettings = {
  name: 'Audio Bot',
  hasAudio: true,
  hasVideo: false,
  enableCaptions: false
};

const publisher = await client.publish({
  settings: publisherSettings,
  onError: onPublisherError,
  onStreamDestroyed: onStreamDestroyed
});

console.log(`Publishing stream: ${publisher.stream.streamId}`);

// onReadyForAudio is registered at connect() time, so capture its resolver first
let signalAudioReady: () => void = () => {};
const audioReady = new Promise<void>((resolve) => {
  signalAudioReady = resolve;
});

await client.connect(applicationId, sessionId, token, {
  sessionSettings,
  onReadyForAudio: () => {
    console.log('Audio system ready - can now add audio');
    signalAudioReady();
  }
});

await client.publish({ settings: publisherSettings });
await audioReady;

// Now safe to add audio
client.addAudio(audioData);

import type { AudioData } from '@vonage/video-connector';

const sampleRate = 48000;
const numberOfChannels = 1;
const numberOfFrames = 960;  // 20ms at 48kHz

// PCM16 LE: 2 bytes per sample per channel
const data = Buffer.alloc(numberOfFrames * numberOfChannels * 2);

const audioData: AudioData = {
  data,
  sampleRate,
  numberOfChannels,
  numberOfFrames
};

const accepted = client.addAudio(audioData);

await client.unpublish();

if (client.isPublishing()) {
  console.log('Still publishing');
}

import type { Session, Stream } from '@vonage/video-connector';

async function onStreamReceived(session: Session, stream: Stream): Promise<void> {
  console.log(`New stream received: ${stream.streamId}`);
  console.log(`From connection: ${stream.connection.connectionId}`);

  try {
    const subscriber = await client.subscribe(stream, {
      settings: {
        subscribeToAudio: true,
        subscribeToVideo: true,
        subscribeToCaptions: true,
        videoSettings: {
          preferredResolution: { width: 640, height: 480 },
          preferredFrameRate: 15
        }
      },
      onError: onSubscriberError,
      onDisconnected: onSubscriberDisconnected,
      onRenderFrame: onRenderFrame,
      onAudioData: onSubscriberAudioData,
      onCaptionText: onCaptionText
    });

    console.log(`Subscribed to stream: ${subscriber.stream.streamId}`);
  } catch (error) {
    // onStreamReceived is a void callback, so handle failures here rather than
    // letting them become an unhandled rejection.
    console.error(`Failed to subscribe to ${stream.streamId}:`, error);
  }
}

import type { Subscriber, VideoFrame } from '@vonage/video-connector';

function onRenderFrame(subscriber: Subscriber, frame: VideoFrame): void {
  const streamId = subscriber.stream.streamId;
  console.log(`Video frame from stream ${streamId}: ${frame.width}x${frame.height}`);
  // Copy before retaining beyond this callback
  const owned = Buffer.from(frame.data);
}

import type { Session, AudioData } from '@vonage/video-connector';

function onAudioData(session: Session, audioData: AudioData): void {
  console.log(
    `Mixed audio from all subscribers: ${audioData.sampleRate}Hz, ` +
    `${audioData.numberOfChannels} channel(s)`
  );
}

import type { Subscriber, CaptionsData } from '@vonage/video-connector';

function onCaptionText(subscriber: Subscriber, captions: CaptionsData): void {
  const status = captions.isFinal ? 'final' : 'interim';
  console.log(`Caption (${status}) from stream ${subscriber.stream.streamId}: ${captions.text}`);
}

import type { Subscriber, AudioData } from '@vonage/video-connector';

function onSubscriberAudioData(subscriber: Subscriber, audioData: AudioData): void {
  console.log(
    `Individual audio from stream ${subscriber.stream.streamId}: ` +
    `${audioData.sampleRate}Hz, ${audioData.numberOfChannels} channel(s)`
  );
}

import type { Session, Stream } from '@vonage/video-connector';

function onStreamDropped(session: Session, stream: Stream): void {
  console.log(`Stream dropped: ${stream.streamId}`);
  client.unsubscribe(stream).catch((error) => {
    console.error('Failed to unsubscribe:', error);
  });
}

import type { Session, AudioData } from '@vonage/video-connector';

function onAudioData(session: Session, audioData: AudioData): void {
  const { sampleRate, numberOfChannels, numberOfFrames, data } = audioData;

  console.log(`Received ${numberOfFrames} frames at ${sampleRate}Hz, ${numberOfChannels} channel(s)`);

  // Read individual samples as 16-bit signed little-endian
  const firstSample = data.readInt16LE(0);

  // Copy if the audio is needed after this callback returns
  const owned = Buffer.from(data);
}

const sampleRate = 48000;
const durationMs = 20;
const numberOfChannels = 1;
const numberOfFrames = (sampleRate * durationMs) / 1000;  // 960 frames

const data = Buffer.alloc(numberOfFrames * numberOfChannels * 2);

// Example: generate a 440Hz sine wave
for (let i = 0; i < numberOfFrames; i++) {
  const sample = Math.round(32767 * 0.5 * Math.sin((2 * Math.PI * 440 * i) / sampleRate));
  data.writeInt16LE(sample, i * 2);
}

client.addAudio({ data, sampleRate, numberOfChannels: 1, numberOfFrames });

import type { Subscriber, VideoFrame } from '@vonage/video-connector';

function onRenderFrame(subscriber: Subscriber, frame: VideoFrame): void {
  const { width, height, format, data } = frame;

  console.log(`Received ${width}x${height} frame in ${format} format`);

  // Copy if the frame is needed after this callback returns
  const owned = Buffer.from(data);
}

import type { VideoFrame } from '@vonage/video-connector';

const width = 640;
const height = 480;

// YUV420P layout:
// Y plane: width * height
// U plane: (width / 2) * (height / 2)
// V plane: (width / 2) * (height / 2)
const ySize = width * height;
const uvSize = (width / 2) * (height / 2);
const data = Buffer.alloc(ySize + 2 * uvSize);

// Fill with mid-gray
data.fill(128);

const videoFrame: VideoFrame = { data, width, height, format: 'YUV420P' };

client.addVideo(videoFrame);

const stats = client.getMediaBufferStats();

if (stats.audio) {
  console.log(`Audio buffer duration: ${stats.audio.durationMs}ms`);
}

if (stats.video) {
  console.log(`Video buffer duration: ${stats.video.durationMs}ms`);
}

if (client.clearMediaBuffers()) {
  console.log('Media buffers cleared successfully');
}

import type { MediaBufferStats } from '@vonage/video-connector';

function onMediaBufferDrained(stats: MediaBufferStats): void {
  console.log('Media buffers drained');

  if (stats.audio) {
    console.log(`Audio buffer: ${stats.audio.durationMs}ms remaining`);
  }

  if (stats.video) {
    console.log(`Video buffer: ${stats.video.durationMs}ms remaining`);
  }
}

const connection = client.getConnection();

if (connection) {
  console.log(`Connection ID: ${connection.connectionId}`);
  console.log(`Connection data: ${connection.data}`);
  console.log(`Created at: ${new Date(connection.creationTime).toISOString()}`);
}

import type { Session, Connection, Stream } from '@vonage/video-connector';

function onSessionError(session: Session, description: string, code: number): void {
  console.error(`Session error: ${description} (Code: ${code})`);
}

function onSessionDisconnected(session: Session): void {
  console.log(`Disconnected from session: ${session.sessionId}`);
}

function onConnectionCreated(session: Session, connection: Connection): void {
  console.log(`Participant joined: ${connection.connectionId}`);
  console.log(`Connection data: ${connection.data}`);
}

function onConnectionDropped(session: Session, connection: Connection): void {
  console.log(`Participant left: ${connection.connectionId}`);
}

function onStreamReceived(session: Session, stream: Stream): void {
  console.log(`Stream received: ${stream.streamId} from ${stream.connection.connectionId}`);
  // Decide whether to subscribe based on your application logic
}

import type { Publisher } from '@vonage/video-connector';

function onPublisherError(publisher: Publisher, description: string, code: number): void {
  console.error(`Publisher error: ${description} (Code: ${code})`);
}

function onStreamDestroyed(publisher: Publisher): void {
  console.log(`Published stream destroyed: ${publisher.stream.streamId}`);
}

import type { Subscriber } from '@vonage/video-connector';

function onSubscriberError(subscriber: Subscriber, description: string, code: number): void {
  console.error(`Subscriber error: ${description} (Code: ${code})`);
}

function onSubscriberDisconnected(subscriber: Subscriber): void {
  console.log(`Unsubscribed from stream: ${subscriber.stream.streamId}`);
}

import { ValidationError } from '@vonage/video-connector';

try {
  await client.publish({ settings: { name: 'Bot', hasAudio: false, hasVideo: false } });
} catch (error) {
  if (error instanceof ValidationError) {
    console.error('Invalid publisher settings:', error.message);
  }
}

try {
  await client.connect(applicationId, sessionId, token, { sessionSettings });
} catch (error) {
  // e.g. "Connection error [1004]: Authentication error"
  // or "Failed to initiate connection. Client may already be connected."
  console.error(error);
}

try {
  await client.connect(applicationId, sessionId, token, { sessionSettings });
  await client.publish({ settings: publisherSettings });
  // ... do work ...
} catch (error) {
  console.error('Application error:', error);
} finally {
  if (client.isPublishing()) {
    await client.unpublish().catch(() => console.warn('Failed to stop publishing'));
  }

  if (client.isConnected()) {
    await client.disconnect().catch(() => console.warn('Failed to disconnect'));
  }
}