SDK Documentation

npm install ultron-live-sdk

// ESM (recommended)
import { UltronLive } from "ultron-live-sdk";

// CommonJS
const { UltronLive } = require("ultron-live-sdk");

<script src="https://cdn.skypack.dev/ultron-live-sdk"></script>

import { UltronLive } from "ultron-live-sdk";

const ultron = new UltronLive({ apiKey: "ulk_your_api_key" });

await ultron.startScreenShare({
  model: "gemini-2.5-flash",
  onTranscript: (text) => console.log("AI:", text),
  onCreditsUpdate: (remaining) => console.log("credits:", remaining),
  onCreditsExhausted: () => alert("Top up to continue"),
});

// later…
ultron.stop();

const ultron = new UltronLive({
  apiKey: "ulk_your_api_key",   // or token: "<jwt>"
  apiVersion: "v1",             // "v1" (default) or "v2"
  model: "gemini-3.1-flash-lite-preview",
  frameInterval: 1500,          // ms between captured frames
  frameQuality: 0.4,            // JPEG quality 0–1
  maxFrameWidth: 1280,
  maxFrameHeight: 720,
});

// Stable (default) — unchanged behavior
const ultron = new UltronLive({ apiKey: "ulk_..." });

// Current — server-side frame filtering, assessment, MCP, realtime tools
const ultronV2 = new UltronLive({ apiKey: "ulk_...", apiVersion: "v2" });

const ultron = new UltronLive({ apiKey: "ulk_...", apiVersion: "v1" });

// 1. Request an OTP (emails a 6-digit code)
await ultron.auth.signup({ email: "user@example.com", firstName: "Ada" }); // new user
// or: await ultron.auth.login({ email: "user@example.com" });            // existing user

// 2. Verify the code → JWT + user (incl. permanent apiKey)
const { data } = await ultron.auth.verifyOtp({ email: "user@example.com", otp: "123456" });
ultron.setToken(data.token);
console.log(data.user.apiKey, data.user.credits);

// 3. Live profile + credits any time
const me = await ultron.auth.getMe();

const { data } = await ultron.memory.get();

data.enabled;              // false = Ultron neither learns nor recalls
data.facts.forEach((f) => {
  console.log(f.text, f.established ? "" : "(seen once — tentative)");
});

// The user correcting Ultron. A user-stated fact outranks anything inferred.
await ultron.memory.addFact({ text: "Prefers short answers", category: "preference" });
await ultron.memory.updateFact(factId, { text: "Uses MongoDB Atlas" });
await ultron.memory.pinFact(factId);      // never evicted
await ultron.memory.deleteFact(factId);
await ultron.memory.clear();              // forget everything

await ultron.memory.setEnabled(false);    // stop remembering entirely

// Distillation runs nightly; run it now to see it work immediately.
await ultron.memory.refresh();

const { data } = await ultron.sessions.list({ limit: 20 });
data.sessions.forEach((s) => console.log(s.startedAt, s.summary));

const detail = await ultron.sessions.get(sessionId, { frames: 100 });
detail.data.summary;           // the close-out digest
detail.data.frames;            // timeline: commentary, tags, image URLs

await ultron.sessions.remove(sessionId);   // Ultron can no longer recall it

await ultron.auth.updateProfile({ firstName: "Ada", dateOfBirth: "1990-12-10" });
await ultron.auth.updateTimezone({ timezone: "Asia/Kolkata" });

// On your server — requires the raw API key.
const { data } = await ultron.auth.sessionTokens.create({
  endUserId: "user_42",
  ttlSeconds: 900,
});
res.json({ token: data.token });

// If a token leaks, kill it — or block that end user entirely.
await ultron.auth.sessionTokens.revoke({ jti: data.jti });
await ultron.auth.sessionTokens.revoke({ endUserId: "user_42" });
await ultron.auth.sessionTokens.unblock("user_42");

// Screen share
await ultron.startScreenShare({
  onTranscript: (text) => render(text),
  onCreditsUpdate: (n) => setCredits(n),
  onError: (e) => console.error(e),
});

// Any MediaStream (camera, <video>.captureStream(), etc.)
const stream = await navigator.mediaDevices.getUserMedia({ video: true });
await ultron.startVideoStream({ stream, onTranscript: console.log });

ultron.stop();

ultron.attachPreview(document.getElementById("preview"));  // attach captured stream to a <video>
const stream = ultron.getPreviewStream();                   // or wire it up yourself

// ElevenLabs with your own key — the backend no longer synthesizes speech.
ultron.configureAudio({ voiceProvider: "elevenlabs", elevenLabsApiKey: "sk_..." });
ultron.setAudioEnabled(true);

const { sessionId } = await ultron.startSession();

const result = await ultron.analyseFrame({
  image: "data:image/jpeg;base64,...",
  history: previousCommentaries,
  sessionId,
});
console.log(result.response, result.creditsRemaining);

const reply = await ultron.chat({
  sessionId,
  message: "What was I working on?",
});

const rtc = ultron.createWebRTC();

rtc.onFrameResult = (r) => render(r.response, r.creditsRemaining);
rtc.onConnected = () => rtc.startSession();
await rtc.connect();

// Send frames (binary preferred — no base64 bloat)
canvas.toBlob((blob) => rtc.sendFrame(blob), "image/jpeg", 0.4);

// Chat over the data channel
rtc.sendChat("What changed on screen?");

rtc.stopSession();
rtc.disconnect();

const ultron = new UltronLive({ apiKey: "ulk_...", apiVersion: "v2" });
const rtc = ultron.createWebRTC();

rtc.onFrameResult = (f) => console.log("frame:", f.response);
rtc.onFlowResult  = (s) => console.log("summary:", s.response);   // tier-2 narrative

// Assessment session — structured verdict instead of commentary
rtc.onAssessmentResult = (r) => console.log("verdict:", r.assessment);
rtc.onConnected = () => rtc.startSession({
  outputType: "json",
  prompt: "Verify the procedure.",
  responseSchema: { type: "object", properties: { verdict: { type: "string" } }, required: ["verdict"] },
});
await rtc.connect();

let seq = 0;
setInterval(() => rtc.sendFrameBase64(base64, { seq: seq++, captureTs: Date.now() }), 150);
// rtc.stopSession()  → emits the final assessment:result
// rtc.finalize()     → verdict mid-stream, keep streaming

const voice = ultron.createRealtimeVoice({ provider: "gemini" });

voice.onReady = () => console.log("ready — start streaming mic");
voice.onTranscriptDelta = (t) => appendCaption(t);
voice.onUserTranscript = (t) => showUserSaid(t);

await voice.connect();                 // waits for relay:ready
voice.sendAudio(base64Pcm16Chunk);     // PCM16 mono, 16kHz (gemini) / 24kHz (openai)
voice.disconnect();

const ultron = new UltronLive({ apiKey: "ulk_...", apiVersion: "v2" });

const result = await ultron.assessment.assessAndWait({
  media: [fileInput.files[0]],            // video, frames, or documents (≤30, ≤18 MB each)
  outputType: "json",
  prompt: "Verify the sample-collection procedure.",
});

// Calibrated verdict (json output):
const v = JSON.parse(result.assessment);
v.verdict;        // 'PASS' | 'FAIL' | 'UNCERTAIN'
v.confidence;     // 0–100 (lowest per-check confidence)
v.needs_review;   // true → route to a human
v.checks;         // [{ name, result, confidence, observed }]

const v = JSON.parse(result.assessment);
if (v.needs_review) routeToHumanReviewer(v);   // low confidence or genuinely uncertain
else applyAutomatically(v.verdict);

await ultron.assessment.assessAndWait({
  media: [file],
  templateId: tpl.id,
  extraChecks: [{ title: "Gloves worn", instruction: "Were gloves worn the whole time?" }],
  skipChecks: [2, "Identity Match"],          // drop question #2 and "Identity Match"
});

// Permanent change to the test's questions:
await ultron.assessment.templates.update(tpl.id, { checks: [...tpl.checks, newCheck] });

await ultron.assessment.assessAndWait({ media: [file], templateId, collectionMode: "off_camera" });
rtc.startSession({ outputType: "json", templateId, collectionMode: "off_camera" });

// Mark collection-step questions so they drop automatically in off-camera mode:
//   checks: [{ title: "Proper Collection", instruction: "…", onCameraOnly: true }, …]

await ultron.assessment.assessFrame(base64DataUrl, { outputType: "json", prompt: "Gauge in green?" });

const { data: templates } = await ultron.assessment.templates.list();
await ultron.assessment.assessAndWait({ media: [file], templateId: templates[0].id, fields: { type: "swab" } });

const ultron = new UltronLive({ apiKey: "ulk_...", apiVersion: "v2" });

const { data: server } = await ultron.mcp.servers.create({
  name: "docs",
  url: "https://example.com/mcp",
  headers: { "X-Workspace": "acme" },
  secrets: [{ key: "Authorization", value: "Bearer abc123" }],   // encrypted at rest
});

await ultron.mcp.servers.test(server.id);     // connect + list tools to validate
const { data: servers } = await ultron.mcp.servers.list();
const { data: tools }   = await ultron.mcp.tools();   // effective tools for the user

ultron.models;                          // ModelMeta[] — full catalogue
ultron.getModelsByProvider("gemini");   // filter by provider
ultron.getModelMeta("gpt-4o");          // metadata for one model
ultron.setModel("gemini-2.5-flash");    // change the default (no restart)

import { UltronAuthError, UltronCreditsError, UltronAPIError } from "ultron-live-sdk";

try {
  await ultron.analyseFrame({ image });
} catch (err) {
  if (err instanceof UltronCreditsError) promptTopUp();        // 402
  else if (err instanceof UltronAuthError) redirectToLogin();  // 401
  else if (err instanceof UltronAPIError) console.error(err.statusCode, err.message);
}