v0.3.0-alpha: Chromium rasterizer (R4), branching, chapters
- Rasterizer: playwright-core + system Chromium renders dom steps to PNG (1280x800, JavaScript disabled — snapshots stay inert). Cached permanently on the step (dom_html immutable). CHROMIUM_PATH configurable. - Video render now includes DOM steps as real frames (verified h264 1280x800, 0 skipped); guides now show rasterized screenshots for DOM steps - Branching: per-step branch hotspots (dashed, labeled) jumping to any target step; lead gates respected on branch jumps - Chapters: demo-level chapter menu in player bar - API: PATCH /api/demos/:id (name, chapters), branches on step patch, both in public JSON - MCP surface: 15 tools (+set_step_branches, +set_chapters) - Docker: chromium + fonts in runtime image, CHROMIUM_PATH preset
This commit is contained in:
parent
a535e15253
commit
928628f49a
11 changed files with 269 additions and 27 deletions
|
|
@ -12,11 +12,12 @@
|
|||
"@modelcontextprotocol/sdk": "^1.12.0",
|
||||
"fastify": "^5.2.0",
|
||||
"pg": "^8.13.1",
|
||||
"zod": "^3.24.1"
|
||||
"zod": "^3.24.1",
|
||||
"playwright-core": "^1.49.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.2",
|
||||
"@types/pg": "^8.11.10",
|
||||
"typescript": "^5.7.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -41,6 +41,17 @@
|
|||
}
|
||||
snap.appendChild(container);
|
||||
|
||||
(step.branches || []).forEach(function (br) {
|
||||
var bh = document.createElement("div");
|
||||
bh.className = "hotspot";
|
||||
bh.style.left = br.x + "%"; bh.style.top = br.y + "%";
|
||||
bh.style.width = br.w + "%"; bh.style.height = br.h + "%";
|
||||
bh.style.borderStyle = "dashed";
|
||||
if (br.label) bh.title = br.label;
|
||||
bh.addEventListener("click", function () { jumpTo(br.target_step_id); });
|
||||
snap.appendChild(bh);
|
||||
});
|
||||
|
||||
if (step.hotspot) {
|
||||
var h = document.createElement("div");
|
||||
h.className = "hotspot";
|
||||
|
|
@ -112,6 +123,19 @@
|
|||
snap.appendChild(overlay);
|
||||
}
|
||||
|
||||
function jumpTo(stepId) {
|
||||
var idx = state.demo.steps.findIndex(function (s) { return s.id === stepId; });
|
||||
if (idx < 0) return;
|
||||
var cur = state.demo.steps[state.i];
|
||||
if (cur.lead_gate && cur.lead_gate.enabled && !gatePassed[cur.id]) {
|
||||
showGate(cur, function () { jumpTo(stepId); });
|
||||
return;
|
||||
}
|
||||
state.i = idx;
|
||||
beacon("advance", stepId);
|
||||
render();
|
||||
}
|
||||
|
||||
function next() {
|
||||
var cur = state.demo.steps[state.i];
|
||||
if (cur.lead_gate && cur.lead_gate.enabled && !gatePassed[cur.id]) {
|
||||
|
|
@ -145,6 +169,18 @@
|
|||
.then(function (demo) {
|
||||
state.demo = demo;
|
||||
titleEl.textContent = demo.name;
|
||||
if (demo.chapters && demo.chapters.length) {
|
||||
var sel = document.createElement("select");
|
||||
sel.style.cssText = "border:1px solid #e2e8f0;border-radius:6px;padding:5px 8px;font:inherit;max-width:180px";
|
||||
var opt0 = document.createElement("option");
|
||||
opt0.textContent = "Chapters"; opt0.value = ""; sel.appendChild(opt0);
|
||||
demo.chapters.forEach(function (c) {
|
||||
var o = document.createElement("option");
|
||||
o.value = c.step_id; o.textContent = c.title; sel.appendChild(o);
|
||||
});
|
||||
sel.addEventListener("change", function () { if (sel.value) { jumpTo(sel.value); sel.value = ""; } });
|
||||
titleEl.parentNode.insertBefore(sel, titleEl.nextSibling);
|
||||
}
|
||||
document.title = demo.name;
|
||||
beacon("view", demo.steps.length ? demo.steps[0].id : null);
|
||||
render();
|
||||
|
|
|
|||
|
|
@ -1,17 +1,26 @@
|
|||
import { pool } from "./db.js";
|
||||
import { rasterizeStep } from "./rasterizer.js";
|
||||
|
||||
interface GuideStep {
|
||||
id: string;
|
||||
position: number;
|
||||
snapshot_type: string;
|
||||
asset_id: string | null;
|
||||
raster_asset_id: string | null;
|
||||
annotation: { title?: string; body?: string } | null;
|
||||
}
|
||||
|
||||
async function stepImageId(st: GuideStep): Promise<string | null> {
|
||||
if (st.snapshot_type === "image") return st.asset_id;
|
||||
if (st.raster_asset_id) return st.raster_asset_id;
|
||||
try { return await rasterizeStep(st.id); } catch { return null; }
|
||||
}
|
||||
|
||||
async function loadGuide(demoId: string): Promise<{ name: string; steps: GuideStep[] } | null> {
|
||||
const d = await pool.query("SELECT name FROM demos WHERE id=$1", [demoId]);
|
||||
if (!d.rowCount) return null;
|
||||
const s = await pool.query(
|
||||
"SELECT position, snapshot_type, asset_id, annotation FROM steps WHERE demo_id=$1 ORDER BY position",
|
||||
"SELECT id, position, snapshot_type, asset_id, raster_asset_id, annotation FROM steps WHERE demo_id=$1 ORDER BY position",
|
||||
[demoId],
|
||||
);
|
||||
return { name: d.rows[0].name, steps: s.rows };
|
||||
|
|
@ -22,12 +31,13 @@ export async function guideMarkdown(demoId: string, baseUrl: string): Promise<st
|
|||
const g = await loadGuide(demoId);
|
||||
if (!g) return null;
|
||||
const lines = [`# ${g.name}`, ""];
|
||||
g.steps.forEach((st, i) => {
|
||||
for (let i = 0; i < g.steps.length; i++) {
|
||||
const st = g.steps[i];
|
||||
lines.push(`## Step ${i + 1}${st.annotation?.title ? `: ${st.annotation.title}` : ""}`, "");
|
||||
if (st.annotation?.body) lines.push(st.annotation.body, "");
|
||||
if (st.snapshot_type === "image" && st.asset_id)
|
||||
lines.push(``, "");
|
||||
});
|
||||
const img = await stepImageId(st);
|
||||
if (img) lines.push(``, "");
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
|
|
@ -36,17 +46,16 @@ export async function guideHtml(demoId: string): Promise<string | null> {
|
|||
if (!g) return null;
|
||||
const esc = (s: string) =>
|
||||
s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
||||
const steps = g.steps
|
||||
.map((st, i) => {
|
||||
const img =
|
||||
st.snapshot_type === "image" && st.asset_id
|
||||
? `<img src="/assets/${st.asset_id}" alt="Step ${i + 1}">`
|
||||
: "";
|
||||
return `<section><h2>Step ${i + 1}${st.annotation?.title ? `: ${esc(st.annotation.title)}` : ""}</h2>${
|
||||
st.annotation?.body ? `<p>${esc(st.annotation.body)}</p>` : ""
|
||||
}${img}</section>`;
|
||||
})
|
||||
.join("\n");
|
||||
const parts: string[] = [];
|
||||
for (let i = 0; i < g.steps.length; i++) {
|
||||
const st = g.steps[i];
|
||||
const imgId = await stepImageId(st);
|
||||
const img = imgId ? `<img src="/assets/${imgId}" alt="Step ${i + 1}">` : "";
|
||||
parts.push(`<section><h2>Step ${i + 1}${st.annotation?.title ? `: ${esc(st.annotation.title)}` : ""}</h2>${
|
||||
st.annotation?.body ? `<p>${esc(st.annotation.body)}</p>` : ""
|
||||
}${img}</section>`);
|
||||
}
|
||||
const steps = parts.join("\n");
|
||||
return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>${esc(g.name)} — Guide</title><style>
|
||||
body{font:16px/1.6 system-ui,sans-serif;color:#0f172a;background:#f8fafc;margin:0}
|
||||
main{max-width:760px;margin:0 auto;padding:40px 20px}
|
||||
|
|
|
|||
|
|
@ -165,6 +165,42 @@ function buildServer(key: AuthedKey): McpServer {
|
|||
},
|
||||
);
|
||||
|
||||
s.tool(
|
||||
"set_step_branches",
|
||||
"Set branch hotspots on a step: extra clickable regions that jump to a target step (percent coordinates). Pass an empty array to clear.",
|
||||
{
|
||||
step_id: z.string().uuid(),
|
||||
branches: z.array(z.object({
|
||||
x: z.number().min(0).max(100), y: z.number().min(0).max(100),
|
||||
w: z.number().min(0.5).max(100), h: z.number().min(0.5).max(100),
|
||||
label: z.string().max(100).optional(),
|
||||
target_step_id: z.string().uuid(),
|
||||
})).max(8),
|
||||
},
|
||||
async ({ step_id, branches }) => {
|
||||
const r = await pool.query("UPDATE steps SET branches=$2 WHERE id=$1 RETURNING id, branches", [
|
||||
step_id, branches.length ? JSON.stringify(branches) : null,
|
||||
]);
|
||||
return text(r.rowCount ? r.rows[0] : { error: "not found" });
|
||||
},
|
||||
);
|
||||
|
||||
s.tool(
|
||||
"set_chapters",
|
||||
"Set the chapter menu for a demo: titled jump points to steps. Pass an empty array to clear.",
|
||||
{
|
||||
demo_id: z.string().uuid(),
|
||||
chapters: z.array(z.object({ title: z.string().max(100), step_id: z.string().uuid() })).max(50),
|
||||
},
|
||||
async ({ demo_id, chapters }) => {
|
||||
const r = await pool.query(
|
||||
"UPDATE demos SET chapters=$2, updated_at=now() WHERE id=$1 RETURNING id, chapters",
|
||||
[demo_id, chapters.length ? JSON.stringify(chapters) : null],
|
||||
);
|
||||
return text(r.rowCount ? r.rows[0] : { error: "not found" });
|
||||
},
|
||||
);
|
||||
|
||||
s.tool(
|
||||
"set_lead_gate",
|
||||
"Enable or disable a lead-capture gate on a step. When enabled, viewers must submit the given fields before advancing past it.",
|
||||
|
|
|
|||
7
apps/api/src/migrations/003_v03.sql
Normal file
7
apps/api/src/migrations/003_v03.sql
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
-- cached rasterization of dom steps (dom_html is immutable after creation, so cache is safe)
|
||||
ALTER TABLE steps ADD COLUMN IF NOT EXISTS raster_asset_id UUID REFERENCES assets(id);
|
||||
-- branching: extra clickable regions that jump to a target step
|
||||
-- [{x,y,w,h,label,target_step_id}]
|
||||
ALTER TABLE steps ADD COLUMN IF NOT EXISTS branches JSONB;
|
||||
-- chapters: [{title, step_id}]
|
||||
ALTER TABLE demos ADD COLUMN IF NOT EXISTS chapters JSONB;
|
||||
93
apps/api/src/rasterizer.ts
Normal file
93
apps/api/src/rasterizer.ts
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import { chromium, type Browser } from "playwright-core";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { pool } from "./db.js";
|
||||
|
||||
const DATA_DIR = process.env.DATA_DIR ?? "/data";
|
||||
const EXEC =
|
||||
process.env.CHROMIUM_PATH ??
|
||||
["/usr/bin/chromium-browser", "/usr/bin/chromium"].find(Boolean)!;
|
||||
|
||||
let browser: Browser | null = null;
|
||||
let launching: Promise<Browser> | null = null;
|
||||
|
||||
async function getBrowser(): Promise<Browser> {
|
||||
if (browser?.isConnected()) return browser;
|
||||
if (!launching) {
|
||||
launching = (async () => {
|
||||
const candidates = process.env.CHROMIUM_PATH
|
||||
? [process.env.CHROMIUM_PATH]
|
||||
: ["/usr/bin/chromium-browser", "/usr/bin/chromium"];
|
||||
let lastErr: unknown;
|
||||
for (const p of candidates) {
|
||||
try {
|
||||
browser = await chromium.launch({
|
||||
executablePath: p,
|
||||
args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],
|
||||
});
|
||||
return browser;
|
||||
} catch (e) { lastErr = e; }
|
||||
}
|
||||
throw lastErr;
|
||||
})().finally(() => { launching = null; });
|
||||
}
|
||||
return launching;
|
||||
}
|
||||
|
||||
async function storeAsset(buf: Buffer, mime: string): Promise<string> {
|
||||
const sha = createHash("sha256").update(buf).digest("hex");
|
||||
const existing = await pool.query("SELECT id FROM assets WHERE sha256=$1", [sha]);
|
||||
if (existing.rowCount) return existing.rows[0].id;
|
||||
const dir = join(DATA_DIR, "assets", sha.slice(0, 2));
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const path = join(dir, sha);
|
||||
writeFileSync(path, buf);
|
||||
const r = await pool.query(
|
||||
"INSERT INTO assets (kind, path, mime, bytes, sha256) VALUES ('image',$1,$2,$3,$4) RETURNING id",
|
||||
[path, mime, buf.length, sha],
|
||||
);
|
||||
return r.rows[0].id;
|
||||
}
|
||||
|
||||
/**
|
||||
* R4: rasterize a dom step to a PNG asset (1280x800 viewport), cached on the
|
||||
* step. dom_html is immutable post-creation, so the cache never invalidates.
|
||||
* The page runs with JavaScript disabled — snapshots are inert documents.
|
||||
*/
|
||||
export async function rasterizeStep(stepId: string): Promise<string | null> {
|
||||
const st = await pool.query(
|
||||
"SELECT snapshot_type, dom_html, raster_asset_id, asset_id FROM steps WHERE id=$1",
|
||||
[stepId],
|
||||
);
|
||||
if (!st.rowCount) return null;
|
||||
const row = st.rows[0];
|
||||
if (row.snapshot_type === "image") return row.asset_id;
|
||||
if (row.raster_asset_id) return row.raster_asset_id;
|
||||
if (!row.dom_html) return null;
|
||||
|
||||
const b = await getBrowser();
|
||||
const ctx = await b.newContext({
|
||||
viewport: { width: 1280, height: 800 },
|
||||
javaScriptEnabled: false,
|
||||
baseURL: process.env.PUBLIC_BASE_URL ?? `http://127.0.0.1:${process.env.PORT ?? 3000}`,
|
||||
});
|
||||
try {
|
||||
const page = await ctx.newPage();
|
||||
// resolve /assets/... references against our own server
|
||||
const base = process.env.PUBLIC_BASE_URL ?? `http://127.0.0.1:${process.env.PORT ?? 3000}`;
|
||||
const html = row.dom_html.replace(/(src|href)="\/assets\//g, `$1="${base}/assets/`);
|
||||
await page.setContent(html, { waitUntil: "networkidle", timeout: 20_000 }).catch(() => {});
|
||||
const buf = await page.screenshot({ type: "png" });
|
||||
const assetId = await storeAsset(Buffer.from(buf), "image/png");
|
||||
await pool.query("UPDATE steps SET raster_asset_id=$2 WHERE id=$1", [stepId, assetId]);
|
||||
return assetId;
|
||||
} finally {
|
||||
await ctx.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function closeRasterizer(): Promise<void> {
|
||||
await browser?.close().catch(() => {});
|
||||
browser = null;
|
||||
}
|
||||
|
|
@ -15,6 +15,16 @@ const stepPatch = z.object({
|
|||
.nullable()
|
||||
.optional(),
|
||||
position: z.number().int().min(0).optional(),
|
||||
branches: z
|
||||
.array(z.object({
|
||||
x: z.number().min(0).max(100), y: z.number().min(0).max(100),
|
||||
w: z.number().min(0.5).max(100), h: z.number().min(0.5).max(100),
|
||||
label: z.string().max(100).optional(),
|
||||
target_step_id: z.string().uuid(),
|
||||
}))
|
||||
.max(8)
|
||||
.nullable()
|
||||
.optional(),
|
||||
lead_gate: z
|
||||
.object({
|
||||
enabled: z.boolean(),
|
||||
|
|
@ -39,7 +49,7 @@ export async function demoRoutes(app: FastifyInstance): Promise<void> {
|
|||
const d = await pool.query("SELECT * FROM demos WHERE id=$1", [id]);
|
||||
if (!d.rowCount) return reply.code(404).send({ error: "not found" });
|
||||
const s = await pool.query(
|
||||
"SELECT id, position, snapshot_type, asset_id, hotspot, annotation, lead_gate FROM steps WHERE demo_id=$1 ORDER BY position",
|
||||
"SELECT id, position, snapshot_type, asset_id, hotspot, annotation, lead_gate, branches FROM steps WHERE demo_id=$1 ORDER BY position",
|
||||
[id],
|
||||
);
|
||||
return { ...d.rows[0], steps: s.rows };
|
||||
|
|
@ -56,6 +66,29 @@ export async function demoRoutes(app: FastifyInstance): Promise<void> {
|
|||
return r.rows[0];
|
||||
});
|
||||
|
||||
app.patch("/api/demos/:id", { preHandler: requireScope("author") }, async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const body = z
|
||||
.object({
|
||||
name: z.string().min(1).max(200).optional(),
|
||||
chapters: z
|
||||
.array(z.object({ title: z.string().max(100), step_id: z.string().uuid() }))
|
||||
.max(50)
|
||||
.nullable()
|
||||
.optional(),
|
||||
})
|
||||
.parse(req.body);
|
||||
const sets: string[] = ["updated_at=now()"];
|
||||
const vals: unknown[] = [];
|
||||
let i = 1;
|
||||
if (body.name !== undefined) { sets.push(`name=$${i++}`); vals.push(body.name); }
|
||||
if (body.chapters !== undefined) { sets.push(`chapters=$${i++}`); vals.push(body.chapters ? JSON.stringify(body.chapters) : null); }
|
||||
vals.push(id);
|
||||
const r = await pool.query(`UPDATE demos SET ${sets.join(",")} WHERE id=$${i} RETURNING id, name, chapters`, vals);
|
||||
if (!r.rowCount) return reply.code(404).send({ error: "not found" });
|
||||
return r.rows[0];
|
||||
});
|
||||
|
||||
app.post("/api/demos/:id/steps", { preHandler: requireScope("author") }, async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const body = z
|
||||
|
|
@ -93,6 +126,7 @@ export async function demoRoutes(app: FastifyInstance): Promise<void> {
|
|||
if (body.hotspot !== undefined) { sets.push(`hotspot=$${i++}`); vals.push(body.hotspot ? JSON.stringify(body.hotspot) : null); }
|
||||
if (body.annotation !== undefined) { sets.push(`annotation=$${i++}`); vals.push(body.annotation ? JSON.stringify(body.annotation) : null); }
|
||||
if (body.lead_gate !== undefined) { sets.push(`lead_gate=$${i++}`); vals.push(body.lead_gate ? JSON.stringify(body.lead_gate) : null); }
|
||||
if (body.branches !== undefined) { sets.push(`branches=$${i++}`); vals.push(body.branches ? JSON.stringify(body.branches) : null); }
|
||||
if (body.position !== undefined) { sets.push(`position=$${i++}`); vals.push(body.position); }
|
||||
if (!sets.length) return reply.code(400).send({ error: "empty patch" });
|
||||
vals.push(stepId);
|
||||
|
|
|
|||
|
|
@ -7,15 +7,15 @@ export async function publicRoutes(app: FastifyInstance): Promise<void> {
|
|||
app.get("/api/public/demo/:token", async (req, reply) => {
|
||||
const { token } = req.params as { token: string };
|
||||
const d = await pool.query(
|
||||
"SELECT id, name, theme FROM demos WHERE share_token=$1 AND status='published'",
|
||||
"SELECT id, name, theme, chapters FROM demos WHERE share_token=$1 AND status='published'",
|
||||
[token],
|
||||
);
|
||||
if (!d.rowCount) return reply.code(404).send({ error: "not found" });
|
||||
const s = await pool.query(
|
||||
"SELECT id, position, snapshot_type, asset_id, dom_html, hotspot, annotation, lead_gate FROM steps WHERE demo_id=$1 ORDER BY position",
|
||||
"SELECT id, position, snapshot_type, asset_id, dom_html, hotspot, annotation, lead_gate, branches FROM steps WHERE demo_id=$1 ORDER BY position",
|
||||
[d.rows[0].id],
|
||||
);
|
||||
return { name: d.rows[0].name, theme: d.rows[0].theme, demo_id: d.rows[0].id, steps: s.rows };
|
||||
return { name: d.rows[0].name, theme: d.rows[0].theme, chapters: d.rows[0].chapters, demo_id: d.rows[0].id, steps: s.rows };
|
||||
});
|
||||
|
||||
// Analytics beacon. R7: no IP persisted, viewer_token only if the link carried one.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { mkdirSync, writeFileSync, readFileSync, rmSync, copyFileSync } from "no
|
|||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { pool } from "./db.js";
|
||||
import { rasterizeStep } from "./rasterizer.js";
|
||||
|
||||
const run = promisify(execFile);
|
||||
const DATA_DIR = process.env.DATA_DIR ?? "/data";
|
||||
|
|
@ -18,12 +19,23 @@ export async function renderVideo(
|
|||
opts: { secondsPerStep?: number; narrationAssetId?: string },
|
||||
): Promise<{ asset_id: string; skipped_dom_steps: number } | { error: string }> {
|
||||
const steps = await pool.query(
|
||||
"SELECT s.snapshot_type, a.path FROM steps s LEFT JOIN assets a ON a.id = s.asset_id WHERE s.demo_id=$1 ORDER BY s.position",
|
||||
"SELECT s.id, s.snapshot_type, a.path FROM steps s LEFT JOIN assets a ON a.id = s.asset_id WHERE s.demo_id=$1 ORDER BY s.position",
|
||||
[demoId],
|
||||
);
|
||||
const imgs = steps.rows.filter((r) => r.snapshot_type === "image" && r.path);
|
||||
const skipped = steps.rows.length - imgs.length;
|
||||
if (!imgs.length) return { error: "no image steps to render (DOM rasterization not yet available)" };
|
||||
const imgs: { path: string }[] = [];
|
||||
let skipped = 0;
|
||||
for (const r of steps.rows) {
|
||||
if (r.snapshot_type === "image" && r.path) { imgs.push({ path: r.path }); continue; }
|
||||
try {
|
||||
const rid = await rasterizeStep(r.id); // R4: dom steps rendered via Chromium
|
||||
if (rid) {
|
||||
const a = await pool.query("SELECT path FROM assets WHERE id=$1", [rid]);
|
||||
if (a.rowCount) { imgs.push({ path: a.rows[0].path }); continue; }
|
||||
}
|
||||
skipped++;
|
||||
} catch { skipped++; }
|
||||
}
|
||||
if (!imgs.length) return { error: "no renderable steps" };
|
||||
|
||||
const secs = Math.min(30, Math.max(1, opts.secondsPerStep ?? 3));
|
||||
const tmp = join(DATA_DIR, "tmp", randomUUID());
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue