From 928628f49a08c44ab513328bfa9cbb6fdafe98b3 Mon Sep 17 00:00:00 2001 From: scot Date: Sat, 18 Jul 2026 05:36:27 +0000 Subject: [PATCH] v0.3.0-alpha: Chromium rasterizer (R4), branching, chapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rasterizer: playwright-core + system Chromium renders dom steps to PNG (1280x800, JavaScript disabled — snapshots stay inert). Cached permanently on the step (dom_html immutable). CHROMIUM_PATH configurable. - Video render now includes DOM steps as real frames (verified h264 1280x800, 0 skipped); guides now show rasterized screenshots for DOM steps - Branching: per-step branch hotspots (dashed, labeled) jumping to any target step; lead gates respected on branch jumps - Chapters: demo-level chapter menu in player bar - API: PATCH /api/demos/:id (name, chapters), branches on step patch, both in public JSON - MCP surface: 15 tools (+set_step_branches, +set_chapters) - Docker: chromium + fonts in runtime image, CHROMIUM_PATH preset --- Dockerfile | 3 +- apps/api/package.json | 5 +- apps/api/public/player.js | 36 +++++++++++ apps/api/src/guide.ts | 41 ++++++++----- apps/api/src/mcp.ts | 36 +++++++++++ apps/api/src/migrations/003_v03.sql | 7 +++ apps/api/src/rasterizer.ts | 93 +++++++++++++++++++++++++++++ apps/api/src/routes/demos.ts | 36 ++++++++++- apps/api/src/routes/publicRoutes.ts | 6 +- apps/api/src/video.ts | 20 +++++-- package-lock.json | 13 ++++ 11 files changed, 269 insertions(+), 27 deletions(-) create mode 100644 apps/api/src/migrations/003_v03.sql create mode 100644 apps/api/src/rasterizer.ts diff --git a/Dockerfile b/Dockerfile index 3bfe518..4685b6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,8 @@ COPY apps ./apps RUN npm run build -w apps/editor && npm run build -w apps/api FROM node:22-alpine -RUN apk add --no-cache ffmpeg +RUN apk add --no-cache ffmpeg chromium nss freetype harfbuzz ca-certificates ttf-freefont +ENV CHROMIUM_PATH=/usr/bin/chromium-browser WORKDIR /app ENV NODE_ENV=production COPY package.json ./ diff --git a/apps/api/package.json b/apps/api/package.json index 407f629..47fd6d1 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -12,11 +12,12 @@ "@modelcontextprotocol/sdk": "^1.12.0", "fastify": "^5.2.0", "pg": "^8.13.1", - "zod": "^3.24.1" + "zod": "^3.24.1", + "playwright-core": "^1.49.1" }, "devDependencies": { "@types/node": "^22.10.2", "@types/pg": "^8.11.10", "typescript": "^5.7.2" } -} +} \ No newline at end of file diff --git a/apps/api/public/player.js b/apps/api/public/player.js index 81da714..54ac10a 100644 --- a/apps/api/public/player.js +++ b/apps/api/public/player.js @@ -41,6 +41,17 @@ } snap.appendChild(container); + (step.branches || []).forEach(function (br) { + var bh = document.createElement("div"); + bh.className = "hotspot"; + bh.style.left = br.x + "%"; bh.style.top = br.y + "%"; + bh.style.width = br.w + "%"; bh.style.height = br.h + "%"; + bh.style.borderStyle = "dashed"; + if (br.label) bh.title = br.label; + bh.addEventListener("click", function () { jumpTo(br.target_step_id); }); + snap.appendChild(bh); + }); + if (step.hotspot) { var h = document.createElement("div"); h.className = "hotspot"; @@ -112,6 +123,19 @@ snap.appendChild(overlay); } + function jumpTo(stepId) { + var idx = state.demo.steps.findIndex(function (s) { return s.id === stepId; }); + if (idx < 0) return; + var cur = state.demo.steps[state.i]; + if (cur.lead_gate && cur.lead_gate.enabled && !gatePassed[cur.id]) { + showGate(cur, function () { jumpTo(stepId); }); + return; + } + state.i = idx; + beacon("advance", stepId); + render(); + } + function next() { var cur = state.demo.steps[state.i]; if (cur.lead_gate && cur.lead_gate.enabled && !gatePassed[cur.id]) { @@ -145,6 +169,18 @@ .then(function (demo) { state.demo = demo; titleEl.textContent = demo.name; + if (demo.chapters && demo.chapters.length) { + var sel = document.createElement("select"); + sel.style.cssText = "border:1px solid #e2e8f0;border-radius:6px;padding:5px 8px;font:inherit;max-width:180px"; + var opt0 = document.createElement("option"); + opt0.textContent = "Chapters"; opt0.value = ""; sel.appendChild(opt0); + demo.chapters.forEach(function (c) { + var o = document.createElement("option"); + o.value = c.step_id; o.textContent = c.title; sel.appendChild(o); + }); + sel.addEventListener("change", function () { if (sel.value) { jumpTo(sel.value); sel.value = ""; } }); + titleEl.parentNode.insertBefore(sel, titleEl.nextSibling); + } document.title = demo.name; beacon("view", demo.steps.length ? demo.steps[0].id : null); render(); diff --git a/apps/api/src/guide.ts b/apps/api/src/guide.ts index ce1c910..239a0e0 100644 --- a/apps/api/src/guide.ts +++ b/apps/api/src/guide.ts @@ -1,17 +1,26 @@ import { pool } from "./db.js"; +import { rasterizeStep } from "./rasterizer.js"; interface GuideStep { + id: string; position: number; snapshot_type: string; asset_id: string | null; + raster_asset_id: string | null; annotation: { title?: string; body?: string } | null; } +async function stepImageId(st: GuideStep): Promise { + if (st.snapshot_type === "image") return st.asset_id; + if (st.raster_asset_id) return st.raster_asset_id; + try { return await rasterizeStep(st.id); } catch { return null; } +} + async function loadGuide(demoId: string): Promise<{ name: string; steps: GuideStep[] } | null> { const d = await pool.query("SELECT name FROM demos WHERE id=$1", [demoId]); if (!d.rowCount) return null; const s = await pool.query( - "SELECT position, snapshot_type, asset_id, annotation FROM steps WHERE demo_id=$1 ORDER BY position", + "SELECT id, position, snapshot_type, asset_id, raster_asset_id, annotation FROM steps WHERE demo_id=$1 ORDER BY position", [demoId], ); return { name: d.rows[0].name, steps: s.rows }; @@ -22,12 +31,13 @@ export async function guideMarkdown(demoId: string, baseUrl: string): Promise { + for (let i = 0; i < g.steps.length; i++) { + const st = g.steps[i]; lines.push(`## Step ${i + 1}${st.annotation?.title ? `: ${st.annotation.title}` : ""}`, ""); if (st.annotation?.body) lines.push(st.annotation.body, ""); - if (st.snapshot_type === "image" && st.asset_id) - lines.push(`![Step ${i + 1}](${baseUrl}/assets/${st.asset_id})`, ""); - }); + const img = await stepImageId(st); + if (img) lines.push(`![Step ${i + 1}](${baseUrl}/assets/${img})`, ""); + } return lines.join("\n"); } @@ -36,17 +46,16 @@ export async function guideHtml(demoId: string): Promise { if (!g) return null; const esc = (s: string) => s.replace(/&/g, "&").replace(//g, ">"); - const steps = g.steps - .map((st, i) => { - const img = - st.snapshot_type === "image" && st.asset_id - ? `Step ${i + 1}` - : ""; - return `

Step ${i + 1}${st.annotation?.title ? `: ${esc(st.annotation.title)}` : ""}

${ - st.annotation?.body ? `

${esc(st.annotation.body)}

` : "" - }${img}
`; - }) - .join("\n"); + const parts: string[] = []; + for (let i = 0; i < g.steps.length; i++) { + const st = g.steps[i]; + const imgId = await stepImageId(st); + const img = imgId ? `Step ${i + 1}` : ""; + parts.push(`

Step ${i + 1}${st.annotation?.title ? `: ${esc(st.annotation.title)}` : ""}

${ + st.annotation?.body ? `

${esc(st.annotation.body)}

` : "" + }${img}
`); + } + const steps = parts.join("\n"); return `${esc(g.name)} — Guide