boxpositron/agcanvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Mermaid overlay support for agents to inject positioned diagrams

Browse Source 
- Agents can send RenderMermaid with Mermaid source + canvas position
  to create SVG texture overlays that coexist with other elements
- MermaidOverlay struct holds source, rendered SVG, SvgRenderer, and
  lazy-loaded egui texture at a specific canvas position/size
- Server handles rendering via mermaid-rs, parses SVG for dimensions,
  sends overlay data through DrawingCommand channel to GUI thread
- Canvas renders overlays as positioned textures between base SVG and
  drawing elements, with proper pan/zoom transforms
- New MCP tool render_mermaid for agent access
- Overlays cleared on undo/redo/checkout to stay consistent with history
- 29 tests passing, clippy clean
This commit is contained in:
David Ibia
2026-02-10 10:44:39 +01:00
parent 5ca1e85209
commit 740fa2f5f9
6 changed files with 266 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
crates/agcanvas-mcp/src/tools.rs
View File

@@ -187,9 +187,27 @@ pub struct BooleanOpParam {
pub stroke_width: Option<f32>,
}
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
pub struct RenderMermaidParam {
#[schemars(description = "Session ID to target. If omitted, uses the active session.")]
pub session_id: Option<String>,
#[schemars(description = "Mermaid diagram source code (e.g., 'flowchart LR\\n A-->B')")]
pub mermaid_source: String,
#[schemars(description = "X position on canvas (default: 0)")]
pub x: Option<f32>,
#[schemars(description = "Y position on canvas (default: 0)")]
pub y: Option<f32>,
#[schemars(description = "Override width (default: natural SVG width)")]
pub width: Option<f32>,
#[schemars(description = "Override height (default: natural SVG height)")]
pub height: Option<f32>,
}
#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
pub struct BatchParam {
#[schemars(description = "JSON array of request objects. Each object must have a 'type' field and the same parameters as individual tool calls. Example: [{\"type\": \"CreateDrawingElement\", \"shape_type\": \"rectangle\", \"x\": 0, \"y\": 0, \"width\": 100, \"height\": 100}, {\"type\": \"CreateDrawingElement\", \"shape_type\": \"ellipse\", \"center_x\": 200, \"center_y\": 200, \"radius_x\": 50, \"radius_y\": 50}]")]
#[schemars(
description = "JSON array of request objects. Each object must have a 'type' field and the same parameters as individual tool calls. Example: [{\"type\": \"CreateDrawingElement\", \"shape_type\": \"rectangle\", \"x\": 0, \"y\": 0, \"width\": 100, \"height\": 100}, {\"type\": \"CreateDrawingElement\", \"shape_type\": \"ellipse\", \"center_x\": 200, \"center_y\": 200, \"radius_x\": 50, \"radius_y\": 50}]"
)]
pub requests_json: String,
}
@@ -527,6 +545,26 @@ impl AgCanvasServer {
self.call_agcanvas(&request).await
}
#[tool(
name = "render_mermaid",
description = "Render a Mermaid diagram (flowchart, sequence, etc.) as an SVG overlay at a specific position on the canvas. The diagram appears as a visual element that can coexist with other shapes and diagrams."
)]
async fn render_mermaid(
&self,
Parameters(params): Parameters<RenderMermaidParam>,
) -> Result<CallToolResult, McpError> {
let request = serde_json::json!({
"type": "RenderMermaid",
"session_id": params.session_id,
"mermaid_source": params.mermaid_source,
"x": params.x,
"y": params.y,
"width": params.width,
"height": params.height,
});
self.call_agcanvas(&request).await
}
#[tool(
description = "Send multiple Augmented Canvas operations in one request. Accepts a JSON array of request objects and returns one response per operation in a BatchResult."
)]

26
crates/agcanvas/src/agent/protocol.rs
View File

@@ -211,6 +211,19 @@ pub enum AgentRequest {
#[serde(default)]
stroke_width: Option<f32>,
},
RenderMermaid {
#[serde(default)]
session_id: Option<String>,
mermaid_source: String,
#[serde(default)]
x: Option<f32>,
#[serde(default)]
y: Option<f32>,
#[serde(default)]
width: Option<f32>,
#[serde(default)]
height: Option<f32>,
},
Batch {
requests: Vec<AgentRequest>,
},
@@ -275,6 +288,11 @@ pub enum AgentResponse {
DrawingElementsCleared {
session_id: String,
},
MermaidRendered {
session_id: String,
overlay_id: String,
svg_source: String,
},
BatchResult {
results: Vec<AgentResponse>,
},
@@ -305,6 +323,14 @@ pub enum DrawingCommand {
Clear {
session_id: String,
},
RenderMermaid {
session_id: String,
overlay_id: String,
mermaid_source: String,
svg_source: String,
position: Pos2,
size: Vec2,
},
}
// ---------------------------------------------------------------------------

75
crates/agcanvas/src/agent/server.rs
View File

@@ -6,12 +6,15 @@ use crate::drawing::{boolean, DrawingElement, Shape, ShapeStyle};
use crate::session::{SessionCreator, SessionStore};
use anyhow::Result;
use futures_util::{SinkExt, StreamExt};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{broadcast, mpsc, RwLock};
use tokio_tungstenite::tungstenite::Message;
use usvg::Tree;
const EVENT_CHANNEL_CAPACITY: usize = 64;
static OVERLAY_ID_COUNTER: AtomicUsize = AtomicUsize::new(0);
pub struct AgentServer {
sessions: Arc<RwLock<SessionStore>>,
@@ -682,6 +685,78 @@ async fn process_single_request(
element,
}
}
AgentRequest::RenderMermaid {
session_id,
mermaid_source,
x,
y,
width,
height,
} => {
let sid = {
let store = sessions.read().await;
match store.resolve_session_id(session_id.as_deref()) {
Some(id) => id,
None => {
return AgentResponse::Error {
message: "No session found".to_string(),
}
}
}
};
let svg_source = match crate::mermaid::render_mermaid_to_svg(&mermaid_source) {
Ok(svg) => svg,
Err(e) => {
return AgentResponse::Error {
message: format!("Failed to render Mermaid: {}", e),
}
}
};
let options = usvg::Options::default();
let tree = match Tree::from_str(&svg_source, &options) {
Ok(tree) => tree,
Err(e) => {
return AgentResponse::Error {
message: format!("Failed to parse Mermaid SVG: {}", e),
}
}
};
let natural_size = tree.size();
let overlay_id = format!(
"mermaid_{}",
OVERLAY_ID_COUNTER.fetch_add(1, Ordering::SeqCst)
);
let position = egui::pos2(x.unwrap_or(0.0), y.unwrap_or(0.0));
let size = egui::vec2(
width.unwrap_or(natural_size.width()),
height.unwrap_or(natural_size.height()),
);
if command_tx
.send(DrawingCommand::RenderMermaid {
session_id: sid.clone(),
overlay_id: overlay_id.clone(),
mermaid_source,
svg_source: svg_source.clone(),
position,
size,
})
.is_err()
{
return AgentResponse::Error {
message: "Failed to enqueue Mermaid render command".to_string(),
};
}
AgentResponse::MermaidRendered {
session_id: sid,
overlay_id,
svg_source,
}
}
AgentRequest::Batch { .. } => AgentResponse::Error {
message: "Nested batch requests are not supported".to_string(),
},

115
crates/agcanvas/src/app.rs
View File

@@ -3,13 +3,14 @@ use crate::canvas::{CanvasInteraction, CanvasState};
use crate::clipboard::ClipboardManager;
use crate::command_palette::{CommandId, CommandPalette};
use crate::drawing::{
draw_creation_preview, draw_elements, draw_marquee, draw_selection, find_handle_at_screen_pos,
screen_to_canvas, DragState, DrawingElement, Shape, ShapeStyle, Tool,
canvas_to_screen, draw_creation_preview, draw_elements, draw_marquee, draw_selection,
find_handle_at_screen_pos, screen_to_canvas, DragState, DrawingElement, Shape, ShapeStyle,
Tool,
};
use crate::history::{ChangeSource, HistoryTree, NodeId};
use crate::mermaid::render_mermaid_to_svg;
use crate::persistence::{self, SavedSession, SavedWorkspace};
use crate::session::{Session, SessionCreator, SessionStore};
use crate::session::{MermaidOverlay, Session, SessionCreator, SessionStore};
use crate::svg::{parse_svg, SvgRenderer};
use egui::{Color32, ColorImage, TextureOptions};
use std::path::Path;
@@ -290,6 +291,43 @@ impl AgCanvasApp {
self.status_message = Some((message, std::time::Instant::now()));
}
fn render_mermaid_overlays_to_texture(&mut self, ctx: &egui::Context) {
let ppp = ctx.pixels_per_point();
let scale = self.active_session().canvas_state.zoom.max(1.0) * ppp;
let session = self.active_session_mut();
let session_id = session.id.clone();
for overlay in &mut session.mermaid_overlays {
if overlay.texture.is_some() {
continue;
}
if let Ok(pixmap) = overlay.renderer.render(scale) {
let size = [pixmap.width() as usize, pixmap.height() as usize];
let pixels: Vec<Color32> = pixmap
.pixels()
.iter()
.map(|p| {
Color32::from_rgba_premultiplied(p.red(), p.green(), p.blue(), p.alpha())
})
.collect();
let image = ColorImage { size, pixels };
overlay.texture = Some(ctx.load_texture(
format!(
"mermaid-{}-{}-{}-{}",
session_id,
overlay.id,
overlay.mermaid_source.len(),
overlay.svg_source.len()
),
image,
TextureOptions::LINEAR,
));
}
}
}
fn handle_mermaid_render(&mut self, ctx: &egui::Context) {
let source = self.mermaid_input.trim().to_string();
if source.is_empty() {
@@ -347,7 +385,9 @@ impl AgCanvasApp {
DrawingCommand::Delete { session_id, id } => {
if let Some(session) = self.sessions.iter_mut().find(|s| s.id == session_id) {
session.drawing_elements.retain(|e| e.id != id);
session.selected_element_ids.retain(|selected_id| selected_id != &id);
session
.selected_element_ids
.retain(|selected_id| selected_id != &id);
session.record_edit(
"Agent: Delete Element",
ChangeSource::Agent { name: None },
@@ -362,6 +402,38 @@ impl AgCanvasApp {
.record_edit("Agent: Clear Canvas", ChangeSource::Agent { name: None });
}
}
DrawingCommand::RenderMermaid {
session_id,
overlay_id,
mermaid_source,
svg_source,
position,
size,
} => {
if let Some(session) = self.sessions.iter_mut().find(|s| s.id == session_id) {
match parse_svg(&svg_source) {
Ok((_, usvg_tree)) => {
let overlay = MermaidOverlay {
id: overlay_id,
mermaid_source,
svg_source,
renderer: SvgRenderer::new(usvg_tree),
texture: None,
position,
size,
};
session.mermaid_overlays.push(overlay);
session.record_edit(
"Agent: Render Mermaid",
ChangeSource::Agent { name: None },
);
}
Err(e) => {
tracing::warn!("Failed to parse Mermaid overlay SVG: {}", e);
}
}
}
}
}
}
}
@@ -1030,6 +1102,14 @@ impl eframe::App for AgCanvasApp {
{
self.render_svg_to_texture(ctx);
}
if self
.active_session()
.mermaid_overlays
.iter()
.any(|overlay| overlay.texture.is_none())
{
self.render_mermaid_overlays_to_texture(ctx);
}
egui::TopBottomPanel::top("menu_bar").show(ctx, |ui| {
egui::menu::bar(ui, |ui| {
@@ -1437,7 +1517,9 @@ impl eframe::App for AgCanvasApp {
let canvas_center = response.rect.center();
if response.dragged_by(egui::PointerButton::Middle) {
self.active_session_mut().canvas_state.pan(response.drag_delta());
self.active_session_mut()
.canvas_state
.pan(response.drag_delta());
}
if response.hovered() {
let scroll_delta = ui.input(|i| i.smooth_scroll_delta.y);
@@ -1458,7 +1540,9 @@ impl eframe::App for AgCanvasApp {
&& response.dragged_by(egui::PointerButton::Primary)
&& ui.input(|i| i.modifiers.command)
{
self.active_session_mut().canvas_state.pan(response.drag_delta());
self.active_session_mut()
.canvas_state
.pan(response.drag_delta());
}
if !self.show_mermaid_dialog && !self.show_text_input {
@@ -1493,6 +1577,21 @@ impl eframe::App for AgCanvasApp {
let offset = self.active_session().canvas_state.offset;
let zoom = self.active_session().canvas_state.zoom;
for overlay in &self.active_session().mermaid_overlays {
if let Some(texture) = &overlay.texture {
let screen_pos =
canvas_to_screen(overlay.position, canvas_center, offset, zoom);
let screen_size = overlay.size * zoom;
let rect = egui::Rect::from_min_size(screen_pos, screen_size);
painter.image(
texture.id(),
rect,
egui::Rect::from_min_max(egui::pos2(0.0, 0.0), egui::pos2(1.0, 1.0)),
Color32::WHITE,
);
}
}
draw_elements(
&painter,
&self.active_session().drawing_elements,
@@ -1505,7 +1604,8 @@ impl eframe::App for AgCanvasApp {
draw_selection(&painter, selected_el, canvas_center, offset, zoom);
}
if let DragState::MarqueeSelecting { start, current } = &self.active_session().drag_state
if let DragState::MarqueeSelecting { start, current } =
&self.active_session().drag_state
{
draw_marquee(&painter, *start, *current, canvas_center, offset, zoom);
}
@@ -1521,6 +1621,7 @@ impl eframe::App for AgCanvasApp {
);
if self.active_session().svg_texture.is_none()
&& self.active_session().mermaid_overlays.is_empty()
&& self.active_session().drawing_elements.is_empty()
{
painter.text(

4
crates/agcanvas/src/drawing/mod.rs
View File

@@ -6,7 +6,7 @@ mod tool;
pub use boolean::BooleanOpType;
pub use element::{DrawingElement, Shape, ShapeStyle};
pub use render::{
draw_creation_preview, draw_elements, draw_marquee, draw_selection, find_handle_at_screen_pos,
screen_to_canvas,
canvas_to_screen, draw_creation_preview, draw_elements, draw_marquee, draw_selection,
find_handle_at_screen_pos, screen_to_canvas,
};
pub use tool::{DragState, Tool};

16
crates/agcanvas/src/session.rs
View File

@@ -77,6 +77,16 @@ pub struct SessionInfo {
pub created_at: i64,
}
pub struct MermaidOverlay {
pub id: String,
pub mermaid_source: String,
pub svg_source: String,
pub renderer: SvgRenderer,
pub texture: Option<TextureHandle>,
pub position: egui::Pos2,
pub size: egui::Vec2,
}
pub struct Session {
pub id: String,
pub name: String,
@@ -88,6 +98,7 @@ pub struct Session {
pub description_text: String,
pub drawing_elements: Vec<DrawingElement>,
pub mermaid_overlays: Vec<MermaidOverlay>,
pub selected_element_ids: Vec<String>,
pub active_tool: Tool,
pub drag_state: DragState,
@@ -110,6 +121,7 @@ impl Session {
svg_source: None,
description_text: String::new(),
drawing_elements: Vec::new(),
mermaid_overlays: Vec::new(),
selected_element_ids: Vec::new(),
active_tool: Tool::default(),
drag_state: DragState::default(),
@@ -145,6 +157,7 @@ impl Session {
self.svg_source = None;
self.description_text.clear();
self.drawing_elements.clear();
self.mermaid_overlays.clear();
self.selected_element_ids.clear();
self.drag_state = DragState::default();
self.canvas_state.reset();
@@ -193,6 +206,7 @@ impl Session {
self.svg_renderer = None;
self.svg_texture = None;
self.description_text.clear();
self.mermaid_overlays.clear();
self.selected_element_ids.clear();
self.drag_state = DragState::default();
}
@@ -205,6 +219,7 @@ impl Session {
self.svg_renderer = None;
self.svg_texture = None;
self.description_text.clear();
self.mermaid_overlays.clear();
self.selected_element_ids.clear();
self.drag_state = DragState::default();
true
@@ -221,6 +236,7 @@ impl Session {
self.svg_renderer = None;
self.svg_texture = None;
self.description_text.clear();
self.mermaid_overlays.clear();
self.selected_element_ids.clear();
self.drag_state = DragState::default();
true