diff --git a/kiosk/src/bundle.rs b/kiosk/src/bundle.rs index 756c078..a9059ab 100644 --- a/kiosk/src/bundle.rs +++ b/kiosk/src/bundle.rs @@ -97,10 +97,41 @@ pub struct BundleCell { pub cooling_timeout_seconds: Option, #[serde(default = "default_fit")] pub fit: String, + #[serde(default)] + pub smart_url: Option, } fn default_fit() -> String { "cover".to_string() } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct SmartUrlConfig { + pub steps: Vec, + #[serde(default)] + pub login_detect_url: Option, + #[serde(default)] + pub session_check_interval_ms: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct SmartUrlStep { + #[serde(rename = "type")] + pub step_type: String, + #[serde(default)] + pub url: Option, + #[serde(default)] + pub selector: Option, + #[serde(default)] + pub value: Option, + #[serde(default)] + pub value_encrypted: Option, + #[serde(default)] + pub delay_ms: Option, + #[serde(default)] + pub timeout_ms: Option, + #[serde(default)] + pub script: Option, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct BundleCamera { pub id: u32, diff --git a/kiosk/src/onvif_events.rs b/kiosk/src/onvif_events.rs index 4760cb1..f9f66e9 100644 --- a/kiosk/src/onvif_events.rs +++ b/kiosk/src/onvif_events.rs @@ -531,6 +531,10 @@ fn forward_event(server: &str, kiosk_key: &str, camera_id: u32, evt: &OnvifEvent /// Decrypt a value encrypted with secrets.encryptForCluster on the server. /// Format: "v1...". AES-256-GCM. /// cluster_key is base64url-encoded 32-byte key. +pub fn decrypt_cluster_public(ciphertext: &str, key: &str) -> Option { + decrypt_cluster(ciphertext, key) +} + fn decrypt_cluster(ciphertext: &str, cluster_key_b64u: &str) -> Option { use aes_gcm::{Aes256Gcm, Key, Nonce, aead::{Aead, KeyInit}}; use base64::Engine; diff --git a/kiosk/src/ui.rs b/kiosk/src/ui.rs index 8ebb8ca..52c4e3a 100644 --- a/kiosk/src/ui.rs +++ b/kiosk/src/ui.rs @@ -1070,7 +1070,14 @@ fn render_layout(display_id: u32, layout_id: u32) { none_cell() } else { let key = format!("web:{url}"); - ensure_web(key, WebSource::Url(url), server_url, kiosk_key).upcast() + let wv = ensure_web(key, WebSource::Url(url), server_url, kiosk_key); + // Smart URL: execute login/navigation steps after page loads. + if let Some(ref smart) = cell.smart_url { + let decrypt_key = server::load_encrypt_key() + .or_else(|| server::load_cluster_key()); + execute_smart_url_steps(&wv, smart, decrypt_key.as_deref()); + } + wv.upcast() } } "none" => none_cell(), @@ -1542,6 +1549,88 @@ fn load_webview_url(webview: &webkit6::WebView, url: &str, server_url: &str, kio webkit6::prelude::WebViewExt::load_uri(webview, url); } +/// Execute smart URL steps on a WebView after loading. Steps run +/// sequentially via JS injection. Used for auto-login, cookie accept, +/// multi-step navigation before showing the final page. +fn execute_smart_url_steps( + webview: &webkit6::WebView, + config: &crate::bundle::SmartUrlConfig, + decrypt_key: Option<&str>, +) { + let mut js_parts: Vec = Vec::new(); + + for step in &config.steps { + match step.step_type.as_str() { + "navigate" => { + if let Some(url) = &step.url { + js_parts.push(format!("window.location.href = {};", js_string_lit(url))); + js_parts.push("await new Promise(r => setTimeout(r, 1000));".to_string()); + } + } + "fill" => { + if let Some(sel) = &step.selector { + let value = step.value.clone().or_else(|| { + step.value_encrypted.as_ref().and_then(|enc| { + decrypt_key.and_then(|k| { + crate::onvif_events::decrypt_cluster_public(enc, k) + }) + }) + }).unwrap_or_default(); + js_parts.push(format!( + "{{ var el = document.querySelector({}); if (el) {{ el.value = {}; el.dispatchEvent(new Event('input', {{bubbles:true}})); }} }}", + js_string_lit(sel), js_string_lit(&value) + )); + } + } + "click" => { + if let Some(sel) = &step.selector { + js_parts.push(format!( + "{{ var el = document.querySelector({}); if (el) el.click(); }}", + js_string_lit(sel) + )); + } + } + "wait" => { + let ms = step.delay_ms.unwrap_or(1000); + js_parts.push(format!("await new Promise(r => setTimeout(r, {ms}));")); + } + "wait_for" => { + if let Some(sel) = &step.selector { + let timeout = step.timeout_ms.unwrap_or(10000); + js_parts.push(format!( + "await new Promise((resolve) => {{ var deadline = Date.now() + {timeout}; (function check() {{ if (document.querySelector({sel})) return resolve(); if (Date.now() > deadline) return resolve(); setTimeout(check, 200); }})(); }});", + sel = js_string_lit(sel) + )); + } + } + "javascript" => { + if let Some(script) = &step.script { + js_parts.push(script.clone()); + } + } + _ => {} + } + } + + if js_parts.is_empty() { return; } + + let full_js = format!("(async () => {{ {} }})();", js_parts.join("\n")); + let wv = webview.clone(); + + // Execute after the page loads — wait for load-changed signal. + use webkit6::prelude::*; + wv.connect_load_changed(move |wv, event| { + if event == webkit6::LoadEvent::Finished { + let js = full_js.clone(); + wv.evaluate_javascript(&js, None, None, None::<>k::gio::Cancellable>, |_| {}); + } + }); +} + +fn js_string_lit(s: &str) -> String { + format!("'{}'", s.replace('\\', "\\\\").replace('\'', "\\'").replace('\n', "\\n")) +} + /// Set a cookie in WebKit's cookie jar so all requests to the server /// carry the kiosk auth token. Name matches what the server's auth_request /// endpoint checks: `betterframe_kiosk_key`. diff --git a/server/src/schemas/wire/smart-url.ts b/server/src/schemas/wire/smart-url.ts new file mode 100644 index 0000000..bac5e5c --- /dev/null +++ b/server/src/schemas/wire/smart-url.ts @@ -0,0 +1,61 @@ +/** + * Smart URL action steps — automated browser sequences for web cells. + * + * When a web cell has smart_url_steps in its options, the kiosk's WebKit + * executes each step in order before displaying the final page. Use cases: + * - Login to a dashboard (navigate → fill form → click → wait → navigate) + * - Accept cookie banners + * - Navigate through multi-step wizards + * - Auto-refresh on session expiry (detect redirect → re-run sequence) + * + * Steps are authored in the admin UI's cell editor and delivered via the + * bundle. Credentials in "fill" steps are encrypted with the per-kiosk + * encryption key. + */ +import * as av from "@anyvali/js"; + +export const SMART_URL_STEP_TYPES = [ + "navigate", // Go to a URL + "fill", // Fill a form field (CSS selector + value) + "click", // Click an element (CSS selector) + "wait", // Wait N milliseconds + "wait_for", // Wait for an element to appear (CSS selector, max timeout) + "javascript", // Execute arbitrary JS (power-user escape hatch) +] as const; + +export const smartUrlStep = av.object( + { + type: av.enum_(SMART_URL_STEP_TYPES), + // "navigate": URL to load + url: av.optional(av.string().maxLength(2048)), + // "fill": CSS selector for the input + value to set + selector: av.optional(av.string().maxLength(512)), + value: av.optional(av.string().maxLength(1024)), + // "fill" with encrypted value (per-kiosk key). Kiosk decrypts at runtime. + value_encrypted: av.optional(av.string().maxLength(2048)), + // "click": CSS selector to click + // (reuses `selector` field) + // "wait": milliseconds + delay_ms: av.optional(av.int().min(0).max(60000)), + // "wait_for": CSS selector + timeout + timeout_ms: av.optional(av.int().min(0).max(60000)), + // "javascript": raw JS to evaluate + script: av.optional(av.string().maxLength(10000)), + }, + { unknownKeys: "strip" }, +); + +export const smartUrlConfig = av.object( + { + steps: av.array(smartUrlStep), + // Re-run the sequence when the page redirects to a login URL. + // Substring match on the current URL — if detected, sequence restarts. + login_detect_url: av.optional(av.string().maxLength(2048)), + // Interval to check for session expiry (ms). 0 = disabled. + session_check_interval_ms: av.optional(av.int().min(0).max(3600000)), + }, + { unknownKeys: "strip" }, +); + +export type SmartUrlStep = av.Infer; +export type SmartUrlConfig = av.Infer; diff --git a/server/src/shared/bundle.ts b/server/src/shared/bundle.ts index dd0a33b..a79d855 100644 --- a/server/src/shared/bundle.ts +++ b/server/src/shared/bundle.ts @@ -44,6 +44,21 @@ export interface BundleCell { html_content: string | null; cooling_timeout_seconds: number | null; fit: "cover" | "contain" | "fill"; + /** Smart URL action steps — automated login/navigation sequence. */ + smart_url?: { + steps: Array<{ + type: string; + url?: string; + selector?: string; + value?: string; + value_encrypted?: string; + delay_ms?: number; + timeout_ms?: number; + script?: string; + }>; + login_detect_url?: string; + session_check_interval_ms?: number; + }; } export interface BundleLayout { @@ -193,6 +208,26 @@ export async function generateBundle( html_content: htmlContent, cooling_timeout_seconds: c.cooling_timeout_seconds, fit: c.fit, + // Smart URL: encrypted credentials use per-kiosk key so each + // kiosk's bundle has uniquely encrypted values. + smart_url: c.options?.["smart_url"] ? (() => { + const raw = c.options["smart_url"] as any; + const steps = Array.isArray(raw.steps) ? raw.steps.map((s: any) => { + const step = { ...s }; + // Encrypt plaintext values with per-kiosk key for transport. + const ek = kioskEncryptKey ?? clusterKey; + if (step.value && step.type === "fill" && ek) { + step.value_encrypted = secrets.encryptForCluster(step.value, ek); + delete step.value; + } + return step; + }) : []; + return { + steps, + login_detect_url: raw.login_detect_url, + session_check_interval_ms: raw.session_check_interval_ms, + }; + })() : undefined, }); } result.push({