Chat+ link previews: a link in a chat shows its page's title, text and picture, YouTube playable in place
X's web chat shows a link as a bare card. An X tab on a chat page may now ask native code (link_preview, x-capture only, chat pages only) for one link's preview. Native code fetches it: https and public hosts only (addresses checked and pinned), never X's hosts, no cookies or referrer, redirects re-checked, page head <= 512 KB, images <= 1.5 MB by magic bytes as data:, 30 fetches a minute, memory-only cache. Only the link leaves the page. On by default (owner, 2026-10-10).
This commit is contained in:
parent
89c9bfc75f
commit
c03f1a989b
21 changed files with 1455 additions and 519 deletions
607
apps/desktop/src-tauri/src/link_preview.rs
Normal file
607
apps/desktop/src-tauri/src/link_preview.rs
Normal file
|
|
@ -0,0 +1,607 @@
|
|||
//! Chat+ link previews (owner-approved network use, 2026-10-10): X's web chat shows a link as a bare
|
||||
//! card (an icon and the domain), so an X tab on a chat page may ask native code for the preview of
|
||||
//! ONE link it shows. Native code fetches that link's page head (or YouTube's oEmbed) itself:
|
||||
//!
|
||||
//! - https only, no user info, default port, a domain name (no IP literal), not X's own hosts;
|
||||
//! - every address the name resolves to must be public (no loopback, private, link-local, CGNAT,
|
||||
//! multicast, documentation or unique-local ranges), and the connection is pinned to those
|
||||
//! addresses, so a DNS answer cannot point the request into the owner's network;
|
||||
//! - no cookies (no cookie store), no referrer, a plain user agent naming the app;
|
||||
//! - redirects followed by hand, at most [`MAX_REDIRECTS`], each hop checked again;
|
||||
//! - only an HTML head is read, at most [`MAX_PAGE_BYTES`]; an image at most [`MAX_IMAGE_BYTES`],
|
||||
//! PNG/JPEG/GIF/WebP by its magic bytes only, handed back as a `data:` URL;
|
||||
//! - the request carries the link and nothing else: no conversation, sender or message text.
|
||||
//!
|
||||
//! Results live in memory only (a bounded cache) and are never written anywhere.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::net::IpAddr;
|
||||
use url::Url;
|
||||
|
||||
pub const MAX_URL_LEN: usize = 2_048;
|
||||
pub const MAX_REDIRECTS: usize = 4;
|
||||
pub const MAX_PAGE_BYTES: usize = 512 * 1024;
|
||||
pub const MAX_IMAGE_BYTES: usize = 1_536 * 1024;
|
||||
pub const MAX_TITLE: usize = 200;
|
||||
pub const MAX_DESCRIPTION: usize = 300;
|
||||
pub const MAX_SITE: usize = 80;
|
||||
pub const USER_AGENT: &str = "Mozilla/5.0 (compatible; wayBetter-LinkPreview/1.0; +https://waybetter.fyi/x/)";
|
||||
|
||||
/// What a chat tab gets back: plain text and, for a page, an image as a `data:` URL. A YouTube link
|
||||
/// also carries its video id (the page offers YouTube's own player, a www.youtube.com frame X allows).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct LinkPreview {
|
||||
pub url: String,
|
||||
pub site: String,
|
||||
pub title: String,
|
||||
pub description: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub image: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub youtube: Option<String>,
|
||||
}
|
||||
|
||||
/// X's chat pages (`/i/chat`, `/messages`): the only pages that may ask for a preview.
|
||||
pub fn is_chat_page(path: &str) -> bool {
|
||||
["/i/chat", "/messages"].iter().any(|root| path == *root || path.starts_with(&format!("{root}/")))
|
||||
}
|
||||
|
||||
/// X's own hosts: their links are X's to show, never fetched from here.
|
||||
fn is_x_host(host: &str) -> bool {
|
||||
["x.com", "twitter.com", "t.co", "twimg.com", "x.co"].iter().any(|own| host == *own || host.ends_with(&format!(".{own}")))
|
||||
}
|
||||
|
||||
/// A link a chat tab may ask about: https, a domain name, default port, no user info, not X.
|
||||
pub fn check_link(value: &str) -> Result<Url, String> {
|
||||
if value.is_empty() || value.len() > MAX_URL_LEN || value.chars().any(|c| c.is_control() || c.is_whitespace()) {
|
||||
return Err("VALIDATION: invalid link".into());
|
||||
}
|
||||
let url = Url::parse(value).map_err(|_| "VALIDATION: invalid link")?;
|
||||
check_url(&url)?;
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn check_url(url: &Url) -> Result<(), String> {
|
||||
if url.scheme() != "https" || !url.username().is_empty() || url.password().is_some() {
|
||||
return Err("VALIDATION: only plain https links".into());
|
||||
}
|
||||
if url.port().is_some_and(|port| port != 443) {
|
||||
return Err("VALIDATION: only the default port".into());
|
||||
}
|
||||
let host = match url.host() {
|
||||
Some(url::Host::Domain(host)) => host.to_ascii_lowercase(),
|
||||
_ => return Err("VALIDATION: the link must name a host".into()),
|
||||
};
|
||||
if !host.contains('.') || host.ends_with(".local") || host.ends_with(".localhost") || host.ends_with(".internal") || host == "localhost" {
|
||||
return Err("VALIDATION: not a public host".into());
|
||||
}
|
||||
if is_x_host(&host) {
|
||||
return Err("UNSUPPORTED: X's own links".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Public unicast only: anything that could reach the owner's machine or network is refused.
|
||||
pub fn is_public(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => {
|
||||
let [a, b, c, _] = v4.octets();
|
||||
!(v4.is_loopback()
|
||||
|| v4.is_private()
|
||||
|| v4.is_link_local()
|
||||
|| v4.is_broadcast()
|
||||
|| v4.is_multicast()
|
||||
|| v4.is_unspecified()
|
||||
|| v4.is_documentation()
|
||||
|| a == 0
|
||||
|| (a == 100 && (64..=127).contains(&b))
|
||||
|| (a == 192 && b == 0 && c == 0)
|
||||
|| (a == 198 && (18..=19).contains(&b))
|
||||
|| a >= 240)
|
||||
}
|
||||
IpAddr::V6(v6) => {
|
||||
if let Some(v4) = v6.to_ipv4_mapped() {
|
||||
return is_public(IpAddr::V4(v4));
|
||||
}
|
||||
let first = v6.segments()[0];
|
||||
!(v6.is_loopback()
|
||||
|| v6.is_unspecified()
|
||||
|| v6.is_multicast()
|
||||
|| (first & 0xfe00) == 0xfc00
|
||||
|| (first & 0xffc0) == 0xfe80
|
||||
|| first == 0x2001 && v6.segments()[1] == 0x0db8
|
||||
|| first == 0x0064 && v6.segments()[1] == 0xff9b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The video id of a YouTube watch, short, shorts or embed link.
|
||||
pub fn youtube_id(url: &Url) -> Option<String> {
|
||||
let host = url.host_str()?.trim_start_matches("www.").trim_start_matches("m.").to_ascii_lowercase();
|
||||
let id = match host.as_str() {
|
||||
"youtu.be" => url.path_segments()?.next()?.to_string(),
|
||||
"youtube.com" | "music.youtube.com" => {
|
||||
let mut segments = url.path_segments()?;
|
||||
match segments.next()? {
|
||||
"watch" => url.query_pairs().find(|(k, _)| k == "v")?.1.into_owned(),
|
||||
"shorts" | "embed" | "live" => segments.next()?.to_string(),
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
(id.len() == 11 && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')).then_some(id)
|
||||
}
|
||||
|
||||
/// Plain text: entities decoded, control characters and runs of white space collapsed, capped.
|
||||
pub fn clean(value: &str, max: usize) -> String {
|
||||
let decoded = decode_entities(value);
|
||||
let mut out = String::new();
|
||||
let mut space = false;
|
||||
for c in decoded.chars() {
|
||||
if c.is_whitespace() || c.is_control() {
|
||||
space = !out.is_empty();
|
||||
continue;
|
||||
}
|
||||
if matches!(c, '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}') {
|
||||
continue;
|
||||
}
|
||||
if space {
|
||||
out.push(' ');
|
||||
space = false;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
if out.chars().count() > max {
|
||||
out = out.chars().take(max.saturating_sub(1)).collect::<String>().trim_end().to_string() + "\u{2026}";
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn decode_entities(value: &str) -> String {
|
||||
let mut out = String::with_capacity(value.len());
|
||||
let mut rest = value;
|
||||
while let Some(at) = rest.find('&') {
|
||||
out.push_str(&rest[..at]);
|
||||
rest = &rest[at..];
|
||||
let end = rest[..rest.len().min(12)].find(';');
|
||||
let decoded = end.and_then(|end| {
|
||||
let name = &rest[1..end];
|
||||
let c = match name {
|
||||
"amp" => Some('&'),
|
||||
"lt" => Some('<'),
|
||||
"gt" => Some('>'),
|
||||
"quot" => Some('"'),
|
||||
"apos" | "#39" => Some('\''),
|
||||
"nbsp" => Some(' '),
|
||||
_ if name.starts_with("#x") || name.starts_with("#X") => u32::from_str_radix(&name[2..], 16).ok().and_then(char::from_u32),
|
||||
_ if name.starts_with('#') => name[1..].parse().ok().and_then(char::from_u32),
|
||||
_ => None,
|
||||
}?;
|
||||
Some((c, end + 1))
|
||||
});
|
||||
match decoded {
|
||||
Some((c, len)) => {
|
||||
out.push(c);
|
||||
rest = &rest[len..];
|
||||
}
|
||||
None => {
|
||||
out.push('&');
|
||||
rest = &rest[1..];
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push_str(rest);
|
||||
out
|
||||
}
|
||||
|
||||
/// The attributes of one tag (`<meta a="b" c='d' e=f>`), names lowercased.
|
||||
fn attributes(tag: &str) -> Vec<(String, String)> {
|
||||
let mut out = Vec::new();
|
||||
let bytes = tag.as_bytes();
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
while i < bytes.len() && (bytes[i].is_ascii_whitespace() || bytes[i] == b'/') {
|
||||
i += 1;
|
||||
}
|
||||
let start = i;
|
||||
while i < bytes.len() && !bytes[i].is_ascii_whitespace() && bytes[i] != b'=' && bytes[i] != b'>' {
|
||||
i += 1;
|
||||
}
|
||||
if start == i {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let name = tag[start..i].to_ascii_lowercase();
|
||||
while i < bytes.len() && bytes[i].is_ascii_whitespace() {
|
||||
i += 1;
|
||||
}
|
||||
let mut value = String::new();
|
||||
if i < bytes.len() && bytes[i] == b'=' {
|
||||
i += 1;
|
||||
while i < bytes.len() && bytes[i].is_ascii_whitespace() {
|
||||
i += 1;
|
||||
}
|
||||
if i < bytes.len() && (bytes[i] == b'"' || bytes[i] == b'\'') {
|
||||
let quote = bytes[i];
|
||||
i += 1;
|
||||
let from = i;
|
||||
while i < bytes.len() && bytes[i] != quote {
|
||||
i += 1;
|
||||
}
|
||||
value = tag[from..i].to_string();
|
||||
i += 1;
|
||||
} else {
|
||||
let from = i;
|
||||
while i < bytes.len() && !bytes[i].is_ascii_whitespace() && bytes[i] != b'>' {
|
||||
i += 1;
|
||||
}
|
||||
value = tag[from..i].to_string();
|
||||
}
|
||||
}
|
||||
out.push((name, value));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// What a page's head says about itself: Open Graph first, then Twitter cards, then `<title>` and
|
||||
/// the description meta. `image` is resolved against `base` and checked like a link.
|
||||
#[derive(Debug, Default, PartialEq)]
|
||||
pub struct PageMeta {
|
||||
pub title: String,
|
||||
pub description: String,
|
||||
pub site: String,
|
||||
pub image: Option<Url>,
|
||||
}
|
||||
|
||||
pub fn parse_head(html: &str, base: &Url) -> PageMeta {
|
||||
let lower = html.to_ascii_lowercase();
|
||||
let head_end = lower.find("</head").unwrap_or(lower.len());
|
||||
let (html, lower) = (&html[..head_end], &lower[..head_end]);
|
||||
let mut found: Vec<(String, String)> = Vec::new();
|
||||
let mut at = 0;
|
||||
while let Some(start) = lower[at..].find("<meta").map(|i| at + i) {
|
||||
let Some(end) = lower[start..].find('>').map(|i| start + i) else { break };
|
||||
let attrs = attributes(&html[start + 5..end]);
|
||||
let key = attrs.iter().find(|(n, _)| n == "property" || n == "name").map(|(_, v)| v.to_ascii_lowercase());
|
||||
let content = attrs.iter().find(|(n, _)| n == "content").map(|(_, v)| v.clone());
|
||||
if let (Some(key), Some(content)) = (key, content) {
|
||||
found.push((key, content));
|
||||
}
|
||||
at = end;
|
||||
}
|
||||
let get = |keys: &[&str]| keys.iter().find_map(|key| found.iter().find(|(k, v)| k == key && !v.trim().is_empty()).map(|(_, v)| v.clone()));
|
||||
let title_tag = lower.find("<title").and_then(|start| {
|
||||
let open = start + lower[start..].find('>')? + 1;
|
||||
let close = open + lower[open..].find("</title")?;
|
||||
Some(html[open..close].to_string())
|
||||
});
|
||||
let title = get(&["og:title", "twitter:title"]).or(title_tag).unwrap_or_default();
|
||||
let description = get(&["og:description", "twitter:description", "description"]).unwrap_or_default();
|
||||
let site = get(&["og:site_name", "application-name"]).unwrap_or_default();
|
||||
let image = get(&["og:image:secure_url", "og:image", "og:image:url", "twitter:image", "twitter:image:src"])
|
||||
.and_then(|value| base.join(decode_entities(value.trim()).as_str()).ok())
|
||||
.filter(|url| check_url(url).is_ok());
|
||||
PageMeta {
|
||||
title: clean(&title, MAX_TITLE),
|
||||
description: clean(&description, MAX_DESCRIPTION),
|
||||
site: clean(&site, MAX_SITE),
|
||||
image,
|
||||
}
|
||||
}
|
||||
|
||||
/// The image type by its first bytes; anything else (SVG included) is refused.
|
||||
pub fn image_mime(bytes: &[u8]) -> Option<&'static str> {
|
||||
match bytes {
|
||||
[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a, ..] => Some("image/png"),
|
||||
[0xff, 0xd8, 0xff, ..] => Some("image/jpeg"),
|
||||
[b'G', b'I', b'F', b'8', b'7' | b'9', b'a', ..] => Some("image/gif"),
|
||||
[b'R', b'I', b'F', b'F', _, _, _, _, b'W', b'E', b'B', b'P', ..] => Some("image/webp"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The host as shown under a preview when the page names no site (`www.` dropped).
|
||||
pub fn display_host(url: &Url) -> String {
|
||||
clean(url.host_str().unwrap_or_default().trim_start_matches("www."), MAX_SITE)
|
||||
}
|
||||
|
||||
#[cfg(feature = "desktop")]
|
||||
pub use fetch::preview;
|
||||
|
||||
#[cfg(feature = "desktop")]
|
||||
mod fetch {
|
||||
use super::*;
|
||||
use base64::Engine;
|
||||
use std::collections::HashMap;
|
||||
use std::net::{SocketAddr, ToSocketAddrs};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const CACHE_ENTRIES: usize = 256;
|
||||
const CACHE_TTL: Duration = Duration::from_secs(30 * 60);
|
||||
/// At most this many fresh fetches per minute for the whole app (cached answers are free).
|
||||
const PER_MINUTE: usize = 30;
|
||||
|
||||
type Cached = (Instant, Result<LinkPreview, String>);
|
||||
|
||||
fn cache() -> &'static Mutex<HashMap<String, Cached>> {
|
||||
static CACHE: OnceLock<Mutex<HashMap<String, Cached>>> = OnceLock::new();
|
||||
CACHE.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
fn allow_fetch() -> bool {
|
||||
static RECENT: OnceLock<Mutex<Vec<Instant>>> = OnceLock::new();
|
||||
let mut recent = RECENT.get_or_init(Default::default).lock().unwrap_or_else(|e| e.into_inner());
|
||||
let now = Instant::now();
|
||||
recent.retain(|at| now.duration_since(*at) < Duration::from_secs(60));
|
||||
if recent.len() >= PER_MINUTE {
|
||||
return false;
|
||||
}
|
||||
recent.push(now);
|
||||
true
|
||||
}
|
||||
|
||||
/// The preview of one link: from the cache, else fetched under the rules above.
|
||||
pub async fn preview(link: &str) -> Result<LinkPreview, String> {
|
||||
let url = check_link(link)?;
|
||||
let key = url.as_str().to_string();
|
||||
if let Some((at, result)) = cache().lock().unwrap_or_else(|e| e.into_inner()).get(&key) {
|
||||
if at.elapsed() < CACHE_TTL {
|
||||
return result.clone();
|
||||
}
|
||||
}
|
||||
if !allow_fetch() {
|
||||
return Err("BUSY: too many previews, try again shortly".into());
|
||||
}
|
||||
let result = match youtube_id(&url) {
|
||||
Some(id) => youtube(&url, id).await,
|
||||
None => page(&url).await,
|
||||
};
|
||||
let mut cache = cache().lock().unwrap_or_else(|e| e.into_inner());
|
||||
if cache.len() >= CACHE_ENTRIES {
|
||||
cache.retain(|_, (at, _)| at.elapsed() < CACHE_TTL);
|
||||
if cache.len() >= CACHE_ENTRIES {
|
||||
cache.clear();
|
||||
}
|
||||
}
|
||||
cache.insert(key, (Instant::now(), result.clone()));
|
||||
result
|
||||
}
|
||||
|
||||
/// Resolves the host off the async runtime and keeps only public addresses (all must be public).
|
||||
async fn public_addrs(url: &Url) -> Result<(String, Vec<SocketAddr>), String> {
|
||||
let host = url.host_str().ok_or("VALIDATION: no host")?.to_string();
|
||||
let lookup = host.clone();
|
||||
let addrs: Vec<SocketAddr> = tauri::async_runtime::spawn_blocking(move || (lookup.as_str(), 443).to_socket_addrs().map(|a| a.collect::<Vec<_>>()))
|
||||
.await
|
||||
.map_err(|_| "NETWORK: lookup failed")?
|
||||
.map_err(|_| "NETWORK: the host does not resolve")?;
|
||||
if addrs.is_empty() || !addrs.iter().all(|addr| is_public(addr.ip())) {
|
||||
return Err("FORBIDDEN: the host is not on the public internet".into());
|
||||
}
|
||||
Ok((host, addrs))
|
||||
}
|
||||
|
||||
fn client(host: &str, addrs: &[SocketAddr]) -> Result<reqwest::Client, String> {
|
||||
if rustls::crypto::CryptoProvider::get_default().is_none() {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
}
|
||||
// No cookie store: nothing is sent or kept. Redirects are followed by hand (each hop checked).
|
||||
reqwest::Client::builder()
|
||||
.user_agent(USER_AGENT)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.referer(false)
|
||||
.https_only(true)
|
||||
.connect_timeout(Duration::from_secs(6))
|
||||
.timeout(Duration::from_secs(12))
|
||||
.resolve_to_addrs(host, addrs)
|
||||
.build()
|
||||
.map_err(|e| format!("NETWORK: {e}"))
|
||||
}
|
||||
|
||||
/// GET with hand-followed redirects; returns the final URL and up to `max` bytes of the body
|
||||
/// (`stop` may end the read early, e.g. once the page head is complete).
|
||||
async fn get(start: &Url, accept: &str, max: usize, stop: fn(&[u8]) -> bool) -> Result<(Url, String, Vec<u8>), String> {
|
||||
let mut url = start.clone();
|
||||
for _ in 0..=MAX_REDIRECTS {
|
||||
check_url(&url)?;
|
||||
let (host, addrs) = public_addrs(&url).await?;
|
||||
let mut response = client(&host, &addrs)?
|
||||
.get(url.clone())
|
||||
.header(reqwest::header::ACCEPT, accept)
|
||||
.header(reqwest::header::ACCEPT_LANGUAGE, "en;q=0.9, *;q=0.5")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("NETWORK: {}", e.without_url()))?;
|
||||
let status = response.status();
|
||||
if status.is_redirection() {
|
||||
let location = response.headers().get(reqwest::header::LOCATION).and_then(|v| v.to_str().ok()).ok_or("NETWORK: redirect without a location")?;
|
||||
url = url.join(location).map_err(|_| "NETWORK: invalid redirect")?;
|
||||
continue;
|
||||
}
|
||||
if !status.is_success() {
|
||||
return Err(format!("NETWORK: the site answered {}", status.as_u16()));
|
||||
}
|
||||
let mime = response.headers().get(reqwest::header::CONTENT_TYPE).and_then(|v| v.to_str().ok()).unwrap_or("").to_ascii_lowercase();
|
||||
if response.content_length().is_some_and(|len| len as usize > max) && !accept.starts_with("text/html") {
|
||||
return Err("VALIDATION: too large".into());
|
||||
}
|
||||
let mut body = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await.map_err(|e| format!("NETWORK: {}", e.without_url()))? {
|
||||
let room = max.saturating_sub(body.len());
|
||||
body.extend_from_slice(&chunk[..chunk.len().min(room)]);
|
||||
if body.len() >= max || stop(&body) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return Ok((url, mime, body));
|
||||
}
|
||||
Err("NETWORK: too many redirects".into())
|
||||
}
|
||||
|
||||
fn head_complete(body: &[u8]) -> bool {
|
||||
let tail = &body[body.len().saturating_sub(16 * 1024)..];
|
||||
tail.windows(7).any(|w| w.eq_ignore_ascii_case(b"</head>"))
|
||||
}
|
||||
|
||||
fn never(_: &[u8]) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
async fn page(url: &Url) -> Result<LinkPreview, String> {
|
||||
let (final_url, mime, body) = get(url, "text/html,application/xhtml+xml;q=0.9", MAX_PAGE_BYTES, head_complete).await?;
|
||||
if !(mime.starts_with("text/html") || mime.starts_with("application/xhtml")) {
|
||||
return Err("UNSUPPORTED: not a web page".into());
|
||||
}
|
||||
let html = String::from_utf8_lossy(&body);
|
||||
let meta = parse_head(&html, &final_url);
|
||||
if meta.title.is_empty() && meta.image.is_none() {
|
||||
return Err("UNSUPPORTED: the page has no preview".into());
|
||||
}
|
||||
let image = match &meta.image {
|
||||
Some(image) => fetch_image(image).await.ok(),
|
||||
None => None,
|
||||
};
|
||||
Ok(LinkPreview {
|
||||
url: url.to_string(),
|
||||
site: if meta.site.is_empty() { display_host(&final_url) } else { meta.site },
|
||||
title: meta.title,
|
||||
description: meta.description,
|
||||
image,
|
||||
youtube: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn fetch_image(url: &Url) -> Result<String, String> {
|
||||
let (_, _, bytes) = get(url, "image/avif,image/webp,image/png,image/jpeg,image/gif;q=0.8", MAX_IMAGE_BYTES, never).await?;
|
||||
if bytes.len() >= MAX_IMAGE_BYTES {
|
||||
return Err("VALIDATION: image too large".into());
|
||||
}
|
||||
let mime = image_mime(&bytes).ok_or("UNSUPPORTED: not an image")?;
|
||||
Ok(format!("data:{mime};base64,{}", base64::engine::general_purpose::STANDARD.encode(&bytes)))
|
||||
}
|
||||
|
||||
/// YouTube: its oEmbed answer (title, channel) and the video's thumbnail; the player is YouTube's
|
||||
/// own www.youtube.com, which X's page allows as a frame, loaded only when the user presses play.
|
||||
async fn youtube(url: &Url, id: String) -> Result<LinkPreview, String> {
|
||||
let mut oembed = Url::parse("https://www.youtube.com/oembed?format=json").expect("static URL");
|
||||
oembed.query_pairs_mut().append_pair("url", &format!("https://www.youtube.com/watch?v={id}"));
|
||||
let (_, _, body) = get(&oembed, "application/json", 64 * 1024, never).await?;
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).map_err(|_| "UNSUPPORTED: no video data")?;
|
||||
let text = |key: &str, max: usize| clean(value.get(key).and_then(|v| v.as_str()).unwrap_or(""), max);
|
||||
// The thumbnail comes as data too: X's chat page does not reliably load it from i.ytimg.com.
|
||||
let thumbnail = Url::parse(&format!("https://i.ytimg.com/vi/{id}/hqdefault.jpg")).expect("checked id");
|
||||
Ok(LinkPreview {
|
||||
url: url.to_string(),
|
||||
site: "YouTube".into(),
|
||||
title: text("title", MAX_TITLE),
|
||||
description: text("author_name", MAX_SITE),
|
||||
image: fetch_image(&thumbnail).await.ok(),
|
||||
youtube: Some(id),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn only_plain_public_https_links() {
|
||||
for ok in ["https://example.com/a?b=c", "https://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://sub.example.co.uk:443/"] {
|
||||
assert!(check_link(ok).is_ok(), "{ok}");
|
||||
}
|
||||
for bad in [
|
||||
"http://example.com/",
|
||||
"https://user:pw@example.com/",
|
||||
"https://example.com:8443/",
|
||||
"https://127.0.0.1/",
|
||||
"https://[::1]/",
|
||||
"https://192.168.0.1/",
|
||||
"https://localhost/",
|
||||
"https://printer.local/",
|
||||
"https://intranet/",
|
||||
"https://x.com/someone/status/1",
|
||||
"https://pbs.twimg.com/media/a.jpg",
|
||||
"https://t.co/abc",
|
||||
"file:///etc/passwd",
|
||||
"javascript:alert(1)",
|
||||
"https://exa mple.com/",
|
||||
"",
|
||||
] {
|
||||
assert!(check_link(bad).is_err(), "{bad}");
|
||||
}
|
||||
assert!(check_link(&format!("https://example.com/{}", "a".repeat(MAX_URL_LEN))).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_and_special_addresses_are_not_public() {
|
||||
for ip in ["127.0.0.1", "10.1.2.3", "172.16.0.1", "192.168.0.192", "169.254.1.1", "100.64.0.1", "0.0.0.0", "224.0.0.1", "255.255.255.255", "198.18.0.1", "::1", "fe80::1", "fd00::1", "::ffff:192.168.0.1", "2001:db8::1", "64:ff9b::a00:1"] {
|
||||
assert!(!is_public(ip.parse().unwrap()), "{ip}");
|
||||
}
|
||||
for ip in ["1.1.1.1", "142.250.185.78", "2606:4700:4700::1111"] {
|
||||
assert!(is_public(ip.parse().unwrap()), "{ip}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_pages_only() {
|
||||
for ok in ["/i/chat", "/i/chat/1-2", "/messages", "/messages/1-2"] {
|
||||
assert!(is_chat_page(ok), "{ok}");
|
||||
}
|
||||
for bad in ["/home", "/i/chatter", "/messagesx", "/someone/status/1", "/"] {
|
||||
assert!(!is_chat_page(bad), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn youtube_ids() {
|
||||
let id = |s: &str| youtube_id(&Url::parse(s).unwrap());
|
||||
assert_eq!(id("https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=1"), Some("dQw4w9WgXcQ".into()));
|
||||
assert_eq!(id("https://youtu.be/dQw4w9WgXcQ?si=x"), Some("dQw4w9WgXcQ".into()));
|
||||
assert_eq!(id("https://m.youtube.com/shorts/dQw4w9WgXcQ"), Some("dQw4w9WgXcQ".into()));
|
||||
assert_eq!(id("https://www.youtube.com/embed/dQw4w9WgXcQ"), Some("dQw4w9WgXcQ".into()));
|
||||
assert_eq!(id("https://www.youtube.com/watch?v=bad\"id"), None);
|
||||
assert_eq!(id("https://www.youtube.com/@channel"), None);
|
||||
assert_eq!(id("https://example.com/watch?v=dQw4w9WgXcQ"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_a_page_head() {
|
||||
let base = Url::parse("https://example.org/play/").unwrap();
|
||||
let html = r#"<!doctype html><html><head><title>Fallback</title>
|
||||
<meta property="og:title" content="Sample & friends">
|
||||
<meta name=description content='The social aquarium'>
|
||||
<meta property="og:site_name" content="Sample">
|
||||
<meta property="og:image" content="/img/card.png">
|
||||
</head><body><meta property="og:title" content="not this"></body></html>"#;
|
||||
let meta = parse_head(html, &base);
|
||||
assert_eq!(meta.title, "Sample & friends");
|
||||
assert_eq!(meta.description, "The social aquarium");
|
||||
assert_eq!(meta.site, "Sample");
|
||||
assert_eq!(meta.image.unwrap().as_str(), "https://example.org/img/card.png");
|
||||
let bare = parse_head("<head><title>\n Only a title \u{202e}</title><meta property='og:image' content='http://insecure.example/a.png'></head>", &base);
|
||||
assert_eq!(bare.title, "Only a title");
|
||||
assert_eq!(bare.image, None, "an image off the link rules is dropped");
|
||||
assert_eq!(parse_head("<head><meta property='og:image' content='https://10.0.0.1/a.png'></head>", &base).image, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_is_plain_and_capped() {
|
||||
assert_eq!(clean("a<b> AB &unknown; &", 50), "a<b> AB &unknown; &");
|
||||
assert_eq!(clean(&"x".repeat(500), 10).chars().count(), 10);
|
||||
assert_eq!(clean("\u{0}\u{7}one\ttwo\n", 50), "one two");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn images_by_magic_bytes_only() {
|
||||
assert_eq!(image_mime(b"\x89PNG\r\n\x1a\nrest"), Some("image/png"));
|
||||
assert_eq!(image_mime(b"\xff\xd8\xff\xe0"), Some("image/jpeg"));
|
||||
assert_eq!(image_mime(b"GIF89a"), Some("image/gif"));
|
||||
assert_eq!(image_mime(b"RIFF\0\0\0\0WEBPVP8 "), Some("image/webp"));
|
||||
assert_eq!(image_mime(b"<svg xmlns="), None);
|
||||
assert_eq!(image_mime(b"<html>"), None);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue