fix(vision): guard image pixel dimensions, not just bytes (#37677)

Anthropic enforces two independent ceilings per image:
1. 5 MB encoded byte size
2. 8000 px longest side

Hermes only guarded #1. A tall screenshot (e.g. 1200x12000 at 0.06 MB)
passes every byte check but fails the pixel check, returning a
non-retryable HTTP 400 that permanently bricks the conversation thread.

Fixes:
- error_classifier: add 'image dimensions exceed' pattern to
  _IMAGE_TOO_LARGE_PATTERNS so the 400 is classified as image_too_large
  and triggers the shrink/retry path instead of falling through to
  non-retryable error.
- conversation_compression: check pixel dimensions (via Pillow) even
  when byte size is under the 4 MB target. If max(dims) > 8000, force
  shrink.
- vision_tools._resize_image_for_vision: add optional max_dimension param.
  When set, images exceeding the pixel cap are downscaled even if they're
  under the byte budget. The resize loop now checks both byte AND pixel
  limits before accepting a candidate.

Closes #37677
This commit is contained in:
kyssta-exe
2026-06-04 06:16:45 -07:00
committed by Teknium
parent f7dabd3019
commit 6bdbe30763
4 changed files with 65 additions and 10 deletions
+33 -6
View File
@@ -342,20 +342,40 @@ def _is_image_size_error(error: Exception) -> bool:
def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None,
max_base64_bytes: int = _RESIZE_TARGET_BYTES) -> str:
max_base64_bytes: int = _RESIZE_TARGET_BYTES,
max_dimension: Optional[int] = None) -> str:
"""Convert an image to a base64 data URL, auto-resizing if too large.
Tries Pillow first to progressively downscale oversized images. If Pillow
is not installed or resizing still exceeds the limit, falls back to the raw
bytes and lets the caller handle the size check.
Args:
max_dimension: If set, images whose longest side exceeds this pixel
count are forcibly downscaled even if they're under the byte
budget. Anthropic enforces an 8000 px per-side cap independently
of the 5 MB byte cap.
Returns the base64 data URL string.
"""
# Quick file-size estimate: base64 expands by ~4/3, plus data URL header.
# Skip the expensive full-read + encode if Pillow can resize directly.
file_size = image_path.stat().st_size
estimated_b64 = (file_size * 4) // 3 + 100 # ~header overhead
if estimated_b64 <= max_base64_bytes:
needs_resize_for_bytes = estimated_b64 > max_base64_bytes
# Check pixel dimensions even if bytes are fine.
needs_resize_for_dims = False
if max_dimension is not None:
try:
from PIL import Image as _PILQuick
with _PILQuick.open(image_path) as _quick_img:
if max(_quick_img.size) > max_dimension:
needs_resize_for_dims = True
except Exception:
pass # can't check; Pillow path below will handle or skip
if not needs_resize_for_bytes and not needs_resize_for_dims:
# Small enough — just encode directly.
data_url = _image_to_base64_data_url(image_path, mime_type=mime_type)
if len(data_url) <= max_base64_bytes:
@@ -373,9 +393,9 @@ def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None,
data_url = _image_to_base64_data_url(image_path, mime_type=mime_type)
return data_url # caller will raise the size error
logger.info("Image file is %.1f MB (estimated base64 %.1f MB, limit %.1f MB), auto-resizing...",
logger.info("Image file is %.1f MB (estimated base64 %.1f MB, limit %.1f MB, max_dimension=%s), auto-resizing...",
file_size / (1024 * 1024), estimated_b64 / (1024 * 1024),
max_base64_bytes / (1024 * 1024))
max_base64_bytes / (1024 * 1024), max_dimension)
mime = mime_type or _determine_mime_type(image_path)
# Choose output format: JPEG for photos (smaller), PNG for transparency
@@ -393,13 +413,20 @@ def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None,
if pil_format == "JPEG" and img.mode in {"RGBA", "P"}:
img = img.convert("RGB")
# Strategy: halve dimensions until base64 fits, up to 4 rounds.
# Strategy: halve dimensions until both base64 fits AND pixel dimensions
# are within limits, up to 4 rounds.
# For JPEG, also try reducing quality at each size step.
# For PNG, quality is irrelevant — only dimension reduction helps.
quality_steps = (85, 70, 50) if pil_format == "JPEG" else (None,)
prev_dims = (img.width, img.height)
candidate = None # will be set on first loop iteration
def _dims_ok(w: int, h: int) -> bool:
"""True if both pixel dimensions are within the limit."""
if max_dimension is None:
return True
return max(w, h) <= max_dimension
for attempt in range(5):
if attempt > 0:
# Proportional scaling: halve the longer side and scale the
@@ -430,7 +457,7 @@ def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None,
img.save(buf, **save_kwargs)
encoded = base64.b64encode(buf.getvalue()).decode("ascii")
candidate = f"data:{out_mime};base64,{encoded}"
if len(candidate) <= max_base64_bytes:
if len(candidate) <= max_base64_bytes and _dims_ok(img.width, img.height):
logger.info("Auto-resized image fits: %.1f MB (quality=%s, %dx%d)",
len(candidate) / (1024 * 1024), q,
img.width, img.height)