fix(vision): cap pixel dimensions proactively at embed time + declare Pillow
Follow-up to the salvaged #37727. That PR fixed the reactive recovery path (classifier + post-failure shrinker) but left the PROACTIVE embed-time guard in vision_tools byte-only — a tall small-byte screenshot (e.g. 1200x12000 at 0.06 MB) still baked into immutable history un-resized, relying on a failed round-trip to trigger reactive shrink. - vision_tools: add _image_exceeds_dimension() + _EMBED_MAX_DIMENSION (7900px); the embed-time cap now fires on bytes OR pixels and passes max_dimension to the resizer, so tall small-byte images are shrunk before they're embedded. - vision_tools: best-effort lazy-install of Pillow (tool.vision) in the resize ImportError fallback so the soft dep self-heals (respects allow_lazy_installs). - error_classifier: add two more Anthropic dimension-cap wording variants. - pyproject + lazy_deps: declare Pillow as the [vision] extra / tool.vision lazy dep (it was undeclared everywhere; without it ALL resize recovery no-ops). - tests: cover _image_exceeds_dimension (tall/small/edge/no-Pillow/corrupt). Co-authored-by: kyssta-exe <kyssta-exe@users.noreply.github.com>
This commit is contained in:
committed by
Teknium
co-authored by
kyssta-exe
parent
6bdbe30763
commit
dd4ba4c2c4
@@ -175,6 +175,12 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = {
|
||||
"uvicorn[standard]==0.41.0",
|
||||
"starlette==1.0.1", # CVE-2026-48710 (BadHost) — keep lazy-install in sync with pyproject [web]
|
||||
),
|
||||
# Vision image-resize recovery (Pillow). Soft dependency: vision_tools and
|
||||
# conversation_compression degrade gracefully without it, but the byte AND
|
||||
# pixel-dimension shrink paths no-op when it's absent, so an oversized
|
||||
# image can brick a session on Anthropic's non-retryable 400. Keep in sync
|
||||
# with pyproject [vision].
|
||||
"tool.vision": ("Pillow==12.2.0",),
|
||||
}
|
||||
|
||||
|
||||
|
||||
+51
-10
@@ -326,6 +326,15 @@ _MAX_BASE64_BYTES = 20 * 1024 * 1024
|
||||
# whether we resize proactively or reactively.
|
||||
_EMBED_TARGET_BYTES = 4 * 1024 * 1024
|
||||
|
||||
# Proactive embed dimension cap (px, longest side). Anthropic enforces an
|
||||
# 8000px per-side ceiling INDEPENDENTLY of the 5 MB byte cap — a tall full-page
|
||||
# screenshot can be well under 5 MB yet far over 8000px (e.g. 1200×12000 at
|
||||
# 0.06 MB), so the byte-only embed check above lets it slip into immutable
|
||||
# history un-resized and the session bricks on a non-retryable 400. We cap at
|
||||
# 7900 (headroom under 8000) so the proactive resize shrinks tall small-byte
|
||||
# images before they are embedded.
|
||||
_EMBED_MAX_DIMENSION = 7900
|
||||
|
||||
# Target size when auto-resizing on API failure (5 MB). After a provider
|
||||
# rejects an image, we downscale to this target and retry once.
|
||||
_RESIZE_TARGET_BYTES = 5 * 1024 * 1024
|
||||
@@ -341,6 +350,24 @@ def _is_image_size_error(error: Exception) -> bool:
|
||||
))
|
||||
|
||||
|
||||
def _image_exceeds_dimension(image_path: Path, max_dimension: int) -> bool:
|
||||
"""True if the image's longest side exceeds ``max_dimension`` px.
|
||||
|
||||
Anthropic enforces an 8000px per-side cap independently of the 5 MB byte
|
||||
cap, so a tall small-byte screenshot can pass every byte check yet trip a
|
||||
non-retryable 400. Returns False (don't force a resize) when Pillow is
|
||||
unavailable or the file can't be read as an image — the byte-based checks
|
||||
still apply, and we never want a missing soft dependency to break the
|
||||
embed path.
|
||||
"""
|
||||
try:
|
||||
from PIL import Image as _PILImage
|
||||
with _PILImage.open(image_path) as _img:
|
||||
return max(_img.size) > max_dimension
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None,
|
||||
max_base64_bytes: int = _RESIZE_TARGET_BYTES,
|
||||
max_dimension: Optional[int] = None) -> str:
|
||||
@@ -388,10 +415,20 @@ def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None,
|
||||
from PIL import Image
|
||||
import io as _io
|
||||
except ImportError:
|
||||
logger.info("Pillow not installed — cannot auto-resize oversized image")
|
||||
if data_url is None:
|
||||
data_url = _image_to_base64_data_url(image_path, mime_type=mime_type)
|
||||
return data_url # caller will raise the size error
|
||||
# Pillow is a lazy-installable soft dependency. Try a best-effort
|
||||
# install (respects security.allow_lazy_installs; no-op if disabled or
|
||||
# offline), then re-import. If it still isn't importable, fall back to
|
||||
# the raw bytes and let the caller raise the size error.
|
||||
try:
|
||||
from tools.lazy_deps import ensure as _ensure_dep
|
||||
_ensure_dep("tool.vision")
|
||||
from PIL import Image
|
||||
import io as _io
|
||||
except Exception:
|
||||
logger.info("Pillow not installed — cannot auto-resize oversized image")
|
||||
if data_url is None:
|
||||
data_url = _image_to_base64_data_url(image_path, mime_type=mime_type)
|
||||
return data_url # caller will raise the size error
|
||||
|
||||
logger.info("Image file is %.1f MB (estimated base64 %.1f MB, limit %.1f MB, max_dimension=%s), auto-resizing...",
|
||||
file_size / (1024 * 1024), estimated_b64 / (1024 * 1024),
|
||||
@@ -696,15 +733,19 @@ async def _vision_analyze_native(
|
||||
|
||||
# Proactive embed cap: this image gets baked into conversation
|
||||
# history and re-sent on every subsequent turn. Anthropic rejects
|
||||
# any single base64 image over 5 MB with a 400, and because history
|
||||
# is immutable, an oversized embed permanently wedges the session —
|
||||
# retries can't clear bytes that are already in the request. Resize
|
||||
# DOWN to the embed target (4 MB, headroom under 5 MB) whenever the
|
||||
# payload exceeds it, not just at the 20 MB hard ceiling.
|
||||
if len(image_data_url) > _EMBED_TARGET_BYTES:
|
||||
# any single base64 image over 5 MB OR over 8000px per side with a
|
||||
# 400, and because history is immutable, an oversized embed
|
||||
# permanently wedges the session — retries can't clear bytes (or
|
||||
# pixels) that are already in the request. Resize DOWN to the embed
|
||||
# target (4 MB / 7900px, headroom under both ceilings) whenever the
|
||||
# payload exceeds either limit, not just at the 20 MB hard ceiling.
|
||||
_over_bytes = len(image_data_url) > _EMBED_TARGET_BYTES
|
||||
_over_dims = _image_exceeds_dimension(temp_image_path, _EMBED_MAX_DIMENSION)
|
||||
if _over_bytes or _over_dims:
|
||||
image_data_url = _resize_image_for_vision(
|
||||
temp_image_path, mime_type=detected_mime_type,
|
||||
max_base64_bytes=_EMBED_TARGET_BYTES,
|
||||
max_dimension=_EMBED_MAX_DIMENSION,
|
||||
)
|
||||
# If even resizing can't get under the absolute hard ceiling,
|
||||
# there's nothing more we can do — reject rather than embed a
|
||||
|
||||
Reference in New Issue
Block a user