seb/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(file-tools): handle UTF-8 BOM in read_file / write_file / patch (#35278)

Browse Source 
Some Windows editors prepend an invisible UTF-8 BOM (U+FEFF) to text
files. We had no awareness of it, so: read_file surfaced a phantom
U+FEFF as the first character; patch matches against the true first
line could miss; and a write/patch round-trip silently stripped the
marker, changing the file's byte signature.

Now:
- read_file / read_file_raw strip a single leading BOM so the model
  never sees it (only on the first chunk — the marker lives at byte 0).
- patch_replace strips the BOM before fuzzy-matching (so an exact
  first-line match works) and its post-write verification compares
  BOM-stripped content.
- write_file restores the BOM when the original file had one and the
  new content doesn't, mirroring the existing line-ending preservation
  (detect on disk via a cheap `head -c 3` probe or reuse pre_content,
  re-prepend across the edit). Guards against double-BOM.

Mid-content U+FEFF is left alone (it's data there, not a file marker).

Tests: TestBomHandling (real LocalEnvironment) — read-strips, raw-read
strips, write preserves, no-BOM-when-original-had-none, no-double-BOM,
patch round-trip preserves, patch matches first line through a BOM,
plus helper unit tests. 208 file-tool tests green.
This commit is contained in:
Teknium
2026-05-30 06:25:50 -07:00
committed by GitHub
parent 5a1aa9e68c
commit 5f84c9144a
2 changed files with 180 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/file_operations.py
+84 -4
View File
@@ -113,6 +113,36 @@ def _normalize_line_endings(text: str, target: str) -> str:
return text
# UTF-8 byte order mark. Some Windows editors (Notepad, older Visual Studio,
# some PowerShell redirects) prepend this invisible 3-byte marker
# (EF BB BF == U+FEFF) to UTF-8 text files. It renders as nothing but is a
# real character at the start of the decoded string, so without handling it:
# - read_file would surface a stray U+FEFF as the first character (the
# model sees a phantom char before `import ...`), and
# - patch matches against the true first line would miss, and write_file
# would silently drop or double the marker on rewrite.
# We strip it on read so the model sees clean content, and restore it on
# write when the original file had one — exactly mirroring the line-ending
# preservation above (detect on disk, preserve across the edit).
_UTF8_BOM = "\ufeff"
def _strip_bom(text: str) -> tuple[str, bool]:
"""Return (text-without-leading-BOM, had_bom).
Only a single leading BOM is stripped; a BOM appearing mid-content is
left alone (it's legitimate data there, not a file marker).
"""
if text and text.startswith(_UTF8_BOM):
return text[len(_UTF8_BOM):], True
return text, False
def _has_bom(text: Optional[str]) -> bool:
"""True if ``text`` begins with a UTF-8 BOM."""
return bool(text) and text.startswith(_UTF8_BOM)
def _is_write_denied(path: str) -> bool:
"""Return True if path is on the write deny list."""
return _shared_is_write_denied(path)
@@ -801,6 +831,22 @@ class ShellFileOperations(FileOperations):
return None
return _detect_line_ending(head_result.stdout)
def _file_has_bom(self, path: str, pre_content: Optional[str] = None) -> bool:
"""Whether the file on disk starts with a UTF-8 BOM.
Uses ``pre_content`` if we already read the file (zero extra exec
calls); otherwise issues a tiny ``head -c 3`` to sample just the
marker. A missing/empty file returns False (new writes get no BOM
unless the caller explicitly includes one).
"""
if pre_content is not None:
return _has_bom(pre_content)
head_cmd = f"head -c 3 {self._escape_shell_arg(path)} 2>/dev/null"
head_result = self._exec(head_cmd)
if head_result.exit_code != 0 or not head_result.stdout:
return False
return _has_bom(head_result.stdout)
def _unified_diff(self, old_content: str, new_content: str, filename: str) -> str:
"""Generate unified diff between old and new content."""
@@ -885,6 +931,11 @@ class ShellFileOperations(FileOperations):
if read_result.exit_code != 0:
return ReadResult(error=f"Failed to read file: {read_result.stdout}")
read_output = _strip_terminal_fence_leaks(read_result.stdout)
# Strip a leading UTF-8 BOM so the model never sees a phantom U+FEFF
# before the first real character. Only meaningful on the first
# chunk (the marker lives at byte 0); later pages can't carry it.
if offset == 1:
read_output, _ = _strip_bom(read_output)
# Get total line count
wc_cmd = f"wc -l < {self._escape_shell_arg(path)}"
@@ -989,8 +1040,14 @@ class ShellFileOperations(FileOperations):
cat_result = self._exec(f"cat {self._escape_shell_arg(path)}")
if cat_result.exit_code != 0:
return ReadResult(error=f"Failed to read file: {cat_result.stdout}")
# Strip a leading UTF-8 BOM so patch's fuzzy matcher operates on
# clean content (a phantom U+FEFF before line 1 would defeat an
# exact first-line match). write_file restores the BOM on the way
# back out — it re-probes the on-disk file, which still has the
# marker — so the round-trip preserves it.
raw_content, _ = _strip_bom(_strip_terminal_fence_leaks(cat_result.stdout))
return ReadResult(
content=_strip_terminal_fence_leaks(cat_result.stdout),
content=raw_content,
file_size=file_size,
)
@@ -1090,6 +1147,18 @@ class ShellFileOperations(FileOperations):
if original_ending == "\r\n":
content = _normalize_line_endings(content, "\r\n")
# ── BOM preservation ──────────────────────────────────────────
# If the file on disk started with a UTF-8 BOM, keep it. read_file
# strips the BOM so the agent never sees it, which means the
# content it hands back to write_file / patch has no BOM either —
# without restoring it here a round-trip would silently strip the
# marker and change the file's byte signature (some Windows
# toolchains key on it). Only prepend when the original had a BOM
# and the new content doesn't already carry one (guards against
# double-BOM if a caller passed raw bytes).
if self._file_has_bom(path, pre_content) and not _has_bom(content):
content = _UTF8_BOM + content
# Snapshot LSP diagnostics for this file (best-effort) so the
# post-write LSP layer can return only diagnostics introduced
# by this specific edit. Mirrors claude-code's
@@ -1193,7 +1262,13 @@ class ShellFileOperations(FileOperations):
return PatchResult(error=f"Failed to read file: {path}")
content = read_result.stdout
# Strip a leading UTF-8 BOM before matching so the fuzzy matcher and
# the diff operate on clean content (a phantom U+FEFF before line 1
# defeats an exact first-line match). write_file restores the BOM on
# the way back out by re-probing the on-disk file, so the round-trip
# preserves the marker.
content, _ = _strip_bom(content)
# Import and use fuzzy matching
from tools.fuzzy_match import fuzzy_find_and_replace
@@ -1242,8 +1317,13 @@ class ShellFileOperations(FileOperations):
# ``new_content`` string has bare LFs. Without this normalization
# every patch on Windows returns a bogus "wrote 39, read 42"
# false-negative even though the edit landed correctly. POSIX
# backends don't translate, so this is a no-op there.
_verify_stdout_normalized = verify_result.stdout.replace("\r\n", "\n").replace("\r", "\n")
# backends don't translate, so this is a no-op there. We also
# strip a leading BOM from the re-read: write_file restored the
# marker on disk but ``new_content`` is the BOM-less string we
# matched against, so the comparison must drop it to stay
# apples-to-apples.
_verify_bomless, _ = _strip_bom(verify_result.stdout)
_verify_stdout_normalized = _verify_bomless.replace("\r\n", "\n").replace("\r", "\n")
_new_content_normalized = new_content.replace("\r\n", "\n").replace("\r", "\n")
if _verify_stdout_normalized != _new_content_normalized:
return PatchResult(error=(