Merge pull request #4076 from cindykrafft/fix/utf16be-first-line

Fix the first line of UTF-16BE input swallowing the second line

CHANGELOG.md

@@ -26,6 +26,7 @@
 - Syntax highlighting for Python files using uv as script runner in shebang #3689 (@janlarres)
 
 ## Bugfixes
+- Fix the first line of UTF-16BE input swallowing the second line, which broke line numbers, `--line-range` and highlighting, see #XXXX (@cindykrafft)
 - Exit quietly with status 0 instead of panicking when stdout is closed early (broken pipe) on auxiliary output paths such as `--completion`, `--config-file`, `--diagnostic`, `--generate-config-file`, `--set-terminal-title` and `cache --build`/`--clear`, matching the main output path since #232, closes #4034, see #4035 (@Mathjk)
 - Track strikethrough (SGR 9 / 29) in the ANSI style tracker so it is re-emitted like bold, dim, italic, and underline, and is no longer dropped after the first highlighted token when displaying ANSI input with syntax highlighting, see #4015 (@mikehasa)
 - Allow boolean flags to be given more than once, so that a flag set in the config file can also be passed on the command line without erroring, see #3912 (@logarithmone1128)

src/input.rs

@@ -291,7 +291,12 @@ impl<'a> InputReader<'a> {
 
         if content_type == Some(ContentType::UTF_16LE) {
             read_utf16_line(&mut reader, &mut first_line, 0x00, 0x0A)?;
-        } else if content_type == Some(ContentType::UTF_16BE) {
+        } else if content_type == Some(ContentType::UTF_16BE)
+            // In UTF-16BE a newline is encoded as 0x00 0x0A, so `read_until(b'\n')` above
+            // may already have read a complete first line. Only keep reading if it did not
+            // (e.g. the 0x0A byte was part of another character such as U+4E0A).
+            && !(first_line.len() % 2 == 0 && first_line.ends_with(&[0x00, 0x0A]))
+        {
             read_utf16_line(&mut reader, &mut first_line, 0x0A, 0x00)?;
         }
 

tests/integration_tests.rs

@@ -2101,6 +2101,19 @@ fn utf16be() {
         .stdout("   1 上一伊刀\n   2 foo bar\n   3 hello world\n");
 }
 
+#[test]
+fn utf16be_first_line_not_merged_with_second() {
+    // The first line of a UTF-16BE file must not swallow the second line
+    bat()
+        .arg("--decorations=always")
+        .arg("--style=numbers")
+        .arg("--color=never")
+        .arg("test_UTF-16BE.txt")
+        .assert()
+        .success()
+        .stdout("   1 hello world\n   2 this is a test\n");
+}
+
 // Regression test for https://github.com/sharkdp/bat/issues/1922
 #[test]
 fn bom_not_stripped_in_loop_through_mode() {