From 4966a32a521aec0bc0f27333199b0334406102a1 Mon Sep 17 00:00:00 2001 From: Charan Rathore <180254320+charan-rathore@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:26:51 +0530 Subject: [PATCH 1/2] gh-99927: Preserve UTF-8 headers under the SMTPUTF8 email policy Signed-off-by: Charan Rathore <180254320+charan-rathore@users.noreply.github.com> --- Lib/email/policy.py | 6 ++++- Lib/test/test_email/test_policy.py | 27 +++++++++++++++++++ ...026-10-16-09-30-00.gh-issue-99927.kx2q.rst | 3 +++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-16-09-30-00.gh-issue-99927.kx2q.rst diff --git a/Lib/email/policy.py b/Lib/email/policy.py index d09d2f5238d40b..85ff79554ed55f 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -132,7 +132,11 @@ def header_source_parse(self, sourcelines): """ name, value = sourcelines[0].split(':', 1) value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') - return (name, value.rstrip('\r\n')) + value = value.rstrip('\r\n') + if self.utf8: + value = value.encode('utf-8', 'surrogateescape').decode( + 'utf-8', 'surrogateescape') + return (name, value) def header_store_parse(self, name, value): """+ diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index 90e8e5580295f9..c30d943826bb96 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -153,6 +153,33 @@ def test_fold_utf8(self): self.assertEqual(p_ascii.fold('Subject', s), expected_ascii) self.assertEqual(p_utf8.fold('Subject', s), expected_utf8) + def test_smtputf8_round_trip_preserves_utf8_headers(self): + odata = textwrap.dedent("""\ + From: Unicôde Nâme + To: Name One , Name Two , Unicôde Nàme + Subject: Unicôde Subjéct + + This message contains Uniçôde Çôntent. + """) + msg = email.message_from_bytes( + odata.encode(), policy=email.policy.SMTPUTF8 + ) + output = msg.as_bytes(policy=email.policy.SMTPUTF8).decode() + + self.assertNotIn("unknown-8bit", output) + self.assertIn( + "Unicôde Nàme ", output + ) + + def test_smtputf8_round_trip_preserves_invalid_header_bytes(self): + msg = email.message_from_bytes( + b"Subject: invalid \xff\n\n", + policy=email.policy.SMTPUTF8, + ) + output = msg.as_bytes(policy=email.policy.SMTPUTF8) + + self.assertIn(b"invalid \xff", output) + def test_fold_zero_max_line_length(self): expected = 'Subject: =?utf-8?q?=C3=A1?=\n' diff --git a/Misc/NEWS.d/next/Library/2026-10-16-09-30-00.gh-issue-99927.kx2q.rst b/Misc/NEWS.d/next/Library/2026-10-16-09-30-00.gh-issue-99927.kx2q.rst new file mode 100644 index 00000000000000..dede5a8b55630e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-16-09-30-00.gh-issue-99927.kx2q.rst @@ -0,0 +1,3 @@ +Fix the :class:`email.policy.SMTPUTF8` policy so UTF-8 header values survive a +parse and regenerate round trip instead of being mangled or demoted to +``unknown-8bit`` bodies. From 21daae5e77fb35dd3f018e72339e25b56e36f722 Mon Sep 17 00:00:00 2001 From: Charan Rathore <180254320+charan-rathore@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:41:08 +0530 Subject: [PATCH 2/2] gh-99927: Move the UTF-8 conversion into the SMTPUTF8 fold path Decoding surrogateescaped header bytes at parse time changed the value seen by every downstream policy: serializing an SMTPUTF8-parsed message with policy=compat32 then encoded-worded whole address headers and destroyed the addr-spec. Perform the conversion only when a utf8 policy actually refolds a source value, preserving structure under compat32. Adds a regression test for the SMTPUTF8-parse/compat32-serialize flow. Signed-off-by: Charan Rathore <180254320+charan-rathore@users.noreply.github.com> --- Lib/email/policy.py | 12 ++++++------ Lib/test/test_email/test_policy.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 85ff79554ed55f..1fb3d57fdb8394 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -132,11 +132,7 @@ def header_source_parse(self, sourcelines): """ name, value = sourcelines[0].split(':', 1) value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') - value = value.rstrip('\r\n') - if self.utf8: - value = value.encode('utf-8', 'surrogateescape').decode( - 'utf-8', 'surrogateescape') - return (name, value) + return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): """+ @@ -229,7 +225,11 @@ def _fold(self, name, value, refold_binary=False): elif refold_binary: refold = _has_surrogates(value) if refold: - return self.header_factory(name, ''.join(lines)).fold(policy=self) + value = ''.join(lines) + if self.utf8: + value = value.encode('utf-8', 'surrogateescape').decode( + 'utf-8', 'surrogateescape') + return self.header_factory(name, value).fold(policy=self) return name + ': ' + self.linesep.join(lines) + self.linesep diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index c30d943826bb96..ea7436c0d8b6de 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -180,6 +180,17 @@ def test_smtputf8_round_trip_preserves_invalid_header_bytes(self): self.assertIn(b"invalid \xff", output) + def test_smtputf8_parse_then_compat32_serialize_preserves_addresses(self): + # gh-99927: parsing with SMTPUTF8 and serializing with compat32 must + # keep address headers structured - regressing here encodes the + # whole header as one encoded word and destroys the addr-spec. + odata = (b"From: Sender \n" + + "To: J\xf6rg \n".encode("utf-8") + + b"Subject: test\n\nbody\n") + msg = email.message_from_bytes(odata, policy=email.policy.SMTPUTF8) + out = msg.as_bytes(policy=email.policy.compat32) + self.assertIn(b"", out) + def test_fold_zero_max_line_length(self): expected = 'Subject: =?utf-8?q?=C3=A1?=\n'