From 0ee847693907ccbdefd316db0d86dc7c339b9722 Mon Sep 17 00:00:00 2001 From: Armaan Sandhu Date: Thu, 15 Oct 2026 21:57:08 -0400 Subject: [PATCH 1/2] gh-159503: Avoid copying the unsent remainder of a bytes write in asyncio selector transports When a large bytes write is queued behind pending data, every partial send re-queued the leftover as b[nbytes:], copying the rest of the object each time. Keep the remainder as a memoryview in _adjust_leftover_buffer() and _write_send(), as write() already does on its first-send path. --- Lib/asyncio/selector_events.py | 4 +-- Lib/test/test_asyncio/test_selector_events.py | 30 +++++++++++++++++++ ...-10-16-01-56-15.gh-issue-159503.WgYiow.rst | 1 + 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index bf20f053ded4353..6f33654b0f62fda 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1143,7 +1143,7 @@ def _adjust_leftover_buffer(self, nbytes: int) -> None: if b_len <= nbytes: nbytes -= b_len else: - buffer.appendleft(b[nbytes:]) + buffer.appendleft(memoryview(b)[nbytes:]) break def _write_send(self): @@ -1155,7 +1155,7 @@ def _write_send(self): n = self._sock.send(buffer) if n != len(buffer): # Not all data was written - self._buffer.appendleft(buffer[n:]) + self._buffer.appendleft(memoryview(buffer)[n:]) self._buffer_size -= n except (BlockingIOError, InterruptedError): self._buffer.appendleft(buffer) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index 74322105a7c7577..2395215891c6220 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -972,6 +972,22 @@ def test_write_sendmsg_partial(self): self.assertTrue(self.loop.writers) self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_partial_bytes_no_copy(self): + # gh-159503: the unsent remainder of a bytes object must not be + # copied, otherwise a large write costs O(n**2). + data = b'data' + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = 2 + + transport = self.socket_transport(sendmsg=True) + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + self.assertIsInstance(transport._buffer[0], memoryview) + self.assertIs(transport._buffer[0].obj, data) + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') def test_write_sendmsg_half_buffer(self): data = [memoryview(b'data1'), memoryview(b'data2')] @@ -1081,6 +1097,20 @@ def test_write_ready_partial(self): self.loop.assert_writer(7, transport._write_ready) self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + def test_write_ready_partial_bytes_no_copy(self): + # gh-159503: the unsent remainder of a bytes object must not be + # copied, otherwise a large write costs O(n**2). + data = b'data' + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + self.assertIsInstance(transport._buffer[0], memoryview) + self.assertIs(transport._buffer[0].obj, data) + def test_write_ready_partial_none(self): data = b'data' self.sock.send.return_value = 0 diff --git a/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst b/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst new file mode 100644 index 000000000000000..02038cf984a1350 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst @@ -0,0 +1 @@ +Fix quadratic copying in :mod:`asyncio` selector socket transports when a large :class:`bytes` write is only partly sent while other data is buffered. The unsent remainder is now kept as a :class:`memoryview` instead of being copied on every partial send. From 12de092b269d7dfbbd4a54823223cd3765103bae Mon Sep 17 00:00:00 2001 From: Armaan Sandhu Date: Fri, 16 Oct 2026 12:20:35 -0400 Subject: [PATCH 2/2] Fold the new checks into the existing partial-write tests Address review: test_write_ready_partial and test_write_sendmsg_partial now run for both bytes and memoryview input and check that the remainder is a view on the original object. Wrap the NEWS entry at 78 columns. --- Lib/test/test_asyncio/test_selector_events.py | 77 +++++++------------ ...-10-16-01-56-15.gh-issue-159503.WgYiow.rst | 5 +- 2 files changed, 31 insertions(+), 51 deletions(-) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index 2395215891c6220..620397a049b7cf4 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -958,35 +958,22 @@ def test_write_sendmsg_full(self): @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') def test_write_sendmsg_partial(self): - - data = memoryview(b'data') - self.sock.sendmsg = mock.Mock() - # Sent partial data - self.sock.sendmsg.return_value = 2 - - transport = self.socket_transport(sendmsg=True) - transport._buffer.append(data) - self.loop._add_writer(7, transport._write_ready) - transport._write_ready() - self.assertTrue(self.sock.sendmsg.called) - self.assertTrue(self.loop.writers) - self.assertEqual(list_to_buffer([b'ta']), transport._buffer) - - @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') - def test_write_sendmsg_partial_bytes_no_copy(self): - # gh-159503: the unsent remainder of a bytes object must not be - # copied, otherwise a large write costs O(n**2). data = b'data' - self.sock.sendmsg = mock.Mock() - self.sock.sendmsg.return_value = 2 - - transport = self.socket_transport(sendmsg=True) - transport._buffer.append(data) - self.loop._add_writer(7, transport._write_ready) - transport._write_ready() - self.assertEqual(list_to_buffer([b'ta']), transport._buffer) - self.assertIsInstance(transport._buffer[0], memoryview) - self.assertIs(transport._buffer[0].obj, data) + for buf in data, memoryview(data): + with self.subTest(type=type(buf).__name__): + self.sock.sendmsg = mock.Mock() + # Sent partial data + self.sock.sendmsg.return_value = 2 + + transport = self.socket_transport(sendmsg=True) + transport._buffer.append(buf) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertTrue(self.loop.writers) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + # gh-159503: the remainder is a view, not a copy + self.assertIs(transport._buffer[0].obj, data) @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') def test_write_sendmsg_half_buffer(self): @@ -1087,29 +1074,19 @@ def test_write_ready_no_data(self): self.assertRaises(AssertionError, transport._write_ready) def test_write_ready_partial(self): - data = memoryview(b'data') - self.sock.send.return_value = 2 - - transport = self.socket_transport() - transport._buffer.append(data) - self.loop._add_writer(7, transport._write_ready) - transport._write_ready() - self.loop.assert_writer(7, transport._write_ready) - self.assertEqual(list_to_buffer([b'ta']), transport._buffer) - - def test_write_ready_partial_bytes_no_copy(self): - # gh-159503: the unsent remainder of a bytes object must not be - # copied, otherwise a large write costs O(n**2). data = b'data' - self.sock.send.return_value = 2 - - transport = self.socket_transport() - transport._buffer.append(data) - self.loop._add_writer(7, transport._write_ready) - transport._write_ready() - self.assertEqual(list_to_buffer([b'ta']), transport._buffer) - self.assertIsInstance(transport._buffer[0], memoryview) - self.assertIs(transport._buffer[0].obj, data) + for buf in data, memoryview(data): + with self.subTest(type=type(buf).__name__): + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport._buffer.append(buf) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + # gh-159503: the remainder is a view, not a copy + self.assertIs(transport._buffer[0].obj, data) def test_write_ready_partial_none(self): data = b'data' diff --git a/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst b/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst index 02038cf984a1350..fca69ffe0087b82 100644 --- a/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst +++ b/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst @@ -1 +1,4 @@ -Fix quadratic copying in :mod:`asyncio` selector socket transports when a large :class:`bytes` write is only partly sent while other data is buffered. The unsent remainder is now kept as a :class:`memoryview` instead of being copied on every partial send. +Fix quadratic copying in :mod:`asyncio` selector socket transports when a +large :class:`bytes` write is only partly sent while other data is buffered. +The unsent remainder is now kept as a :class:`memoryview` instead of being +copied on every partial send.