From 55bb9a77841d579e795f2150e9fe742c541d6226 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Oct 2026 02:18:56 +0200 Subject: [PATCH 1/5] gh-158897: Optimize io.BufferedReader.readline() by calling memchr() Optimize io.BufferedReader.readline() when lines are close to the buffer size (128 kB by default) or longer than the buffer size. Replace the C loop searching for the newline byte in the buffer with a memchr() call which is more efficient. --- ...026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst | 4 ++++ Modules/_io/bufferedio.c | 16 +++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst diff --git a/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst new file mode 100644 index 00000000000000..e3b46b6554c417 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst @@ -0,0 +1,4 @@ +Optimize :meth:`io.BufferedReader.readline` when lines are close to the +buffer size (128 kB by default) or longer than the buffer size. Replace the +C loop searching for the newline byte in the buffer with a :c:func`!memchr` +call which is more efficient. Patch by Victor Stinner. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index d6318af1263d9c..611719428d3b36 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1280,16 +1280,14 @@ _buffered_readline(buffered *self, Py_ssize_t limit) n = limit; } start = self->buffer; - const char *end = start + n; - s = start; - while (s < end) { - if (*s++ == '\n') { - if (PyBytesWriter_WriteBytes(writer, start, s - start) < 0) { - goto error; - } - self->pos = s - start; - goto found; + const char *newline = memchr(start, '\n', n); + if (newline != NULL) { + n = newline - start; + if (PyBytesWriter_WriteBytes(writer, start, n) < 0) { + goto error; } + self->pos = n; + goto found; } if (PyBytesWriter_WriteBytes(writer, start, n) < 0) { From ae3e408357926dc38416a1a89df055d5c8e87cdf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Oct 2026 02:37:32 +0200 Subject: [PATCH 2/5] Fix Changelog syntax --- .../next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst index e3b46b6554c417..82c3779617d473 100644 --- a/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst +++ b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst @@ -1,4 +1,4 @@ -Optimize :meth:`io.BufferedReader.readline` when lines are close to the +Optimize :meth:`!io.BufferedReader.readline` when lines are close to the buffer size (128 kB by default) or longer than the buffer size. Replace the C loop searching for the newline byte in the buffer with a :c:func`!memchr` call which is more efficient. Patch by Victor Stinner. From 461b878d58c8a4e31e24d833e5ad35f4783c6247 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Oct 2026 02:37:40 +0200 Subject: [PATCH 3/5] Oops, fix off-by-one error --- Modules/_io/bufferedio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 611719428d3b36..44c3f56caee18b 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1282,7 +1282,7 @@ _buffered_readline(buffered *self, Py_ssize_t limit) start = self->buffer; const char *newline = memchr(start, '\n', n); if (newline != NULL) { - n = newline - start; + n = newline - start + 1; if (PyBytesWriter_WriteBytes(writer, start, n) < 0) { goto error; } From fc422eda955212a561825e962114bb36aad14b9a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Oct 2026 02:39:55 +0200 Subject: [PATCH 4/5] Fix Changelog syntax, again! --- .../next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst index 82c3779617d473..0837fd9d6987d1 100644 --- a/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst +++ b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst @@ -1,4 +1,4 @@ Optimize :meth:`!io.BufferedReader.readline` when lines are close to the buffer size (128 kB by default) or longer than the buffer size. Replace the -C loop searching for the newline byte in the buffer with a :c:func`!memchr` +C loop searching for the newline byte in the buffer with a :c:func:`!memchr` call which is more efficient. Patch by Victor Stinner. From df7148a295843f3266badc7730da097476a81f5a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 7 Oct 2026 14:43:10 +0200 Subject: [PATCH 5/5] Document 4x faster case --- .../Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst index 0837fd9d6987d1..4524077daefe9a 100644 --- a/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst +++ b/Misc/NEWS.d/next/Library/2026-10-07-02-29-45.gh-issue-158897.jH3FwQ.rst @@ -1,4 +1,5 @@ Optimize :meth:`!io.BufferedReader.readline` when lines are close to the buffer size (128 kB by default) or longer than the buffer size. Replace the C loop searching for the newline byte in the buffer with a :c:func:`!memchr` -call which is more efficient. Patch by Victor Stinner. +call which is more efficient. In the good case, it makes ``readline()`` up to +4x faster! Patch by Victor Stinner.