From 8cc5076098dbeb84e5861c1cda149049b4309af5 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Mon, 19 Oct 2026 09:45:13 +0530 Subject: [PATCH] REFACTOR: Speed up output-converter batching and add CI coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/profiler_benchmarks/README.md | 20 +- eng/profiler_benchmarks/report.py | 2 + eng/profiler_benchmarks/workloads.py | 48 ++ mssql_python/cursor.py | 145 ++++- mssql_python/pybind/ddbc_bindings.cpp | 66 +- mssql_python/pybind/ddbc_bindings.h | 20 + mssql_python/pybind/output_converter.hpp | 258 ++++++++ mssql_python/row.py | 9 +- tests/test_036_profiler_ci.py | 175 +++++- tests/test_output_converter_bytes.py | 748 +++++++++++++++++++++++ 10 files changed, 1440 insertions(+), 51 deletions(-) create mode 100644 mssql_python/pybind/output_converter.hpp create mode 100644 tests/test_output_converter_bytes.py diff --git a/eng/profiler_benchmarks/README.md b/eng/profiler_benchmarks/README.md index 2fdeb4d91..cecf29f87 100644 --- a/eng/profiler_benchmarks/README.md +++ b/eng/profiler_benchmarks/README.md @@ -12,7 +12,7 @@ python -m eng.profiler_benchmarks.controller --base main --candidate HEAD \ python -m eng.profiler_benchmarks.report profiler-results/report.json ``` -The fixed registry has 22 tasks. `--scenarios` runs a local subset, but subset +The fixed registry has 24 tasks. `--scenarios` runs a local subset, but subset reports remain incomplete and cannot produce a verdict. `lob_varchar_256k_fetchall` fetches one 256 KiB `VARCHAR(MAX)` value to exercise @@ -26,6 +26,24 @@ validation (including zero), EOF validation, and diagnostic checks are outside the timed fetch window. Both revisions run the same workload; the report's improvement thresholds are unchanged. +`output_converter_fetchall` and `output_converter_fetchmany1000` measure separate +fetch APIs over 10,000 ordered integer/NVARCHAR rows from the existing fixture. +An `SQL_WVARCHAR` converter decodes UTF-16LE bytes and uppercases the text. +Both retain all converted rows through the timer boundary; the batched case +includes the terminal empty `fetchmany(1000)` call and accumulating its batches. +The timed callback only performs UTF-16LE decode and uppercase conversion. +Exact output values/types, count/order, and diagnostics are checked after +collection; focused contract tests check callback bytes using a fake cursor. +Registration, query execution, expected-value construction, validation, cursor +cleanup, and restoring/removing the converter are outside the fetch timer. +Converter state is restored even if setup, fetching, or validation fails. + +These CI workloads use the fixture's `unicode_row_{i}` text rather than the +separate profiling-OFF study's fixed-width text. +Reports calculate fresh paired changes from profiling-ON measurements; +they do not embed historical speedups or promise to reproduce production-latency +results. The two API rows use the existing comparison policy below. + ## Measurement contract CI uses the PR merge's first parent as the exact base. It reuses the diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 07f176f18..cb29a6b99 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -39,6 +39,8 @@ "cte": "Common table expression queries", "lob_varchar_256k_fetchall": "256 KiB VARCHAR(MAX) / fetchall()", "scalar_fetchval": "10,000 scalar values / fetchval() (debug disabled)", + "output_converter_fetchall": "10,000 output-converted rows / fetchall()", + "output_converter_fetchmany1000": "10,000 output-converted rows / fetchmany(1000)", } CASES = tuple(TASK_NAMES) MAX_BYTES = 8 * 1024 * 1024 diff --git a/eng/profiler_benchmarks/workloads.py b/eng/profiler_benchmarks/workloads.py index 9059ab6c9..cb4308e58 100644 --- a/eng/profiler_benchmarks/workloads.py +++ b/eng/profiler_benchmarks/workloads.py @@ -194,6 +194,52 @@ def scalar_fetchval(conn, table, ctx): ctx.disable() +def output_converter_fetch(conn, table, ctx, batched=False): + """Time fetching with a UTF-16LE converter; validate retained rows afterward.""" + from mssql_python import SQL_WVARCHAR + + texts = [f"unicode_row_{index}" for index in range(10_000)] + expected = [(index, text.upper()) for index, text in enumerate(texts)] + api = "fetchmany(1000)" if batched else "fetchall()" + + def converter(raw): + return raw.decode("utf-16le").upper() + + with conn.cursor() as cursor: + previous = conn.get_output_converter(SQL_WVARCHAR) + try: + conn.add_output_converter(SQL_WVARCHAR, converter) + cursor.execute(f"SELECT TOP (10000) int_col, nvarchar_col FROM {table} ORDER BY id") + rows = [] + ctx.enable() + start = time.perf_counter() + if batched: + while batch := cursor.fetchmany(1000): + rows.extend(batch) + else: + rows = cursor.fetchall() + wall_ms = (time.perf_counter() - start) * 1000 + cpp, py = ctx.collect() + assert [tuple(row) for row in rows] == expected + assert all(type(row[0]) is int and type(row[1]) is str for row in rows) + assert not cursor.messages, "Clean converter fetch unexpectedly produced diagnostics" + return dict( + title=f"Output converter / {api}", + wall_ms=wall_ms, + cpp=cpp, + py=py, + detail=f"Rows: 10000; types: int,nvarchar; converter: UTF-16LE upper; API: {api}", + ) + finally: + try: + ctx.disable() + finally: + if previous is None: + conn.remove_output_converter(SQL_WVARCHAR) + else: + conn.add_output_converter(SQL_WVARCHAR, previous) + + def registry(): """Keep every PR #552 scenario, including its existing timing boundaries.""" result = dict(scenarios.SCENARIOS) @@ -208,4 +254,6 @@ def registry(): result.update((name, (partial(query, sql=sql), False)) for name, sql in QUERIES.items()) result["lob_varchar_256k_fetchall"] = (lob_fetch, False) result["scalar_fetchval"] = (scalar_fetchval, True) + result["output_converter_fetchall"] = (output_converter_fetch, True) + result["output_converter_fetchmany1000"] = (partial(output_converter_fetch, batched=True), True) return result diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index d4b85e56b..fda78f397 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -60,6 +60,11 @@ ddbc_sql_const.SQL_TIME.value: ddbc_sql_const.SQL_TYPE_TIME.value, ddbc_sql_const.SQL_TIMESTAMP.value: ddbc_sql_const.SQL_TYPE_TIMESTAMP.value, } +_CONVERTER_ROW_BUILDER = ddbc_bindings._ConverterRowBuilder(Row, ddbc_bindings) + + +def _native_converter_row(): + return _CONVERTER_ROW_BUILDER.matches(Row) def _string_only_output_converter(converter): @@ -2844,19 +2849,34 @@ def fetchone(self) -> Union[None, Row]: self._refresh_decoding_cache() char_enc = self._cached_char_encoding wchar_enc = self._cached_wchar_encoding + native_converters = self._cached_converter_map + deferred_text = False + if native_converters and _native_converter_row(): + deferred_text = True # Fetch raw data row_data = [] try: with perf_phase("py::fetchone::cpp_call"): - ret = ddbc_bindings.DDBCSQLFetchOne( - self.hstmt, - row_data, - char_enc, - wchar_enc, - self._cached_char_ctype, - self.messages, - ) + if deferred_text: + ret = ddbc_bindings.DDBCSQLFetchOne( + self.hstmt, + row_data, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + native_converters, + ) + else: + ret = ddbc_bindings.DDBCSQLFetchOne( + self.hstmt, + row_data, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + ) check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) @@ -2877,7 +2897,21 @@ def fetchone(self) -> Union[None, Row]: # Get column and converter maps column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() + if deferred_text and not converter_map: + ddbc_bindings.restore_output_converter_text(row_data) + deferred_text = False with perf_phase("py::fetchone::row_wrap"): + if deferred_text: + return _CONVERTER_ROW_BUILDER.one( + row_data, + Row, + converter_map, + column_map, + self, + self._uuid_str_indices, + column_map_lower, + self._cached_result_columns, + ) if not converter_map and not self._uuid_str_indices: return Row._fast_create( row_data, column_map, self, column_map_lower, self._cached_result_columns @@ -2919,20 +2953,36 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: self._refresh_decoding_cache() char_enc = self._cached_char_encoding wchar_enc = self._cached_wchar_encoding + native_converters = self._cached_converter_map + deferred_text = False + if native_converters and _native_converter_row(): + deferred_text = True # Fetch raw data rows_data = [] try: with perf_phase("py::fetchmany::cpp_call"): - ret = ddbc_bindings.DDBCSQLFetchMany( - self.hstmt, - rows_data, - size, - char_enc, - wchar_enc, - self._cached_char_ctype, - self.messages, - ) + if deferred_text: + ret = ddbc_bindings.DDBCSQLFetchMany( + self.hstmt, + rows_data, + size, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + native_converters, + ) + else: + ret = ddbc_bindings.DDBCSQLFetchMany( + self.hstmt, + rows_data, + size, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + ) check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) @@ -2950,10 +3000,23 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: # Get column and converter maps column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() - + if deferred_text and not converter_map: + for row_data in rows_data: + ddbc_bindings.restore_output_converter_text(row_data) + deferred_text = False # Convert raw data to Row objects uuid_idx = self._uuid_str_indices with perf_phase("py::fetchmany::row_wrap"): + if deferred_text: + return _CONVERTER_ROW_BUILDER.many( + rows_data, + globals(), + converter_map, + column_map, + self, + uuid_idx, + column_map_lower, + ) if not converter_map and not uuid_idx: return ddbc_bindings.construct_rows( rows_data, @@ -2994,19 +3057,34 @@ def fetchall(self) -> List[Row]: self._refresh_decoding_cache() char_enc = self._cached_char_encoding wchar_enc = self._cached_wchar_encoding + native_converters = self._cached_converter_map + deferred_text = False + if native_converters and _native_converter_row(): + deferred_text = True # Fetch raw data rows_data = [] try: with perf_phase("py::fetchall::cpp_call"): - ret = ddbc_bindings.DDBCSQLFetchAll( - self.hstmt, - rows_data, - char_enc, - wchar_enc, - self._cached_char_ctype, - self.messages, - ) + if deferred_text: + ret = ddbc_bindings.DDBCSQLFetchAll( + self.hstmt, + rows_data, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + native_converters, + ) + else: + ret = ddbc_bindings.DDBCSQLFetchAll( + self.hstmt, + rows_data, + char_enc, + wchar_enc, + self._cached_char_ctype, + self.messages, + ) # Check for errors check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) @@ -3024,10 +3102,23 @@ def fetchall(self) -> List[Row]: # Get column and converter maps column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() - + if deferred_text and not converter_map: + for row_data in rows_data: + ddbc_bindings.restore_output_converter_text(row_data) + deferred_text = False # Convert raw data to Row objects uuid_idx = self._uuid_str_indices with perf_phase("py::fetchall::row_wrap"): + if deferred_text: + return _CONVERTER_ROW_BUILDER.many( + rows_data, + globals(), + converter_map, + column_map, + self, + uuid_idx, + column_map_lower, + ) if not converter_map and not uuid_idx: return ddbc_bindings.construct_rows( rows_data, diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 4e4523955..ab9c68119 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -3437,7 +3437,8 @@ static inline bool IsLobOrVariantColumn(SQLSMALLINT dataType, SQLULEN columnSize SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, py::list& row, const std::string& charEncoding = "utf-16le", const std::string& wcharEncoding = "utf-16le", - int charCtype = SQL_C_WCHAR, py::handle messages = {}) { + int charCtype = SQL_C_WCHAR, py::handle messages = {}, + py::handle converters = {}) { PERF_TIMER("SQLGetData_wrap"); // Note: wcharEncoding parameter is reserved for future use // Currently WCHAR data always uses UTF-16LE for Windows compatibility @@ -3613,9 +3614,15 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p // null termination. This preserves embedded NULs and avoids // any risk of reading past the valid range if the driver // omits the terminator. - row.append(FetchText::from_utf16_native( - reinterpret_cast(dataBuffer.data()), - static_cast(numCharsInData * sizeof(SQLWCHAR)))); + const char* textData = + reinterpret_cast(dataBuffer.data()); + Py_ssize_t textSize = + static_cast(numCharsInData * sizeof(SQLWCHAR)); + py::object text; + if (OutputConverters::enabled(converters, i - 1)) + text = OutputConverters::text(textData, textSize); + row.append(text ? text + : FetchText::from_utf16_native(textData, textSize)); LOG("SQLGetData: CHAR column %d fetched as WCHAR, " "length=%lu", i, (unsigned long)numCharsInData); @@ -3792,9 +3799,16 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p // null termination. This preserves embedded NULs and avoids // any risk of reading past the valid range if the driver // omits the terminator. - row.append(FetchText::from_utf16_native( - reinterpret_cast(dataBuffer.data()), - static_cast(numCharsInData * sizeof(SQLWCHAR)))); + const char* textData = + reinterpret_cast(dataBuffer.data()); + Py_ssize_t textSize = + static_cast(numCharsInData * sizeof(SQLWCHAR)); + py::object text; + if (dataType != SQL_SS_VARIANT && + OutputConverters::enabled(converters, i - 1)) + text = OutputConverters::text(textData, textSize); + row.append(text ? text + : FetchText::from_utf16_native(textData, textSize)); LOG("SQLGetData: Appended NVARCHAR string " "length=%lu for column %d", (unsigned long)numCharsInData, i); @@ -4447,7 +4461,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& py::list& rows, SQLUSMALLINT numCols, SQLULEN& numRowsFetched, const std::vector& lobColumns, const std::string& charEncoding = "utf-16le", int charCtype = SQL_C_WCHAR, - py::handle messages = {}) { + py::handle messages = {}, py::handle converters = {}) { PERF_TIMER("FetchBatchData"); LOG("FetchBatchData: Fetching data in batches"); SQLRETURN ret; @@ -4536,6 +4550,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& columnInfosExt[col].isLob = columnInfos[col].isLob; columnInfosExt[col].charEncoding = effectiveCharEnc; columnInfosExt[col].messages = messages.ptr(); + columnInfosExt[col].converterText = OutputConverters::enabled(converters, col); columnInfosExt[col].isUtf8 = (effectiveCharEnc == "utf-8"); // Set useWideChar for SQL_CHAR/VARCHAR columns when charCtype is SQL_C_WCHAR SQLSMALLINT dt = columnInfos[col].dataType; @@ -4968,7 +4983,7 @@ struct FetchStateGuard { SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetchSize, const std::string& charEncoding = "utf-16le", const std::string& wcharEncoding = "utf-16le", int charCtype = SQL_C_WCHAR, - py::handle messages = {}) { + py::handle messages = {}, py::handle converters = {}) { PERF_TIMER("FetchMany_wrap"); // Issue #531: upgrade SQL_C_CHAR + utf-8 to SQL_C_WCHAR on Windows so the // driver does lossless UTF-16 conversion instead of returning ACP bytes. @@ -5046,7 +5061,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch py::list row; ret = SQLGetData_wrap(StatementHandle, numCols, row, charEncoding, wcharEncoding, - charCtype, messages); + charCtype, messages, converters); CheckFetchError(StatementHandle, ret); rows.append(row); numRowsFetched++; @@ -5068,7 +5083,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch fetchStateGuard.configure(&numRowsFetched, fetchSize); ret = FetchBatchData(hStmt, buffers, columnNames, rows, numCols, numRowsFetched, lobColumns, - charEncoding, charCtype, messages); + charEncoding, charCtype, messages, converters); CheckFetchError(StatementHandle, ret); if (!SQL_SUCCEEDED(ret) && ret != SQL_NO_DATA) { LOG("FetchMany_wrap: Error when fetching data - SQLRETURN=%d", ret); @@ -6116,7 +6131,7 @@ SQLRETURN FetchArrowBatch_wrap(SqlHandlePtr StatementHandle, py::list& capsules, SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, const std::string& charEncoding = "utf-16le", const std::string& wcharEncoding = "utf-16le", int charCtype = SQL_C_WCHAR, - py::handle messages = {}) { + py::handle messages = {}, py::handle converters = {}) { PERF_TIMER("FetchAll_wrap"); // Issue #531: upgrade SQL_C_CHAR + utf-8 to SQL_C_WCHAR on Windows so the // driver does lossless UTF-16 conversion instead of returning ACP bytes. @@ -6185,7 +6200,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, py::list row; ret = SQLGetData_wrap(StatementHandle, numCols, row, charEncoding, wcharEncoding, - charCtype, messages); + charCtype, messages, converters); CheckFetchError(StatementHandle, ret); rows.append(row); } @@ -6248,7 +6263,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, while (ret != SQL_NO_DATA) { ret = FetchBatchData(hStmt, buffers, columnNames, rows, numCols, numRowsFetched, lobColumns, - charEncoding, charCtype, messages); + charEncoding, charCtype, messages, converters); CheckFetchError(StatementHandle, ret); if (!SQL_SUCCEEDED(ret) && ret != SQL_NO_DATA) { LOG("FetchAll_wrap: Error when fetching data - SQLRETURN=%d", ret); @@ -6280,7 +6295,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, SQLRETURN FetchOne_wrap(SqlHandlePtr StatementHandle, py::list& row, const std::string& charEncoding = "utf-16le", const std::string& wcharEncoding = "utf-16le", int charCtype = SQL_C_WCHAR, - py::handle messages = {}) { + py::handle messages = {}, py::handle converters = {}) { PERF_TIMER("FetchOne_wrap"); // Issue #531: upgrade SQL_C_CHAR + utf-8 to SQL_C_WCHAR on Windows so the // driver does lossless UTF-16 conversion instead of returning ACP bytes. @@ -6308,7 +6323,7 @@ SQLRETURN FetchOne_wrap(SqlHandlePtr StatementHandle, py::list& row, // Retrieve column count SQLSMALLINT colCount = SQLNumResultCols_wrap(StatementHandle, messages); ret = SQLGetData_wrap(StatementHandle, colCount, row, charEncoding, wcharEncoding, - charCtype, messages); + charCtype, messages, converters); if (!SQL_SUCCEEDED(ret)) { LOG("FetchOne_wrap: Error retrieving data with SQLGetData - SQLRETURN=%d", ret); return ret; @@ -6512,20 +6527,22 @@ PYBIND11_MODULE(ddbc_bindings, m) { py::arg("ColumnMetadata"), py::arg("messages") = py::none()); m.def("DDBCSQLGetData", &SQLGetData_wrap, "Retrieve data from the result set", py::arg("StatementHandle"), py::arg("colCount"), py::arg("row"), py::arg("charEncoding"), - py::arg("wcharEncoding"), py::arg("charCtype"), py::arg("messages") = py::none()); + py::arg("wcharEncoding"), py::arg("charCtype"), py::arg("messages") = py::none(), + py::arg("converters") = py::none()); m.def("DDBCSQLMoreResults", &SQLMoreResults_wrap, "Check for more results in the result set"); m.def("DDBCSQLFetchOne", &FetchOne_wrap, "Fetch one row from the result set", py::arg("StatementHandle"), py::arg("row"), py::arg("charEncoding") = "utf-16le", py::arg("wcharEncoding") = "utf-16le", py::arg("charCtype") = SQL_C_WCHAR, - py::arg("messages") = py::none()); + py::arg("messages") = py::none(), py::arg("converters") = py::none()); m.def("DDBCSQLFetchMany", &FetchMany_wrap, py::arg("StatementHandle"), py::arg("rows"), py::arg("fetchSize"), py::arg("charEncoding") = "utf-16le", py::arg("wcharEncoding") = "utf-16le", py::arg("charCtype") = SQL_C_WCHAR, - py::arg("messages") = py::none(), "Fetch many rows from the result set"); + py::arg("messages") = py::none(), py::arg("converters") = py::none(), + "Fetch many rows from the result set"); m.def("DDBCSQLFetchAll", &FetchAll_wrap, "Fetch all rows from the result set", py::arg("StatementHandle"), py::arg("rows"), py::arg("charEncoding") = "utf-16le", py::arg("wcharEncoding") = "utf-16le", py::arg("charCtype") = SQL_C_WCHAR, - py::arg("messages") = py::none()); + py::arg("messages") = py::none(), py::arg("converters") = py::none()); m.def("DDBCSQLFetchArrowBatch", &FetchArrowBatch_wrap, "Fetch an arrow batch of given length from the result set", py::arg("StatementHandle"), py::arg("capsules"), py::arg("arrowBatchSize"), py::arg("charCtype"), @@ -6632,6 +6649,15 @@ PYBIND11_MODULE(ddbc_bindings, m) { py::arg("column_map"), py::arg("cursor"), py::arg("column_map_lower") = py::none(), py::arg("column_names") = py::none()); + m.def("apply_output_converters", &OutputConverters::apply, + "Apply a cached converter map, preserving owned text bytes and exception fallback"); + m.def("restore_output_converter_text", &OutputConverters::restore, + "Restore deferred text before using custom Row hooks or a changed converter map"); + py::class_(m, "_ConverterRowBuilder") + .def(py::init()) + .def("matches", &OutputConverters::RowBuilder::matches) + .def("one", &OutputConverters::RowBuilder::one) + .def("many", &OutputConverters::RowBuilder::many); // Expose logger bridge function to Python m.def("update_log_level", &mssql_python::logging::LoggerBridge::updateLevel, diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 748ecc5ca..9405d87f4 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -59,6 +59,7 @@ using py::literals::operator""_a; // Include logger bridge for LOG macros #include "logger_bridge.hpp" #include "fetch_text.hpp" +#include "output_converter.hpp" #if defined(__APPLE__) || defined(__linux__) #include @@ -465,6 +466,7 @@ struct ColumnInfoExt { bool useWideChar; // True when charCtype == SQL_C_WCHAR (VARCHAR fetched as UTF-16) std::string charEncoding; // Effective decoding encoding for SQL_C_CHAR data PyObject* messages = nullptr; // Borrowed from the enclosing native fetch call. + bool converterText = false; }; // Forward declare FetchLobColumnData (defined in ddbc_bindings.cpp) - MUST be @@ -596,6 +598,15 @@ inline void ProcessChar(PyObject* row, ColumnBuffers& buffers, const void* colIn SQLUSMALLINT col, SQLULEN rowIdx, SQLHSTMT hStmt) { const ColumnInfoExt* colInfo = static_cast(colInfoPtr); SQLLEN dataLen = buffers.indicators[col - 1][rowIdx]; + if (colInfo->converterText && colInfo->useWideChar && !colInfo->isLob && dataLen >= 0 && + static_cast(dataLen / sizeof(SQLWCHAR)) < colInfo->fetchBufferSize) { + const auto* data = &buffers.wcharBuffers[col - 1][rowIdx * colInfo->fetchBufferSize]; + py::object text = OutputConverters::text(reinterpret_cast(data), dataLen); + if (text) { + PyList_SET_ITEM(row, col - 1, text.release().ptr()); + return; + } + } // Handle empty strings if (dataLen == 0) { @@ -709,6 +720,15 @@ inline void ProcessWChar(PyObject* row, ColumnBuffers& buffers, const void* colI SQLUSMALLINT col, SQLULEN rowIdx, SQLHSTMT hStmt) { const ColumnInfoExt* colInfo = static_cast(colInfoPtr); SQLLEN dataLen = buffers.indicators[col - 1][rowIdx]; + if (colInfo->converterText && !colInfo->isLob && dataLen >= 0 && + static_cast(dataLen / sizeof(SQLWCHAR)) < colInfo->fetchBufferSize) { + const auto* data = &buffers.wcharBuffers[col - 1][rowIdx * colInfo->fetchBufferSize]; + py::object text = OutputConverters::text(reinterpret_cast(data), dataLen); + if (text) { + PyList_SET_ITEM(row, col - 1, text.release().ptr()); + return; + } + } // Handle empty strings if (dataLen == 0) { diff --git a/mssql_python/pybind/output_converter.hpp b/mssql_python/pybind/output_converter.hpp new file mode 100644 index 000000000..4adf37521 --- /dev/null +++ b/mssql_python/pybind/output_converter.hpp @@ -0,0 +1,258 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#pragma once + +#include "fetch_text.hpp" +#include "py_ref.hpp" + +#include +#include + +namespace OutputConverters { + +inline constexpr const char* text_capsule_name = "mssql_python.output_converter.utf16le"; + +inline bool enabled(py::handle converters, Py_ssize_t column) { + return converters && PyList_CheckExact(converters.ptr()) && + column < PyList_GET_SIZE(converters.ptr()) && + PyList_GET_ITEM(converters.ptr(), column) != Py_None; +} + +// The capsule owns callback-ready bytes, not a view of an ODBC buffer. Only +// well-formed text takes this path; malformed data retains each fetcher's +// existing decoding/error behavior. +inline py::object text(const char* data, Py_ssize_t size) { + if (!PY_LITTLE_ENDIAN || size < 0 || size % 2 != 0) return {}; + const auto* bytes = reinterpret_cast(data); + for (Py_ssize_t i = 0; i < size; i += 2) { + unsigned int unit = bytes[i] | (bytes[i + 1] << 8); + if (unit >= 0xD800 && unit <= 0xDBFF) { + if (i + 3 >= size) return {}; + unsigned int next = bytes[i + 2] | (bytes[i + 3] << 8); + if (next < 0xDC00 || next > 0xDFFF) return {}; + i += 2; + } else if (unit >= 0xDC00 && unit <= 0xDFFF) { + return {}; + } + } + py::object raw = steal(PyBytes_FromStringAndSize(data, size)); + if (!raw) throw py::error_already_set(); + py::object result = steal(PyCapsule_New(raw.ptr(), text_capsule_name, [](PyObject* capsule) { + Py_DECREF(static_cast(PyCapsule_GetPointer(capsule, text_capsule_name))); + })); + if (!result) throw py::error_already_set(); + raw.release(); + return result; +} + +inline PyObject* raw_text(py::handle value) { + return PyCapsule_IsValid(value.ptr(), text_capsule_name) + ? static_cast(PyCapsule_GetPointer(value.ptr(), text_capsule_name)) + : nullptr; +} + +inline py::object original(py::handle value) { + PyObject* raw = raw_text(value); + if (!raw) return borrow(value.ptr()); + return FetchText::from_utf16_native(PyBytes_AS_STRING(raw), PyBytes_GET_SIZE(raw)); +} + +inline py::list restore(py::list values) { + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(values.ptr()); ++i) { + if (raw_text(PyList_GET_ITEM(values.ptr(), i))) { + py::object value = original(py::handle(PyList_GET_ITEM(values.ptr(), i))); + if (PyList_SetItem(values.ptr(), i, value.release().ptr()) < 0) + throw py::error_already_set(); + } + } + return values; +} + +inline py::list apply(py::list values, py::list converters) { + py::object result = steal(PyList_GetSlice(values.ptr(), 0, PyList_GET_SIZE(values.ptr()))); + if (!result) throw py::error_already_set(); + Py_ssize_t i = 0; + for (; i < PyList_GET_SIZE(values.ptr()) && i < PyList_GET_SIZE(converters.ptr()); ++i) { + py::object value = borrow(PyList_GET_ITEM(values.ptr(), i)); + py::object converter = borrow(PyList_GET_ITEM(converters.ptr(), i)); + int active = PyObject_IsTrue(converter.ptr()); + if (active < 0) throw py::error_already_set(); + bool replaced = false; + if (active && !value.is_none()) { + try { + PyObject* raw = raw_text(value); + py::object argument; + if (raw) { + argument = borrow(raw); + } else if (PyUnicode_CheckExact(value.ptr())) { + argument = steal(PyUnicode_AsEncodedString(value.ptr(), "utf-16-le", "strict")); + } else if (PyUnicode_Check(value.ptr())) { + argument = steal(PyObject_CallMethod(value.ptr(), "encode", "s", "utf-16-le")); + } else { + argument = value; + } + if (!argument) throw py::error_already_set(); + py::object converted = steal(PyObject_CallOneArg(converter.ptr(), argument.ptr())); + if (!converted) throw py::error_already_set(); + if (PyList_SetItem(result.ptr(), i, converted.release().ptr()) < 0) + throw py::error_already_set(); + replaced = true; + } catch (py::error_already_set& error) { + // Match Row's existing Exception fallback, but not BaseException. + if (!error.matches(PyExc_Exception)) throw; + error.restore(); + PyErr_Clear(); + } + } + if (!replaced && i < PyList_GET_SIZE(result.ptr()) && + raw_text(PyList_GET_ITEM(result.ptr(), i))) { + py::object restored = original(py::handle(PyList_GET_ITEM(result.ptr(), i))); + if (PyList_SetItem(result.ptr(), i, restored.release().ptr()) < 0) + throw py::error_already_set(); + } + } + for (; i < PyList_GET_SIZE(result.ptr()); ++i) { + if (raw_text(PyList_GET_ITEM(result.ptr(), i))) { + py::object restored = original(py::handle(PyList_GET_ITEM(result.ptr(), i))); + if (PyList_SetItem(result.ptr(), i, restored.release().ptr()) < 0) + throw py::error_already_set(); + } + } + return py::reinterpret_borrow(result); +} + +// Python owns this object's references. Unlike a static type cache, its lifetime +// is bounded by the importing interpreter. Attribute writes still use normal +// descriptor dispatch (including descriptors inherited from a native Row base). +#if defined(__GNUG__) && !defined(_WIN32) +class __attribute__((visibility("hidden"))) RowBuilder { +#else +class RowBuilder { +#endif + py::object row_class_; + py::object bindings_; + py::object apply_; + std::array hook_names_{ + py::str("__init__"), py::str("_apply_output_converters_optimized"), + py::str("__new__"), py::str("__getattribute__"), py::str("__setattr__")}; + std::array hooks_; + std::array code_; + std::array field_names_{ + py::str("_values"), py::str("_column_map"), py::str("_cursor"), + py::str("_column_map_lower"), py::str("_column_names")}; + py::str apply_name_{"apply_output_converters"}; + py::str row_name_{"Row"}; + unsigned int verified_version_ = 0; + + py::object construct(py::list values, py::object row_class, py::object converters, + py::object column_map, py::object cursor, py::object uuid_indices, + py::object lower_map, py::object names) { + // Do not mutate even private-looking lists: a native bridge wrapper may + // retain them. The original-value snapshot is also needed after callback + // failures and in-place edits to the converter map. + if (matches(row_class) && uuid_indices.is_none() && + PyList_CheckExact(values.ptr()) && PyList_CheckExact(converters.ptr()) && + PyList_GET_SIZE(converters.ptr()) != 0) { + auto* type = reinterpret_cast(row_class.ptr()); + // A native base with a custom allocator/new must keep its constructor. + if (type->tp_new == PyBaseObject_Type.tp_new) { + py::tuple args(0); + py::object row = steal(type->tp_new(type, args.ptr(), nullptr)); + if (!row) throw py::error_already_set(); + py::list converted = apply(values, py::reinterpret_borrow(converters)); + std::array fields{ + converted.ptr(), column_map.ptr(), cursor.ptr(), lower_map.ptr(), names.ptr()}; + for (size_t i = 0; i < fields.size(); ++i) { + if (PyObject_SetAttr(row.ptr(), field_names_[i].ptr(), fields[i]) < 0) + throw py::error_already_set(); + } + return row; + } + } + + restore(values); + return row_class(values, column_map, py::arg("cursor") = cursor, + py::arg("converter_map") = converters, + py::arg("uuid_str_indices") = uuid_indices, + py::arg("column_map_lower") = lower_map, + py::arg("column_names") = names); + } + +public: + RowBuilder(py::object row_class, py::object bindings) + : row_class_(std::move(row_class)), bindings_(std::move(bindings)) { + if (!PyType_Check(row_class_.ptr()) || !PyModule_Check(bindings_.ptr())) + throw py::type_error("expected a Row type and bindings module"); + if (row_class_.ptr() != py::module_::import("mssql_python.row").attr("Row").ptr()) + throw py::type_error("expected the canonical Row class"); + for (size_t i = 0; i < hooks_.size(); ++i) + hooks_[i] = row_class_.attr(hook_names_[i]); + for (size_t i = 0; i < code_.size(); ++i) { + if (PyFunction_Check(hooks_[i].ptr())) + code_[i] = borrow(PyFunction_GetCode(hooks_[i].ptr())); + } + apply_ = bindings_.attr(apply_name_); + } + + bool matches(py::handle row_class) { + if (row_class.ptr() != row_class_.ptr() || + Py_TYPE(bindings_.ptr()) != &PyModule_Type) + return false; + PyObject* current_apply = PyDict_GetItemWithError( + PyModule_GetDict(bindings_.ptr()), apply_name_.ptr()); + if (!current_apply) { + if (PyErr_Occurred()) throw py::error_already_set(); + return false; + } + if (current_apply != apply_.ptr()) return false; + // Changing function.__code__ does not invalidate the owning type's tag. + for (size_t i = 0; i < code_.size(); ++i) { + if (code_[i] && PyFunction_GetCode(hooks_[i].ptr()) != code_[i].ptr()) + return false; + } + + auto* type = reinterpret_cast(row_class.ptr()); + // CPython invalidates this tag for mutations to the type or its bases. + // Zero is not a valid cached tag; recheck the hooks whenever invalidated. + if (type->tp_version_tag && type->tp_version_tag == verified_version_) return true; + verified_version_ = 0; + for (size_t i = 0; i < hooks_.size(); ++i) { + if (row_class_.attr(hook_names_[i]).ptr() != hooks_[i].ptr()) return false; + } + verified_version_ = type->tp_version_tag; + return true; + } + + py::object one(py::list values, py::object row_class, py::object converters, + py::object column_map, py::object cursor, py::object uuid_indices, + py::object lower_map, py::object names) { + return construct(values, row_class, converters, column_map, cursor, uuid_indices, + lower_map, names); + } + + py::list many(py::list rows_data, py::dict cursor_globals, py::object converters, + py::object column_map, py::object cursor, py::object uuid_indices, + py::object lower_map) { + py::list result; + // Retain ordinary list-iteration behavior if a wrapper/callback holds and + // edits rows_data. Resolve Row and column names again between callbacks. + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(rows_data.ptr()); ++i) { + py::object values = borrow(PyList_GET_ITEM(rows_data.ptr(), i)); + PyObject* selected = PyDict_GetItemWithError(cursor_globals.ptr(), row_name_.ptr()); + if (!selected) { + if (!PyErr_Occurred()) PyErr_SetString(PyExc_NameError, "name 'Row' is not defined"); + throw py::error_already_set(); + } + py::object row_class = borrow(selected); + py::object names = cursor.attr("_cached_result_columns"); + if (!PyList_Check(values.ptr())) + throw py::type_error("native converter rows must contain lists"); + result.append(construct(py::reinterpret_borrow(values), row_class, converters, + column_map, cursor, uuid_indices, lower_map, names)); + } + return result; + } +}; + +} // namespace OutputConverters diff --git a/mssql_python/row.py b/mssql_python/row.py index ccec53787..7556b8b51 100644 --- a/mssql_python/row.py +++ b/mssql_python/row.py @@ -9,6 +9,7 @@ import uuid as _uuid from collections.abc import Mapping from typing import Any +from mssql_python import ddbc_bindings from mssql_python.logging import logger @@ -86,6 +87,8 @@ def __init__( cursor: Optional cursor reference (for backward compatibility and lowercase access) converter_map: Pre-computed converter map (shared across rows for performance). An empty sequence skips converters; None enables the connection fallback. + Built-in cursor conversion can perform the same initialization in native + code; overridden Row hooks retain this Python constructor. uuid_str_indices: Tuple of column indices whose uuid.UUID values should be converted to str. Pre-computed once per result set when native_uuid=False. None means no conversion (native_uuid=True, the default). @@ -199,12 +202,16 @@ def _apply_output_converters_optimized(self, values, converter_map): Apply output converters using pre-computed converter map for optimal performance. Args: - values: Raw values from the database + values: Raw values from the database. Eligible native text cells own + UTF-16LE bytes until conversion; failures restore the original str. converter_map: Pre-computed list of converters (one per column, None if no converter) Returns: List of converted values """ + if type(values) is list and type(converter_map) is list: + return ddbc_bindings.apply_output_converters(values, converter_map) + converted_values = list(values) for i, (value, converter) in enumerate(zip(values, converter_map)): diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 66ca343b0..cf6888daa 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -16,6 +16,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock from urllib.error import URLError +import weakref import zipfile import zlib @@ -148,7 +149,9 @@ def test_phase_call_changes_are_reported_without_summing_nested_totals(report): assert row["phases"] == [(4.0, "ddbc::query")] -@pytest.mark.parametrize("case", ["nan", "missing", "environment", "work", "few", "prefix", "zero"]) +@pytest.mark.parametrize( + "case", ["nan", "missing", "environment", "work", "few", "prefix", "zero", "empty-native"] +) def test_reject_invalid_or_incomparable_data(report, case): sample = report["pairs"][0]["candidate"] if case == "nan": @@ -167,6 +170,8 @@ def test_reject_invalid_or_incomparable_data(report, case): sample["scenarios"]["select"]["cpp"] = { "not-native": sample["scenarios"]["select"]["cpp"]["ddbc::query"] } + elif case == "empty-native": + sample["scenarios"]["output_converter_fetchall"]["cpp"] = {} with pytest.raises(ValueError): reporting.validate(report) @@ -690,13 +695,179 @@ def __exit__(self, *args): def test_report_cases_match_the_executed_workload_registry(): _, workloads = controller.load_suite() assert tuple(workloads.registry()) == reporting.CASES - assert len(reporting.CASES) == 22 + assert len(reporting.CASES) == 24 assert workloads.registry()["scalar_fetchval"] == (workloads.scalar_fetchval, True) + assert workloads.registry()["output_converter_fetchall"] == ( + workloads.output_converter_fetch, + True, + ) + batch, needs_table = workloads.registry()["output_converter_fetchmany1000"] + assert needs_table and batch.func is workloads.output_converter_fetch + assert batch.keywords == {"batched": True} assert [name for name in reporting.CASES if name.startswith("lob_")] == [ "lob_varchar_256k_fetchall", ] +@pytest.mark.parametrize("batched", (False, True), ids=("fetchall", "fetchmany1000")) +def test_output_converter_workload_boundaries_and_report(monkeypatch, report, batched): + from mssql_python import SQL_WVARCHAR + + cursor = MagicMock() + cursor.messages = [] + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + previous = (lambda raw: raw) if batched else None + connection.get_output_converter.return_value = previous + context = MagicMock() + counters = {"ddbc::FetchBatchData": dict(calls=1, total_us=1000, min_us=1000, max_us=1000)} + context.collect.return_value = (counters, {}) + references = [] + callback_inputs = [] + index = 0 + + class RetainedRow(list): + def __iter__(self): + context.collect.assert_called_once() + return super().__iter__() + + def fetch(size=10_000): + nonlocal index + assert clock.call_count == 1 + context.collect.assert_not_called() + converter = connection.add_output_converter.call_args.args[1] + rows = [] + for i in range(index, min(index + size, 10_000)): + raw = f"unicode_row_{i}".encode("utf-16le") + callback_inputs.append(raw) + rows.append(RetainedRow((i, converter(raw)))) + index += len(rows) + references.extend(weakref.ref(row) for row in rows) + return rows + + cursor.fetchall.side_effect = fetch + cursor.fetchmany.side_effect = fetch + + def tick(): + cursor.execute.assert_called_once_with( + "SELECT TOP (10000) int_col, nvarchar_col FROM #perf_test ORDER BY id" + ) + connection.get_output_converter.assert_called_once_with(SQL_WVARCHAR) + assert connection.add_output_converter.call_count == 1 + assert connection.add_output_converter.call_args.args[0] == SQL_WVARCHAR + connection.remove_output_converter.assert_not_called() + context.enable.assert_called_once() + context.collect.assert_not_called() + if clock.call_count == 1: + assert not references + else: + assert len(references) == 10_000 and all(ref() is not None for ref in references) + return 1.0 if clock.call_count == 1 else 1.1 + + clock = MagicMock(side_effect=tick) + monkeypatch.setattr(benchmark_workloads.time, "perf_counter", clock) + result = benchmark_workloads.output_converter_fetch( + connection, "#perf_test", context, batched=batched + ) + assert clock.call_count == 2 + assert result["wall_ms"] == pytest.approx(100) + assert result["cpp"] is counters + assert callback_inputs == [f"unicode_row_{i}".encode("utf-16le") for i in range(10_000)] + api = "fetchmany(1000)" if batched else "fetchall()" + assert result["detail"] == ( + f"Rows: 10000; types: int,nvarchar; converter: UTF-16LE upper; API: {api}" + ) + if batched: + assert [call.args for call in cursor.fetchmany.call_args_list] == [(1000,)] * 11 + cursor.fetchall.assert_not_called() + connection.add_output_converter.assert_called_with(SQL_WVARCHAR, previous) + assert connection.add_output_converter.call_count == 2 + connection.remove_output_converter.assert_not_called() + else: + cursor.fetchall.assert_called_once_with() + cursor.fetchmany.assert_not_called() + connection.remove_output_converter.assert_called_once_with(SQL_WVARCHAR) + cursor.fetchone.assert_not_called() + connection.clear_output_converters.assert_not_called() + context.collect.assert_called_once() + context.disable.assert_called_once() + connection.cursor.return_value.__exit__.assert_called_once() + + clear_slowdowns(report) + name = "output_converter_fetchmany1000" if batched else "output_converter_fetchall" + for pair in report["pairs"]: + pair["candidate"]["scenarios"][name]["wall_ms"] = 7 + reporting.validate(report) + row = next(row for row in reporting.comparisons(report) if row["name"] == name) + assert row["status"] == "improvement" and row["change_pct"] == pytest.approx(-30) + assert ( + f"| {reporting.TASK_NAMES[name]} | 10.000 ms | 7.000 ms | -30.0% | consistent improvement |" + in reporting.render([report], "c" * 40, 42) + ) + + +@pytest.mark.parametrize( + ("batched", "problem"), + ( + (False, "setup"), + (True, "fetch"), + (False, "output"), + (True, "output-type"), + (True, "missing"), + (False, "warning"), + (True, "collect"), + ), +) +def test_output_converter_workload_rejects_invalid_results_and_restores(batched, problem): + from mssql_python import SQL_WVARCHAR + + cursor = MagicMock() + cursor.messages = [("01000", "unexpected")] if problem == "warning" else [] + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + previous = lambda raw: raw + connection.get_output_converter.return_value = previous + context = MagicMock() + context.collect.return_value = ({}, {}) + if problem == "setup": + cursor.execute.side_effect = RuntimeError("setup failed") + if problem == "collect": + context.collect.side_effect = RuntimeError("collect failed") + index = 0 + + def fetch(size=10_000): + nonlocal index + if problem == "fetch": + raise RuntimeError("fetch failed") + converter = connection.add_output_converter.call_args.args[1] + rows = [] + for i in range(index, min(index + size, 10_000)): + raw = f"unicode_row_{i}".encode("utf-16le") + converted = converter(raw) + rows.append( + ( + float(i) if problem == "output-type" else i, + "wrong" if problem == "output" else converted, + ) + ) + index += len(rows) + return rows[1:] if problem == "missing" else rows + + cursor.fetchall.side_effect = fetch + cursor.fetchmany.side_effect = fetch + expected_error = RuntimeError if problem in ("setup", "fetch", "collect") else AssertionError + with pytest.raises(expected_error): + benchmark_workloads.output_converter_fetch( + connection, "#perf_test", context, batched=batched + ) + context.disable.assert_called_once() + connection.add_output_converter.assert_called_with(SQL_WVARCHAR, previous) + assert connection.add_output_converter.call_count == 2 + connection.remove_output_converter.assert_not_called() + connection.clear_output_converters.assert_not_called() + connection.cursor.return_value.__exit__.assert_called_once() + + def test_scalar_fetchval_workload_times_only_fetch_and_reaches_eof(monkeypatch, report): monkeypatch.setattr("mssql_python.logging.logger", SimpleNamespace(is_debug_enabled=False)) cursor = MagicMock() diff --git a/tests/test_output_converter_bytes.py b/tests/test_output_converter_bytes.py new file mode 100644 index 000000000..68788bbe1 --- /dev/null +++ b/tests/test_output_converter_bytes.py @@ -0,0 +1,748 @@ +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +Output converter byte representation, fallback, ordering and ownership. +""" + +import decimal +import importlib +import inspect +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest +import mssql_python +from mssql_python import ddbc_bindings +from mssql_python.row import Row + +METHODS = ("fetchone", "fetchmany", "fetchall") +TEXT = "\ufeffA\x00\u00e9\u03a9\U0001f642\ufffe" + + +def _fetch(cursor, method): + if method == "fetchall": + return cursor.fetchall() + rows = [] + if method == "fetchone": + while (row := cursor.fetchone()) is not None: + rows.append(row) + else: + while batch := cursor.fetchmany(1000): + rows.extend(batch) + return rows + + +@pytest.fixture +def connection(conn_str): + connection = mssql_python.connect(conn_str) + try: + yield connection + finally: + connection.close() + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("width", (1, 32)) +def test_converter_native_bytes_and_row_major_order(connection, method, width): + inputs = [] + raw_values = [] + native_apply = ddbc_bindings.apply_output_converters + bridge_name = { + "fetchone": "DDBCSQLFetchOne", + "fetchmany": "DDBCSQLFetchMany", + "fetchall": "DDBCSQLFetchAll", + }[method] + bridge = getattr(ddbc_bindings, bridge_name) + + def converter(raw): + assert type(raw) is bytes + inputs.append(raw) + return raw.decode("utf-16-le") + "!" + + def observe(*args): + status = bridge(*args) + if method == "fetchone": + raw_values.extend(args[1]) + else: + for values in args[1]: + raw_values.extend(values) + return status + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, converter) + columns = ", ".join(f"CAST(? AS NVARCHAR(100)) AS c{i}" for i in range(width)) + expected = [TEXT + str(i) for i in range(width)] + with connection.cursor() as cursor: + cursor.execute( + f"SELECT {columns} FROM (VALUES (1), (2), (3)) AS v(n) ORDER BY n", + expected, + ) + with patch.object(ddbc_bindings, bridge_name, side_effect=observe): + rows = _fetch(cursor, method) + assert [list(row) for row in rows] == [[value + "!" for value in expected]] * 3 + assert inputs == [value.encode("utf-16-le") for value in expected] * 3 + assert all(type(value).__name__ == "PyCapsule" for value in raw_values) + # The ODBC statement/buffers are gone; retained cells still own the same bytes. + assert native_apply(raw_values, [lambda raw: raw] * len(raw_values)) == inputs + + +@pytest.mark.parametrize("method", METHODS) +def test_converter_empty_nul_and_bom_values(connection, method): + expected = ["", "\x00", "\x00\x00", "\ufeff", "\ufffe", "\U0001f642"] + calls = [] + connection.add_output_converter(str, lambda raw: calls.append(raw) or raw) + with connection.cursor() as cursor: + for value in expected: + cursor.execute("SELECT CAST(? AS NVARCHAR(10))", [value]) + assert _fetch(cursor, method)[0][0] == value.encode("utf-16-le") + assert calls == [value.encode("utf-16-le") for value in expected] + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("fail", (False, True)) +def test_converter_releases_owned_payload_after_wrapping(connection, method, fail): + inputs = [] + + def converter(raw): + inputs.append(raw) + if fail: + raise ValueError("keep original") + return raw.decode("utf-16-le") + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, converter) + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10)) FROM (VALUES (1), (2), (3)) AS v(n)") + rows = _fetch(cursor, method) + assert [row[0] for row in rows] == ["abc"] * 3 + assert all(type(row[0]) is str for row in rows) + assert len(inputs) == 3 + # Only the callback's list owns these non-interned bytes after fetch returns. + # Measure outside pytest's rewritten assertion, which retains its operands. + first_refs = sys.getrefcount(inputs[0]) + second_refs = sys.getrefcount(inputs[1]) + third_refs = sys.getrefcount(inputs[2]) + assert (first_refs, second_refs, third_refs) == (2, 2, 2) + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("sqltype", ("VARCHAR(100)", "NVARCHAR(MAX)", "XML")) +def test_converter_failures_keep_original_text(connection, method, sqltype): + observed = [] + + def converter(raw): + observed.append(raw) + raise ValueError("converter failure") + + connection.add_output_converter(str, converter) + value = "text" if sqltype == "XML" else "abc" + with connection.cursor() as cursor: + cursor.execute(f"SELECT CAST(? AS {sqltype})", [value]) + row = _fetch(cursor, method)[0] + assert type(row[0]) is str + assert row[0] == value + assert observed == [value.encode("utf-16-le")] + + +@pytest.mark.parametrize("method", METHODS) +def test_converter_null_bypass_and_arbitrary_results(connection, method): + inputs = [] + result = object() + + def converter(value): + inputs.append(value) + return result + + for sqltype in ( + mssql_python.SQL_WVARCHAR, + mssql_python.SQL_VARBINARY, + mssql_python.SQL_GUID, + mssql_python.SQL_SS_XML, + mssql_python.SQL_DECIMAL, + ): + connection.add_output_converter(sqltype, converter) + with connection.cursor() as cursor: + cursor.execute( + "SELECT CAST(NULL AS NVARCHAR(10)), CAST(NULL AS VARBINARY(10)), " + "CAST(NULL AS UNIQUEIDENTIFIER), CAST(NULL AS XML), CAST(NULL AS DECIMAL(10,2)), " + "CAST(N'abc' AS NVARCHAR(10)), CAST(0x0102 AS VARBINARY(2)), " + "CAST(1.25 AS DECIMAL(10,2))" + ) + row = _fetch(cursor, method)[0] + assert list(row[:5]) == [None] * 5 + assert all(value is result for value in row[5:]) + assert inputs == [b"a\x00b\x00c\x00", b"\x01\x02", decimal.Decimal("1.25")] + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("mutation", ("replace", "remove", "clear")) +def test_converter_mutation_during_fetch_preserves_batch_snapshot(connection, method, mutation): + inputs = [] + + def original(raw): + inputs.append(raw) + if mutation == "replace": + connection.add_output_converter(mssql_python.SQL_WVARCHAR, lambda value: "new") + elif mutation == "remove": + connection.remove_output_converter(mssql_python.SQL_WVARCHAR) + else: + connection.clear_output_converters() + return "old" + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, original) + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10)) FROM (VALUES (1), (2), (3)) AS v(n)") + rows = _fetch(cursor, method) + expected = ["old"] * 3 + if method == "fetchone": + expected = ["old"] + (["new"] if mutation == "replace" else ["abc"]) * 2 + assert [row[0] for row in rows] == expected + assert len(inputs) == (1 if method == "fetchone" else 3) + connection.clear_output_converters() + cursor.execute("SELECT CAST(N'reused' AS NVARCHAR(10))") + assert cursor.fetchone()[0] == "reused" + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("hook", ("constructor", "converter")) +def test_custom_row_hooks_receive_original_values(connection, method, hook, monkeypatch): + cursor_module = importlib.import_module("mssql_python.cursor") + seen = [] + + class CustomRow(Row): + def __init__(self, values, *args, **kwargs): + seen.append(list(values)) + super().__init__(values, *args, **kwargs) + + original = Row._apply_output_converters_optimized + + def custom_convert(self, values, converters): + seen.append(list(values)) + return original(self, values, converters) + + if hook == "constructor": + monkeypatch.setattr(cursor_module, "Row", CustomRow) + else: + monkeypatch.setattr(Row, "_apply_output_converters_optimized", custom_convert) + connection.add_output_converter(mssql_python.SQL_WVARCHAR, lambda raw: raw) + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10))") + assert _fetch(cursor, method)[0][0] == b"a\x00b\x00c\x00" + assert seen == [["abc"]] + + +@pytest.mark.parametrize("method", ("fetchmany", "fetchall")) +def test_converter_can_change_row_constructor_between_rows(connection, method, monkeypatch): + cursor_module = importlib.import_module("mssql_python.cursor") + seen = [] + + class CustomRow(Row): + def __init__(self, values, *args, **kwargs): + seen.append(list(values)) + super().__init__(values, *args, **kwargs) + + def converter(raw): + monkeypatch.setattr(cursor_module, "Row", CustomRow) + return raw + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, converter) + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10)) FROM (VALUES (1), (2)) AS v(n)") + assert [row[0] for row in _fetch(cursor, method)] == [b"a\x00b\x00c\x00"] * 2 + assert seen == [["abc"]] + + +@pytest.mark.parametrize("method", ("fetchmany", "fetchall")) +def test_converter_can_clear_cached_map_between_rows(connection, method): + calls = [] + with connection.cursor() as cursor: + + def converter(raw): + calls.append(raw) + cursor._cached_converter_map.clear() + return raw + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, converter) + cursor.execute( + "SELECT CAST(N'abc' AS NVARCHAR(10)), CAST(N'def' AS NVARCHAR(10)) " + "FROM (VALUES (1), (2)) AS v(n)" + ) + assert [list(row) for row in _fetch(cursor, method)] == [ + [b"a\x00b\x00c\x00", "def"], + ["abc", "def"], + ] + assert calls == [b"a\x00b\x00c\x00"] + + +@pytest.mark.parametrize("method", METHODS) +def test_converter_builtin_row_wrapping_avoids_python_dispatch(connection, method): + calls = [] + connection.add_output_converter( + mssql_python.SQL_WVARCHAR, lambda raw: calls.append(raw) or raw.decode("utf-16-le") + ) + python_row_calls = [] + encodes = [] + original_init = Row.__init__.__code__ + original_apply = Row._apply_output_converters_optimized.__code__ + + def profile(frame, event, argument): + if event == "call" and frame.f_code in (original_init, original_apply): + python_row_calls.append(frame.f_code.co_name) + if event == "c_call" and getattr(argument, "__name__", None) == "encode": + encodes.append(argument) + + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10)) FROM (VALUES (1), (2), (3)) AS v(n)") + previous = sys.getprofile() + try: + sys.setprofile(profile) + rows = _fetch(cursor, method) + finally: + sys.setprofile(previous) + assert [row[0] for row in rows] == ["abc"] * 3 + assert calls == [b"a\x00b\x00c\x00"] * 3 + assert python_row_calls == [] + assert encodes == [] + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("hook", ("__init__", "__new__", "__getattribute__", "__setattr__")) +def test_converter_row_type_mutation_invalidates_guard(connection, method, hook, monkeypatch): + if hook == "__new__": + # Deleting a patched __new__ does not restore CPython's original tp_new. + # Isolate the native slot state, not just the Python attribute. + code = ( + "import runpy, sys; " + "runpy.run_path(sys.argv[1])['_check_converter_new_mutation']" + "(sys.argv[2], sys.argv[3])" + ) + result = subprocess.run( + [ + sys.executable, + "-c", + code, + str(Path(__file__).resolve()), + method, + str(Path(ddbc_bindings.module.__file__).resolve()), + ], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + else: + _check_converter_row_type_mutation(connection, method, hook, monkeypatch) + + cursor_module = importlib.import_module("mssql_python.cursor") + assert cursor_module._CONVERTER_ROW_BUILDER.matches(Row) + assert Row(["abc"], {}, converter_map=[lambda raw: raw])[0] == b"a\x00b\x00c\x00" + + +def _check_converter_new_mutation(method, expected_native): + import os + + assert Path(ddbc_bindings.module.__file__).resolve() == Path(expected_native) + try: + connection = mssql_python.connect(os.environ["DB_CONNECTION_STRING"], timeout=5) + except mssql_python.Error as failure: + raise AssertionError( + f"Connection failed: {type(failure).__name__}; connection details withheld" + ) from None + with connection, pytest.MonkeyPatch.context() as monkeypatch: + _check_converter_row_type_mutation(connection, method, "__new__", monkeypatch) + + +def _check_converter_row_type_mutation(connection, method, hook, monkeypatch): + cursor_module = importlib.import_module("mssql_python.cursor") + builder = cursor_module._CONVERTER_ROW_BUILDER + assert builder.matches(Row) + observed = [] + original = getattr(Row, hook) + + if hook == "__init__": + + def replacement(self, values, *args, **kwargs): + observed.append(list(values)) + original(self, values, *args, **kwargs) + + elif hook == "__new__": + + def replacement(cls, *args, **kwargs): + observed.append(list(args[0])) + return original(cls) + + replacement = staticmethod(replacement) + elif hook == "__getattribute__": + + def replacement(self, name): + observed.append(name) + return original(self, name) + + else: + + def replacement(self, name, value): + observed.append(name) + original(self, name, value) + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, lambda raw: raw) + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10))") + with monkeypatch.context() as change: + change.setattr(Row, hook, replacement) + assert not builder.matches(Row) + assert _fetch(cursor, method)[0][0] == b"a\x00b\x00c\x00" + assert observed + assert builder.matches(Row) + + +@pytest.mark.parametrize("method", ("fetchmany", "fetchall")) +def test_converter_changed_helper_between_rows_uses_python_fallback( + connection, method, monkeypatch +): + calls = [] + original_apply = Row._apply_output_converters_optimized + + def replacement(self, values, converters): + calls.append(list(values)) + return original_apply(self, values, converters) + + def converter(raw): + monkeypatch.setattr(Row, "_apply_output_converters_optimized", replacement) + return raw + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, converter) + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10)) FROM (VALUES (1), (2)) AS v(n)") + assert [row[0] for row in _fetch(cursor, method)] == [b"a\x00b\x00c\x00"] * 2 + assert calls == [["abc"]] + + +@pytest.mark.parametrize("method", METHODS) +def test_converter_native_helper_override_is_not_bypassed(connection, method): + connection.add_output_converter(mssql_python.SQL_WVARCHAR, lambda raw: raw) + calls = [] + original_apply = ddbc_bindings.apply_output_converters + + def replacement(values, converters): + calls.append(list(values)) + return original_apply(values, converters) + + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10))") + with patch.object(ddbc_bindings, "apply_output_converters", replacement): + assert _fetch(cursor, method)[0][0] == b"a\x00b\x00c\x00" + assert calls == [["abc"]] + + +@pytest.mark.parametrize("method", METHODS) +def test_converter_changed_function_code_invalidates_guard(connection, method, monkeypatch): + cursor_module = importlib.import_module("mssql_python.cursor") + builder = cursor_module._CONVERTER_ROW_BUILDER + assert builder.matches(Row) + + def replacement(self, values, converters): + return ["patched"] * len(values) + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, lambda raw: raw) + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10))") + with monkeypatch.context() as change: + change.setattr(Row._apply_output_converters_optimized, "__code__", replacement.__code__) + assert not builder.matches(Row) + assert _fetch(cursor, method)[0][0] == "patched" + assert builder.matches(Row) + + +def test_converter_builder_releases_python_references(): + tracked = (Row, ddbc_bindings, Row.__init__, Row._apply_output_converters_optimized) + before = [sys.getrefcount(value) for value in tracked] + for _ in range(10): + builder = ddbc_bindings._ConverterRowBuilder(Row, ddbc_bindings) + assert builder.matches(Row) + del builder + after = [sys.getrefcount(value) for value in tracked] + assert after == before + + +def test_converter_builder_rejects_unrelated_initializers(): + class CustomRow(Row): + pass + + with pytest.raises(TypeError, match="canonical Row"): + ddbc_bindings._ConverterRowBuilder(CustomRow, ddbc_bindings) + with pytest.raises(TypeError, match="Row type"): + ddbc_bindings._ConverterRowBuilder(None, ddbc_bindings) + + +@pytest.mark.parametrize("method", ("fetchmany", "fetchall")) +def test_converter_column_names_are_read_between_callbacks(connection, method): + names = ("replacement",) + calls = [] + with connection.cursor() as cursor: + + def converter(raw): + calls.append(raw) + cursor._cached_result_columns = names + return raw + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, converter) + cursor.execute( + "SELECT CAST(N'abc' AS NVARCHAR(10)) AS original FROM (VALUES (1), (2)) AS v(n)" + ) + rows = _fetch(cursor, method) + assert rows[0]._column_names == ("original",) + assert rows[0]._column_names is not names + assert rows[1]._column_names is names + assert calls == [b"a\x00b\x00c\x00"] * 2 + + +@pytest.mark.parametrize("method", METHODS) +def test_converter_removed_during_fetch_keeps_no_converter_row_path(connection, method): + bridge_name = { + "fetchone": "DDBCSQLFetchOne", + "fetchmany": "DDBCSQLFetchMany", + "fetchall": "DDBCSQLFetchAll", + }[method] + bridge = getattr(ddbc_bindings, bridge_name) + connection.add_output_converter(mssql_python.SQL_WVARCHAR, lambda raw: raw) + + def remove_converter(*args): + status = bridge(*args) + if args[1]: + connection.remove_output_converter(mssql_python.SQL_WVARCHAR) + return status + + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10))") + with ( + patch.object(ddbc_bindings, bridge_name, remove_converter), + patch.object(Row, "_fast_create", wraps=Row._fast_create) as fast_one, + patch.object( + ddbc_bindings, "construct_rows", wraps=ddbc_bindings.construct_rows + ) as fast_many, + ): + assert _fetch(cursor, method)[0][0] == "abc" + assert fast_one.call_count == int(method == "fetchone") + expected_batches = {"fetchone": [], "fetchmany": [1, 0], "fetchall": [1]} + assert [len(call.args[0]) for call in fast_many.call_args_list] == expected_batches[method] + + +@pytest.mark.parametrize("method", ("fetchmany", "fetchall")) +def test_converter_row_field_descriptor_failure_releases_partial_batch( + connection, method, monkeypatch +): + import gc + import weakref + + references = [] + descriptor = inspect.getattr_static(Row, "_column_map") + + class FailingColumnMap: + def __get__(self, instance, owner): + return descriptor.__get__(instance, owner) + + def __set__(self, instance, value): + references.append(weakref.ref(instance)) + if len(references) == 2: + raise ValueError("injected column-map failure") + descriptor.__set__(instance, value) + + connection.add_output_converter(mssql_python.SQL_WVARCHAR, lambda raw: raw) + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10)) FROM (VALUES (1), (2), (3)) AS v(n)") + with monkeypatch.context() as change: + change.setattr(Row, "_column_map", FailingColumnMap()) + with pytest.raises(ValueError, match="injected column-map failure"): + _fetch(cursor, method) + gc.collect() + assert len(references) == 2 + assert all(reference() is None for reference in references) + cursor.execute("SELECT CAST(N'reuse' AS NVARCHAR(10))") + assert cursor.fetchone()[0] == "reuse".encode("utf-16-le") + + +@pytest.mark.parametrize("method", METHODS) +def test_converter_bridge_retained_inputs_are_not_mutated(connection, method): + bridge_name = { + "fetchone": "DDBCSQLFetchOne", + "fetchmany": "DDBCSQLFetchMany", + "fetchall": "DDBCSQLFetchAll", + }[method] + bridge = getattr(ddbc_bindings, bridge_name) + retained = [] + connection.add_output_converter(mssql_python.SQL_WVARCHAR, lambda raw: "converted") + + def observe(*args): + status = bridge(*args) + if method == "fetchone": + if args[1]: + retained.append(args[1]) + else: + retained.extend(args[1]) + return status + + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(N'abc' AS NVARCHAR(10)) FROM (VALUES (1), (2)) AS v(n)") + with patch.object(ddbc_bindings, bridge_name, observe): + assert [row[0] for row in _fetch(cursor, method)] == ["converted"] * 2 + assert len(retained) == 2 + for values in retained: + assert type(values[0]).__name__ == "PyCapsule" + assert ddbc_bindings.restore_output_converter_text(values) == ["abc"] + + +@pytest.mark.parametrize("method", METHODS) +def test_no_converter_does_not_defer_text(connection, method): + with connection.cursor() as cursor: + cursor.execute("SELECT CAST(? AS NVARCHAR(100))", [TEXT]) + with patch.object( + ddbc_bindings, + "apply_output_converters", + side_effect=AssertionError("unexpected converter dispatch"), + ): + assert _fetch(cursor, method)[0][0] == TEXT + + +@pytest.mark.parametrize("method", METHODS) +def test_converter_respects_narrow_decoding_settings(connection, method): + observed = [] + connection.setdecoding(mssql_python.SQL_CHAR, encoding="latin-1") + connection.setencoding(encoding="utf-8") + connection.add_output_converter(str, lambda raw: observed.append(raw) or raw) + with connection.cursor() as cursor: + cursor.execute("SELECT CONVERT(VARCHAR(1), 0xE9)") + row = _fetch(cursor, method)[0] + assert observed == ["\u00e9".encode("utf-16-le")] + assert row[0] == observed[0] + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("raw_utf16", ("0x00D8", "0x00DC", "0x00D84100")) +def test_malformed_text_retains_legacy_fetch_behavior(connection, method, raw_utf16): + outcomes = [] + for native in (False, True): + calls = [] + connection.add_output_converter( + mssql_python.SQL_WVARCHAR, lambda raw: calls.append(raw) or raw + ) + with connection.cursor() as cursor: + cursor.execute(f"SELECT CAST({raw_utf16} AS NVARCHAR(10))") + with patch("mssql_python.cursor._native_converter_row", return_value=native): + try: + values = [tuple(row) for row in _fetch(cursor, method)] + except UnicodeError as error: + values = type(error) + outcomes.append((values, calls)) + assert outcomes[0] == outcomes[1] + + +@pytest.mark.parametrize( + ("values_type", "converters_type"), + ((list, list), (tuple, list), (list, tuple)), + ids=("native-lists", "tuple-values", "tuple-converters"), +) +def test_native_converter_preserves_input_and_exception_semantics(values_type, converters_type): + expected = ["abc", b"\x01", 42, None, "\ud800"] + values = values_type(expected) + calls = [] + + def converter(value): + calls.append(value) + raise OverflowError("keep original") + + row = Row(values, {}, converter_map=converters_type([converter] * len(values))) + assert row._values == expected + assert row._values is not values + assert calls == [b"a\x00b\x00c\x00", b"\x01", 42] + assert values == values_type(expected) + + +@pytest.mark.parametrize("result", (None, []), ids=("none", "empty-list")) +def test_native_converter_preserves_falsey_results(result): + assert Row(["abc"], {}, converter_map=[lambda raw: result])[0] is result + + +def test_native_converter_preserves_falsey_and_null_truth_checks(): + events = [] + + class FalseConverter: + def __bool__(self): + events.append("bool") + return False + + def __call__(self, value): + pytest.fail("falsey converter was called") + + converter = FalseConverter() + assert Row(["abc", None], {}, converter_map=[converter, converter])._values == ["abc", None] + assert events == ["bool", "bool"] + + class RaisingConverter: + def __bool__(self): + raise ValueError("converter truth failure") + + with pytest.raises(ValueError, match="converter truth failure"): + Row([None], {}, converter_map=[RaisingConverter()]) + + +def test_native_converter_preserves_string_subclass_encoding(): + class Text(str): + def encode(self, encoding): + assert encoding == "utf-16-le" + return b"custom" + + assert Row([Text("abc")], {}, converter_map=[lambda raw: raw])[0] == b"custom" + + +def test_native_converter_does_not_swallow_baseexception(): + def converter(raw): + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + Row(["abc"], {}, converter_map=[converter]) + + +def test_native_converter_list_mutation_keeps_original_fallback_snapshot(): + values = ["abc", "def"] + converters = [] + + def first(raw): + if len(values) == 2: + values[1] = "changed" + values.append("new") + converters.append(first) + return raw + + def failing(raw): + raise ValueError("original fallback") + + converters.extend((first, failing)) + assert Row(values, {}, converter_map=converters)._values == [b"a\x00b\x00c\x00", "def"] + + +def test_native_converter_reference_cleanup_in_subprocess(): + source = """ +import gc +import sys +from mssql_python.row import Row + +class Value: + pass + +value = Value() +def converter(raw): + return value + +refs = sys.getrefcount(value) +for _ in range(1000): + row = Row(["abc", None, 42], {}, converter_map=[converter] * 3) + assert row[0] is value and row[1] is None and row[2] is value + del row +gc.collect() +assert sys.getrefcount(value) == refs +""" + result = subprocess.run( + [sys.executable, "-c", source], capture_output=True, text=True, timeout=30 + ) + assert result.returncode == 0, result.stdout + result.stderr