From c2a065e670877bdb0d3268487a5ff9b22ed583dd Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 15 Oct 2026 14:20:31 +0200 Subject: [PATCH 1/2] test(core): parse optimization status regardless of forced color With FORCE_COLOR set, as in an interactive terminal, console.log colors the numbers it prints, so the `done ` regex never matched and both optimization tests failed. Print a single interpolated string, which is never colored. Co-Authored-By: Claude --- packages/core/tests/optimization.integ.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/tests/optimization.integ.test.ts b/packages/core/tests/optimization.integ.test.ts index e6437210..9be36aca 100644 --- a/packages/core/tests/optimization.integ.test.ts +++ b/packages/core/tests/optimization.integ.test.ts @@ -21,7 +21,7 @@ describe("optimization helpers", () => { `const target = () => 1; const before = %GetOptimizationStatus(target); require(${corePath}).optimizeFunctionSync(target); - console.log("done", before, %GetOptimizationStatus(target));`, + console.log(\`done \${before} \${%GetOptimizationStatus(target)}\`);`, ); const [, before, after] = stdout.match(/done (\d+) (\d+)/) ?? []; expect(stdout).toContain("done"); @@ -34,7 +34,7 @@ describe("optimization helpers", () => { const before = %GetOptimizationStatus(target); require(${corePath}) .optimizeFunction(target) - .then(() => console.log("done", before, %GetOptimizationStatus(target)));`, + .then(() => console.log(\`done \${before} \${%GetOptimizationStatus(target)}\`));`, ); const [, before, after] = stdout.match(/done (\d+) (\d+)/) ?? []; expect(stdout).toContain("done"); From b9435120ce6dbe0f6aa55c20844c04041b0adb9e Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 15 Oct 2026 14:20:34 +0200 Subject: [PATCH 2/2] fix(vitest-plugin)!: disable vitest 5 export getter tracking under codspeed Vitest 5 wraps every module export getter with a diagnostic counter (a Map lookup and write per access) unless benchmark.suppressExportGetterWarnings is set. That counter runs inside the measured code, inflating results: a bench reading an export 100k times measures 109M instructions on Vitest 5 versus 38M on Vitest 4. Default the option to true when the plugin is active, keeping any value the user sets explicitly. Plain `vitest bench` runs are unaffected. With this, the same bench measures the same instruction count on Vitest 4 and 5. Drop the import hoisting from the benches and examples: it only worked around the tracker. Closes COD-3892 Closes #98 Co-Authored-By: Claude --- .../with-typescript-esm/src/fibonacci.bench.ts | 6 +----- examples/with-vitest-v5/src/fibonacci.bench.ts | 10 +--------- packages/vitest-plugin/benches/flat.bench.ts | 6 +----- packages/vitest-plugin/benches/parsePr.bench.ts | 6 +----- packages/vitest-plugin/src/__tests__/index.test.ts | 14 ++++++++++++-- packages/vitest-plugin/src/index.ts | 3 ++- packages/vitest-plugin/src/vitestBackend.ts | 11 ++++++++++- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/examples/with-typescript-esm/src/fibonacci.bench.ts b/examples/with-typescript-esm/src/fibonacci.bench.ts index 4c00d98c..c1ad321d 100644 --- a/examples/with-typescript-esm/src/fibonacci.bench.ts +++ b/examples/with-typescript-esm/src/fibonacci.bench.ts @@ -1,9 +1,5 @@ import { describe, test } from "vitest"; -import { recursiveFibonacci as recursiveFibonacciExport } from "./fibonacci"; - -// Imports are getters under Vite's module runner; read them once, outside the -// measured loop. -const recursiveFibonacci = recursiveFibonacciExport; +import { recursiveFibonacci } from "./fibonacci"; describe("recursiveFibonacci", () => { test("fibo 30", async ({ bench }) => { diff --git a/examples/with-vitest-v5/src/fibonacci.bench.ts b/examples/with-vitest-v5/src/fibonacci.bench.ts index 5108c589..b8b596c5 100644 --- a/examples/with-vitest-v5/src/fibonacci.bench.ts +++ b/examples/with-vitest-v5/src/fibonacci.bench.ts @@ -1,13 +1,5 @@ import { describe, test } from "vitest"; -import { - iterativeFibonacci as iterativeFibonacciExport, - recursiveFibonacci as recursiveFibonacciExport, -} from "./fibonacci"; - -// Imports are getters under Vite's module runner; read them once, outside the -// measured loop. -const recursiveFibonacci = recursiveFibonacciExport; -const iterativeFibonacci = iterativeFibonacciExport; +import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci"; describe("fibonacci", () => { test("fibo 15", async ({ bench }) => { diff --git a/packages/vitest-plugin/benches/flat.bench.ts b/packages/vitest-plugin/benches/flat.bench.ts index dd409924..0251df28 100644 --- a/packages/vitest-plugin/benches/flat.bench.ts +++ b/packages/vitest-plugin/benches/flat.bench.ts @@ -1,9 +1,5 @@ import { describe, test } from "vitest"; -import parsePrExport from "./parsePr"; - -// Imports are getters under Vite's module runner; read it once, outside the -// measured loop. -const parsePr = parsePrExport; +import parsePr from "./parsePr"; const LONG_BODY = new Array(1_000) diff --git a/packages/vitest-plugin/benches/parsePr.bench.ts b/packages/vitest-plugin/benches/parsePr.bench.ts index 4f078785..e9da3915 100644 --- a/packages/vitest-plugin/benches/parsePr.bench.ts +++ b/packages/vitest-plugin/benches/parsePr.bench.ts @@ -1,9 +1,5 @@ import { describe, test } from "vitest"; -import parsePrExport from "./parsePr"; - -// Imports are getters under Vite's module runner; read it once, outside the -// measured loop. -const parsePr = parsePrExport; +import parsePr from "./parsePr"; const LONG_BODY = new Array(1_000) diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index 5c691280..d7459fea 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -1,6 +1,7 @@ import { getV8Flags } from "@codspeed/core"; import { fromPartial } from "@total-typescript/shoehorn"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { type ViteUserConfig } from "vitest/config"; import codspeedPlugin from "../index"; const coreMocks = vi.hoisted(() => { @@ -34,10 +35,10 @@ function apply(mode: string) { return plugin.apply({}, fromPartial({ mode })); } -function config() { +function config(userConfig: ViteUserConfig = {}) { if (typeof plugin.config !== "function") throw new Error("config is not a function"); - return plugin.config.call({} as never, {}, fromPartial({})); + return plugin.config.call({} as never, userConfig, fromPartial({})); } describe("codSpeedPlugin", () => { @@ -148,9 +149,18 @@ describe("codSpeedPlugin", () => { provider: expect.stringContaining( "packages/vitest-plugin/src/v5/provider.ts", ), + suppressExportGetterWarnings: true, }, }, }); }); + + it("should keep an explicit suppressExportGetterWarnings from the user", () => { + const result = config({ + test: { benchmark: { suppressExportGetterWarnings: false } }, + }) as ViteUserConfig; + + expect(result.test?.benchmark?.suppressExportGetterWarnings).toBe(false); + }); }); }); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 171d9e86..cb3af1ae 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -37,7 +37,7 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(): ViteUserConfig { + config(userConfig): ViteUserConfig { return { test: { pool: "forks", @@ -45,6 +45,7 @@ export default function codspeedPlugin(): Plugin { ...resolveVitestBackend().getBenchmarkTestConfig( getV8Flags(), resolveFile, + (userConfig as ViteUserConfig).test, ), }, }; diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index b702e4d1..46c171d7 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -10,6 +10,7 @@ export interface VitestBackend { getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, + userTestConfig: ViteUserConfig["test"], ): ViteUserConfig["test"]; } @@ -36,10 +37,18 @@ class V5Backend implements VitestBackend { getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, + userTestConfig: ViteUserConfig["test"], ): ViteUserConfig["test"] { + // Vitest's export getter tracker runs inside the measured code; keep it off + // unless the user explicitly configured it. + const suppressExportGetterWarnings = + userTestConfig?.benchmark?.suppressExportGetterWarnings ?? true; return { execArgv: v8Flags, - benchmark: { provider: resolveFile("v5/provider") }, + benchmark: { + provider: resolveFile("v5/provider"), + suppressExportGetterWarnings, + }, }; } }