Skip to content

Commit bde98b6

Browse files
jonahkaganjonahkagan
authored
feat(coverage): thresholds to support maximum uncovered items (#7061)
1 parent 5f8d209 commit bde98b6

File tree

5 files changed

+122
-30
lines changed

5 files changed

+122
-30
lines changed

‎docs/config/index.md‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1481,7 +1481,26 @@ Do not show files with 100% statement, branch, and function coverage.
14811481

14821482
#### coverage.thresholds
14831483

1484-
Options for coverage thresholds
1484+
Options for coverage thresholds.
1485+
1486+
If a threshold is set to a positive number, it will be interpreted as the minimum percentage of coverage required. For example, setting the lines threshold to `90` means that 90% of lines must be covered.
1487+
1488+
If a threshold is set to a negative number, it will be treated as the maximum number of uncovered items allowed. For example, setting the lines threshold to `-10` means that no more than 10 lines may be uncovered.
1489+
1490+
<!-- eslint-skip -->
1491+
```ts
1492+
{
1493+
coverage: {
1494+
thresholds: {
1495+
// Requires 90% function coverage
1496+
functions: 90,
1497+
1498+
// Require that no more than 10 lines are uncovered
1499+
lines: -10,
1500+
}
1501+
}
1502+
}
1503+
```
14851504

14861505
##### coverage.thresholds.lines
14871506

@@ -1490,7 +1509,6 @@ Options for coverage thresholds
14901509
- **CLI:** `--coverage.thresholds.lines=<number>`
14911510

14921511
Global threshold for lines.
1493-
See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information.
14941512

14951513
##### coverage.thresholds.functions
14961514

@@ -1499,7 +1517,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol
14991517
- **CLI:** `--coverage.thresholds.functions=<number>`
15001518

15011519
Global threshold for functions.
1502-
See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information.
15031520

15041521
##### coverage.thresholds.branches
15051522

@@ -1508,7 +1525,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol
15081525
- **CLI:** `--coverage.thresholds.branches=<number>`
15091526

15101527
Global threshold for branches.
1511-
See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information.
15121528

15131529
##### coverage.thresholds.statements
15141530

@@ -1517,7 +1533,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol
15171533
- **CLI:** `--coverage.thresholds.statements=<number>`
15181534

15191535
Global threshold for statements.
1520-
See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information.
15211536

15221537
##### coverage.thresholds.perFile
15231538

@@ -1535,7 +1550,7 @@ Check thresholds per file.
15351550
- **Available for providers:** `'v8' | 'istanbul'`
15361551
- **CLI:** `--coverage.thresholds.autoUpdate=<boolean>`
15371552

1538-
Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds.
1553+
Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is better than the configured thresholds.
15391554
This option helps to maintain thresholds when coverage is improved.
15401555

15411556
##### coverage.thresholds.100

‎packages/vitest/src/utils/coverage.ts‎

Lines changed: 56 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -363,25 +363,54 @@ export class BaseCoverageProvider<Options extends ResolvedCoverageOptions<'istan
363363
for (const thresholdKey of THRESHOLD_KEYS) {
364364
const threshold = thresholds[thresholdKey]
365365

366-
if (threshold !== undefined) {
366+
if (threshold === undefined) {
367+
continue
368+
}
369+
370+
/**
371+
* Positive thresholds are treated as minimum coverage percentages (X means: X% of lines must be covered),
372+
* while negative thresholds are treated as maximum uncovered counts (-X means: X lines may be uncovered).
373+
*/
374+
if (threshold >= 0) {
367375
const coverage = summary.data[thresholdKey].pct
368376

369377
if (coverage < threshold) {
370378
process.exitCode = 1
371379

372-
/*
380+
/**
373381
* Generate error message based on perFile flag:
374382
* - ERROR: Coverage for statements (33.33%) does not meet threshold (85%) for src/math.ts
375383
* - ERROR: Coverage for statements (50%) does not meet global threshold (85%)
376384
*/
377-
let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${
378-
name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"`
385+
let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"`
379386
} threshold (${threshold}%)`
380387

381388
if (this.options.thresholds?.perFile && file) {
382389
errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}`
383390
}
384391

392+
this.ctx.logger.error(errorMessage)
393+
}
394+
}
395+
else {
396+
const uncovered = summary.data[thresholdKey].total - summary.data[thresholdKey].covered
397+
const absoluteThreshold = threshold * -1
398+
399+
if (uncovered > absoluteThreshold) {
400+
process.exitCode = 1
401+
402+
/**
403+
* Generate error message based on perFile flag:
404+
* - ERROR: Uncovered statements (33) exceed threshold (30) for src/math.ts
405+
* - ERROR: Uncovered statements (33) exceed global threshold (30)
406+
*/
407+
let errorMessage = `ERROR: Uncovered ${thresholdKey} (${uncovered}) exceed ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"`
408+
} threshold (${absoluteThreshold})`
409+
410+
if (this.options.thresholds?.perFile && file) {
411+
errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}`
412+
}
413+
385414
this.ctx.logger.error(errorMessage)
386415
}
387416
}
@@ -416,12 +445,30 @@ export class BaseCoverageProvider<Options extends ResolvedCoverageOptions<'istan
416445

417446
for (const key of THRESHOLD_KEYS) {
418447
const threshold = thresholds[key] ?? 100
419-
const actual = Math.min(
420-
...summaries.map(summary => summary[key].pct),
421-
)
448+
/**
449+
* Positive thresholds are treated as minimum coverage percentages (X means: X% of lines must be covered),
450+
* while negative thresholds are treated as maximum uncovered counts (-X means: X lines may be uncovered).
451+
*/
452+
if (threshold >= 0) {
453+
const actual = Math.min(
454+
...summaries.map(summary => summary[key].pct),
455+
)
422456

423-
if (actual > threshold) {
424-
thresholdsToUpdate.push([key, actual])
457+
if (actual > threshold) {
458+
thresholdsToUpdate.push([key, actual])
459+
}
460+
}
461+
else {
462+
const absoluteThreshold = threshold * -1
463+
const actual = Math.max(
464+
...summaries.map(summary => summary[key].total - summary[key].covered),
465+
)
466+
467+
if (actual < absoluteThreshold) {
468+
// If everything was covered, set new threshold to 100% (since a threshold of 0 would be considered as 0%)
469+
const updatedThreshold = actual === 0 ? 100 : actual * -1
470+
thresholdsToUpdate.push([key, updatedThreshold])
471+
}
425472
}
426473
}
427474

‎test/coverage-test/fixtures/configs/vitest.config.thresholds-auto-update.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,14 @@ export default defineConfig({
99
// Global ones
1010
lines: 0.1,
1111
functions: 0.2,
12-
branches: 0.3,
13-
statements: 0.4,
12+
branches: -1000,
13+
statements: -2000,
1414

1515
'**/src/math.ts': {
1616
branches: 0.1,
1717
functions: 0.2,
18-
lines: 0.3,
19-
statements: 0.4
18+
lines: -1000,
19+
statements: -2000,
2020
}
2121
}
2222
}

‎test/coverage-test/test/threshold-auto-update.test.ts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ test('thresholds.autoUpdate updates thresholds', async () => {
2020
// Global ones
2121
lines: 0.1,
2222
functions: 0.2,
23-
branches: 0.3,
24-
statements: 0.4,
23+
branches: -1000,
24+
statements: -2000,
2525
2626
'**/src/math.ts': {
2727
branches: 0.1,
2828
functions: 0.2,
29-
lines: 0.3,
30-
statements: 0.4
29+
lines: -1000,
30+
statements: -2000,
3131
}
3232
}
3333
}
@@ -56,13 +56,13 @@ test('thresholds.autoUpdate updates thresholds', async () => {
5656
lines: 55.55,
5757
functions: 33.33,
5858
branches: 100,
59-
statements: 55.55,
59+
statements: -8,
6060
6161
'**/src/math.ts': {
6262
branches: 100,
6363
functions: 25,
64-
lines: 50,
65-
statements: 50
64+
lines: -6,
65+
statements: -6,
6666
}
6767
}
6868
}
@@ -84,13 +84,13 @@ test('thresholds.autoUpdate updates thresholds', async () => {
8484
lines: 33.33,
8585
functions: 33.33,
8686
branches: 100,
87-
statements: 33.33,
87+
statements: -4,
8888
8989
'**/src/math.ts': {
9090
branches: 100,
9191
functions: 25,
92-
lines: 25,
93-
statements: 25
92+
lines: -3,
93+
statements: -3,
9494
}
9595
}
9696
}

‎test/coverage-test/test/threshold-failure.test.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { expect } from 'vitest'
22
import { sum } from '../fixtures/src/math'
33
import { coverageTest, isV8Provider, normalizeURL, runVitest, test } from '../utils'
44

5-
test('failing thresholds', async () => {
5+
test('failing percentage thresholds', async () => {
66
const { exitCode, stderr } = await runVitest({
77
include: [normalizeURL(import.meta.url)],
88
coverage: {
@@ -28,6 +28,36 @@ test('failing thresholds', async () => {
2828
expect(stderr).toContain('ERROR: Coverage for functions (25%) does not meet "**/fixtures/src/math.ts" threshold (100%)')
2929
})
3030

31+
test('failing absolute thresholds', async () => {
32+
const { exitCode, stderr } = await runVitest({
33+
include: [normalizeURL(import.meta.url)],
34+
coverage: {
35+
all: false,
36+
include: ['**/fixtures/src/math.ts'],
37+
thresholds: {
38+
'**/fixtures/src/math.ts': {
39+
branches: -1,
40+
functions: -2,
41+
lines: -5,
42+
statements: -1,
43+
},
44+
},
45+
},
46+
}, { throwOnError: false })
47+
48+
expect(exitCode).toBe(1)
49+
50+
if (isV8Provider()) {
51+
expect(stderr).toContain('ERROR: Uncovered lines (6) exceed "**/fixtures/src/math.ts" threshold (5)')
52+
expect(stderr).toContain('ERROR: Uncovered functions (3) exceed "**/fixtures/src/math.ts" threshold (2)')
53+
expect(stderr).toContain('ERROR: Uncovered statements (6) exceed "**/fixtures/src/math.ts" threshold (1)')
54+
}
55+
else {
56+
expect(stderr).toContain('ERROR: Uncovered functions (3) exceed "**/fixtures/src/math.ts" threshold (2)')
57+
expect(stderr).toContain('ERROR: Uncovered statements (3) exceed "**/fixtures/src/math.ts" threshold (1)')
58+
}
59+
})
60+
3161
coverageTest('cover some lines, but not too much', () => {
3262
expect(sum(1, 2)).toBe(3)
3363
})

0 commit comments

Comments
 (0)