Skip to content

Commit 7efe327

Browse files
yangw-devyangw-dev
committed
add workflow to dispatch
ghstack-source-id: 633724d Pull-Request: #161585
1 parent 1f0cd3e commit 7efe327

File tree

9 files changed

+229
-31
lines changed

9 files changed

+229
-31
lines changed

‎.ci/lumen_cli/cli/lib/common/gh_summary.py‎

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
from __future__ import annotations
22

33
import logging
4-
import os
54
import textwrap
5+
import xml.etree.ElementTree as ET
66
from pathlib import Path
77
from typing import TYPE_CHECKING
88

9+
from cli.lib.common.envs_helper import get_env
910
from cli.lib.common.utils import get_wheels
1011
from jinja2 import Template
1112

@@ -16,6 +17,23 @@
1617

1718
logger = logging.getLogger(__name__)
1819

20+
21+
# ---- Template (title + per-command failures) ----
22+
_TPL_FAIL_BY_CMD = Template(
23+
textwrap.dedent("""\
24+
## {{ title }}
25+
26+
{%- for section in sections if section.failures %}
27+
### Test Command: {{ section.label }}
28+
29+
{%- for f in section.failures %}
30+
- {{ f }}
31+
{%- endfor %}
32+
33+
{%- endfor %}
34+
""")
35+
)
36+
1937
_TPL_CONTENT = Template(
2038
textwrap.dedent("""\
2139
## {{ title }}
@@ -53,8 +71,17 @@
5371

5472

5573
def gh_summary_path() -> Path | None:
56-
"""Return the Path to the GitHub step summary file, or None if not set."""
57-
p = os.environ.get("GITHUB_STEP_SUMMARY")
74+
"""Return the Path to the GitHub step summary file,
75+
if TEMP_GITHUB_STEP_SUMMARY is set, use that instead,
76+
this happens when run jobs in docker container
77+
the github flow need to make sure to output the summary to github step summary after
78+
"""
79+
p = get_env("GITHUB_STEP_SUMMARY")
80+
overrides = get_env("TEMP_GITHUB_STEP_SUMMARY")
81+
if overrides:
82+
p = overrides
83+
if not p or not Path(p).exists():
84+
return None
5885
return Path(p) if p else None
5986

6087

@@ -141,3 +168,62 @@ def render_content(
141168
tpl = _TPL_CONTENT
142169
md = tpl.render(title=title, content=content, lang=lang)
143170
return md
171+
172+
173+
def summarize_failures_by_test_command(
174+
xml_and_labels: Iterable[tuple[str | Path, str]],
175+
*,
176+
title: str = "Pytest Failures by Test Command",
177+
dedupe_within_command: bool = True,
178+
) -> bool:
179+
"""
180+
Render a single Markdown block summarizing failures grouped by test command.
181+
Returns True if anything was written, False otherwise.
182+
"""
183+
sections: list[dict] = []
184+
185+
for xml_path, label in xml_and_labels:
186+
xmlp = Path(xml_path)
187+
if not xmlp.exists():
188+
logger.warning("XML %s not found, skipping", xmlp)
189+
continue
190+
failed = _parse_failed(xmlp)
191+
if dedupe_within_command:
192+
failed = sorted(set(failed))
193+
194+
# collect even if empty; we'll filter in the template render
195+
sections.append({"label": label, "failures": failed})
196+
197+
# If *all* sections are empty or we collected nothing, skip writing.
198+
if not sections or all(not s["failures"] for s in sections):
199+
return False
200+
201+
md = _TPL_FAIL_BY_CMD.render(title=title, sections=sections).rstrip() + "\n"
202+
return write_gh_step_summary(md)
203+
204+
205+
def _to_name_from_testcase(tc: ET.Element) -> str:
206+
name = tc.attrib.get("name", "")
207+
file_attr = tc.attrib.get("file")
208+
if file_attr:
209+
return f"{file_attr}:{name}"
210+
211+
classname = tc.attrib.get("classname", "")
212+
parts = classname.split(".") if classname else []
213+
if len(parts) >= 1:
214+
mod_parts = parts[:-1] if len(parts) >= 2 else parts
215+
mod_path = "/".join(mod_parts) + ".py" if mod_parts else "unknown.py"
216+
return f"{mod_path}:{name}"
217+
return f"unknown.py:{name or 'unknown_test'}"
218+
219+
220+
def _parse_failed(xml_path: Path) -> list[str]:
221+
if not xml_path.exists():
222+
return []
223+
tree = ET.parse(xml_path)
224+
root = tree.getroot()
225+
failed: list[str] = []
226+
for tc in root.iter("testcase"):
227+
if any(x.tag in {"failure", "error"} for x in tc):
228+
failed.append(_to_name_from_testcase(tc))
229+
return failed

‎.ci/lumen_cli/cli/lib/common/path_helper.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,17 @@ def get_path(path: Union[str, Path], resolve: bool = False) -> Path:
1717
return result.resolve() if resolve else result
1818

1919

20-
def ensure_dir_exists(path: Union[str, Path]) -> Path:
21-
"""Create directory if it doesn't exist."""
22-
path_obj = get_path(path)
23-
path_obj.mkdir(parents=True, exist_ok=True)
24-
return path_obj
20+
def ensure_path(path: Union[str, Path], is_file: bool = False) -> Path:
21+
"""Ensure directory or file exists.
22+
If is_file=True, create parent dirs and touch the file.
23+
"""
24+
p = Path(path)
25+
if is_file:
26+
p.parent.mkdir(parents=True, exist_ok=True)
27+
p.touch(exist_ok=True)
28+
else:
29+
p.mkdir(parents=True, exist_ok=True)
30+
return p
2531

2632

2733
def remove_dir(path: Union[str, Path, None]) -> None:
@@ -36,7 +42,7 @@ def remove_dir(path: Union[str, Path, None]) -> None:
3642
def force_create_dir(path: Union[str, Path]) -> Path:
3743
"""Remove directory if exists, then create fresh empty directory."""
3844
remove_dir(path)
39-
return ensure_dir_exists(path)
45+
return ensure_path(path)
4046

4147

4248
def copy(src: Union[str, Path], dst: Union[str, Path]) -> None:

‎.ci/lumen_cli/cli/lib/common/utils.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,16 @@
44

55
import logging
66
import os
7+
import secrets
78
import shlex
89
import subprocess
910
import sys
1011
from contextlib import contextmanager
1112
from pathlib import Path
1213
from typing import Optional
1314

15+
from cli.lib.common.path_helper import ensure_path
16+
1417

1518
logger = logging.getLogger(__name__)
1619

@@ -137,3 +140,42 @@ def get_wheels(
137140
relpath = str((Path(dirpath) / fname).relative_to(root))
138141
items.append({"pkg": pkg, "relpath": relpath})
139142
return items
143+
144+
145+
def attach_junitxml_if_pytest(
146+
cmd: str,
147+
dir: Optional[Path],
148+
prefix: str,
149+
*,
150+
ensure_unique: bool = False,
151+
resolve_xml: bool = False,
152+
) -> tuple[str, Optional[Path]]:
153+
"""
154+
Append --junitxml=<ABS_PATH> to a pytest command string.
155+
The XML filename is <prefix>_<random-hex>.xml.
156+
157+
- dir: target folder (will be created), if None, skip the junitxml attachment
158+
- prefix: filename prefix (e.g., "junit" -> junit_ab12cd34.xml)
159+
- ensure_unique: if True, regenerate a hash with 8 characters
160+
161+
Returns: (amended_cmd, abs_xml_path)
162+
"""
163+
if "pytest" not in cmd:
164+
return cmd, None
165+
if dir is None:
166+
return cmd, None
167+
ensure_path(dir)
168+
169+
file_name_prefix = f"{prefix}"
170+
if ensure_unique:
171+
file_name_prefix += f"_{unique_hex(8)}"
172+
xml_path = dir / f"{file_name_prefix}_junit_pytest.xml"
173+
if resolve_xml:
174+
xml_path = xml_path.resolve()
175+
176+
return f"{cmd} --junitxml={xml_path.as_posix()}", xml_path
177+
178+
179+
def unique_hex(length: int = 8) -> str:
180+
"""Return a random hex string of `length` characters."""
181+
return secrets.token_hex((length + 1) // 2)[:length]

‎.ci/lumen_cli/cli/lib/core/vllm/lib.py‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
import logging
22
import os
33
import textwrap
4-
from typing import Any
4+
from pathlib import Path
5+
from typing import Any, Optional
56

67
from cli.lib.common.gh_summary import write_gh_step_summary
78
from cli.lib.common.git_helper import clone_external_repo
89
from cli.lib.common.pip_helper import pip_install_packages
9-
from cli.lib.common.utils import run_command, temp_environ, working_directory
10+
from cli.lib.common.utils import (
11+
attach_junitxml_if_pytest,
12+
run_command,
13+
temp_environ,
14+
working_directory,
15+
)
1016
from jinja2 import Template
1117

1218

@@ -186,6 +192,9 @@ def run_test_plan(
186192
tests_map: dict[str, Any],
187193
shard_id: int = 0,
188194
num_shards: int = 0,
195+
*,
196+
test_summary_path: Optional[Path] = None,
197+
test_summary_result: Optional[list[tuple[str, str]]] = None,
189198
):
190199
"""
191200
a method to run list of tests based on the test plan.
@@ -198,7 +207,6 @@ def run_test_plan(
198207
tests = tests_map[test_plan]
199208
pkgs = tests.get("package_install", [])
200209
title = tests.get("title", "unknown test")
201-
202210
is_parallel = check_parallelism(tests, title, shard_id, num_shards)
203211
if is_parallel:
204212
title = title.replace("%N", f"{shard_id}/{num_shards}")
@@ -212,7 +220,15 @@ def run_test_plan(
212220
temp_environ(tests.get("env_vars", {})),
213221
):
214222
failures = []
215-
for step in tests["steps"]:
223+
for idx, step in enumerate(tests["steps"]):
224+
# generate xml report for each test for test summary if needed
225+
step, xml_file_path = attach_junitxml_if_pytest(
226+
cmd=step, dir=test_summary_path, prefix=f"{test_plan}_{idx}"
227+
)
228+
if xml_file_path and xml_file_path.exists() and test_summary_result:
229+
test_summary_result.append((title, str(xml_file_path)))
230+
else:
231+
logger.info("No test report will be generate for %s", step)
216232
logger.info("Running step: %s", step)
217233
if is_parallel:
218234
step = replace_buildkite_placeholders(step, shard_id, num_shards)

‎.ci/lumen_cli/cli/lib/core/vllm/vllm_build.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
)
2121
from cli.lib.common.path_helper import (
2222
copy,
23-
ensure_dir_exists,
23+
ensure_path,
2424
force_create_dir,
2525
get_path,
2626
is_path_exist,
@@ -165,7 +165,7 @@ def run(self):
165165
self.cp_torch_whls_if_exist(inputs)
166166

167167
# make sure the output dir to store the build artifacts exist
168-
ensure_dir_exists(Path(inputs.output_dir))
168+
ensure_path(Path(inputs.output_dir))
169169

170170
cmd = self._generate_docker_build_cmd(inputs)
171171
logger.info("Running docker build: \n %s", cmd)

‎.ci/lumen_cli/cli/lib/core/vllm/vllm_test.py‎

Lines changed: 47 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,24 @@
1111

1212
from cli.lib.common.cli_helper import BaseRunner
1313
from cli.lib.common.envs_helper import env_path_field, env_str_field, get_env
14+
from cli.lib.common.gh_summary import (
15+
gh_summary_path,
16+
summarize_failures_by_test_command,
17+
)
1418
from cli.lib.common.path_helper import copy, remove_dir
1519
from cli.lib.common.pip_helper import (
1620
pip_install_first_match,
1721
pip_install_packages,
1822
pkg_exists,
1923
run_python,
2024
)
21-
from cli.lib.common.utils import run_command, working_directory
22-
from cli.lib.core.vllm.lib import clone_vllm, run_test_plan, sample_vllm_test_library
25+
from cli.lib.common.utils import ensure_path, run_command, working_directory
26+
from cli.lib.core.vllm.lib import (
27+
clone_vllm,
28+
run_test_plan,
29+
sample_vllm_test_library,
30+
summarize_build_info,
31+
)
2332

2433

2534
logger = logging.getLogger(__name__)
@@ -41,13 +50,20 @@ class VllmTestParameters:
4150
"VLLM_WHEELS_PATH", "./dist/external/vllm/wheels"
4251
)
4352

53+
# generate a file to store test summary
54+
test_summary_file: Path = env_path_field(
55+
"TEMP_GITHUB_STEP_SUMMARY", "./generated_step_summary.md"
56+
)
57+
4458
torch_cuda_arch_list: str = env_str_field("TORCH_CUDA_ARCH_LIST", "8.9")
4559

4660
def __post_init__(self):
4761
if not self.torch_whls_path.exists():
4862
raise ValueError("missing torch_whls_path")
4963
if not self.vllm_whls_path.exists():
5064
raise ValueError("missing vllm_whls_path")
65+
if self.test_summary_file:
66+
ensure_path(self.test_summary_file, is_file=True)
5167

5268

5369
class TestInpuType(Enum):
@@ -91,33 +107,54 @@ def prepare(self):
91107
logger.info("Display VllmTestParameters %s", params)
92108
self._set_envs(params)
93109

94-
clone_vllm(dst=self.work_directory)
110+
vllm_commit = clone_vllm(dst=self.work_directory)
95111
with working_directory(self.work_directory):
96112
remove_dir(Path("vllm"))
97113
self._install_wheels(params)
98114
self._install_dependencies()
99115
# verify the torches are not overridden by test dependencies
100116
check_versions()
117+
return vllm_commit
101118

102119
def run(self):
103120
"""
104121
main function to run vllm test
105122
"""
106-
self.prepare()
107-
with working_directory(self.work_directory):
108-
if self.test_type == TestInpuType.TEST_PLAN:
109-
if self.num_shards > 1:
123+
vllm_commit = self.prepare()
124+
125+
# prepare test summary
126+
test_summary_path = Path("tmp_pytest_report").resolve()
127+
ensure_path(test_summary_path)
128+
test_summary_result = []
129+
130+
try:
131+
with working_directory(self.work_directory):
132+
if self.test_type == TestInpuType.TEST_PLAN:
110133
run_test_plan(
111134
self.test_plan,
112135
"vllm",
113136
sample_vllm_test_library(),
114137
self.shard_id,
115138
self.num_shards,
139+
test_summary_path=test_summary_path,
140+
test_summary_result=test_summary_result,
116141
)
117142
else:
118-
run_test_plan(self.test_plan, "vllm", sample_vllm_test_library())
119-
else:
120-
raise ValueError(f"Unknown test type {self.test_type}")
143+
raise ValueError(f"Unknown test type {self.test_type}")
144+
except Exception as e:
145+
logger.error("Failed to run vllm test: %s", e)
146+
raise e
147+
finally:
148+
self.vllm_test_gh_summary(vllm_commit, test_summary_result)
149+
150+
def vllm_test_gh_summary(
151+
self, vllm_commit: str, test_summary_results: list[tuple[str, str]]
152+
):
153+
if not gh_summary_path():
154+
return logger.info("Skipping, not detect GH Summary env var....")
155+
logger.info("Generate GH Summary ...")
156+
summarize_build_info(vllm_commit)
157+
summarize_failures_by_test_command(test_summary_results)
121158

122159
def _install_wheels(self, params: VllmTestParameters):
123160
logger.info("Running vllm test with inputs: %s", params)

0 commit comments

Comments
 (0)