{% macro render_analysis_mode(project_name) %} {% set strict = project_strictness[project_name] %} {{ 'strict' if strict else 'non-strict' }} {% endmacro %} {% macro render_diagnostic(diag, change_type, project_name) %}
:{{ diag.line }}:{{ diag.column }}
[{{ diag.level }}] {{ diag.lint_name }} - {{ diag.message|e }}
{% endmacro %} {% macro render_flaky_location(loc, change_type, project_name) %}
:{{ loc.line }}:{{ loc.column }}
flaky
{% for v in loc.variants %}
({{ v.count }}/{{ loc.flaky_runs }})
[{{ v.diagnostic.level }}] {{ v.diagnostic.lint_name }} - {{ v.diagnostic.message|e }}
{% endfor %}
{% endmacro %} {% macro render_flaky_changed(change, project_name) %}
:{{ change.old.line }}:{{ change.old.column }}
flaky
old
({{ change.old.flaky_runs }} runs)
{% for v in change.old.variants %}
({{ v.count }}/{{ change.old.flaky_runs }})
[{{ v.diagnostic.level }}] {{ v.diagnostic.lint_name }} - {{ v.diagnostic.message|e }}
{% endfor %}
new
({{ change.new.flaky_runs }} runs)
{% for v in change.new.variants %}
({{ v.count }}/{{ change.new.flaky_runs }})
[{{ v.diagnostic.level }}] {{ v.diagnostic.lint_name }} - {{ v.diagnostic.message|e }}
{% endfor %}
{% endmacro %} {% macro render_exit_statuses(statuses, runs, summarized_panic_messages=()) %} {% if not statuses %} not present {% endif %} {% for status in statuses %}
({{ status.count }}/{{ runs }}) {% if status.return_code is none %} timeout {% else %} exit code {{ status.return_code }} {% endif %}
{% set summarized_messages = namespace(remaining=summarized_panic_messages | list) %} {% for variant in status.panic_messages %} {% if variant.count == status.count and variant.message in summarized_messages.remaining %} {% set _ = summarized_messages.remaining.remove(variant.message) %} {% else %}
panic ({{ variant.count }}/{{ runs }} runs) 
{{ variant.message | e }}
{% endif %} {% endfor %} {% for variant in status.get("stderr", []) %}
stderr ({{ variant.count }}/{{ runs }} runs) 
{{ variant.message | e }}
{% endfor %} {% endfor %} {% endmacro %}

Ecosystem report

Old: {{ old_branch_info }} {{ old_commit[:8] }} → New: {{ new_branch_info }} {{ new_commit[:8] }}
{% if statistics.merged_by_lint or statistics.merged_by_project %}
{% if statistics.merged_by_lint %}
{% for lint_data in statistics.merged_by_lint %} {% endfor %}
Lint Rule Removed Added Changed
Total {{ statistics.total_removed }} {{ statistics.total_added }} {{ statistics.total_changed }}
{{ lint_data.lint_name }} {{ lint_data.removed }} {{ lint_data.added }} {{ lint_data.changed }}
{% endif %} {% if statistics.merged_by_project %}
{% for project_data in statistics.merged_by_project[:20] %} {% endfor %} {% if statistics.merged_by_project|length > 20 %} {% for project_data in statistics.merged_by_project[20:] %} {% endfor %} {% endif %}
Project Removed Added Changed
Total {{ statistics.total_removed }} {{ statistics.total_added }} {{ statistics.total_changed }}
{{ project_data.project_name }}{{ render_analysis_mode(project_data.project_name) }}{% if project_data.is_flaky %} flaky{% endif %} {{ project_data.removed }} {{ project_data.added }} {{ project_data.changed }}
{% endif %}
{% endif %}
{% if project_kinds %} {% endif %}
{% if diffs.failed_projects %} {% set new_failure_projects = diffs.failed_projects | selectattr('failure_status', 'equalto', 'new') | list %} {% set new_panic_projects = diffs.failed_projects | selectattr('failure_status', 'equalto', 'new_panics') | list %} {% set fixed_failure_projects = diffs.failed_projects | selectattr('failure_status', 'equalto', 'fixed') | list %} {% if new_failure_projects %}
❌ New ecosystem failure{{ 's' if new_failure_projects|length != 1 else '' }} introduced in: {% for project in new_failure_projects %}{{ project.project }} ({{ failure_descriptor(project, 'new') }}){{ ", " if not loop.last }}{% endfor %}
{% endif %} {% if new_panic_projects %}
❌ Panic regression{{ 's' if new_panic_projects|length != 1 else '' }} introduced in: {% for project in new_panic_projects %}{{ project.project }} ({{ project.introduced_panic_messages|length }} new panic message{{ 's' if project.introduced_panic_messages|length != 1 else '' }}){{ ", " if not loop.last }}{% endfor %}
{% endif %} {% if fixed_failure_projects %}
🎉 Fixed ecosystem failure{{ 's' if fixed_failure_projects|length != 1 else '' }} in: {% for project in fixed_failure_projects %}{{ project.project }} ({{ failure_descriptor(project, 'fixed') }}){{ ", " if not loop.last }}{% endfor %}
{% endif %}

Failed Projects

{% for project in diffs.failed_projects %} {% set failure_status = project.failure_status %} {% set old_summarized_panic_messages = project.fixed_panic_messages + project.old_persistent_panic_messages %} {% set new_summarized_panic_messages = project.introduced_panic_messages + project.new_persistent_panic_messages %} {% if project.old_panic_messages or project.new_panic_messages %} {% endif %} {% endfor %}
Project Status Old Status New Status Old Outcomes New Outcomes
{{ project.project }}{{ render_analysis_mode(project.project) }}
{{ project.project_location }} 		{{ failure_status_labels[failure_status] }} {% if project.old_status == 'timeout' %} timeout {% elif project.old_status == 'abnormal exit' %} abnormal exit {% else %} success {% endif %} {% if project.new_status == 'timeout' %} timeout {% elif project.new_status == 'abnormal exit' %} abnormal exit {% else %} success {% endif %} {{ render_exit_statuses(project.old_exit_statuses, project.old_runs, summarized_panic_messages=old_summarized_panic_messages) }} {{ render_exit_statuses(project.new_exit_statuses, project.new_runs, summarized_panic_messages=new_summarized_panic_messages) }}
{% if project.introduced_panic_messages %}
❌ New panic message{{ 's' if project.introduced_panic_messages|length != 1 }} (introduced by this PR) 
{{ project.introduced_panic_messages | join('\n') | e }}
{% endif %} {% if project.fixed_panic_messages %} {% set fixed_panics_are_improvement = failure_status in ['fixed', 'reduced'] %}
{{ '🎉 Fixed' if fixed_panics_are_improvement else '➖ Previous' }} panic message{{ 's' if project.fixed_panic_messages|length != 1 }} (no longer present) 
{{ project.fixed_panic_messages | join('\n') | e }}
{% endif %} {% if project.persistent_panic_messages %}
➖ Persistent panic message{{ 's' if project.persistent_panic_messages|length != 1 }} (present on both baseline and PR) {% if project.old_persistent_panic_messages != project.new_persistent_panic_messages %} Baseline 
{{ project.old_persistent_panic_messages | join('\n') | e }}
PR 
{{ project.new_persistent_panic_messages | join('\n') | e }}
{% else %} 
{{ project.persistent_panic_messages | join('\n') | e }}
{% endif %}
{% endif %}
{% endif %} {% if diffs.flaky_exit_status_changes %}

Flaky Exit Status Changes

{% for project in diffs.flaky_exit_status_changes %} {% endfor %}
Project Baseline outcomes PR outcomes
{{ project.project }}{{ render_analysis_mode(project.project) }} flaky
{{ project.project_location }} 		{{ render_exit_statuses(project.old, project.old_runs) }} {{ render_exit_statuses(project.new, project.new_runs) }}
{% endif %} {% if diffs.added_projects %}

Added Projects

{% for project in diffs.added_projects %}

{{ project.project }}{{ render_analysis_mode(project.project) }} ({{ project.project_location }})

Observed outcomes: {{ render_exit_statuses(project.exit_statuses, project.exit_status_runs) }}
{% set file_groups = project.diagnostics|groupby('path') %} {% for path, diagnostics in file_groups %}

{{ path }}

{% for diag in diagnostics %} {{ render_diagnostic(diag, "added", project.project) }} {% endfor %}
{% endfor %}
{% endfor %} {% endif %} {% if diffs.removed_projects %}

Removed Projects

{% for project in diffs.removed_projects %}

{{ project.project }}{{ render_analysis_mode(project.project) }} ({{ project.project_location }})

Observed outcomes: {{ render_exit_statuses(project.exit_statuses, project.exit_status_runs) }}
{% set file_groups = project.diagnostics|groupby('path') %} {% for path, diagnostics in file_groups %}

{{ path }}

{% for diag in diagnostics %} {{ render_diagnostic(diag, "removed", project.project) }} {% endfor %}
{% endfor %}
{% endfor %} {% endif %} {% if diffs.modified_projects %} {% for project in diffs.modified_projects %}

{{ project.project }}{{ render_analysis_mode(project.project) }} ({{ project.project_location }})

{% if project.diffs.added_files %} {% for file_data in project.diffs.added_files %}

{{ file_data.path }}

{% for diag in file_data.diagnostics %} {{ render_diagnostic(diag, "added", project.project) }} {% endfor %} {# Flaky locations in this added file #} {% if project.flaky_file_diffs is defined and file_data.path in project.flaky_file_diffs %} {% for loc in project.flaky_file_diffs[file_data.path].added %} {{ render_flaky_location(loc, "added", project.project) }} {% endfor %} {% endif %}
{% endfor %} {% endif %} {% if project.diffs.removed_files %} {% for file_data in project.diffs.removed_files %}

{{ file_data.path }}

{% for diag in file_data.diagnostics %} {{ render_diagnostic(diag, "removed", project.project) }} {% endfor %} {# Flaky locations in this removed file #} {% if project.flaky_file_diffs is defined and file_data.path in project.flaky_file_diffs %} {% for loc in project.flaky_file_diffs[file_data.path].removed %} {{ render_flaky_location(loc, "removed", project.project) }} {% endfor %} {% endif %}
{% endfor %} {% endif %} {% if project.diffs.modified_files %} {% for file_data in project.diffs.modified_files %}

{{ file_data.path }}

{% if file_data.diffs.added_lines %} {% for line_data in file_data.diffs.added_lines %} {% for diag in line_data.diagnostics %} {{ render_diagnostic(diag, "added", project.project) }} {% endfor %} {% endfor %} {% endif %} {% if file_data.diffs.removed_lines %} {% for line_data in file_data.diffs.removed_lines %} {% for diag in line_data.diagnostics %} {{ render_diagnostic(diag, "removed", project.project) }} {% endfor %} {% endfor %} {% endif %} {% if file_data.diffs.modified_lines %} {% for line_data in file_data.diffs.modified_lines %} {% for diff_item in line_data.text_diffs %}
:{{ diff_item.old.line }}:{{ diff_item.old.column }}
{% if diff_item.old.message == diff_item.new.message %}
[{{ diff_item.old.level }}] {{ diff_item.old.lint_name }} - {{ diff_item.old.message|e }}
Old column: {{ diff_item.old.column }} → New column: {{ diff_item.new.column }}
{% else %}
[{{ diff_item.old.level }}] {{ diff_item.old.lint_name }} - {{ diff_item.old.message|e }}
[{{ diff_item.new.level }}] {{ diff_item.new.lint_name }} - {{ diff_item.new.message|e }}
{% endif %}
{% endfor %} {% for diag in line_data.removed %} {{ render_diagnostic(diag, "removed", project.project) }} {% endfor %} {% for diag in line_data.added %} {{ render_diagnostic(diag, "added", project.project) }} {% endfor %} {% endfor %} {% endif %} {# Flaky locations in this modified file #} {% if project.flaky_file_diffs is defined and file_data.path in project.flaky_file_diffs %} {% set ffd = project.flaky_file_diffs[file_data.path] %} {% for loc in ffd.added %} {{ render_flaky_location(loc, "added", project.project) }} {% endfor %} {% for loc in ffd.removed %} {{ render_flaky_location(loc, "removed", project.project) }} {% endfor %} {% for change in ffd.changed %} {{ render_flaky_changed(change, project.project) }} {% endfor %} {% endif %}
{% endfor %} {% endif %} {# Files that only have flaky changes (no stable diagnostic changes) #} {% if project.flaky_file_diffs is defined %} {% set stable_files = [] %} {% for f in project.diffs.added_files %}{% if stable_files.append(f.path) %}{% endif %}{% endfor %} {% for f in project.diffs.removed_files %}{% if stable_files.append(f.path) %}{% endif %}{% endfor %} {% for f in project.diffs.modified_files %}{% if stable_files.append(f.path) %}{% endif %}{% endfor %} {% for flaky_path, ffd in project.flaky_file_diffs.items()|sort %} {% if flaky_path not in stable_files %}

{{ flaky_path }}

{% for loc in ffd.added %} {{ render_flaky_location(loc, "added", project.project) }} {% endfor %} {% for loc in ffd.removed %} {{ render_flaky_location(loc, "removed", project.project) }} {% endfor %} {% for change in ffd.changed %} {{ render_flaky_changed(change, project.project) }} {% endfor %}
{% endif %} {% endfor %} {% endif %}
{% endfor %} {% endif %}