{# Copyright 2025 Google LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #}{% extends 'base.html' %} {% import 'macros.html' as macros %} {% block content %} {% if index_css_content %} {% endif %} {% if index_js_content %} {% endif %}
{% for benchmark in benchmarks %} {% endfor %}
Benchmark Status Build Rate Crash Rate Bugs Program Counter Coverage Line Coverage Diff
{{ loop.index }}
{{ benchmark.project }}
                                    {{ benchmark.signature }}
                                
{{ benchmark.status }} {{ benchmark.result.build_success_rate|percent}} {{ benchmark.result.crash_rate|percent }} {{ benchmark.result.found_bug }} {{ benchmark.result.max_coverage |percent }} {{ benchmark.result.max_line_coverage_diff|percent }}
{{ macros.expand_collapse_buttons('project-summary-expand-all', 'project-summary-collapse-all') }}
{% set ns_projects = namespace( sum_successful_benchmarks=0, sum_total_coverage_gain=0, sum_total_relative_gain=0, sum_ofg_total_covered_lines=0, sum_ofg_new_covered_lines=0, sum_existing_covered_lines=0, sum_total_project_lines=0) %} {% for project in projects %} {% set ns_projects.sum_successful_benchmarks = ns_projects.sum_successful_benchmarks + project.count %} {% set ns_projects.sum_ofg_total_covered_lines = ns_projects.sum_ofg_total_covered_lines + project.coverage_ofg_total_covered_lines %} {% set ns_projects.sum_ofg_new_covered_lines = ns_projects.sum_ofg_new_covered_lines + project.coverage_ofg_total_new_covered_lines %} {% set ns_projects.sum_existing_covered_lines = ns_projects.sum_existing_covered_lines + project.coverage_existing_total_covered_lines %} {% set ns_projects.sum_total_project_lines = ns_projects.sum_total_project_lines + project.coverage_existing_total_lines %} {% endfor %} {% if ns_projects.sum_total_project_lines > 0 %} {% set ns_projects.sum_total_coverage_gain = ns_projects.sum_ofg_new_covered_lines / ns_projects.sum_total_project_lines %} {% else %} {% set ns_projects.sum_total_coverage_gain = 0 %} {% endif %} {% if ns_projects.sum_existing_covered_lines > 0 %} {% set ns_projects.sum_total_relative_gain = ns_projects.sum_ofg_new_covered_lines / ns_projects.sum_existing_covered_lines %} {% else %} {% set ns_projects.sum_total_relative_gain = 0 %} {% endif %}
Project Total benchmarks Successful benchmarks Total coverage gain Total relative gain OSS-Fuzz-Gen total covered lines OSS-Fuzz-Gen new covered lines Existing covered lines Total project lines
{{ project.count }} {{ project.success }} {{ project.coverage_gain|percent }}% {{ project.coverage_relative_gain | percent }}% {{project.coverage_ofg_total_covered_lines }} {{ project.coverage_ofg_total_new_covered_lines }} {{ project.coverage_existing_total_covered_lines }} {{ project.coverage_existing_total_lines }}
Total {{ accumulated_results.total_runs }} {{ ns_projects.sum_successful_benchmarks }} {{ ns_projects.sum_total_coverage_gain|percent }}% {{ ns_projects.sum_total_relative_gain|percent }}% {{ ns_projects.sum_ofg_total_covered_lines }} {{ ns_projects.sum_ofg_new_covered_lines }} {{ ns_projects.sum_existing_covered_lines }} {{ ns_projects.sum_total_project_lines }}
{{ macros.expand_collapse_buttons('crashes-expand-all', 'crashes-collapse-all') }} {% set projects_with_crashes = {} %} {% for bug_sample in samples_with_bugs %} {% if bug_sample.benchmark.project not in projects_with_crashes %} {% set _ = projects_with_crashes.update({bug_sample.benchmark.project: {}}) %} {% endif %} {% if bug_sample.benchmark.id not in projects_with_crashes[bug_sample.benchmark.project] %} {% set _ = projects_with_crashes[bug_sample.benchmark.project].update({bug_sample.benchmark.id: []}) %} {% endif %} {% set _ = projects_with_crashes[bug_sample.benchmark.project][bug_sample.benchmark.id].append(bug_sample) %} {% endfor %} {% set ns_projects_with_crashes = namespace( sum_total_crashes=0, sum_total_benchmarks=0, sum_true_positives=0, sum_false_positives=0) %} {% for project_name, project_benchmarks in projects_with_crashes.items() %} {% set project_loop = loop %} {% set project_bugs = [] %} {% for benchmark_bugs in project_benchmarks.values() %} {% set _ = project_bugs.extend(benchmark_bugs) %} {% endfor %} {% set ns_projects_with_crashes.sum_total_crashes = ns_projects_with_crashes.sum_total_crashes + project_bugs|length %} {% set ns_projects_with_crashes.sum_total_benchmarks = ns_projects_with_crashes.sum_total_benchmarks + project_benchmarks|length %} {% set ns_projects_with_crashes.sum_true_positives = ns_projects_with_crashes.sum_true_positives + project_bugs|selectattr('sample.result.is_semantic_error', 'equalto', false)|list|length %} {% set ns_projects_with_crashes.sum_false_positives = ns_projects_with_crashes.sum_false_positives + project_bugs|selectattr('sample.result.is_semantic_error', 'equalto', true)|list|length %} {% endfor %}
Project Total crashes Total benchmarks True positives False positives
{{ project_bugs|length }} {{ project_benchmarks|length }} {{ project_bugs|selectattr('sample.result.is_semantic_error', 'equalto', false)|list|length }} {{ project_bugs|selectattr('sample.result.is_semantic_error', 'equalto', true)|list|length }}
Total {{ ns_projects_with_crashes.sum_total_crashes }} {{ ns_projects_with_crashes.sum_total_benchmarks }} {{ ns_projects_with_crashes.sum_true_positives }} {{ ns_projects_with_crashes.sum_false_positives }}

Per Project

{% set ns_proj = namespace(total_cycles=0, total_duration=0, count=0) %} {% for project_name, pdata in unified_data.items() %} {% set cycles = (pdata.get('average_cycles_per_benchmark') or 0) | float %} {% set duration = (pdata.get('average_trial_duration_sec') or 0) | float %} {% set ns_proj.total_cycles = ns_proj.total_cycles + cycles %} {% set ns_proj.total_duration = ns_proj.total_duration + duration %} {% set ns_proj.count = ns_proj.count + 1 %} {% endfor %} {% if ns_proj.count > 0 %} {% endif %}
Project Avg cycles/benchmark Avg trial duration (min)
{{ project_name }} {{ cycles }} {{ (duration / 60) | round(2) }}
Average {{ (ns_proj.total_cycles / ns_proj.count) | round(2) }} {{ ((ns_proj.total_duration / ns_proj.count) / 60) | round(2) }}

Per Benchmark

{% set ns_bench = namespace(total_cycles=0, total_duration=0, count=0) %} {% for project_name, pdata in unified_data.items() %} {% if pdata.benchmarks %} {% for bench_id, bdata in pdata.benchmarks.items() %} {% set cycles = (bdata.get('avg_cycles_per_sample') or 0) | float %} {% set duration = (bdata.get('avg_trial_duration_sec') or 0) | float %} {% set ns_bench.total_cycles = ns_bench.total_cycles + cycles %} {% set ns_bench.total_duration = ns_bench.total_duration + duration %} {% set ns_bench.count = ns_bench.count + 1 %} {% endfor %} {% endif %} {% endfor %} {% if ns_bench.count > 0 %} {% endif %}
Project Benchmark Average cycles/sample Average trial duration (min)
{{ project_name }}
                                                {{ bench_id }}
                                            
{{ cycles }} {{ (duration / 60) | round(2) }}
Average - {{ (ns_bench.total_cycles / ns_bench.count) | round(2) }} {{ ((ns_bench.total_duration / ns_bench.count) / 60) | round(2) }}

Per Sample

{% set ns_sample = namespace(total_cycles=0, total_duration=0, count=0) %} {% for project_name, pdata in unified_data.items() %} {% if pdata.benchmarks %} {% for bench_id, bdata in pdata.benchmarks.items() %} {% if bdata.samples %} {% for s in bdata.samples %} {% set cycles = (s.get('cycles_count') or 0) | int %} {% set duration = (s.get('trial_avg_duration_sec') or 0) | float %} {% set ns_sample.total_cycles = ns_sample.total_cycles + cycles %} {% set ns_sample.total_duration = ns_sample.total_duration + duration %} {% set ns_sample.count = ns_sample.count + 1 %} {% endfor %} {% endif %} {% endfor %} {% endif %} {% endfor %} {% if ns_sample.count > 0 %} {% endif %}
Project Benchmark Sample Cycles Trial duration (min)
{{ project_name }}
                                                        {{ bench_id }}
                                                    
                                                        Sample {{ s.sample }}
                                                    
{{ cycles }} {{ (duration / 60) | round(2) }}
Average - - {{ (ns_sample.total_cycles / ns_sample.count) | round(2) }} {{ ((ns_sample.total_duration / ns_sample.count) / 60) | round(2) }}
{% for language in coverage_language_gains['coverage_gains_per_language'] %} {% endfor %}
Language OSS-Fuzz total lines OSS-Fuzz coverage lines Experiment new coverage lines Increase of total Increase of covered
{{language}} {{ coverage_language_gains['oss_fuzz_language_status'].get(language, {}).get('total', 'N/A') }} {{ coverage_language_gains['oss_fuzz_language_status'].get(language, {}).get('covered', 'N/A') }} {{ coverage_language_gains['coverage_gains_per_language'].get(language, 'N/A') }} {{ coverage_language_gains['comperative_coverage_gains'].get(language, {}).get('total_coverage_increase', 'N/A') }}% {{ coverage_language_gains['comperative_coverage_gains'].get(language, {}).get('relative_coverage_increase', 'N/A') }}%

Table of Contents

{% endblock %}