{
  "annotations": [
    {
      "kind": "AnnotationQuery",
      "spec": {
        "builtIn": true,
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "query": {
          "datasource": {
            "name": "-- Grafana --"
          },
          "group": "grafana",
          "kind": "DataQuery",
          "spec": {},
          "version": "v0"
        }
      }
    }
  ],
  "cursorSync": "Crosshair",
  "editable": true,
  "elements": {
    "panel-1": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "app": "grafana-assistant-app",
                      "expr": "sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"})",
                      "instant": false,
                      "legendFormat": "{{cluster}}",
                      "queryType": "range",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "The current number of workflow jobs assigned by the GitHub Actions service to ARC scale sets, by cluster.",
        "id": 1,
        "links": [],
        "title": "Assigned Jobs [cluster, scale_set]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic-by-name",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": false,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.9,
                  "drawStyle": "line",
                  "fillOpacity": 0,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "off"
                  }
                },
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": 0
                    }
                  ]
                }
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-2": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "app": "grafana-assistant-app",
                      "expr": "(100 * (1 - (sum by(cluster) (gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}) / sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"})))) and on(cluster) (sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}) > 0)",
                      "instant": false,
                      "legendFormat": "{{cluster}}",
                      "queryType": "range",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Percent of jobs assigned to the scale set that are not yet running on a runner.",
        "id": 2,
        "links": [],
        "title": "% of Assigned Jobs Not Running [cluster, scale_set]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic-by-name",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": false,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.9,
                  "drawStyle": "line",
                  "fillOpacity": 0,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "dashed+area"
                  }
                },
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "rgba(255, 255, 255, 0)",
                      "value": 0
                    },
                    {
                      "color": "#EAB839",
                      "value": 10
                    },
                    {
                      "color": "red",
                      "value": 20
                    }
                  ]
                },
                "unit": "percent"
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-9": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "app": "grafana-assistant-app",
                      "expr": "(100 * clamp_min(sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}) - sum by(cluster) (gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}) - 600 * max_over_time((clamp_min(sum by(cluster) (deriv(gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}[5m])) + sum by(cluster) (rate(gha_completed_jobs_total{cluster=~\"$cluster\", name=~\"$scale_set\"}[5m])), 0))[28d:5m]), 0) / sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"})) and on(cluster) (sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}) > 0)",
                      "instant": false,
                      "legendFormat": "{{cluster}}",
                      "queryType": "range",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Percent of assigned jobs that, based on historical peak runner start rate over the last 28 days, should already be running by now. A 10-minute grace window is applied: jobs that fit within what the system could start in 10 minutes at peak observed throughput are not counted. Reads near 0% when the system is ramping at expected speed; climbs when jobs are accumulating faster than capacity has historically allowed.",
        "id": 9,
        "links": [],
        "title": "Startup Throughput Adjusted % of Assigned Jobs Not Running [cluster, scale_set]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic-by-name",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": false,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.9,
                  "drawStyle": "line",
                  "fillOpacity": 0,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "dashed+area"
                  }
                },
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "rgba(255, 255, 255, 0)",
                      "value": 0
                    },
                    {
                      "color": "#EAB839",
                      "value": 10
                    },
                    {
                      "color": "red",
                      "value": 20
                    }
                  ]
                },
                "unit": "percent"
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-3": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "app": "grafana-assistant-app",
                      "expr": "sum by(cluster) (gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"})",
                      "instant": false,
                      "legendFormat": "{{cluster}}",
                      "queryType": "range",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "The total number of jobs running.",
        "id": 3,
        "links": [],
        "title": "Running Jobs [cluster, scale_set]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic-by-name",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": false,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.9,
                  "drawStyle": "line",
                  "fillOpacity": 0,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "off"
                  }
                },
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": 0
                    }
                  ]
                }
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-4": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "app": "grafana-assistant-app",
                      "expr": "(100 * (avg_over_time((sum by(cluster) (gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}))[3h:]) - avg_over_time((sum by(cluster) (gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"} offset 1w))[3h:])) / avg_over_time((sum by(cluster) (gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"} offset 1w))[3h:])) and on(cluster) (avg_over_time((sum by(cluster) (gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"} offset 1w))[3h:]) > 0) unless on(cluster) (label_replace(vector(1), \"cluster\", \"meta-prod-aws-uw1\", \"\", \"\") and on() (count(count by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\"})) == count(count by(cluster) (gha_assigned_jobs{cluster=~\".*arc.*production.*|meta-prod-aws-.*|lf-prod-aws-.*\"}))))",
                      "instant": false,
                      "legendFormat": "{{cluster}}",
                      "queryType": "range",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Week-over-week percentage change in the 3-hour rolling average of running jobs. meta-prod-aws-uw1 is hidden when all clusters are selected (its low traffic makes the week-over-week percentage too spiky to be useful) and reappears when an explicit cluster subset that includes it is selected.",
        "id": 4,
        "links": [],
        "title": "Running Jobs Week-over-Week % Change, 3h Avg [cluster, scale_set]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic-by-name",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": true,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.9,
                  "drawStyle": "line",
                  "fillOpacity": 0,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "dashed+area"
                  }
                },
                "decimals": 2,
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "red",
                      "value": null
                    },
                    {
                      "color": "#EAB839",
                      "value": -35
                    },
                    {
                      "color": "rgba(255, 255, 255, 0)",
                      "value": -20
                    },
                    {
                      "color": "#EAB839",
                      "value": 80
                    },
                    {
                      "color": "red",
                      "value": 100
                    }
                  ]
                },
                "unit": "percent"
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-5": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "app": "grafana-assistant-app",
                      "expr": "(100 * (avg_over_time((sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}))[3h:]) - avg_over_time((sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"} offset 1w))[3h:])) / avg_over_time((sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"} offset 1w))[3h:])) and on(cluster) (avg_over_time((sum by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"} offset 1w))[3h:]) > 0) unless on(cluster) (label_replace(vector(1), \"cluster\", \"meta-prod-aws-uw1\", \"\", \"\") and on() (count(count by(cluster) (gha_assigned_jobs{cluster=~\"$cluster\"})) == count(count by(cluster) (gha_assigned_jobs{cluster=~\".*arc.*production.*|meta-prod-aws-.*|lf-prod-aws-.*\"}))))",
                      "instant": false,
                      "legendFormat": "{{cluster}}",
                      "queryType": "range",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Week-over-week percentage change in the 3-hour rolling average of assigned jobs. meta-prod-aws-uw1 is hidden when all clusters are selected (its low traffic makes the week-over-week percentage too spiky to be useful) and reappears when an explicit cluster subset that includes it is selected.",
        "id": 5,
        "links": [],
        "title": "Assigned Jobs Week-over-Week % Change, 3h Avg [cluster, scale_set]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic-by-name",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": true,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.9,
                  "drawStyle": "line",
                  "fillOpacity": 0,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "dashed+area"
                  }
                },
                "decimals": 2,
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "red",
                      "value": null
                    },
                    {
                      "color": "#EAB839",
                      "value": -35
                    },
                    {
                      "color": "rgba(255, 255, 255, 0)",
                      "value": -20
                    },
                    {
                      "color": "#EAB839",
                      "value": 80
                    },
                    {
                      "color": "red",
                      "value": 100
                    }
                  ]
                },
                "unit": "percent"
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-6": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "builder",
                      "expr": "clamp_min(sum by(cluster, name) (gha_assigned_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}) - sum by(cluster, name) (gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}), 0)",
                      "legendFormat": "{{cluster}} / {{name}}",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Estimated jobs waiting for a runner by scale set/machine type. Calculated as ARC assigned jobs minus ARC running jobs.",
        "id": 6,
        "links": [],
        "title": "Assigned Jobs Waiting to Run by Scale Set [cluster, scale_set]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": false,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.6,
                  "drawStyle": "line",
                  "fillOpacity": 8,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "never",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "off"
                  }
                },
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": 0
                    },
                    {
                      "color": "yellow",
                      "value": 1
                    },
                    {
                      "color": "orange",
                      "value": 10
                    },
                    {
                      "color": "red",
                      "value": 50
                    }
                  ]
                }
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [
                  "lastNotNull",
                  "max"
                ],
                "displayMode": "table",
                "placement": "right",
                "showLegend": true,
                "sortBy": "Last *",
                "sortDesc": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "multi",
                "sort": "desc"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-7": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "ceczcsck1b20wb"
                    },
                    "group": "grafana-clickhouse-datasource",
                    "kind": "DataQuery",
                    "spec": {
                      "editorType": "sql",
                      "format": 0,
                      "meta": {
                        "builderOptions": {
                          "columns": [],
                          "database": "",
                          "limit": 1000,
                          "mode": "list",
                          "queryType": "table",
                          "table": ""
                        }
                      },
                      "pluginVersion": "4.17.0",
                      "queryType": "timeseries",
                      "rawSql": "SELECT\n  toStartOfInterval(created_at, INTERVAL 5 MINUTE) AS time,\n  uniqExact(id) AS jobs\nFROM default.workflow_job\nWHERE match(repository_full_name, '^(${repository:regex})$')\n  AND arrayExists(label -> match(label, '^(mt|lf)-(rel-)?l-'), labels)\n  AND (\n    (runner_group_name = 'meta-prod-aws-ue2' AND match('meta-prod-aws-ue2', '^(${cluster:regex})$'))\n    OR (runner_group_name = 'meta-prod-aws-uw1' AND match('meta-prod-aws-uw1', '^(${cluster:regex})$'))\n    OR (runner_group_name = 'meta-prod-aws-ue1' AND match('meta-prod-aws-ue1', '^(${cluster:regex})$'))\n    OR (runner_group_name = 'lf-prod-aws-ue1' AND match('lf-prod-aws-ue1', '^(${cluster:regex})$'))\n    OR (runner_group_name = 'lf-prod-aws-ue2' AND match('lf-prod-aws-ue2', '^(${cluster:regex})$'))\n  )\n  AND $__timeFilter(created_at)\nGROUP BY time\nORDER BY time ASC"
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Count of unique GitHub Actions workflow jobs picked up by OSDC production runners for the selected repositories and clusters, aggregated in 5-minute buckets.",
        "id": 7,
        "links": [],
        "title": "Jobs Submitted to OSDC Runners, per 5min [cluster, repository]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic",
                  "seriesBy": "last"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": false,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.6,
                  "drawStyle": "bars",
                  "fillOpacity": 70,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 1,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "off"
                  }
                },
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": 0
                    }
                  ]
                }
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": false
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-8": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "app": "grafana-assistant-app",
                      "expr": "clamp_min(300 * (sum by(cluster) (deriv(gha_running_jobs{cluster=~\"$cluster\", name=~\"$scale_set\"}[5m])) + sum by(cluster) (rate(gha_completed_jobs_total{cluster=~\"$cluster\", name=~\"$scale_set\"}[5m]))), 0)",
                      "instant": false,
                      "interval": "5m",
                      "legendFormat": "{{cluster}}",
                      "queryType": "range",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Number of new jobs starting to run per 5-minute bucket. Computed as the derivative of running jobs plus the completion rate (since running = started - completed), scaled to a 5-minute window. A flat or low value while jobs are queued indicates the system is not adding runners fast enough; a high value indicates active ramp-up.",
        "id": 8,
        "links": [],
        "title": "Runner Start Rate, Jobs per 5min [cluster, scale_set]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "color": {
                  "mode": "palette-classic-by-name",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisBorderShow": false,
                  "axisCenteredZero": false,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.9,
                  "drawStyle": "line",
                  "fillOpacity": 0,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "off"
                  }
                },
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": 0
                    }
                  ]
                },
                "unit": "short"
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    },
    "panel-10": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "ceczcsck1b20wb"
                    },
                    "group": "grafana-clickhouse-datasource",
                    "kind": "DataQuery",
                    "spec": {
                      "editorType": "sql",
                      "format": 0,
                      "meta": {
                        "builderOptions": {
                          "columns": [],
                          "database": "",
                          "limit": 1000,
                          "mode": "list",
                          "queryType": "table",
                          "table": ""
                        }
                      },
                      "pluginVersion": "4.17.0",
                      "queryType": "timeseries",
                      "rawSql": "SELECT\n    bucket_time AS time,\n    dateDiff('second', j_created_at, bucket_time) / 60.0 AS queue_minutes\nFROM (\n    SELECT\n        argMax(created_at, started_at) AS j_created_at,\n        argMax(status, started_at) AS j_status,\n        argMax(runner_group_name, started_at) AS j_runner_group_name,\n        argMax(labels, started_at) AS j_labels,\n        max(_inserted_at) AS j_last_insert,\n        if(argMax(status, started_at) IN ('queued', 'waiting'), now64(9), argMax(started_at, started_at)) AS j_queue_end\n    FROM default.workflow_job\n    PREWHERE created_at <= $__toTime\n        AND created_at >= $__fromTime - INTERVAL 24 HOUR\n    WHERE arrayExists(x -> startsWith(x, 'mt-l-') OR startsWith(x, 'mt-rel-l-') OR startsWith(x, 'lf-l-') OR startsWith(x, 'lf-rel-l-'), labels)\n        AND match(repository_full_name, '^(${repository:regex})$')\n    GROUP BY id\n    HAVING NOT arrayExists(x -> position(x, 'h100') > 0 OR position(x, 'b200') > 0 OR position(x, 'gb200') > 0 OR position(x, 'a100') > 0, j_labels)\n        AND arrayExists(x -> match(x, '^(${runner_name_prefix:regex})$'), j_labels)\n        AND (\n          (j_runner_group_name = 'meta-prod-aws-ue2' AND match('meta-prod-aws-ue2', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = 'meta-prod-aws-uw1' AND match('meta-prod-aws-uw1', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = 'meta-prod-aws-ue1' AND match('meta-prod-aws-ue1', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = 'lf-prod-aws-ue1' AND match('lf-prod-aws-ue1', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = 'lf-prod-aws-ue2' AND match('lf-prod-aws-ue2', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = '' AND j_status IN ('queued', 'waiting') AND j_last_insert > now() - INTERVAL 3 HOUR)\n        )\n        AND j_queue_end >= $__fromTime\n) ARRAY JOIN\n    arrayDistinct(arrayConcat(\n        arrayFilter(\n            t -> t >= j_created_at AND t >= $__fromTime AND t <= $__toTime,\n            arrayMap(x -> toDateTime64(x, 9),\n                range(\n                    toUInt32(intDiv(toUInt32(greatest(j_created_at, $__fromTime)), 300) * 300),\n                    toUInt32(intDiv(toUInt32(least(j_queue_end, $__toTime)), 300) * 300) + 300,\n                    300\n                )\n            )\n        ),\n        if(j_queue_end >= $__fromTime AND j_queue_end <= $__toTime, [j_queue_end], [])\n    )) AS bucket_time\nWHERE bucket_time <= j_queue_end\nORDER BY bucket_time"
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Per-job queue time distribution over time for OSDC production runners, excluding capacity-constrained GPU runners (h100, b200, gb200, a100). Each 5-minute bucket between a job's created_at and started_at (or now, if still queued) contributes a point at its current accumulated queue duration, so a long-queued job shows rising values across every bucket of its wait instead of a single point at start. Y-axis log scale, minutes.",
        "id": 10,
        "links": [],
        "title": "Queue Time Distribution (excl. capacity-constrained instances) [cluster, repository, runner_name_prefix]",
        "vizConfig": {
          "group": "heatmap",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "custom": {
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "scaleDistribution": {
                    "type": "linear"
                  }
                }
              },
              "overrides": []
            },
            "options": {
              "calculate": true,
              "calculation": {
                "xBuckets": {
                  "mode": "size",
                  "value": ""
                },
                "yBuckets": {
                  "mode": "count",
                  "scale": {
                    "log": 10,
                    "type": "log"
                  },
                  "value": "30"
                }
              },
              "cellGap": 0,
              "cellValues": {
                "unit": "short"
              },
              "color": {
                "exponent": 0.5,
                "fill": "#96D98D",
                "mode": "opacity",
                "reverse": false,
                "scale": "exponential",
                "steps": 64
              },
              "exemplars": {
                "color": "rgba(255,0,255,0.7)"
              },
              "filterValues": {
                "le": 1e-9
              },
              "legend": {
                "show": true
              },
              "rowsFrame": {
                "layout": "auto"
              },
              "showValue": "never",
              "tooltip": {
                "mode": "single",
                "show": true,
                "yHistogram": false
              },
              "yAxis": {
                "axisLabel": "Queue (min)",
                "axisPlacement": "left",
                "decimals": 1,
                "min": 0.01,
                "reverse": false,
                "unit": "m"
              }
            },
            "version": ""
          }
        }
      }
    },
    "panel-11": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "ceczcsck1b20wb"
                    },
                    "group": "grafana-clickhouse-datasource",
                    "kind": "DataQuery",
                    "spec": {
                      "editorType": "sql",
                      "format": 0,
                      "meta": {
                        "builderOptions": {
                          "columns": [],
                          "database": "",
                          "limit": 1000,
                          "mode": "list",
                          "queryType": "table",
                          "table": ""
                        }
                      },
                      "pluginVersion": "4.17.0",
                      "queryType": "timeseries",
                      "rawSql": "SELECT\n    bucket_time AS time,\n    dateDiff('second', j_created_at, bucket_time) / 60.0 AS queue_minutes\nFROM (\n    SELECT\n        argMax(created_at, started_at) AS j_created_at,\n        argMax(status, started_at) AS j_status,\n        argMax(runner_group_name, started_at) AS j_runner_group_name,\n        argMax(labels, started_at) AS j_labels,\n        max(_inserted_at) AS j_last_insert,\n        if(argMax(status, started_at) IN ('queued', 'waiting'), now64(9), argMax(started_at, started_at)) AS j_queue_end\n    FROM default.workflow_job\n    PREWHERE created_at <= $__toTime\n        AND created_at >= $__fromTime - INTERVAL 24 HOUR\n    WHERE arrayExists(x -> startsWith(x, 'mt-l-') OR startsWith(x, 'mt-rel-l-') OR startsWith(x, 'lf-l-') OR startsWith(x, 'lf-rel-l-'), labels)\n        AND match(repository_full_name, '^(${repository:regex})$')\n    GROUP BY id\n    HAVING arrayExists(x -> position(x, 'h100') > 0 OR position(x, 'b200') > 0 OR position(x, 'gb200') > 0 OR position(x, 'a100') > 0, j_labels)\n        AND arrayExists(x -> match(x, '^(${runner_name_prefix:regex})$'), j_labels)\n        AND (\n          (j_runner_group_name = 'meta-prod-aws-ue2' AND match('meta-prod-aws-ue2', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = 'meta-prod-aws-uw1' AND match('meta-prod-aws-uw1', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = 'meta-prod-aws-ue1' AND match('meta-prod-aws-ue1', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = 'lf-prod-aws-ue1' AND match('lf-prod-aws-ue1', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = 'lf-prod-aws-ue2' AND match('lf-prod-aws-ue2', '^(${cluster:regex})$'))\n          OR (j_runner_group_name = '' AND j_status IN ('queued', 'waiting') AND j_last_insert > now() - INTERVAL 3 HOUR)\n        )\n        AND j_queue_end >= $__fromTime\n) ARRAY JOIN\n    arrayDistinct(arrayConcat(\n        arrayFilter(\n            t -> t >= j_created_at AND t >= $__fromTime AND t <= $__toTime,\n            arrayMap(x -> toDateTime64(x, 9),\n                range(\n                    toUInt32(intDiv(toUInt32(greatest(j_created_at, $__fromTime)), 300) * 300),\n                    toUInt32(intDiv(toUInt32(least(j_queue_end, $__toTime)), 300) * 300) + 300,\n                    300\n                )\n            )\n        ),\n        if(j_queue_end >= $__fromTime AND j_queue_end <= $__toTime, [j_queue_end], [])\n    )) AS bucket_time\nWHERE bucket_time <= j_queue_end\nORDER BY bucket_time"
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Per-job queue time distribution over time for OSDC production runners, ONLY capacity-constrained GPU runners (h100, b200, gb200, a100). Each 5-minute bucket between a job's created_at and started_at (or now, if still queued) contributes a point at its current accumulated queue duration, so a long-queued job shows rising values across every bucket of its wait instead of a single point at start. Y-axis log scale, minutes.",
        "id": 11,
        "links": [],
        "title": "Queue Time Distribution (capacity-constrained instances only) [cluster, repository, runner_name_prefix]",
        "vizConfig": {
          "group": "heatmap",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "custom": {
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "scaleDistribution": {
                    "type": "linear"
                  }
                }
              },
              "overrides": []
            },
            "options": {
              "calculate": true,
              "calculation": {
                "xBuckets": {
                  "mode": "size",
                  "value": ""
                },
                "yBuckets": {
                  "mode": "count",
                  "scale": {
                    "log": 10,
                    "type": "log"
                  },
                  "value": "30"
                }
              },
              "cellGap": 0,
              "cellValues": {
                "unit": "short"
              },
              "color": {
                "exponent": 0.5,
                "fill": "#5794F2",
                "mode": "opacity",
                "reverse": false,
                "scale": "exponential",
                "steps": 64
              },
              "exemplars": {
                "color": "rgba(255,0,255,0.7)"
              },
              "filterValues": {
                "le": 1e-9
              },
              "legend": {
                "show": true
              },
              "rowsFrame": {
                "layout": "auto"
              },
              "showValue": "never",
              "tooltip": {
                "mode": "single",
                "show": true,
                "yHistogram": false
              },
              "yAxis": {
                "axisLabel": "Queue (min)",
                "axisPlacement": "left",
                "decimals": 1,
                "min": 0.01,
                "reverse": false,
                "unit": "m"
              }
            },
            "version": ""
          }
        }
      }
    },
    "panel-12": {
      "kind": "Panel",
      "spec": {
        "data": {
          "kind": "QueryGroup",
          "spec": {
            "queries": [
              {
                "kind": "PanelQuery",
                "spec": {
                  "hidden": false,
                  "query": {
                    "datasource": {
                      "name": "grafanacloud-prom"
                    },
                    "group": "prometheus",
                    "kind": "DataQuery",
                    "spec": {
                      "expr": "count by(cluster) (kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"hf-cache\", container=\"rclone\", reason=\"OOMKilled\"} == 1) or vector(0)",
                      "instant": false,
                      "legendFormat": "{{cluster}}",
                      "queryType": "range",
                      "range": true
                    },
                    "version": "v0"
                  },
                  "refId": "A"
                }
              }
            ],
            "queryOptions": {},
            "transformations": []
          }
        },
        "description": "Count of hf-cache rclone mounts whose container's LAST termination was OOMKilled (kube_pod_container_status_last_terminated_reason). This is a sticky signal: it stays set after the pod recovers and only clears when the pod is replaced (node recycle / rollout) — so it counts mounts that OOM'd recently, INCLUDING ones already back to Running. It is an upper bound / early-warning indicator, NOT a live 'currently broken' count (that would need a per-pod not-ready or waiting-reason metric, which is not currently shipped to Mimir for the hf-cache namespace). A sustained rise still means the small-tier OOM is active and jobs on those nodes can fail to start.",
        "id": 12,
        "links": [],
        "title": "hf-cache Mounts OOM-killed — recent, incl. recovered [cluster]",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
                "decimals": 0,
                "color": {
                  "mode": "palette-classic-by-name",
                  "seriesBy": "max"
                },
                "custom": {
                  "axisSoftMax": 5,
                  "axisBorderShow": false,
                  "axisCenteredZero": false,
                  "axisColorMode": "text",
                  "axisLabel": "",
                  "axisPlacement": "auto",
                  "barAlignment": 0,
                  "barWidthFactor": 0.9,
                  "drawStyle": "line",
                  "fillOpacity": 0,
                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
                  "lineInterpolation": "linear",
                  "lineStyle": {
                    "fill": "solid"
                  },
                  "lineWidth": 2,
                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
                  "showPoints": "auto",
                  "showValues": false,
                  "spanNulls": false,
                  "stacking": {
                    "group": "A",
                    "mode": "none"
                  },
                  "thresholdsStyle": {
                    "mode": "line+area"
                  }
                },
                "thresholds": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": 0
                    },
                    {
                      "color": "red",
                      "value": 1
                    }
                  ]
                }
              },
              "overrides": []
            },
            "options": {
              "annotations": {
                "clustering": -1,
                "multiLane": false
              },
              "legend": {
                "calcs": [],
                "displayMode": "list",
                "placement": "bottom",
                "showLegend": true
              },
              "tooltip": {
                "hideZeros": false,
                "mode": "single",
                "sort": "none"
              }
            }
          },
          "version": "13.1.0-25668120414"
        }
      }
    }
  },
  "layout": {
    "kind": "GridLayout",
    "spec": {
      "items": [
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-7"
            },
            "height": 10,
            "width": 24,
            "x": 0,
            "y": 0
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-1"
            },
            "height": 10,
            "width": 8,
            "x": 0,
            "y": 10
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-5"
            },
            "height": 10,
            "width": 8,
            "x": 8,
            "y": 10
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-2"
            },
            "height": 7,
            "width": 8,
            "x": 16,
            "y": 10
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-9"
            },
            "height": 7,
            "width": 8,
            "x": 16,
            "y": 17
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-8"
            },
            "height": 6,
            "width": 8,
            "x": 16,
            "y": 24
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-3"
            },
            "height": 10,
            "width": 8,
            "x": 0,
            "y": 20
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-4"
            },
            "height": 10,
            "width": 8,
            "x": 8,
            "y": 20
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-6"
            },
            "height": 10,
            "width": 24,
            "x": 0,
            "y": 30
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-10"
            },
            "height": 8,
            "width": 12,
            "x": 0,
            "y": 40
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-11"
            },
            "height": 8,
            "width": 12,
            "x": 12,
            "y": 40
          }
        },
        {
          "kind": "GridLayoutItem",
          "spec": {
            "element": {
              "kind": "ElementReference",
              "name": "panel-12"
            },
            "height": 8,
            "width": 24,
            "x": 0,
            "y": 48
          }
        }
      ]
    }
  },
  "links": [],
  "liveNow": false,
  "preferences": {
    "layout": {
      "kind": "GridLayout",
      "spec": {
        "items": []
      }
    }
  },
  "preload": false,
  "tags": [],
  "timeSettings": {
    "autoRefresh": "1m",
    "autoRefreshIntervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "fiscalYearStartMonth": 0,
    "from": "now-24h",
    "hideTimepicker": false,
    "timezone": "browser",
    "to": "now"
  },
  "title": "OSDC",
  "variables": [
    {
      "kind": "QueryVariable",
      "spec": {
        "allowCustomValue": false,
        "current": {
          "text": [
            "All"
          ],
          "value": [
            "$__all"
          ]
        },
        "definition": "label_values(gha_assigned_jobs{cluster=~\".*arc.*production.*|meta-prod-aws-.*|lf-prod-aws-.*\"},cluster)",
        "hide": "dontHide",
        "includeAll": true,
        "label": "Cluster",
        "multi": true,
        "name": "cluster",
        "options": [],
        "query": {
          "datasource": {
            "name": "grafanacloud-prom"
          },
          "group": "prometheus",
          "kind": "DataQuery",
          "spec": {
            "qryType": 1,
            "query": "label_values(gha_assigned_jobs{cluster=~\".*arc.*production.*|meta-prod-aws-.*|lf-prod-aws-.*\"},cluster)",
            "refId": "PrometheusVariableQueryEditor-VariableQuery"
          },
          "version": "v0"
        },
        "refresh": "onDashboardLoad",
        "regex": "",
        "regexApplyTo": "value",
        "skipUrlSync": false,
        "sort": "disabled"
      }
    },
    {
      "kind": "QueryVariable",
      "spec": {
        "allValue": ".*",
        "allowCustomValue": false,
        "current": {
          "text": "All",
          "value": [
            "$__all"
          ]
        },
        "definition": "SELECT DISTINCT extract(runner_name, '^(.+)-[^-]+-runner-[^-]+$') AS prefix FROM default.workflow_job WHERE arrayExists(x -> startsWith(x, 'mt-l-') OR startsWith(x, 'mt-rel-l-') OR startsWith(x, 'lf-l-') OR startsWith(x, 'lf-rel-l-'), labels) AND started_at > now() - INTERVAL 7 DAY AND runner_group_name IN ('meta-prod-aws-ue2', 'meta-prod-aws-uw1', 'meta-prod-aws-ue1', 'lf-prod-aws-ue1', 'lf-prod-aws-ue2') AND extract(runner_name, '^(.+)-[^-]+-runner-[^-]+$') != '' ORDER BY prefix",
        "hide": "dontHide",
        "includeAll": true,
        "label": "Runner name",
        "multi": true,
        "name": "runner_name_prefix",
        "options": [],
        "query": {
          "datasource": {
            "name": "ceczcsck1b20wb"
          },
          "group": "grafana-clickhouse-datasource",
          "kind": "DataQuery",
          "spec": {
            "editorType": "sql",
            "format": 1,
            "queryType": "table",
            "rawSql": "SELECT DISTINCT extract(runner_name, '^(.+)-[^-]+-runner-[^-]+$') AS prefix FROM default.workflow_job WHERE arrayExists(x -> startsWith(x, 'mt-l-') OR startsWith(x, 'mt-rel-l-') OR startsWith(x, 'lf-l-') OR startsWith(x, 'lf-rel-l-'), labels) AND started_at > now() - INTERVAL 7 DAY AND runner_group_name IN ('meta-prod-aws-ue2', 'meta-prod-aws-uw1', 'meta-prod-aws-ue1', 'lf-prod-aws-ue1', 'lf-prod-aws-ue2') AND extract(runner_name, '^(.+)-[^-]+-runner-[^-]+$') != '' ORDER BY prefix"
          },
          "version": "v0"
        },
        "refresh": "onDashboardLoad",
        "regex": "",
        "regexApplyTo": "value",
        "skipUrlSync": false,
        "sort": "alphabeticalAsc"
      }
    },
    {
      "kind": "QueryVariable",
      "spec": {
        "allValue": ".*",
        "allowCustomValue": false,
        "current": {
          "text": "All",
          "value": [
            "$__all"
          ]
        },
        "definition": "label_values(gha_assigned_jobs{cluster=~\"${cluster:regex}\", name=~\"(${runner_name_prefix:regex})-.*\"},name)",
        "hide": "dontHide",
        "includeAll": true,
        "label": "Scale set",
        "multi": true,
        "name": "scale_set",
        "options": [],
        "query": {
          "datasource": {
            "name": "grafanacloud-prom"
          },
          "group": "prometheus",
          "kind": "DataQuery",
          "spec": {
            "qryType": 1,
            "query": "label_values(gha_assigned_jobs{cluster=~\"${cluster:regex}\", name=~\"(${runner_name_prefix:regex})-.*\"},name)",
            "refId": "PrometheusVariableQueryEditor-VariableQuery"
          },
          "version": "v0"
        },
        "refresh": "onDashboardLoad",
        "regex": "",
        "regexApplyTo": "value",
        "skipUrlSync": false,
        "sort": "disabled"
      }
    },
    {
      "kind": "QueryVariable",
      "spec": {
        "allValue": ".*",
        "allowCustomValue": false,
        "current": {
          "text": "All",
          "value": [
            "$__all"
          ]
        },
        "definition": "query_result(group by(full_name) (label_join(gha_completed_jobs_total{cluster=~\"${cluster:regex}\"}, \"full_name\", \"/\", \"organization\", \"repository\")))",
        "hide": "dontHide",
        "includeAll": true,
        "label": "Repository",
        "multi": true,
        "name": "repository",
        "options": [],
        "query": {
          "datasource": {
            "name": "grafanacloud-prom"
          },
          "group": "prometheus",
          "kind": "DataQuery",
          "spec": {
            "qryType": 3,
            "query": "query_result(group by(full_name) (label_join(gha_completed_jobs_total{cluster=~\"${cluster:regex}\"}, \"full_name\", \"/\", \"organization\", \"repository\")))",
            "refId": "PrometheusVariableQueryEditor-VariableQuery"
          },
          "version": "v0"
        },
        "refresh": "onDashboardLoad",
        "regex": "/full_name=\"([^\"]+)\"/",
        "regexApplyTo": "value",
        "skipUrlSync": false,
        "sort": "alphabeticalAsc"
      }
    }
  ]
}
