<mxfile host="65bd71144e">
    <diagram name="dense-to-sparse" id="dense-to-sparse">
        <mxGraphModel dx="2542" dy="1437" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1168" pageHeight="628" math="0" shadow="0">
            <root>
                <mxCell id="0"/>
                <mxCell id="1" parent="0"/>
                <mxCell id="title_l" value="Dense Transformer Block" style="text;html=1;align=center;fontSize=38;fontStyle=1;fontColor=#1F4E79;" parent="1" vertex="1">
                    <mxGeometry x="24" width="500" height="32" as="geometry"/>
                </mxCell>
                <mxCell id="title_r" value="Sparse Transformer Block" style="text;html=1;align=center;fontSize=38;fontStyle=1;fontColor=#1F4E79;" parent="1" vertex="1">
                    <mxGeometry x="644" width="500" height="32" as="geometry"/>
                </mxCell>
                <mxCell id="box_l" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFF2CC;strokeColor=#D6B656;strokeWidth=2;" parent="1" vertex="1">
                    <mxGeometry x="24" y="64" width="500" height="540" as="geometry"/>
                </mxCell>
                <mxCell id="box_r" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#D5E8D4;strokeColor=#82B366;strokeWidth=2;" parent="1" vertex="1">
                    <mxGeometry x="644" y="64" width="500" height="540" as="geometry"/>
                </mxCell>
                <mxCell id="l_in" value="hidden_states" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#666;fontSize=26;" parent="1" vertex="1">
                    <mxGeometry x="124" y="80" width="300" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="l_attn" value="Self-Attention" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#DAE8FC;strokeColor=#6C8EBF;fontSize=26;" parent="1" vertex="1">
                    <mxGeometry x="124" y="140" width="300" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="l_norm1" value="Residual + RMSNorm" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#E1D5E7;strokeColor=#9673A6;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="124" y="200" width="300" height="44" as="geometry"/>
                </mxCell>
                <mxCell id="l_mlp" value="Dense MLP" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFE6CC;strokeColor=#D79B00;fontSize=26;fontStyle=1;verticalAlign=top;" parent="1" vertex="1">
                    <mxGeometry x="84" y="264" width="380" height="170" as="geometry"/>
                </mxCell>
                <mxCell id="l_w13" value="&lt;font style=&quot;font-size: 21px;&quot;&gt;gate_proj, up_proj&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#D79B00;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="104" y="310" width="340" height="36" as="geometry"/>
                </mxCell>
                <mxCell id="l_w2" value="&lt;font style=&quot;font-size: 21px;&quot;&gt;down_proj&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#D79B00;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="104" y="369" width="340" height="36" as="geometry"/>
                </mxCell>
                <mxCell id="l_norm2" value="Residual + RMSNorm" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#E1D5E7;strokeColor=#9673A6;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="124" y="454" width="300" height="46" as="geometry"/>
                </mxCell>
                <mxCell id="l_out" value="output" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#666;fontSize=26;" parent="1" vertex="1">
                    <mxGeometry x="124" y="520" width="300" height="50" as="geometry"/>
                </mxCell>
                <mxCell id="r_in" value="hidden_states" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#666;fontSize=26;" parent="1" vertex="1">
                    <mxGeometry x="744" y="80" width="300" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="r_attn" value="Self-Attention" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#DAE8FC;strokeColor=#6C8EBF;fontSize=26;" parent="1" vertex="1">
                    <mxGeometry x="744" y="140" width="300" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="r_norm1" value="Residual + RMSNorm" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#E1D5E7;strokeColor=#9673A6;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="744" y="200" width="300" height="44" as="geometry"/>
                </mxCell>
                <mxCell id="r_experts" value="Sparse MoE" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFE6CC;strokeColor=#D79B00;fontSize=23;fontStyle=1;verticalAlign=top;" parent="1" vertex="1">
                    <mxGeometry x="684" y="264" width="420" height="170" as="geometry"/>
                </mxCell>
                <mxCell id="e0" value="E0" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#D79B00;fontSize=22;" parent="1" vertex="1">
                    <mxGeometry x="699" y="354" width="44" height="60" as="geometry"/>
                </mxCell>
                <mxCell id="e1" value="E1" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#D79B00;fontSize=22;" parent="1" vertex="1">
                    <mxGeometry x="749" y="354" width="44" height="60" as="geometry"/>
                </mxCell>
                <mxCell id="4" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="e2" target="r_experts">
                    <mxGeometry relative="1" as="geometry"/>
                </mxCell>
                <mxCell id="e2" value="E2" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;fontSize=22;fontColor=#333333;" parent="1" vertex="1">
                    <mxGeometry x="799" y="354" width="44" height="60" as="geometry"/>
                </mxCell>
                <mxCell id="e3" value="E3" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#D79B00;fontSize=22;" parent="1" vertex="1">
                    <mxGeometry x="849" y="354" width="44" height="60" as="geometry"/>
                </mxCell>
                <mxCell id="e4" value="E4" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#D79B00;fontSize=22;" parent="1" vertex="1">
                    <mxGeometry x="899" y="354" width="44" height="60" as="geometry"/>
                </mxCell>
                <mxCell id="5" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="e5" target="r_experts">
                    <mxGeometry relative="1" as="geometry">
                        <mxPoint x="890.0000000000002" y="430" as="targetPoint"/>
                    </mxGeometry>
                </mxCell>
                <mxCell id="e5" value="E5" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;fontSize=22;fontColor=#333333;" parent="1" vertex="1">
                    <mxGeometry x="949" y="354" width="44" height="60" as="geometry"/>
                </mxCell>
                <mxCell id="e6" value="E6" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#D79B00;fontSize=22;" parent="1" vertex="1">
                    <mxGeometry x="999" y="354" width="44" height="60" as="geometry"/>
                </mxCell>
                <mxCell id="e7" value="E7" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#D79B00;fontSize=22;" parent="1" vertex="1">
                    <mxGeometry x="1049" y="354" width="44" height="60" as="geometry"/>
                </mxCell>
                <mxCell id="r_norm2" value="Residual + RMSNorm" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#E1D5E7;strokeColor=#9673A6;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="743" y="454" width="300" height="46" as="geometry"/>
                </mxCell>
                <mxCell id="r_out" value="output" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffffff;strokeColor=#666;fontSize=26;" parent="1" vertex="1">
                    <mxGeometry x="743" y="520" width="300" height="50" as="geometry"/>
                </mxCell>
                <mxCell id="3" style="edgeStyle=none;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="r_router" target="e5">
                    <mxGeometry relative="1" as="geometry"/>
                </mxCell>
                <mxCell id="r_router" value="&lt;span style=&quot;font-weight: normal;&quot;&gt;&lt;font style=&quot;font-size: 20px;&quot;&gt;Router&lt;/font&gt;&lt;/span&gt;" style="rounded=0;whiteSpace=wrap;html=1;fontSize=23;fontStyle=1;" parent="1" vertex="1">
                    <mxGeometry x="744" y="304" width="300" height="30" as="geometry"/>
                </mxCell>
                <mxCell id="2" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="r_router" target="e2">
                    <mxGeometry width="50" height="50" relative="1" as="geometry">
                        <mxPoint x="560" y="470" as="sourcePoint"/>
                        <mxPoint x="610" y="420" as="targetPoint"/>
                    </mxGeometry>
                </mxCell>
            </root>
        </mxGraphModel>
    </diagram>
</mxfile>