<mxfile host="65bd71144e">
    <diagram name="mixtral-decoder-swap" id="mixtral-decoder-swap">
        <mxGraphModel dx="3716" dy="1369" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1168" pageHeight="668" math="0" shadow="0">
            <root>
                <mxCell id="0"/>
                <mxCell id="1" parent="0"/>
                <mxCell id="title_hf" value="HF Transformer Block" style="text;html=1;align=center;fontSize=38;fontStyle=1;fontColor=#1F4E79;" parent="1" vertex="1">
                    <mxGeometry x="24" y="10" width="500" height="32" as="geometry"/>
                </mxCell>
                <mxCell id="title_te" value="TE Transformer Block" style="text;html=1;align=center;fontSize=38;fontStyle=1;fontColor=#1F4E79;" parent="1" vertex="1">
                    <mxGeometry x="644" y="10" width="500" height="32" as="geometry"/>
                </mxCell>
                <mxCell id="hf_box" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFF2CC;strokeColor=#D6B656;strokeWidth=2;" parent="1" vertex="1">
                    <mxGeometry x="24" y="64" width="500" height="580" as="geometry"/>
                </mxCell>
                <mxCell id="te_box" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#D5E8D4;strokeColor=#82B366;strokeWidth=2;" parent="1" vertex="1">
                    <mxGeometry x="644" y="64" width="500" height="580" as="geometry"/>
                </mxCell>
                <mxCell id="hf_in_ln" value="input_layernorm" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;fontSize=26;" parent="1" vertex="1">
                    <mxGeometry x="124" y="84" width="300" height="44" as="geometry"/>
                </mxCell>
                <mxCell id="hf_attn_g" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=none;strokeColor=#6C8EBF;dashed=1;verticalAlign=top;fontStyle=2;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="44" y="74" width="460" height="220" as="geometry"/>
                </mxCell>
                <mxCell id="hf_q" value="q_proj" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#DAE8FC;strokeColor=#6C8EBF;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="64" y="174" width="130" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="hf_k" value="k_proj" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#DAE8FC;strokeColor=#6C8EBF;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="209" y="174" width="130" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="hf_v" value="v_proj" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#DAE8FC;strokeColor=#6C8EBF;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="354" y="174" width="130" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="hf_o" value="o_proj" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#DAE8FC;strokeColor=#6C8EBF;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="164" y="244" width="220" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="hf_post_ln" value="post_attention_layernorm" style="rounded=0;whiteSpace=wrap;html=1;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="124" y="314" width="300" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="hf_moe_g" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=none;strokeColor=#B85450;dashed=1;verticalAlign=top;fontStyle=2;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="44" y="374" width="460" height="250" as="geometry"/>
                </mxCell>
                <mxCell id="hf_gate" value="mlp.gate" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#F8CECC;strokeColor=#B85450;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="124" y="404" width="300" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="hf_gu" value="&lt;span style=&quot;font-weight: normal;&quot;&gt;mlp.experts.gate_up_proj&lt;/span&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFE6CC;strokeColor=#D79B00;fontSize=23;fontStyle=1;" parent="1" vertex="1">
                    <mxGeometry x="64" y="459" width="420" height="70" as="geometry"/>
                </mxCell>
                <mxCell id="hf_dn" value="&lt;span style=&quot;font-weight: normal;&quot;&gt;mlp.experts.down_proj&lt;/span&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFE6CC;strokeColor=#D79B00;fontSize=23;fontStyle=1;" parent="1" vertex="1">
                    <mxGeometry x="64" y="544" width="420" height="65" as="geometry"/>
                </mxCell>
                <mxCell id="te_attn_g" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=none;strokeColor=#82B366;dashed=1;verticalAlign=top;fontStyle=2;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="664" y="94" width="460" height="200" as="geometry"/>
                </mxCell>
                <mxCell id="te_lnq" value="&lt;font style=&quot;font-size: 21px;&quot;&gt;layernorm_qkv.layer_norm&lt;br&gt;query, key, value&lt;br&gt;self_attention.proj&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="684" y="140" width="420" height="120" as="geometry"/>
                </mxCell>
                <mxCell id="te_post_ln" value="post_attention_layernorm" style="rounded=0;whiteSpace=wrap;html=1;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="744" y="314" width="300" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="te_moe_g" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=none;strokeColor=#B85450;dashed=1;verticalAlign=top;fontStyle=2;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="664" y="374" width="460" height="250" as="geometry"/>
                </mxCell>
                <mxCell id="te_gate" value="mlp.gate" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#F8CECC;strokeColor=#B85450;fontSize=23;" parent="1" vertex="1">
                    <mxGeometry x="744" y="404" width="300" height="40" as="geometry"/>
                </mxCell>
                <mxCell id="te_gu" value="&lt;span style=&quot;font-weight: normal;&quot;&gt;mlp.experts_gate_up&lt;/span&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFE6CC;strokeColor=#D79B00;fontSize=23;fontStyle=1;" parent="1" vertex="1">
                    <mxGeometry x="684" y="459" width="420" height="70" as="geometry"/>
                </mxCell>
                <mxCell id="te_dn" value="&lt;span style=&quot;font-weight: normal;&quot;&gt;mlp.experts_down&lt;/span&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#FFE6CC;strokeColor=#D79B00;fontSize=23;fontStyle=1;" parent="1" vertex="1">
                    <mxGeometry x="684" y="544" width="420" height="65" as="geometry"/>
                </mxCell>
                <mxCell id="e1" style="endArrow=classic;html=1;strokeColor=#8ea8ce;strokeWidth=2;exitX=1;exitY=0.5;entryX=0;entryY=0.5;" parent="1" source="hf_in_ln" target="te_lnq" edge="1">
                    <mxGeometry relative="1" as="geometry"/>
                </mxCell>
                <mxCell id="e3" style="endArrow=classic;html=1;strokeColor=#6C8EBF;strokeWidth=2;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" target="te_lnq" edge="1">
                    <mxGeometry relative="1" as="geometry">
                        <mxPoint x="504" y="194" as="sourcePoint"/>
                        <mxPoint x="604" y="219" as="targetPoint"/>
                    </mxGeometry>
                </mxCell>
                <mxCell id="e5" style="endArrow=classic;html=1;strokeColor=#6C8EBF;strokeWidth=2;exitX=1;exitY=0.5;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="hf_o" target="te_lnq" edge="1">
                    <mxGeometry relative="1" as="geometry"/>
                </mxCell>
                <mxCell id="e6" style="endArrow=classic;html=1;strokeColor=#9673A6;strokeWidth=2;exitX=1;exitY=0.5;entryX=0;entryY=0.5;" parent="1" source="hf_post_ln" target="te_post_ln" edge="1">
                    <mxGeometry relative="1" as="geometry"/>
                </mxCell>
                <mxCell id="e7" style="endArrow=classic;html=1;strokeColor=#B85450;strokeWidth=2;exitX=1;exitY=0.5;entryX=0;entryY=0.5;" parent="1" source="hf_gate" target="te_gate" edge="1">
                    <mxGeometry relative="1" as="geometry"/>
                </mxCell>
                <mxCell id="e8" style="endArrow=classic;html=1;strokeColor=#D79B00;strokeWidth=2;exitX=1;exitY=0.5;entryX=0;entryY=0.5;" parent="1" source="hf_gu" target="te_gu" edge="1">
                    <mxGeometry relative="1" as="geometry"/>
                </mxCell>
                <mxCell id="e9" style="endArrow=classic;html=1;strokeColor=#D79B00;strokeWidth=2;exitX=1;exitY=0.5;entryX=0;entryY=0.5;" parent="1" source="hf_dn" target="te_dn" edge="1">
                    <mxGeometry relative="1" as="geometry"/>
                </mxCell>
            </root>
        </mxGraphModel>
    </diagram>
</mxfile>