Update 消息队列.md

补充延迟消息、死信队列和 Kafka。
This commit is contained in:
CyC2018
2026-10-26 22:04:09 +08:00
parent 303ef42716
commit 6abd9bc0f8
5 changed files with 329 additions and 1 deletions
+111
View File
@@ -0,0 +1,111 @@
<svg xmlns="http://www.w3.org/2000/svg" width="760" height="370" viewBox="0 0 760 370">
<style>
text { font-family: "PingFang SC", "Microsoft YaHei", Helvetica, Arial, sans-serif; fill: #3F3F3F; font-size: 12px; }
.title { font-weight: bold; font-size: 15px; fill: #333; text-anchor: middle; }
.b { font-weight: bold; }
.c { text-anchor: middle; }
.e { text-anchor: end; }
.sub { font-size: 11px; fill: #6E6E6E; }
.note { fill: #C0605A; }
.ok { fill: #6A9F68; }
.mono { font-family: Menlo, Consolas, monospace; }
.arrow { stroke: #555; stroke-width: 1.1; fill: none; marker-end: url(#a); }
.red { stroke: #C0605A; marker-end: url(#r); }
.green { stroke: #6A9F68; marker-end: url(#g); }
.dash { stroke-dasharray: 5 4; }
.line { stroke: #555; stroke-width: 1.1; fill: none; }
.frame { fill: none; stroke: #999; stroke-width: 1; stroke-dasharray: 4 3; }
.shadow { fill: #DADADA; }
</style>
<defs>
<marker id="a" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M1,1.5 L9.5,5 L1,8.5 z" fill="#555"/>
</marker>
<marker id="r" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M1,1.5 L9.5,5 L1,8.5 z" fill="#C0605A"/>
</marker>
<marker id="g" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M1,1.5 L9.5,5 L1,8.5 z" fill="#6A9F68"/>
</marker>
</defs>
<rect width="100%" height="100%" fill="#FFFFFF"/>
<text x="380" y="28" class="title">Kafka:按 key 选分区,组内每个分区只交给一个消费者</text>
<rect x="24" y="70" width="150" height="128" rx="5" fill="#FFFFFF" stroke="#BBBBBB" stroke-width="1.2"/>
<line x1="24" y1="84" x2="174" y2="84" stroke="#BBBBBB" stroke-width="1"/>
<circle cx="33" cy="77" r="2.2" fill="#BBBBBB"/>
<circle cx="41" cy="77" r="2.2" fill="#BBBBBB"/>
<circle cx="49" cy="77" r="2.2" fill="#BBBBBB"/>
<text x="33" y="100" class="sub mono">key value</text>
<text x="33" y="116" class="sub mono">1024 下单</text>
<text x="33" y="132" class="sub mono">1025 下单</text>
<text x="33" y="148" class="sub mono">1024 支付</text>
<text x="33" y="164" class="sub mono">1026 下单</text>
<text x="33" y="180" class="sub mono">1024 发货</text>
<text x="33" y="196" class="sub mono">1025 支付</text>
<text x="99" y="216" class="c sub b">生产者按顺序发送</text>
<text x="99" y="232" class="c sub mono">hash(key) % 3 选分区</text>
<text x="290" y="98" class="e sub b">分区 0</text>
<rect x="300" y="74" width="54" height="40" rx="3" fill="#C6E4FB" stroke="#7FB2D6" stroke-width="1"/>
<text x="327" y="98" class="c sub mono" style="fill:#444"></text>
<text x="327" y="91" class="c sub mono">1026</text>
<text x="327" y="107" class="c sub">下单</text>
<text x="327" y="70" class="c sub mono" style="font-size:10px">0</text>
<rect x="354" y="74" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<rect x="408" y="74" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<rect x="462" y="74" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<rect x="516" y="74" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<line x1="178" y1="130" x2="246" y2="94" class="arrow "/>
<text x="290" y="172" class="e sub b">分区 1</text>
<rect x="300" y="148" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<rect x="354" y="148" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<rect x="408" y="148" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<rect x="462" y="148" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<rect x="516" y="148" width="54" height="40" rx="3" fill="none" stroke="#D6D6D6" stroke-dasharray="3 2"/>
<line x1="178" y1="130" x2="246" y2="168" class="arrow "/>
<text x="290" y="246" class="e sub b">分区 2</text>
<rect x="300" y="222" width="54" height="40" rx="3" fill="#E6E3FA" stroke="#B8B0DE" stroke-width="1"/>
<text x="327" y="246" class="c sub mono" style="fill:#444"></text>
<text x="327" y="239" class="c sub mono">1024</text>
<text x="327" y="255" class="c sub">下单</text>
<text x="327" y="218" class="c sub mono" style="font-size:10px">0</text>
<rect x="354" y="222" width="54" height="40" rx="3" fill="#C6E4FB" stroke="#7FB2D6" stroke-width="1"/>
<text x="381" y="246" class="c sub mono" style="fill:#444"></text>
<text x="381" y="239" class="c sub mono">1025</text>
<text x="381" y="255" class="c sub">下单</text>
<text x="381" y="218" class="c sub mono" style="font-size:10px">1</text>
<rect x="408" y="222" width="54" height="40" rx="3" fill="#E6E3FA" stroke="#B8B0DE" stroke-width="1"/>
<text x="435" y="246" class="c sub mono" style="fill:#444"></text>
<text x="435" y="239" class="c sub mono">1024</text>
<text x="435" y="255" class="c sub">支付</text>
<text x="435" y="218" class="c sub mono" style="font-size:10px">2</text>
<rect x="462" y="222" width="54" height="40" rx="3" fill="#E6E3FA" stroke="#B8B0DE" stroke-width="1"/>
<text x="489" y="246" class="c sub mono" style="fill:#444"></text>
<text x="489" y="239" class="c sub mono">1024</text>
<text x="489" y="255" class="c sub">发货</text>
<text x="489" y="218" class="c sub mono" style="font-size:10px">3</text>
<rect x="516" y="222" width="54" height="40" rx="3" fill="#C6E4FB" stroke="#7FB2D6" stroke-width="1"/>
<text x="543" y="246" class="c sub mono" style="fill:#444"></text>
<text x="543" y="239" class="c sub mono">1025</text>
<text x="543" y="255" class="c sub">支付</text>
<text x="543" y="218" class="c sub mono" style="font-size:10px">4</text>
<line x1="178" y1="130" x2="246" y2="242" class="arrow "/>
<text x="300" y="296" class="sub">订单 1024 的三条消息都在分区 2,顺序不变</text>
<text x="290" y="70" class="e sub">偏移量</text>
<rect x="590" y="56" width="150" height="268" rx="8" class="frame"/>
<text x="600" y="74" class="b">消费者组 billing</text>
<rect x="612" y="84" width="108" height="34" rx="6" fill="#C6ECE9" stroke="#7DBCB7"/>
<text x="666" y="105.5" class="c sub b">consumer-1</text>
<line x1="574" y1="94" x2="610" y2="101" class="arrow "/>
<rect x="612" y="140" width="108" height="34" rx="6" fill="#C6ECE9" stroke="#7DBCB7"/>
<text x="666" y="161.5" class="c sub b">consumer-2</text>
<line x1="574" y1="168" x2="610" y2="157" class="arrow "/>
<rect x="612" y="196" width="108" height="34" rx="6" fill="#C6ECE9" stroke="#7DBCB7"/>
<text x="666" y="217.5" class="c sub b">consumer-3</text>
<line x1="574" y1="242" x2="610" y2="213" class="arrow "/>
<rect x="612" y="252" width="108" height="34" rx="6" fill="#EAE9E7" stroke="#BFBEBA"/>
<text x="666" y="273.5" class="c sub b">consumer-4</text>
<text x="666" y="304" class="c sub note">没有分到分区</text>
<text x="24" y="334" class="sub">本机 3 个节点的 Kafka 4.4 实测:key 为 1024 和 1025 的消息都落在分区 2,1026 落在分区 0;</text>
<text x="24" y="352" class="sub">4 个消费者分 3 个分区,consumer-4 空闲。停掉 consumer-3 后触发重平衡,分区 2 改由 consumer-4 消费。</text>
</svg>

After

Width:  |  Height:  |  Size: 7.3 KiB

+85
View File
@@ -0,0 +1,85 @@
<svg xmlns="http://www.w3.org/2000/svg" width="760" height="480" viewBox="0 0 760 480">
<style>
text { font-family: "PingFang SC", "Microsoft YaHei", Helvetica, Arial, sans-serif; fill: #3F3F3F; font-size: 12px; }
.title { font-weight: bold; font-size: 15px; fill: #333; text-anchor: middle; }
.b { font-weight: bold; }
.c { text-anchor: middle; }
.e { text-anchor: end; }
.sub { font-size: 11px; fill: #6E6E6E; }
.note { fill: #C0605A; }
.ok { fill: #6A9F68; }
.mono { font-family: Menlo, Consolas, monospace; }
.arrow { stroke: #555; stroke-width: 1.1; fill: none; marker-end: url(#a); }
.red { stroke: #C0605A; marker-end: url(#r); }
.green { stroke: #6A9F68; marker-end: url(#g); }
.dash { stroke-dasharray: 5 4; }
.line { stroke: #555; stroke-width: 1.1; fill: none; }
.frame { fill: none; stroke: #999; stroke-width: 1; stroke-dasharray: 4 3; }
.shadow { fill: #DADADA; }
</style>
<defs>
<marker id="a" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M1,1.5 L9.5,5 L1,8.5 z" fill="#555"/>
</marker>
<marker id="r" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M1,1.5 L9.5,5 L1,8.5 z" fill="#C0605A"/>
</marker>
<marker id="g" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M1,1.5 L9.5,5 L1,8.5 z" fill="#6A9F68"/>
</marker>
</defs>
<rect width="100%" height="100%" fill="#FFFFFF"/>
<text x="380" y="28" class="title">零拷贝:sendfile 让数据不经过用户空间</text>
<text x="24" y="70" class="b">read + write:4 次拷贝,2 次系统调用带来 4 次用户态与内核态的切换</text>
<rect x="24" y="80" width="712" height="46" rx="4" fill="#F6F6F6" stroke="#E2E2E2"/>
<text x="36" y="107" class="sub b">用户空间</text>
<rect x="24" y="130" width="712" height="46" rx="4" fill="#FFFFFF" stroke="#E2E2E2"/>
<text x="36" y="157" class="sub b">内核空间</text>
<rect x="24" y="180" width="712" height="46" rx="4" fill="#F6F6F6" stroke="#E2E2E2"/>
<text x="36" y="207" class="sub b">硬件</text>
<rect x="120" y="188" width="104" height="30" rx="6" fill="#EAE9E7" stroke="#BFBEBA"/>
<text x="172" y="207.5" class="c sub b">磁盘</text>
<rect x="250" y="138" width="104" height="30" rx="6" fill="#C6E4FB" stroke="#7FB2D6"/>
<text x="302" y="157.5" class="c sub b">页缓存</text>
<rect x="380" y="88" width="104" height="30" rx="6" fill="#E6E3FA" stroke="#B8B0DE"/>
<text x="432" y="107.5" class="c sub b">应用的缓冲区</text>
<rect x="510" y="138" width="104" height="30" rx="6" fill="#C6E4FB" stroke="#7FB2D6"/>
<text x="562" y="157.5" class="c sub b">Socket 缓冲区</text>
<rect x="630" y="188" width="104" height="30" rx="6" fill="#EAE9E7" stroke="#BFBEBA"/>
<text x="682" y="207.5" class="c sub b">网卡</text>
<line x1="200" y1="188" x2="262" y2="168" class="arrow "/>
<text x="224" y="174" class="e sub">① DMA</text>
<line x1="330" y1="138" x2="392" y2="118" class="arrow "/>
<text x="354" y="124" class="e sub note">② CPU</text>
<line x1="472" y1="118" x2="530" y2="138" class="arrow "/>
<text x="508" y="124" class="sub note">③ CPU</text>
<line x1="600" y1="168" x2="660" y2="188" class="arrow "/>
<text x="636" y="174" class="sub">④ DMA</text>
<text x="300" y="108" class="c sub mono">read()</text>
<text x="600" y="108" class="c sub mono">write()</text>
<text x="24" y="258" class="b">sendfile:数据留在内核中,网卡支持 SG-DMA 时 CPU 不拷贝数据</text>
<rect x="24" y="268" width="712" height="46" rx="4" fill="#F6F6F6" stroke="#E2E2E2"/>
<text x="36" y="295" class="sub b">用户空间</text>
<rect x="24" y="318" width="712" height="46" rx="4" fill="#FFFFFF" stroke="#E2E2E2"/>
<text x="36" y="345" class="sub b">内核空间</text>
<rect x="24" y="368" width="712" height="46" rx="4" fill="#F6F6F6" stroke="#E2E2E2"/>
<text x="36" y="395" class="sub b">硬件</text>
<rect x="120" y="376" width="104" height="30" rx="6" fill="#EAE9E7" stroke="#BFBEBA"/>
<text x="172" y="395.5" class="c sub b">磁盘</text>
<rect x="250" y="326" width="104" height="30" rx="6" fill="#C6E4FB" stroke="#7FB2D6"/>
<text x="302" y="345.5" class="c sub b">页缓存</text>
<rect x="510" y="326" width="104" height="30" rx="6" fill="#C6E4FB" stroke="#7FB2D6"/>
<text x="562" y="345.5" class="c sub b">Socket 缓冲区</text>
<rect x="630" y="376" width="104" height="30" rx="6" fill="#EAE9E7" stroke="#BFBEBA"/>
<text x="682" y="395.5" class="c sub b">网卡</text>
<text x="380" y="296" class="c sub mono">应用只调用一次 sendfile()</text>
<line x1="200" y1="376" x2="262" y2="356" class="arrow "/>
<text x="224" y="362" class="e sub">① DMA</text>
<line x1="354" y1="341" x2="508" y2="341" class="arrow dash"/>
<text x="431" y="336" class="c sub">只追加地址和长度</text>
<path d="M330,356 C420,402 560,402 628,394" class="arrow green"/>
<text x="545" y="384" class="c sub ok">② DMA:网卡直接从页缓存读取</text>
<text x="24" y="450" class="sub">Kafka 的 broker 把日志文件发给消费者时使用 sendfile。启用 TLS 后数据要在用户空间加密,就不再使用零拷贝。</text>
<text x="24" y="468" class="sub">网卡不支持 SG-DMA 时,sendfile 仍要由 CPU 把页缓存中的数据拷贝到 Socket 缓冲区,共 3 次拷贝。</text>
</svg>

After

Width:  |  Height:  |  Size: 5.4 KiB